From 062465170ddb4520ca6fbc64a0f6e27e786de101 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Sat, 30 Mar 2013 16:25:04 -0500 Subject: [PATCH 1/1] Fixes, comments --- src/dentry.h | 5 +- src/export_image.c | 5 +- src/extract_image.c | 54 ++++++++++++++---- src/hardlink.c | 50 +++++++++++------ src/lookup_table.c | 48 +++------------- src/lookup_table.h | 15 +++-- src/metadata_resource.c | 19 +++---- src/mount_image.c | 41 ++++++-------- src/resource.c | 80 +++++++++++++++++++++++++-- src/symlink.c | 3 +- src/verify.c | 3 +- src/wim.c | 4 +- src/wimlib_internal.h | 28 ++++++++-- src/write.c | 118 ++++++++++++++++++++++++++++------------ 14 files changed, 311 insertions(+), 162 deletions(-) diff --git a/src/dentry.h b/src/dentry.h index 28fdeaf2..299abc9f 100644 --- a/src/dentry.h +++ b/src/dentry.h @@ -265,7 +265,10 @@ struct wim_inode { * link_count of them) */ struct list_head i_dentry; - struct hlist_node i_hlist; + union { + struct hlist_node i_hlist; + struct list_head i_list; + }; union { /* Used during image extraction to build a list of inodes that diff --git a/src/export_image.c b/src/export_image.c index 7e6d2789..784938e9 100644 --- a/src/export_image.c +++ b/src/export_image.c @@ -104,7 +104,6 @@ wimlib_export_image(WIMStruct *src_wim, int ret; struct wim_lookup_table *joined_tab, *src_wim_tab_save; struct wim_image_metadata *src_imd; - struct hlist_node *cur_node; struct list_head lte_list_head; struct wim_inode *inode; @@ -209,7 +208,7 @@ wimlib_export_image(WIMStruct *src_wim, for_lookup_table_entry(src_wim->lookup_table, lte_zero_out_refcnt, NULL); src_imd = wim_get_current_image_metadata(src_wim); INIT_LIST_HEAD(<e_list_head); - hlist_for_each_entry(inode, cur_node, &src_imd->inode_list, i_hlist) { + image_for_each_inode(inode, src_imd) { ret = inode_allocate_needed_ltes(inode, src_wim->lookup_table, dest_wim->lookup_table, @@ -235,7 +234,7 @@ wimlib_export_image(WIMStruct *src_wim, /* All memory allocations have been taken care of, so it's no longer * possible for this function to fail. Go ahead and update the lookup * table of the destination WIM and the boot index, if needed. */ - hlist_for_each_entry(inode, cur_node, &src_imd->inode_list, i_hlist) { + image_for_each_inode(inode, src_imd) { inode_move_ltes_to_table(inode, src_wim->lookup_table, dest_wim->lookup_table, diff --git a/src/extract_image.c b/src/extract_image.c index 85b5af8a..dff7dd4f 100644 --- a/src/extract_image.c +++ b/src/extract_image.c @@ -268,7 +268,7 @@ extract_regular_file_unlinked(struct wim_dentry *dentry, } ret = extract_wim_resource_to_fd(lte, out_fd, wim_resource_size(lte)); - if (ret != 0) { + if (ret) { ERROR("Failed to extract resource to `%s'", output_path); goto out; } @@ -586,7 +586,7 @@ calculate_bytes_to_extract(struct list_head *stream_list, u64 num_streams = 0; /* For each stream to be extracted... */ - list_for_each_entry(lte, stream_list, staging_list) { + list_for_each_entry(lte, stream_list, extraction_list) { if (extract_flags & (WIMLIB_EXTRACT_FLAG_SYMLINK | WIMLIB_EXTRACT_FLAG_HARDLINK)) { @@ -614,7 +614,7 @@ maybe_add_stream_for_extraction(struct wim_lookup_table_entry *lte, { if (++lte->out_refcnt == 1) { INIT_LIST_HEAD(<e->inode_list); - list_add_tail(<e->staging_list, stream_list); + list_add_tail(<e->extraction_list, stream_list); } } @@ -653,18 +653,17 @@ inode_find_streams_for_extraction(struct wim_inode *inode, } static void -find_streams_for_extraction(struct hlist_head *inode_list, +find_streams_for_extraction(struct wim_image_metadata *imd, struct list_head *stream_list, struct wim_lookup_table *lookup_table, int extract_flags) { struct wim_inode *inode; - struct hlist_node *cur; struct wim_dentry *dentry; for_lookup_table_entry(lookup_table, lte_zero_out_refcnt, NULL); INIT_LIST_HEAD(stream_list); - hlist_for_each_entry(inode, cur, inode_list, i_hlist) { + image_for_each_inode(inode, imd) { if (!inode->i_resolved) inode_resolve_ltes(inode, lookup_table); inode_for_each_dentry(dentry, inode) @@ -713,7 +712,7 @@ apply_stream_list(struct list_head *stream_list, * sequential reading of the WIM can be implemented. */ /* For each distinct stream to be extracted */ - list_for_each_entry(lte, stream_list, staging_list) { + list_for_each_entry(lte, stream_list, extraction_list) { /* For each inode that contains the stream */ list_for_each_entry(inode, <e->inode_list, i_lte_inode_list) { /* For each dentry that points to the inode */ @@ -745,6 +744,41 @@ apply_stream_list(struct list_head *stream_list, return 0; } +static int +sort_stream_list_by_wim_position(struct list_head *stream_list) +{ + struct list_head *cur; + size_t num_streams; + struct wim_lookup_table_entry **array; + size_t i; + size_t array_size; + + num_streams = 0; + list_for_each(cur, stream_list) + num_streams++; + array_size = num_streams * sizeof(array[0]); + array = MALLOC(array_size); + if (!array) { + ERROR("Failed to allocate %zu bytes to sort stream entries", + array_size); + return WIMLIB_ERR_NOMEM; + } + cur = stream_list->next; + for (i = 0; i < num_streams; i++) { + array[i] = container_of(cur, struct wim_lookup_table_entry, extraction_list); + cur = cur->next; + } + + qsort(array, num_streams, sizeof(array[0]), cmp_streams_by_wim_position); + + INIT_LIST_HEAD(stream_list); + for (i = 0; i < num_streams; i++) + list_add_tail(&array[i]->extraction_list, stream_list); + FREE(array); + return 0; +} + + /* Extracts the image @image from the WIM @w to the directory or NTFS volume * @target. */ static int @@ -754,7 +788,6 @@ extract_single_image(WIMStruct *w, int image, { int ret; struct list_head stream_list; - struct hlist_head *inode_list; struct apply_args args; const struct apply_operations *ops; @@ -792,10 +825,9 @@ extract_single_image(WIMStruct *w, int image, if (ret) goto out; - inode_list = &wim_get_current_image_metadata(w)->inode_list; - /* Build a list of the streams that need to be extracted */ - find_streams_for_extraction(inode_list, &stream_list, + find_streams_for_extraction(wim_get_current_image_metadata(w), + &stream_list, w->lookup_table, extract_flags); /* Calculate the number of bytes of data that will be extracted */ diff --git a/src/hardlink.c b/src/hardlink.c index 08f62250..c7c5a2f2 100644 --- a/src/hardlink.c +++ b/src/hardlink.c @@ -60,7 +60,7 @@ init_inode_table(struct wim_inode_table *table, size_t capacity) } table->num_entries = 0; table->capacity = capacity; - INIT_HLIST_HEAD(&table->extra_inodes); + INIT_LIST_HEAD(&table->extra_inodes); return 0; } @@ -87,7 +87,7 @@ inode_table_insert(struct wim_dentry *dentry, void *_table) /* A dentry with a hard link group ID of 0 indicates that it's * in a hard link group by itself. Add it to the list of extra * inodes rather than inserting it into the hash lists. */ - hlist_add_head(&d_inode->i_hlist, &table->extra_inodes); + list_add_tail(&d_inode->i_list, &table->extra_inodes); wimlib_assert(d_inode->i_dentry.next == &dentry->d_alias); wimlib_assert(d_inode->i_dentry.prev == &dentry->d_alias); @@ -148,6 +148,22 @@ inode_table_get_inode(struct wim_inode_table *table, u64 ino, u64 devno) return inode; } +void +inode_ref_streams(struct wim_inode *inode) +{ + for (unsigned i = 0; i <= inode->i_num_ads; i++) { + struct wim_lookup_table_entry *lte; + lte = inode_stream_lte_resolved(inode, i); + if (lte) + lte->refcnt++; + } +} + +void +inode_add_link(struct wim_inode *inode, struct wim_dentry *dentry) +{ +} + /* Given a directory entry with the name @name for the file with the inode * number @ino and device number @devno, create a new WIM dentry with an * associated inode, where the inode is shared if an inode with the same @ino @@ -172,6 +188,8 @@ inode_table_new_dentry(struct wim_inode_table *table, const tchar *name, free_dentry(dentry); return WIMLIB_ERR_NOMEM; } + if (inode->i_nlink > 1) + inode_ref_streams(inode); dentry->d_inode = inode; inode_add_dentry(dentry, inode); *dentry_ret = dentry; @@ -251,7 +269,7 @@ inodes_consistent(const struct wim_inode * restrict ref_inode, /* Fix up a "true" inode and check for inconsistencies */ static int -fix_true_inode(struct wim_inode *inode, struct hlist_head *inode_list) +fix_true_inode(struct wim_inode *inode, struct list_head *inode_list) { struct wim_dentry *dentry; struct wim_dentry *ref_dentry = NULL; @@ -273,7 +291,7 @@ fix_true_inode(struct wim_inode *inode, struct hlist_head *inode_list) ref_inode = ref_dentry->d_inode; ref_inode->i_nlink = 1; - hlist_add_head(&ref_inode->i_hlist, inode_list); + list_add_tail(&ref_inode->i_list, inode_list); list_del(&inode->i_dentry); list_add(&ref_inode->i_dentry, &ref_dentry->d_alias); @@ -314,7 +332,7 @@ fix_true_inode(struct wim_inode *inode, struct hlist_head *inode_list) * group remaining. */ static int -fix_nominal_inode(struct wim_inode *inode, struct hlist_head *inode_list, +fix_nominal_inode(struct wim_inode *inode, struct list_head *inode_list, bool *ino_changes_needed) { struct wim_dentry *dentry; @@ -429,29 +447,28 @@ next_dentry_2: hlist_for_each_entry_safe(inode, cur, tmp, &true_inodes, i_hlist) { ret = fix_true_inode(inode, inode_list); - if (ret != 0) + if (ret) return ret; } return 0; } static int -fix_inodes(struct wim_inode_table *table, struct hlist_head *inode_list, +fix_inodes(struct wim_inode_table *table, struct list_head *inode_list, bool *ino_changes_needed) { struct wim_inode *inode; struct hlist_node *cur, *tmp; int ret; - INIT_HLIST_HEAD(inode_list); + INIT_LIST_HEAD(inode_list); for (u64 i = 0; i < table->capacity; i++) { hlist_for_each_entry_safe(inode, cur, tmp, &table->array[i], i_hlist) { ret = fix_nominal_inode(inode, inode_list, ino_changes_needed); - if (ret != 0) + if (ret) return ret; } } - hlist_for_each_safe(cur, tmp, &table->extra_inodes) - hlist_add_head(cur, inode_list); + list_splice_tail(&table->extra_inodes, inode_list); return 0; } @@ -484,7 +501,7 @@ fix_inodes(struct wim_inode_table *table, struct hlist_head *inode_list, * is returned in the hlist @inode_list. */ int -dentry_tree_fix_inodes(struct wim_dentry *root, struct hlist_head *inode_list) +dentry_tree_fix_inodes(struct wim_dentry *root, struct list_head *inode_list) { struct wim_inode_table inode_tab; int ret; @@ -504,11 +521,10 @@ dentry_tree_fix_inodes(struct wim_dentry *root, struct hlist_head *inode_list) if (ret == 0 && ino_changes_needed) { u64 cur_ino = 1; - struct hlist_node *tmp; struct wim_inode *inode; WARNING("Re-assigning inode numbers due to inode inconsistencies"); - hlist_for_each_entry(inode, tmp, inode_list, i_hlist) { + list_for_each_entry(inode, inode_list, i_list) { if (inode->i_nlink > 1) inode->i_ino = cur_ino++; else @@ -522,13 +538,13 @@ dentry_tree_fix_inodes(struct wim_dentry *root, struct hlist_head *inode_list) * the inodes to a single list @head. */ void inode_table_prepare_inode_list(struct wim_inode_table *table, - struct hlist_head *head) + struct list_head *head) { struct wim_inode *inode; struct hlist_node *cur, *tmp; u64 cur_ino = 1; - INIT_HLIST_HEAD(head); + INIT_LIST_HEAD(head); for (size_t i = 0; i < table->capacity; i++) { hlist_for_each_entry_safe(inode, cur, tmp, &table->array[i], i_hlist) { @@ -536,7 +552,7 @@ inode_table_prepare_inode_list(struct wim_inode_table *table, inode->i_ino = cur_ino++; else inode->i_ino = 0; - hlist_add_head(&inode->i_hlist, head); + list_add_tail(&inode->i_list, head); } INIT_HLIST_HEAD(&table->array[i]); } diff --git a/src/lookup_table.c b/src/lookup_table.c index 5b57e3ac..4d62ca5b 100644 --- a/src/lookup_table.c +++ b/src/lookup_table.c @@ -215,7 +215,7 @@ finalize_lte(struct wim_lookup_table_entry *lte) #ifdef WITH_FUSE if (lte->resource_location == RESOURCE_IN_STAGING_FILE) { unlink(lte->staging_file_name); - list_del(<e->staging_list); + list_del(<e->unhashed_list); } #endif free_lookup_table_entry(lte); @@ -275,7 +275,7 @@ for_lookup_table_entry(struct wim_lookup_table *table, return 0; } -static int +int cmp_streams_by_wim_position(const void *p1, const void *p2) { const struct wim_lookup_table_entry *lte1, *lte2; @@ -289,40 +289,6 @@ cmp_streams_by_wim_position(const void *p1, const void *p2) return 0; } -int -sort_stream_list_by_wim_position(struct list_head *stream_list) -{ - struct list_head *cur; - size_t num_streams; - struct wim_lookup_table_entry **array; - size_t i; - size_t array_size; - - num_streams = 0; - list_for_each(cur, stream_list) - num_streams++; - array_size = num_streams * sizeof(array[0]); - array = MALLOC(array_size); - if (!array) { - ERROR("Failed to allocate %zu bytes to sort stream entries", - array_size); - return WIMLIB_ERR_NOMEM; - } - cur = stream_list->next; - for (i = 0; i < num_streams; i++) { - array[i] = container_of(cur, struct wim_lookup_table_entry, staging_list); - cur = cur->next; - } - - qsort(array, num_streams, sizeof(array[0]), cmp_streams_by_wim_position); - - INIT_LIST_HEAD(stream_list); - for (i = 0; i < num_streams; i++) - list_add_tail(&array[i]->staging_list, stream_list); - FREE(array); - return 0; -} - static int add_lte_to_array(struct wim_lookup_table_entry *lte, @@ -667,9 +633,13 @@ print_lookup_table_entry(const struct wim_lookup_table_entry *lte, FILE *out) tfprintf(out, T("Part Number = %hu\n"), lte->part_number); tfprintf(out, T("Reference Count = %u\n"), lte->refcnt); - tfprintf(out, T("Hash = 0x")); - print_hash(lte->hash, out); - tputc(T('\n'), out); + if (lte->unhashed) { + tfprintf(out, T("(Unhashed, back ptr at %p)\n"), lte->my_ptr); + } else { + tfprintf(out, T("Hash = 0x")); + print_hash(lte->hash, out); + tputc(T('\n'), out); + } tfprintf(out, T("Flags = ")); u8 flags = lte->resource_entry.flags; diff --git a/src/lookup_table.h b/src/lookup_table.h index 62d02d69..633c874e 100644 --- a/src/lookup_table.h +++ b/src/lookup_table.h @@ -139,7 +139,6 @@ struct wim_lookup_table_entry { u16 resource_location : 5; u8 unique_size : 1; u8 unhashed : 1; - u8 is_ads : 1; /* (On-disk field) * Number of times this lookup table entry is referenced by dentries. @@ -160,6 +159,10 @@ struct wim_lookup_table_entry { * table. */ size_t hash_short; + /* Unhashed entries only (unhashed == 1): this points directly + * to the pointer to this 'struct wim_lookup_table_entry' + * contained in a 'struct wim_ads_entry' or 'struct wim_inode'. + * */ struct wim_lookup_table_entry **my_ptr; }; @@ -226,7 +229,11 @@ struct wim_lookup_table_entry { * * This field is also used to make other lists of lookup table entries. * */ - struct list_head staging_list; + union { + struct list_head unhashed_list; + struct list_head staging_list; + struct list_head extraction_list; + }; }; static inline u64 @@ -317,7 +324,7 @@ for_lookup_table_entry(struct wim_lookup_table *table, void *arg); extern int -sort_stream_list_by_wim_position(struct list_head *stream_list); +cmp_streams_by_wim_position(const void *p1, const void *p2); extern int for_lookup_table_entry_pos_sorted(struct wim_lookup_table *table, @@ -481,7 +488,7 @@ lookup_table_insert_unhashed(struct wim_lookup_table *table, struct wim_lookup_table_entry **my_ptr) { lte->unhashed = 1; - list_add_tail(<e->staging_list, table->unhashed_streams); + list_add_tail(<e->unhashed_list, table->unhashed_streams); lte->my_ptr = my_ptr; *my_ptr = lte; } diff --git a/src/metadata_resource.c b/src/metadata_resource.c index a9e2e386..91b1121f 100644 --- a/src/metadata_resource.c +++ b/src/metadata_resource.c @@ -51,7 +51,6 @@ read_metadata_resource(WIMStruct *w, struct wim_image_metadata *imd) struct wim_dentry *dentry; const struct wim_lookup_table_entry *metadata_lte; u64 metadata_len; - struct hlist_head inode_list; metadata_lte = imd->metadata_lte; metadata_len = wim_resource_size(metadata_lte); @@ -151,7 +150,7 @@ read_metadata_resource(WIMStruct *w, struct wim_image_metadata *imd) goto out_free_dentry_tree; /* Build hash table that maps hard link group IDs to dentry sets */ - ret = dentry_tree_fix_inodes(dentry, &inode_list); + ret = dentry_tree_fix_inodes(dentry, &imd->inode_list); if (ret) goto out_free_dentry_tree; @@ -159,16 +158,13 @@ read_metadata_resource(WIMStruct *w, struct wim_image_metadata *imd) DEBUG("Running miscellaneous verifications on the dentry tree"); for_lookup_table_entry(w->lookup_table, lte_zero_real_refcnt, NULL); ret = for_dentry_in_tree(dentry, verify_dentry, w); - if (ret != 0) + if (ret) goto out_free_dentry_tree; } DEBUG("Done reading image metadata"); imd->root_dentry = dentry; - imd->inode_list = inode_list; - if (imd->inode_list.first) - imd->inode_list.first->pprev = &imd->inode_list.first; INIT_LIST_HEAD(&imd->unhashed_streams); goto out_free_buf; out_free_dentry_tree: @@ -203,16 +199,15 @@ write_wim_resource_from_buffer(const void *buf, u64 buf_size, * write_wim_resource(). */ struct wim_lookup_table_entry lte; int ret; - lte.resource_entry.original_size = buf_size; lte.resource_location = RESOURCE_IN_ATTACHED_BUFFER; lte.attached_buffer = (void*)buf; + lte.resource_entry.original_size = buf_size; + lte.resource_entry.flags = 0; lte.unhashed = 1; - zero_out_hash(lte.hash); ret = write_wim_resource(<e, out_fp, out_ctype, out_res_entry, 0); - if (ret) - return ret; - copy_hash(hash, lte.hash); - return 0; + if (ret == 0) + copy_hash(hash, lte.hash); + return ret; } /* Write the metadata resource for the current WIM image. */ diff --git a/src/mount_image.c b/src/mount_image.c index 4700cb52..2d0d56c6 100644 --- a/src/mount_image.c +++ b/src/mount_image.c @@ -99,7 +99,7 @@ struct wimfs_context { u64 next_ino; /* List of inodes in the mounted image */ - struct hlist_head *image_inode_list; + struct list_head *image_inode_list; /* Name and message queue descriptors for message queues between the * filesystem daemon process and the unmount process. These are used @@ -346,7 +346,7 @@ create_dentry(struct fuse_context *fuse_ctx, const char *path, } } dentry_add_child(parent, new); - hlist_add_head(&new->d_inode->i_hlist, wimfs_ctx->image_inode_list); + list_add_tail(&new->d_inode->i_list, wimfs_ctx->image_inode_list); if (dentry_ret) *dentry_ret = new; return 0; @@ -802,19 +802,19 @@ rebuild_wim(struct wimfs_context *ctx, int write_flags, wimlib_progress_func_t progress_func) { int ret; - struct wim_lookup_table_entry *lte, *tmp; + struct wim_lookup_table_entry *lte; WIMStruct *w = ctx->wim; struct wim_image_metadata *imd = wim_get_current_image_metadata(ctx->wim); DEBUG("Closing all staging file descriptors."); - list_for_each_entry_safe(lte, tmp, &imd->unhashed_streams, staging_list) { + image_for_each_unhashed_stream(lte, imd) { ret = inode_close_fds(lte->lte_inode); if (ret) return ret; } DEBUG("Freeing entries for zero-length streams"); - list_for_each_entry_safe(lte, tmp, &imd->unhashed_streams, staging_list) { + image_for_each_unhashed_stream(lte, imd) { if (wim_resource_size(lte) == 0) { *lte->my_ptr = NULL; free_lookup_table_entry(lte); @@ -1671,9 +1671,7 @@ wimfs_link(const char *to, const char *from) struct wim_dentry *from_dentry, *from_dentry_parent; const char *link_name; struct wim_inode *inode; - struct wim_lookup_table_entry *lte; WIMStruct *w = wimfs_get_WIMStruct(); - u16 i; int ret; inode = wim_pathname_to_inode(w, to); @@ -1698,15 +1696,10 @@ wimfs_link(const char *to, const char *from) if (ret) return -ENOMEM; - inode_add_dentry(from_dentry, inode); - from_dentry->d_inode = inode; inode->i_nlink++; - - for (i = 0; i <= inode->i_num_ads; i++) { - lte = inode_stream_lte_resolved(inode, i); - if (lte) - lte->refcnt++; - } + inode_ref_streams(inode); + from_dentry->d_inode = inode; + inode_add_dentry(from_dentry, inode); dentry_add_child(from_dentry_parent, from_dentry); return 0; } @@ -2418,7 +2411,6 @@ wimlib_mount_image(WIMStruct *wim, int image, const char *dir, struct wim_lookup_table *joined_tab, *wim_tab_save; struct wim_image_metadata *imd; struct wimfs_context ctx; - struct hlist_node *cur_node; struct wim_inode *inode; DEBUG("Mount: wim = %p, image = %d, dir = %s, flags = %d, ", @@ -2428,7 +2420,7 @@ wimlib_mount_image(WIMStruct *wim, int image, const char *dir, return WIMLIB_ERR_INVALID_PARAM; ret = verify_swm_set(wim, additional_swms, num_additional_swms); - if (ret != 0) + if (ret) return ret; if ((mount_flags & WIMLIB_MOUNT_FLAG_READWRITE) && (wim->hdr.total_parts != 1)) { @@ -2440,7 +2432,7 @@ wimlib_mount_image(WIMStruct *wim, int image, const char *dir, ret = new_joined_lookup_table(wim, additional_swms, num_additional_swms, &joined_tab); - if (ret != 0) + if (ret) return ret; wim_tab_save = wim->lookup_table; wim->lookup_table = joined_tab; @@ -2448,12 +2440,12 @@ wimlib_mount_image(WIMStruct *wim, int image, const char *dir, if (mount_flags & WIMLIB_MOUNT_FLAG_READWRITE) { ret = wim_run_full_verifications(wim); - if (ret != 0) + if (ret) goto out; } ret = select_wim_image(wim, image); - if (ret != 0) + if (ret) goto out; DEBUG("Selected image %d", image); @@ -2476,7 +2468,7 @@ wimlib_mount_image(WIMStruct *wim, int image, const char *dir, if (mount_flags & WIMLIB_MOUNT_FLAG_READWRITE) { ret = lock_wim(wim, wim->fp); - if (ret != 0) + if (ret) goto out; } @@ -2494,12 +2486,13 @@ wimlib_mount_image(WIMStruct *wim, int image, const char *dir, ctx.image_inode_list = &imd->inode_list; ctx.default_uid = getuid(); ctx.default_gid = getgid(); + ctx.wim->lookup_table->unhashed_streams = &imd->unhashed_streams; if (mount_flags & WIMLIB_MOUNT_FLAG_STREAM_INTERFACE_WINDOWS) ctx.default_lookup_flags = LOOKUP_FLAG_ADS_OK; DEBUG("Unlinking message queues in case they already exist"); ret = set_message_queue_names(&ctx, dir); - if (ret != 0) + if (ret) goto out_unlock; unlink_message_queues(&ctx); @@ -2537,7 +2530,7 @@ wimlib_mount_image(WIMStruct *wim, int image, const char *dir, if ((mount_flags & WIMLIB_MOUNT_FLAG_READWRITE)) { /* Read-write mount. Make the staging directory */ ret = make_staging_dir(&ctx, staging_dir); - if (ret != 0) + if (ret) goto out_free_dir_copy; } else { /* Read-only mount */ @@ -2570,7 +2563,7 @@ wimlib_mount_image(WIMStruct *wim, int image, const char *dir, * assign inode numbers */ DEBUG("Resolving lookup table entries and assigning inode numbers"); ctx.next_ino = 1; - hlist_for_each_entry(inode, cur_node, &imd->inode_list, i_hlist) { + image_for_each_inode(inode, imd) { inode_resolve_ltes(inode, wim->lookup_table); inode->i_ino = ctx.next_ino++; } diff --git a/src/resource.c b/src/resource.c index 344ba0d2..7a5d1718 100644 --- a/src/resource.c +++ b/src/resource.c @@ -537,6 +537,8 @@ read_partial_wim_resource(const struct wim_lookup_table_entry *lte, cb, ctx_or_buf); } else { + offset += lte->resource_entry.offset; + if (fseeko(wim_fp, offset, SEEK_SET)) { ERROR_WITH_ERRNO("Failed to seek to offset %"PRIu64 " in WIM", offset); @@ -654,12 +656,21 @@ read_buffer_prefix(const struct wim_lookup_table_entry *lte, void *ctx_or_buf, int _ignored_flags) { const void *inbuf = lte->attached_buffer; + int ret; + if (cb) { - return cb(inbuf, size, ctx_or_buf); + while (size) { + size_t chunk_size = min(WIM_CHUNK_SIZE, size); + ret = cb(inbuf, chunk_size, ctx_or_buf); + if (ret) + return ret; + size -= chunk_size; + inbuf += chunk_size; + } } else { memcpy(ctx_or_buf, inbuf, size); - return 0; } + return 0; } typedef int (*read_resource_prefix_handler_t)(const struct wim_lookup_table_entry *lte, @@ -668,6 +679,23 @@ typedef int (*read_resource_prefix_handler_t)(const struct wim_lookup_table_entr void *ctx_or_buf, int flags); +/* + * Read the first @size bytes from a generic "resource", which may be located in + * the WIM (compressed or uncompressed), in an external file, or directly in an + * in-memory buffer. + * + * Feed the data either to a callback function (cb != NULL, passing it + * ctx_or_buf), or write it directly into a buffer (cb == NULL, ctx_or_buf + * specifies the buffer, which must have room for @size bytes). + * + * When using a callback function, it is called with chunks up to 32768 bytes in + * size until the resource is exhausted. + * + * If the resource is located in a WIM file, @flags can be + * WIMLIB_RESOURCE_FLAG_MULTITHREADED if it must be safe to access the resource + * concurrently by multiple threads, or WIMLIB_RESOURCE_FLAG_RAW if the raw + * compressed data is to be supplied instead of the uncompressed data. + */ int read_resource_prefix(const struct wim_lookup_table_entry *lte, u64 size, consume_data_callback_t cb, void *ctx_or_buf, @@ -703,6 +731,22 @@ read_full_resource_into_buf(const struct wim_lookup_table_entry *lte, thread_safe ? WIMLIB_RESOURCE_FLAG_MULTITHREADED : 0); } +struct extract_ctx { + SHA_CTX sha_ctx; + consume_data_callback_t extract_chunk; + void *extract_chunk_arg; +}; + +static int +extract_chunk_sha1_wrapper(const void *chunk, size_t chunk_size, + void *_ctx) +{ + struct extract_ctx *ctx = _ctx; + + sha1_update(&ctx->sha_ctx, chunk, chunk_size); + return ctx->extract_chunk(chunk, chunk_size, ctx->extract_chunk_arg); +} + /* Extracts the first @size bytes of a WIM resource to somewhere. In the * process, the SHA1 message digest of the resource is checked if the full * resource is being extracted. @@ -715,8 +759,36 @@ extract_wim_resource(const struct wim_lookup_table_entry *lte, consume_data_callback_t extract_chunk, void *extract_chunk_arg) { - return read_resource_prefix(lte, size, extract_chunk, - extract_chunk_arg, 0); + int ret; + if (size == wim_resource_size(lte)) { + /* Do SHA1 */ + struct extract_ctx ctx; + ctx.extract_chunk = extract_chunk; + ctx.extract_chunk_arg = extract_chunk_arg; + sha1_init(&ctx.sha_ctx); + ret = read_resource_prefix(lte, size, + extract_chunk_sha1_wrapper, + &ctx, 0); + if (ret == 0) { + u8 hash[SHA1_HASH_SIZE]; + sha1_final(hash, &ctx.sha_ctx); + if (!hashes_equal(hash, lte->hash)) { + #ifdef ENABLE_ERROR_MESSAGES + ERROR_WITH_ERRNO("Invalid SHA1 message digest " + "on the following WIM resource:"); + print_lookup_table_entry(lte, stderr); + if (lte->resource_location == RESOURCE_IN_WIM) + ERROR("The WIM file appears to be corrupt!"); + ret = WIMLIB_ERR_INVALID_RESOURCE_HASH; + #endif + } + } + } else { + /* Don't do SHA1 */ + ret = read_resource_prefix(lte, size, extract_chunk, + extract_chunk_arg, 0); + } + return ret; } static int diff --git a/src/symlink.c b/src/symlink.c index bd370e30..538b3e85 100644 --- a/src/symlink.c +++ b/src/symlink.c @@ -219,7 +219,7 @@ inode_set_symlink(struct wim_inode *inode, ret = make_symlink_reparse_data_buf(target, &symlink_buf_len, &symlink_buf); - if (ret != 0) + if (ret) return ret; DEBUG("Made symlink reparse data buf (len = %zu, name len = %zu)", @@ -243,7 +243,6 @@ inode_set_symlink(struct wim_inode *inode, lte->resource_location = RESOURCE_IN_ATTACHED_BUFFER; lte->attached_buffer = symlink_buf; lte->resource_entry.original_size = symlink_buf_len; - lte->resource_entry.size = symlink_buf_len; copy_hash(lte->hash, symlink_buf_hash); } diff --git a/src/verify.c b/src/verify.c index f9808ada..6da6b6a9 100644 --- a/src/verify.c +++ b/src/verify.c @@ -175,10 +175,9 @@ image_run_full_verifications(WIMStruct *w) { struct wim_image_metadata *imd; struct wim_inode *inode; - struct hlist_node *cur; imd = wim_get_current_image_metadata(w); - hlist_for_each_entry(inode, cur, &imd->inode_list, i_hlist) + image_for_each_inode(inode, imd) inode->i_verified = 0; return for_dentry_in_tree(imd->root_dentry, verify_dentry, w); } diff --git a/src/wim.c b/src/wim.c index ed20145e..96fba59f 100644 --- a/src/wim.c +++ b/src/wim.c @@ -533,7 +533,7 @@ destroy_image_metadata(struct wim_image_metadata *imd, imd->metadata_lte = NULL; } INIT_LIST_HEAD(&imd->unhashed_streams); - INIT_HLIST_HEAD(&imd->inode_list); + INIT_LIST_HEAD(&imd->inode_list); } void @@ -574,7 +574,7 @@ new_image_metadata() imd = CALLOC(1, sizeof(*imd)); if (imd) { imd->refcnt = 1; - INIT_HLIST_HEAD(&imd->inode_list); + INIT_LIST_HEAD(&imd->inode_list); INIT_LIST_HEAD(&imd->unhashed_streams); DEBUG("Created new image metadata (refcnt=1)"); } else { diff --git a/src/wimlib_internal.h b/src/wimlib_internal.h index 18f02738..91f99ebf 100644 --- a/src/wimlib_internal.h +++ b/src/wimlib_internal.h @@ -239,10 +239,12 @@ struct wim_security_data { /* Metadata for a WIM image */ struct wim_image_metadata { + /* Number of WIMStruct's that are sharing this image metadata (from + * calls to wimlib_export_image().) */ unsigned long refcnt; /* Pointer to the root dentry of the image. */ - struct wim_dentry *root_dentry; + struct wim_dentry *root_dentry; /* Pointer to the security data of the image. */ struct wim_security_data *security_data; @@ -251,9 +253,14 @@ struct wim_image_metadata { */ struct wim_lookup_table_entry *metadata_lte; - /* Linked list of inodes of this image */ - struct hlist_head inode_list; + /* Linked list of 'struct wim_inode's for this image. */ + struct list_head inode_list; + /* Linked list of 'struct wim_lookup_table_entry's for this image that + * are referred to in the dentry tree, but have not had a SHA1 message + * digest calculated yet and therefore have not been inserted into the + * WIM's lookup table. This list is added to during wimlib_add_image() + * and wimlib_mount_image() (read-write only). */ struct list_head unhashed_streams; /* 1 iff the dentry tree has been modified. If this is the case, the @@ -356,6 +363,12 @@ resource_is_compressed(const struct resource_entry *entry) return (entry->flags & WIM_RESHDR_FLAG_COMPRESSED); } +#define image_for_each_inode(inode, imd) \ + list_for_each_entry(inode, &imd->inode_list, i_list) + +#define image_for_each_unhashed_stream(lte, imd) \ + list_for_each_entry(lte, &imd->unhashed_streams, unhashed_list) + /* add_image.c */ extern bool @@ -392,7 +405,7 @@ struct wim_inode_table { * number with some other inode until assign_inode_numbers() is * called. */ - struct hlist_head extra_inodes; + struct list_head extra_inodes; }; extern int @@ -402,9 +415,12 @@ extern int inode_table_new_dentry(struct wim_inode_table *table, const tchar *name, u64 ino, u64 devno, struct wim_dentry **dentry_ret); +extern void +inode_ref_streams(struct wim_inode *inode); + extern void inode_table_prepare_inode_list(struct wim_inode_table *table, - struct hlist_head *head); + struct list_head *head); static inline void destroy_inode_table(struct wim_inode_table *table) @@ -414,7 +430,7 @@ destroy_inode_table(struct wim_inode_table *table) extern int -dentry_tree_fix_inodes(struct wim_dentry *root, struct hlist_head *inode_list); +dentry_tree_fix_inodes(struct wim_dentry *root, struct list_head *inode_list); /* header.c */ diff --git a/src/write.c b/src/write.c index 55dbbc12..ccad7f07 100644 --- a/src/write.c +++ b/src/write.c @@ -187,9 +187,9 @@ get_compress_func(int out_ctype) * * @chunk: Uncompressed data of the chunk. * @chunk_size: Size of the chunk (<= WIM_CHUNK_SIZE) - * @out_fp: FILE * to write tho chunk to. - * @out_ctype: Compression type to use when writing the chunk (ignored if no - * chunk table provided) + * @out_fp: FILE * to write the chunk to. + * @compress: Compression function to use (NULL if writing uncompressed + * data). * @chunk_tab: Pointer to chunk table being created. It is updated with the * offset of the chunk we write. * @@ -202,7 +202,7 @@ write_wim_resource_chunk(const void *chunk, unsigned chunk_size, { const u8 *out_chunk; unsigned out_chunk_size; - if (chunk_tab) { + if (compress) { u8 *compressed_chunk = alloca(chunk_size); out_chunk_size = compress(chunk, chunk_size, compressed_chunk); @@ -404,7 +404,6 @@ write_wim_resource(struct wim_lookup_table_entry *lte, /* Doing a raw write: The new compressed size is the same as * the compressed size in the other WIM. */ new_size = lte->resource_entry.size; - out_res_entry->flags = lte->resource_entry.flags; } else if (out_ctype == WIMLIB_COMPRESSION_TYPE_NONE) { /* Using WIMLIB_COMPRESSION_TYPE_NONE: The new compressed size * is the original size. */ @@ -644,23 +643,31 @@ do_write_stream_list(struct list_head *my_resources, list_del(<e->write_streams_list); if (lte->unhashed && !lte->unique_size) { struct wim_lookup_table_entry *duplicate_lte; + struct wim_lookup_table_entry **my_ptr; + my_ptr = lte->my_ptr; ret = sha1_resource(lte); if (ret) return ret; - list_del(<e->staging_list); - duplicate_lte = __lookup_resource(lookup_table, lte->hash); if (duplicate_lte) { bool new_stream = (duplicate_lte->out_refcnt == 0); duplicate_lte->refcnt += lte->refcnt; duplicate_lte->out_refcnt += lte->refcnt; + *my_ptr = duplicate_lte; free_lookup_table_entry(lte); - if (new_stream) + if (new_stream) { lte = duplicate_lte; - else - continue; + DEBUG("Stream of length %"PRIu64" is duplicate " + "with one already in WIM", + wim_resource_size(duplicate_lte)); + } else { + DEBUG("Discarding duplicate stream of length %"PRIu64, + wim_resource_size(duplicate_lte)); + goto skip_to_progress; + } + } else { lookup_table_insert(lookup_table, lte); lte->out_refcnt = lte->refcnt; @@ -678,10 +685,10 @@ do_write_stream_list(struct list_head *my_resources, if (ret) return ret; if (lte->unhashed) { - wimlib_assert(__lookup_resource(lookup_table, lte->hash) == NULL); lookup_table_insert(lookup_table, lte); lte->unhashed = 0; } + skip_to_progress: do_write_streams_progress(progress, progress_func, wim_resource_size(lte)); @@ -1418,10 +1425,11 @@ static int stream_size_table_insert(struct wim_lookup_table_entry *lte, void *_tab) { struct stream_size_table *tab = _tab; - size_t pos = hash_u64(wim_resource_size(lte)) % tab->capacity; + size_t pos; struct wim_lookup_table_entry *hashed_lte; struct hlist_node *tmp; + pos = hash_u64(wim_resource_size(lte)) % tab->capacity; lte->unique_size = 1; hlist_for_each_entry(hashed_lte, tmp, &tab->array[pos], hash_list_2) { if (wim_resource_size(hashed_lte) == wim_resource_size(lte)) { @@ -1452,11 +1460,12 @@ inode_find_streams_to_write(struct wim_inode *inode, for (unsigned i = 0; i <= inode->i_num_ads; i++) { lte = inode_stream_lte(inode, i, table); if (lte) { - if (lte->out_refcnt == 0) + if (lte->out_refcnt == 0) { + if (lte->unhashed) + stream_size_table_insert(lte, tab); list_add_tail(<e->write_streams_list, stream_list); + } lte->out_refcnt += inode->i_nlink; - if (lte->unhashed) - stream_size_table_insert(lte, tab); } } return 0; @@ -1468,11 +1477,20 @@ image_find_streams_to_write(WIMStruct *w) struct wim_image_metadata *imd; struct find_streams_ctx *ctx; struct wim_inode *inode; - struct hlist_node *cur; + struct wim_lookup_table_entry *lte; ctx = w->private; imd = wim_get_current_image_metadata(w); - hlist_for_each_entry(inode, cur, &imd->inode_list, i_hlist) { + + image_for_each_unhashed_stream(lte, imd) { + lte->out_refcnt = 0; + wimlib_assert(lte->unhashed); + wimlib_assert(lte->my_ptr != NULL); + } + + /* Go through this image's inodes to find any streams that have not been + * found yet. */ + image_for_each_inode(inode, imd) { inode_find_streams_to_write(inode, w->lookup_table, &ctx->stream_list, &ctx->stream_size_tab); @@ -1480,31 +1498,61 @@ image_find_streams_to_write(WIMStruct *w) return 0; } +/* Given a WIM that from which one or all of the images is being written, build + * the list of unique streams ('struct wim_lookup_table_entry's) that must be + * written, plus any unhashed streams that need to be written but may be + * identical to other hashed or unhashed streams being written. These unhashed + * streams are checksummed while the streams are being written. To aid this + * process, the member @unique_size is set to 1 on streams that have a unique + * size and therefore must be written. + * + * The out_refcnt member of each 'struct wim_lookup_table_entry' is set to + * indicate the number of times the stream is referenced in only the streams + * that are being written; this may still be adjusted later when unhashed + * streams are being resolved. + */ static int -write_wim_streams(WIMStruct *w, int image, int write_flags, - unsigned num_threads, - wimlib_progress_func_t progress_func) +prepare_stream_list(WIMStruct *wim, int image, struct list_head *stream_list) { - struct find_streams_ctx ctx; int ret; + struct find_streams_ctx ctx; - for_lookup_table_entry(w->lookup_table, lte_zero_out_refcnt, NULL); + for_lookup_table_entry(wim->lookup_table, lte_zero_out_refcnt, NULL); ret = init_stream_size_table(&ctx.stream_size_tab, 9001); if (ret) return ret; - for_lookup_table_entry(w->lookup_table, stream_size_table_insert, + for_lookup_table_entry(wim->lookup_table, stream_size_table_insert, &ctx.stream_size_tab); - INIT_LIST_HEAD(&ctx.stream_list); - w->private = &ctx; - for_image(w, image, image_find_streams_to_write); + wim->private = &ctx; + for_image(wim, image, image_find_streams_to_write); destroy_stream_size_table(&ctx.stream_size_tab); - ret = write_stream_list(&ctx.stream_list, - w->lookup_table, - w->out_fp, - wimlib_get_compression_type(w), write_flags, - num_threads, progress_func); - return ret; + + INIT_LIST_HEAD(stream_list); + list_splice(&ctx.stream_list, stream_list); + return 0; +} + +/* Writes the streams for the specified @image in @wim to @wim->out_fp. + */ +static int +write_wim_streams(WIMStruct *wim, int image, int write_flags, + unsigned num_threads, + wimlib_progress_func_t progress_func) +{ + int ret; + struct list_head stream_list; + + ret = prepare_stream_list(wim, image, &stream_list); + if (ret) + return ret; + return write_stream_list(&stream_list, + wim->lookup_table, + wim->out_fp, + wimlib_get_compression_type(wim), + write_flags, + num_threads, + progress_func); } /* @@ -1767,19 +1815,19 @@ wimlib_write(WIMStruct *w, const tchar *path, } ret = begin_write(w, path, write_flags); - if (ret != 0) + if (ret) goto out; ret = write_wim_streams(w, image, write_flags, num_threads, progress_func); - if (ret != 0) + if (ret) goto out; if (progress_func) progress_func(WIMLIB_PROGRESS_MSG_WRITE_METADATA_BEGIN, NULL); ret = for_image(w, image, write_metadata_resource); - if (ret != 0) + if (ret) goto out; if (progress_func) -- 2.43.0