From 13c6ce3160fce7c40008d1d182325c8b42450d1e Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Sat, 30 Mar 2013 14:29:55 -0500 Subject: [PATCH] refcnt image metadata; calculate full path on-demand --- src/add_image.c | 76 +++++-------- src/delete_image.c | 76 ++++++------- src/dentry.c | 64 +++++------ src/dentry.h | 15 +-- src/export_image.c | 42 +++---- src/extract_image.c | 21 ++-- src/hardlink.c | 2 +- src/join.c | 2 +- src/lookup_table.c | 17 +-- src/lookup_table.h | 63 ++++++----- src/metadata_resource.c | 18 ++- src/mount_image.c | 80 +++++--------- src/ntfs-apply.c | 73 +++++++------ src/ntfs-capture.c | 16 +-- src/resource.c | 13 ++- src/security.c | 20 ++-- src/split.c | 4 +- src/verify.c | 24 ++-- src/wim.c | 119 +++++++++++++++----- src/wimlib.h | 4 +- src/wimlib_internal.h | 54 +++++---- src/win32.c | 8 +- src/write.c | 236 ++++++++++++++++++++++++++++++++-------- src/xml.c | 2 +- 24 files changed, 610 insertions(+), 439 deletions(-) diff --git a/src/add_image.c b/src/add_image.c index 02ca2f25..904f3545 100644 --- a/src/add_image.c +++ b/src/add_image.c @@ -54,49 +54,36 @@ * Adds the dentry tree and security data for a new image to the image metadata * array of the WIMStruct. */ -int +static int add_new_dentry_tree(WIMStruct *w, struct wim_dentry *root_dentry, struct wim_security_data *sd) { - struct wim_lookup_table_entry *metadata_lte; - struct wim_image_metadata *imd; struct wim_image_metadata *new_imd; - - wimlib_assert(root_dentry != NULL); - - DEBUG("Reallocating image metadata array for image_count = %u", - w->hdr.image_count + 1); - imd = CALLOC((w->hdr.image_count + 1), sizeof(struct wim_image_metadata)); - - if (!imd) { - ERROR("Failed to allocate memory for new image metadata array"); - goto err; - } - - memcpy(imd, w->image_metadata, - w->hdr.image_count * sizeof(struct wim_image_metadata)); + int ret; + struct wim_lookup_table_entry *metadata_lte; metadata_lte = new_lookup_table_entry(); if (!metadata_lte) - goto err_free_imd; + return WIMLIB_ERR_NOMEM; metadata_lte->resource_entry.flags = WIM_RESHDR_FLAG_METADATA; + metadata_lte->unhashed = 1; - new_imd = &imd[w->hdr.image_count]; + new_imd = new_image_metadata(); + if (!new_imd) { + free_lookup_table_entry(metadata_lte); + return WIMLIB_ERR_NOMEM; + } new_imd->root_dentry = root_dentry; new_imd->metadata_lte = metadata_lte; new_imd->security_data = sd; new_imd->modified = 1; - FREE(w->image_metadata); - w->image_metadata = imd; - w->hdr.image_count++; - return 0; -err_free_imd: - FREE(imd); -err: - return WIMLIB_ERR_NOMEM; + ret = append_image_metadata(w, new_imd); + if (ret) + put_image_metadata(new_imd, NULL); + return ret; } @@ -126,8 +113,7 @@ unix_capture_regular_file(const char *path, lte->file_on_disk = file_on_disk; lte->resource_location = RESOURCE_IN_FILE_ON_DISK; lte->resource_entry.original_size = size; - lookup_table_insert_unhashed(lookup_table, lte); - inode->i_lte = lte; + lookup_table_insert_unhashed(lookup_table, lte, &inode->i_lte); } return 0; } @@ -371,12 +357,10 @@ unix_build_dentry_tree_recursive(struct wim_dentry **root_ret, else ret = unix_capture_symlink(path, inode, lookup_table); out: - if (ret == 0) { + if (ret == 0) *root_ret = root; - } else { + else free_dentry_tree(root, lookup_table); - lookup_table_free_unhashed_streams(lookup_table); - } return ret; } @@ -849,6 +833,7 @@ wimlib_add_image_multisource(WIMStruct *w, struct wim_security_data *sd; struct wim_image_metadata *imd; struct wim_inode_table inode_table; + struct list_head unhashed_streams; int ret; struct sd_set sd_set; @@ -930,7 +915,6 @@ wimlib_add_image_multisource(WIMStruct *w, goto out_destroy_inode_table; } sd->total_length = 8; - sd->refcnt = 1; sd_set.sd = sd; sd_set.rb_root.rb_node = NULL; @@ -945,10 +929,9 @@ wimlib_add_image_multisource(WIMStruct *w, goto out_free_security_data; } - - DEBUG("Building dentry tree."); + INIT_LIST_HEAD(&unhashed_streams); + w->lookup_table->unhashed_streams = &unhashed_streams; root_dentry = NULL; - for (size_t i = 0; i < num_sources; i++) { int flags; union wimlib_progress_info progress; @@ -1006,32 +989,29 @@ wimlib_add_image_multisource(WIMStruct *w, goto out_free_dentry_tree; } - DEBUG("Calculating full paths of dentries."); - ret = for_dentry_in_tree(root_dentry, calculate_dentry_full_path, NULL); - if (ret) - goto out_free_dentry_tree; - ret = add_new_dentry_tree(w, root_dentry, sd); if (ret) goto out_free_dentry_tree; - imd = &w->image_metadata[w->hdr.image_count - 1]; + imd = w->image_metadata[w->hdr.image_count - 1]; + INIT_LIST_HEAD(&imd->unhashed_streams); + list_splice(&unhashed_streams, &imd->unhashed_streams); DEBUG("Assigning hard link group IDs"); inode_table_prepare_inode_list(&inode_table, &imd->inode_list); ret = xml_add_image(w, name); if (ret) - goto out_destroy_imd; + goto out_put_imd; if (add_image_flags & WIMLIB_ADD_IMAGE_FLAG_BOOT) wimlib_set_boot_idx(w, w->hdr.image_count); + ret = 0; goto out_destroy_inode_table; -out_destroy_imd: - destroy_image_metadata(&w->image_metadata[w->hdr.image_count - 1], - w->lookup_table); - w->hdr.image_count--; +out_put_imd: + put_image_metadata(w->image_metadata[--w->hdr.image_count], + w->lookup_table); goto out_destroy_inode_table; out_free_branch: free_dentry_tree(branch, w->lookup_table); diff --git a/src/delete_image.c b/src/delete_image.c index 389d2309..c95819e9 100644 --- a/src/delete_image.c +++ b/src/delete_image.c @@ -31,63 +31,63 @@ WIMLIBAPI int wimlib_delete_image(WIMStruct *w, int image) { - int i; int ret; + int first, last; if (w->hdr.total_parts != 1) { ERROR("Deleting an image from a split WIM is not supported."); return WIMLIB_ERR_SPLIT_UNSUPPORTED; } - if (image == WIMLIB_ALL_IMAGES) { - for (i = w->hdr.image_count; i >= 1; i--) { - ret = wimlib_delete_image(w, i); - if (ret != 0) - return ret; - } - return 0; - } - if (!w->all_images_verified) { ret = wim_run_full_verifications(w); - if (ret != 0) + if (ret) return ret; } - DEBUG("Deleting image %d", image); + if (image == WIMLIB_ALL_IMAGES) { + last = w->hdr.image_count; + first = 1; + } else { + last = image; + first = image; + } - /* Even if the dentry tree is not allocated, we must select it (and - * therefore allocate it) so that we can decrement the reference counts - * in the lookup table. */ - ret = select_wim_image(w, image); - if (ret != 0) - return ret; + for (image = last; image >= first; image--) { + DEBUG("Deleting image %d", image); - /* Free the dentry tree, any lookup table entries that have their refcnt - * decremented to 0, and the security data. */ - destroy_image_metadata(&w->image_metadata[image - 1], w->lookup_table); + /* Even if the dentry tree is not allocated, we must select it (and + * therefore allocate it) so that we can decrement the reference counts + * in the lookup table. */ + ret = select_wim_image(w, image); + if (ret) + return ret; - /* Get rid of the empty slot in the image metadata array. */ - memmove(&w->image_metadata[image - 1], &w->image_metadata[image], - (w->hdr.image_count - image) * sizeof(struct wim_image_metadata)); + /* Unless the image metadata is shared by another WIMStruct, free the + * dentry tree, any lookup table entries that have their refcnt + * decremented to 0, and the security data. */ + put_image_metadata(w->image_metadata[image - 1], w->lookup_table); - /* Decrement the image count. */ - if (--w->hdr.image_count == 0) { - FREE(w->image_metadata); - w->image_metadata = NULL; - } + /* Get rid of the empty slot in the image metadata array. */ + memmove(w->image_metadata[image - 1], + &w->image_metadata[image], + (w->hdr.image_count - image) * sizeof(w->image_metadata[0])); - /* Fix the boot index. */ - if (w->hdr.boot_idx == image) - w->hdr.boot_idx = 0; - else if (w->hdr.boot_idx > image) - w->hdr.boot_idx--; + /* Decrement the image count. */ + --w->hdr.image_count; - w->current_image = WIMLIB_NO_IMAGE; + /* Fix the boot index. */ + if (w->hdr.boot_idx == image) + w->hdr.boot_idx = 0; + else if (w->hdr.boot_idx > image) + w->hdr.boot_idx--; - /* Remove the image from the XML information. */ - xml_delete_image(&w->wim_info, image); + w->current_image = WIMLIB_NO_IMAGE; - w->deletion_occurred = 1; + /* Remove the image from the XML information. */ + xml_delete_image(&w->wim_info, image); + + w->deletion_occurred = 1; + } return 0; } diff --git a/src/dentry.c b/src/dentry.c index 2ed77da7..16d9b416 100644 --- a/src/dentry.c +++ b/src/dentry.c @@ -275,14 +275,15 @@ for_dentry_in_tree_depth(struct wim_dentry *root, /* Calculate the full path of @dentry. The full path of its parent must have * already been calculated, or it must be the root dentry. */ -int -calculate_dentry_full_path(struct wim_dentry *dentry, void *ignore) +static int +calculate_dentry_full_path(struct wim_dentry *dentry) { tchar *full_path; u32 full_path_nbytes; + int ret; - wimlib_assert(dentry_is_root(dentry) || - dentry->parent->full_path != NULL); + if (dentry->_full_path) + return 0; if (dentry_is_root(dentry)) { full_path = TSTRDUP(T("/")); @@ -290,7 +291,7 @@ calculate_dentry_full_path(struct wim_dentry *dentry, void *ignore) return WIMLIB_ERR_NOMEM; full_path_nbytes = 1 * sizeof(tchar); } else { - const struct wim_dentry *parent; + struct wim_dentry *parent; tchar *parent_full_path; u32 parent_full_path_nbytes; size_t filename_nbytes; @@ -300,7 +301,12 @@ calculate_dentry_full_path(struct wim_dentry *dentry, void *ignore) parent_full_path = T(""); parent_full_path_nbytes = 0; } else { - parent_full_path = parent->full_path; + if (!parent->_full_path) { + ret = calculate_dentry_full_path(parent); + if (ret) + return ret; + } + parent_full_path = parent->_full_path; parent_full_path_nbytes = parent->full_path_nbytes; } @@ -336,12 +342,18 @@ calculate_dentry_full_path(struct wim_dentry *dentry, void *ignore) sizeof(tchar) + 1]); #endif } - FREE(dentry->full_path); - dentry->full_path = full_path; + dentry->_full_path = full_path; dentry->full_path_nbytes= full_path_nbytes; return 0; } +tchar * +dentry_full_path(struct wim_dentry *dentry) +{ + calculate_dentry_full_path(dentry); + return dentry->_full_path; +} + static int increment_subdir_offset(struct wim_dentry *dentry, void *subdir_offset_p) { @@ -559,10 +571,15 @@ get_parent_dentry(WIMStruct *w, const tchar *path) /* Prints the full path of a dentry. */ int -print_dentry_full_path(struct wim_dentry *dentry, void *ignore) +print_dentry_full_path(struct wim_dentry *dentry, void *_ignore) { - if (dentry->full_path) - tprintf(T("%"TS"\n"), dentry->full_path); + tchar *full_path = dentry_full_path(dentry); + if (!full_path) + return WIMLIB_ERR_NOMEM; + tprintf(T("%"TS"\n"), full_path); + FREE(full_path); + dentry->_full_path = 0; + dentry->full_path_nbytes = 0; return 0; } @@ -628,8 +645,8 @@ print_dentry(struct wim_dentry *dentry, void *lookup_table) wimlib_printf(T("Filename = \"%"WS"\"\n"), dentry->file_name); if (dentry_has_short_name(dentry)) wimlib_printf(T("Short Name \"%"WS"\"\n"), dentry->short_name); - if (dentry->full_path) - tprintf(T("Full Path = \"%"TS"\"\n"), dentry->full_path); + if (dentry->_full_path) + tprintf(T("Full Path = \"%"TS"\"\n"), dentry->_full_path); lte = inode_stream_lte(dentry->d_inode, 0, lookup_table); if (lte) { @@ -666,7 +683,6 @@ static void dentry_common_init(struct wim_dentry *dentry) { memset(dentry, 0, sizeof(struct wim_dentry)); - dentry->refcnt = 1; } struct wim_inode * @@ -843,20 +859,12 @@ free_dentry(struct wim_dentry *dentry) { FREE(dentry->file_name); FREE(dentry->short_name); - FREE(dentry->full_path); + FREE(dentry->_full_path); if (dentry->d_inode) put_inode(dentry->d_inode); FREE(dentry); } -void -put_dentry(struct wim_dentry *dentry) -{ - wimlib_assert(dentry->refcnt != 0); - if (--dentry->refcnt == 0) - free_dentry(dentry); -} - /* This function is passed as an argument to for_dentry_in_tree_depth() in order * to free a directory tree. */ static int @@ -875,8 +883,7 @@ do_free_dentry(struct wim_dentry *dentry, void *__lookup_table) lte_decrement_refcnt(lte, lookup_table); } } - - put_dentry(dentry); + free_dentry(dentry); return 0; } @@ -896,13 +903,6 @@ free_dentry_tree(struct wim_dentry *root, struct wim_lookup_table *lookup_table) for_dentry_in_tree_depth(root, do_free_dentry, lookup_table); } -int -increment_dentry_refcnt(struct wim_dentry *dentry, void *ignore) -{ - dentry->refcnt++; - return 0; -} - /* * Links a dentry into the directory tree. * diff --git a/src/dentry.h b/src/dentry.h index 601a40c5..28fdeaf2 100644 --- a/src/dentry.h +++ b/src/dentry.h @@ -186,10 +186,6 @@ struct wim_dentry { * points. */ u64 subdir_offset; - /* Number of references to the dentry tree itself, as in multiple - * WIMStructs */ - u32 refcnt; - /* Pointer to the UTF-16LE short filename (malloc()ed buffer) */ utf16lechar *short_name; @@ -197,7 +193,7 @@ struct wim_dentry { utf16lechar *file_name; /* Full path of this dentry */ - tchar *full_path; + tchar *_full_path; }; #define rbnode_dentry(node) container_of(node, struct wim_dentry, rb_node) @@ -346,9 +342,6 @@ for_dentry_in_tree_depth(struct wim_dentry *root, int (*visitor)(struct wim_dentry*, void*), void *args); -extern int -calculate_dentry_full_path(struct wim_dentry *dentry, void *ignore); - extern void calculate_subdir_offsets(struct wim_dentry *dentry, u64 *subdir_offset_p); @@ -379,6 +372,9 @@ print_dentry(struct wim_dentry *dentry, void *lookup_table); extern int print_dentry_full_path(struct wim_dentry *entry, void *ignore); +extern tchar * +dentry_full_path(struct wim_dentry *dentry); + extern struct wim_inode * new_timeless_inode(); @@ -404,9 +400,6 @@ extern void free_dentry_tree(struct wim_dentry *root, struct wim_lookup_table *lookup_table); -extern int -increment_dentry_refcnt(struct wim_dentry *dentry, void *ignore); - extern void unlink_dentry(struct wim_dentry *dentry); diff --git a/src/export_image.c b/src/export_image.c index b62f67c5..7e6d2789 100644 --- a/src/export_image.c +++ b/src/export_image.c @@ -62,10 +62,6 @@ inode_move_ltes_to_table(struct wim_inode *inode, { struct wim_lookup_table_entry *src_lte, *dest_lte; unsigned i; - struct wim_dentry *dentry; - - inode_for_each_dentry(dentry, inode) - dentry->refcnt++; for (i = 0; i <= inode->i_num_ads; i++) { src_lte = inode_stream_lte_unresolved(inode, i, src_lookup_table); @@ -92,7 +88,7 @@ inode_move_ltes_to_table(struct wim_inode *inode, } /* - * Copies an image, or all the images, from a WIM file, into another WIM file. + * Exports an image, or all the images, from a WIM file, into another WIM file. */ WIMLIBAPI int wimlib_export_image(WIMStruct *src_wim, @@ -106,7 +102,6 @@ wimlib_export_image(WIMStruct *src_wim, wimlib_progress_func_t progress_func) { int ret; - struct wim_security_data *sd; struct wim_lookup_table *joined_tab, *src_wim_tab_save; struct wim_image_metadata *src_imd; struct hlist_node *cur_node; @@ -155,7 +150,7 @@ wimlib_export_image(WIMStruct *src_wim, additional_swms, num_additional_swms, progress_func); - if (ret != 0) + if (ret) return ret; } return 0; @@ -188,21 +183,21 @@ wimlib_export_image(WIMStruct *src_wim, } ret = verify_swm_set(src_wim, additional_swms, num_additional_swms); - if (ret != 0) + if (ret) return ret; if (num_additional_swms) { ret = new_joined_lookup_table(src_wim, additional_swms, num_additional_swms, &joined_tab); - if (ret != 0) + if (ret) return ret; src_wim_tab_save = src_wim->lookup_table; src_wim->lookup_table = joined_tab; } ret = select_wim_image(src_wim, src_image); - if (ret != 0) { + if (ret) { ERROR("Could not select image %d from the WIM `%"TS"' " "to export it", src_image, src_wim->filename); goto out; @@ -219,32 +214,27 @@ wimlib_export_image(WIMStruct *src_wim, src_wim->lookup_table, dest_wim->lookup_table, <e_list_head); - if (ret != 0) + if (ret) goto out_free_ltes; } ret = xml_export_image(src_wim->wim_info, src_image, &dest_wim->wim_info, dest_name, dest_description); - if (ret != 0) + if (ret) goto out_free_ltes; - sd = src_imd->security_data; - ret = add_new_dentry_tree(dest_wim, src_imd->root_dentry, sd); - if (ret != 0) + ret = append_image_metadata(dest_wim, src_imd); + if (ret) goto out_xml_delete_image; - dest_wim->image_metadata[ - dest_wim->hdr.image_count - 1].inode_list = src_imd->inode_list; - if (src_imd->inode_list.first) - src_imd->inode_list.first->pprev = NULL; + /* The `struct image_metadata' is now referenced by both the @src_wim + * and the @dest_wim. */ + src_imd->refcnt++; /* All memory allocations have been taken care of, so it's no longer - * possible for this function to fail. Go ahead and increment the - * reference counts of the dentry tree and security data, then update - * the lookup table of the destination WIM and the boot index, if - * needed. */ - sd->refcnt++; + * possible for this function to fail. Go ahead and update the lookup + * table of the destination WIM and the boot index, if needed. */ hlist_for_each_entry(inode, cur_node, &src_imd->inode_list, i_hlist) { inode_move_ltes_to_table(inode, src_wim->lookup_table, @@ -253,10 +243,9 @@ wimlib_export_image(WIMStruct *src_wim, } if (export_flags & WIMLIB_EXPORT_FLAG_BOOT) - wimlib_set_boot_idx(dest_wim, dest_wim->hdr.image_count); + dest_wim->hdr.boot_idx = dest_wim->hdr.image_count; ret = 0; goto out; - out_xml_delete_image: xml_delete_image(&dest_wim->wim_info, dest_wim->hdr.image_count + 1); out_free_ltes: @@ -265,7 +254,6 @@ out_free_ltes: list_for_each_entry_safe(lte, tmp, <e_list_head, staging_list) free_lookup_table_entry(lte); } - out: if (num_additional_swms) { free_lookup_table(src_wim->lookup_table); diff --git a/src/extract_image.c b/src/extract_image.c index a24f0ab5..85b5af8a 100644 --- a/src/extract_image.c +++ b/src/extract_image.c @@ -112,7 +112,7 @@ extract_regular_file_linked(struct wim_dentry *dentry, size_t i; num_path_components = - get_num_path_components(dentry->full_path) - 1; + get_num_path_components(dentry_full_path(dentry)) - 1; num_output_dir_path_components = get_num_path_components(args->target); @@ -333,7 +333,7 @@ extract_symlink(struct wim_dentry *dentry, if (ret <= 0) { ERROR("Could not read the symbolic link from dentry `%s'", - dentry->full_path); + dentry_full_path(dentry)); return WIMLIB_ERR_INVALID_DENTRY; } ret = symlink(target, output_path); @@ -499,10 +499,12 @@ apply_dentry_normal(struct wim_dentry *dentry, void *arg) if (dentry_is_root(dentry)) { output_path = (tchar*)args->target; } else { + if (!dentry_full_path(dentry)) + return WIMLIB_ERR_NOMEM; output_path = alloca(len * sizeof(tchar) + dentry->full_path_nbytes + sizeof(tchar)); memcpy(output_path, args->target, len * sizeof(tchar)); - memcpy(output_path + len, dentry->full_path, dentry->full_path_nbytes); + memcpy(output_path + len, dentry->_full_path, dentry->full_path_nbytes); len += dentry->full_path_nbytes / sizeof(tchar); output_path[len] = T('\0'); } @@ -526,10 +528,12 @@ apply_dentry_timestamps_normal(struct wim_dentry *dentry, void *arg) if (dentry_is_root(dentry)) { output_path = (tchar*)args->target; } else { + if (!dentry_full_path(dentry)) + return WIMLIB_ERR_NOMEM; output_path = alloca(len * sizeof(tchar) + dentry->full_path_nbytes + sizeof(tchar)); memcpy(output_path, args->target, len * sizeof(tchar)); - memcpy(output_path + len, dentry->full_path, dentry->full_path_nbytes); + memcpy(output_path + len, dentry->_full_path, dentry->full_path_nbytes); len += dentry->full_path_nbytes / sizeof(tchar); output_path[len] = T('\0'); } @@ -553,13 +557,16 @@ maybe_apply_dentry(struct wim_dentry *dentry, void *arg) if (dentry->is_extracted) return 0; + if (!dentry_full_path(dentry)) + return WIMLIB_ERR_NOMEM; + if (args->extract_flags & WIMLIB_EXTRACT_FLAG_NO_STREAMS) if (inode_unnamed_lte_resolved(dentry->d_inode)) return 0; if ((args->extract_flags & WIMLIB_EXTRACT_FLAG_VERBOSE) && args->progress_func) { - args->progress.extract.cur_path = dentry->full_path; + args->progress.extract.cur_path = dentry_full_path(dentry); args->progress_func(WIMLIB_PROGRESS_MSG_EXTRACT_DENTRY, &args->progress); } @@ -782,10 +789,10 @@ extract_single_image(WIMStruct *w, int image, ops = &normal_apply_operations; ret = select_wim_image(w, image); - if (ret != 0) + if (ret) goto out; - inode_list = &w->image_metadata[image - 1].inode_list; + inode_list = &wim_get_current_image_metadata(w)->inode_list; /* Build a list of the streams that need to be extracted */ find_streams_for_extraction(inode_list, &stream_list, diff --git a/src/hardlink.c b/src/hardlink.c index ff42f26f..08f62250 100644 --- a/src/hardlink.c +++ b/src/hardlink.c @@ -184,7 +184,7 @@ print_inode_dentries(const struct wim_inode *inode) { struct wim_dentry *dentry; inode_for_each_dentry(dentry, inode) - tprintf(T("`%"TS"'\n"), dentry->full_path); + print_dentry_full_path(dentry, NULL); } #endif diff --git a/src/join.c b/src/join.c index e566ba28..0d0df255 100644 --- a/src/join.c +++ b/src/join.c @@ -150,7 +150,7 @@ join_wims(WIMStruct **swms, unsigned num_swms, /* Copy the metadata resources from the first SWM part */ joined_wim->hdr.image_count = swms[0]->hdr.image_count; for (i = 0; i < joined_wim->hdr.image_count; i++) { - ret = copy_resource(swms[0]->image_metadata[i].metadata_lte, + ret = copy_resource(swms[0]->image_metadata[i]->metadata_lte, joined_wim); if (ret) return ret; diff --git a/src/lookup_table.c b/src/lookup_table.c index 9502db47..5b57e3ac 100644 --- a/src/lookup_table.c +++ b/src/lookup_table.c @@ -232,7 +232,8 @@ lte_decrement_refcnt(struct wim_lookup_table_entry *lte, wimlib_assert(lte != NULL); wimlib_assert(lte->refcnt != 0); if (--lte->refcnt == 0) { - lookup_table_unlink(table, lte); + if (!lte->unhashed) + lookup_table_unlink(table, lte); #ifdef WITH_FUSE if (lte->num_opened_fds == 0) #endif @@ -267,7 +268,7 @@ for_lookup_table_entry(struct wim_lookup_table *table, { wimlib_assert2(!(lte->resource_entry.flags & WIM_RESHDR_FLAG_METADATA)); ret = visitor(lte, arg); - if (ret != 0) + if (ret) return ret; } } @@ -495,7 +496,7 @@ read_lookup_table(WIMStruct *w) w->current_image + 1, cur_entry->resource_entry.offset); w->image_metadata[ - w->current_image++].metadata_lte = cur_entry; + w->current_image++]->metadata_lte = cur_entry; } else { /* Lookup table entry for a stream that is not a * metadata resource */ @@ -597,7 +598,7 @@ write_lookup_table(WIMStruct *w, int image, struct resource_entry *out_res_entry for (int i = start_image; i <= end_image; i++) { struct wim_lookup_table_entry *metadata_lte; - metadata_lte = w->image_metadata[i - 1].metadata_lte; + metadata_lte = w->image_metadata[i - 1]->metadata_lte; metadata_lte->out_refcnt = 1; metadata_lte->output_resource_entry.flags |= WIM_RESHDR_FLAG_METADATA; ret = write_lookup_table_entry(metadata_lte, out); @@ -922,11 +923,13 @@ lookup_table_total_stream_size(struct wim_lookup_table *table) } void -lookup_table_free_unhashed_streams(struct wim_lookup_table *table) +free_lte_list(struct list_head *list) { struct wim_lookup_table_entry *lte, *tmp; - list_for_each_entry_safe(lte, tmp, table->unhashed_streams, staging_list) + list_for_each_entry_safe(lte, tmp, list, staging_list) { + DEBUG("%p", lte); free_lookup_table_entry(lte); - INIT_LIST_HEAD(table->unhashed_streams); +} + INIT_LIST_HEAD(list); } diff --git a/src/lookup_table.h b/src/lookup_table.h index ab545f2d..62d02d69 100644 --- a/src/lookup_table.h +++ b/src/lookup_table.h @@ -136,7 +136,10 @@ struct wim_lookup_table_entry { u16 part_number; /* See enum resource_location above */ - u16 resource_location; + u16 resource_location : 5; + u8 unique_size : 1; + u8 unhashed : 1; + u8 is_ads : 1; /* (On-disk field) * Number of times this lookup table entry is referenced by dentries. @@ -156,20 +159,17 @@ struct wim_lookup_table_entry { * the full 20 byte hash just to insert the entry in a hash * table. */ size_t hash_short; - }; - - union { - #ifdef WITH_FUSE - u16 num_opened_fds; - #endif - /* This field is used for the special hardlink or symlink image - * extraction mode. In these mode, all identical files are linked - * together, and @extracted_file will be set to the filename of the - * first extracted file containing this stream. */ - tchar *extracted_file; + struct wim_lookup_table_entry **my_ptr; }; + /* When a WIM file is written, out_refcnt starts at 0 and is incremented + * whenever the file resource pointed to by this lookup table entry + * needs to be written. The file resource only need to be written when + * out_refcnt is nonzero, since otherwise it is not referenced by any + * dentries. */ + u32 out_refcnt; + /* Pointers to somewhere where the stream is actually located. See the * comments for the @resource_location field above. */ union { @@ -187,15 +187,20 @@ struct wim_lookup_table_entry { * RESOURCE_IN_STAGING_FILE) */ struct wim_inode *lte_inode; - /* When a WIM file is written, out_refcnt starts at 0 and is incremented - * whenever the file resource pointed to by this lookup table entry - * needs to be written. The file resource only need to be written when - * out_refcnt is nonzero, since otherwise it is not referenced by any - * dentries. */ - u32 out_refcnt; - u32 real_refcnt; + union { + #ifdef WITH_FUSE + u16 num_opened_fds; + #endif + + /* This field is used for the special hardlink or symlink image + * extraction mode. In these mode, all identical files are linked + * together, and @extracted_file will be set to the filename of the + * first extracted file containing this stream. */ + tchar *extracted_file; + }; + union { /* When a WIM file is written, @output_resource_entry is filled * in with the resource entry for the output WIM. This will not @@ -208,6 +213,11 @@ struct wim_lookup_table_entry { struct list_head msg_list; struct list_head inode_list; + + struct { + struct hlist_node hash_list_2; + struct list_head write_streams_list; + }; }; /* List of lookup table entries that correspond to streams that have @@ -349,13 +359,6 @@ inode_unresolve_ltes(struct wim_inode *inode); extern int write_lookup_table_entry(struct wim_lookup_table_entry *lte, void *__out); -static inline struct resource_entry* -wim_metadata_resource_entry(WIMStruct *w) -{ - return &w->image_metadata[ - w->current_image - 1].metadata_lte->resource_entry; -} - static inline struct wim_lookup_table_entry * inode_stream_lte_resolved(const struct wim_inode *inode, unsigned stream_idx) { @@ -474,12 +477,16 @@ lookup_table_total_stream_size(struct wim_lookup_table *table); static inline void lookup_table_insert_unhashed(struct wim_lookup_table *table, - struct wim_lookup_table_entry *lte) + struct wim_lookup_table_entry *lte, + struct wim_lookup_table_entry **my_ptr) { + lte->unhashed = 1; list_add_tail(<e->staging_list, table->unhashed_streams); + lte->my_ptr = my_ptr; + *my_ptr = lte; } extern void -lookup_table_free_unhashed_streams(struct wim_lookup_table *table); +free_lte_list(struct list_head *list); #endif diff --git a/src/metadata_resource.c b/src/metadata_resource.c index 9b5a8305..a9e2e386 100644 --- a/src/metadata_resource.c +++ b/src/metadata_resource.c @@ -105,7 +105,7 @@ read_metadata_resource(WIMStruct *w, struct wim_image_metadata *imd) wimlib_assert(imd->security_data == NULL); ret = read_security_data(buf, metadata_len, &imd->security_data); - if (ret != 0) + if (ret) goto out_free_buf; dentry_offset = (imd->security_data->total_length + 7) & ~7; @@ -147,18 +147,12 @@ read_metadata_resource(WIMStruct *w, struct wim_image_metadata *imd) /* Now read the entire directory entry tree into memory. */ DEBUG("Reading dentry tree"); ret = read_dentry_tree(buf, metadata_len, dentry); - if (ret != 0) - goto out_free_dentry_tree; - - /* Calculate the full paths in the dentry tree. */ - DEBUG("Calculating dentry full paths"); - ret = for_dentry_in_tree(dentry, calculate_dentry_full_path, NULL); - if (ret != 0) + if (ret) goto out_free_dentry_tree; /* Build hash table that maps hard link group IDs to dentry sets */ ret = dentry_tree_fix_inodes(dentry, &inode_list); - if (ret != 0) + if (ret) goto out_free_dentry_tree; if (!w->all_images_verified) { @@ -175,6 +169,7 @@ read_metadata_resource(WIMStruct *w, struct wim_image_metadata *imd) imd->inode_list = inode_list; if (imd->inode_list.first) imd->inode_list.first->pprev = &imd->inode_list.first; + INIT_LIST_HEAD(&imd->unhashed_streams); goto out_free_buf; out_free_dentry_tree: free_dentry_tree(dentry, NULL); @@ -199,7 +194,7 @@ recalculate_security_data_length(struct wim_security_data *sd) * uncompressed data rather a lookup table entry; also writes the SHA1 hash of * the buffer to @hash. */ static int -write_wim_resource_from_buffer(const u8 *buf, u64 buf_size, +write_wim_resource_from_buffer(const void *buf, u64 buf_size, FILE *out_fp, int out_ctype, struct resource_entry *out_res_entry, u8 hash[SHA1_HASH_SIZE]) @@ -211,6 +206,7 @@ write_wim_resource_from_buffer(const u8 *buf, u64 buf_size, lte.resource_entry.original_size = buf_size; lte.resource_location = RESOURCE_IN_ATTACHED_BUFFER; lte.attached_buffer = (void*)buf; + lte.unhashed = 1; zero_out_hash(lte.hash); ret = write_wim_resource(<e, out_fp, out_ctype, out_res_entry, 0); if (ret) @@ -279,7 +275,7 @@ write_metadata_resource(WIMStruct *w) /* Get the lookup table entry for the metadata resource so we can update * it. */ - lte = w->image_metadata[w->current_image - 1].metadata_lte; + lte = wim_get_current_image_metadata(w)->metadata_lte; /* Write the metadata resource to the output WIM using the proper * compression type. The lookup table entry for the metadata resource diff --git a/src/mount_image.c b/src/mount_image.c index 128e1736..4700cb52 100644 --- a/src/mount_image.c +++ b/src/mount_image.c @@ -98,9 +98,6 @@ struct wimfs_context { * filesystem anyway. */ u64 next_ino; - /* List of lookup table entries for files in the staging directory */ - struct list_head staging_list; - /* List of inodes in the mounted image */ struct hlist_head *image_inode_list; @@ -128,7 +125,6 @@ init_wimfs_context(struct wimfs_context *ctx) memset(ctx, 0, sizeof(*ctx)); ctx->unmount_to_daemon_mq = (mqd_t)-1; ctx->daemon_to_unmount_mq = (mqd_t)-1; - INIT_LIST_HEAD(&ctx->staging_list); } #define WIMFS_CTX(fuse_ctx) ((struct wimfs_context*)(fuse_ctx)->private_data) @@ -296,7 +292,7 @@ close_wimfs_fd(struct wimfs_fd *fd) fd->f_inode->i_ino, fd->f_inode->i_num_opened_fds, fd->f_inode->i_num_allocated_fds); ret = lte_put_fd(fd->f_lte, fd); - if (ret != 0) + if (ret) return ret; inode_put_fd(fd->f_inode, fd); @@ -382,7 +378,7 @@ remove_dentry(struct wim_dentry *dentry, lte_decrement_refcnt(lte, lookup_table); } unlink_dentry(dentry); - put_dentry(dentry); + free_dentry(dentry); } static mode_t @@ -651,15 +647,21 @@ extract_resource_to_staging_dir(struct wim_inode *inode, new_lte->staging_file_name = staging_file_name; new_lte->lte_inode = inode; - if (stream_id == 0) - inode->i_lte = new_lte; - else - for (u16 i = 0; i < inode->i_num_ads; i++) - if (inode->i_ads_entries[i].stream_id == stream_id) - inode->i_ads_entries[i].lte = new_lte; + struct wim_lookup_table_entry **my_ptr; + + if (stream_id == 0) { + my_ptr = &inode->i_lte; + } else { + for (u16 i = 0; ; i++) { + wimlib_assert(i < inode->i_num_ads); + if (inode->i_ads_entries[i].stream_id == stream_id) { + my_ptr = &inode->i_ads_entries[i].lte; + break; + } + } + } - lookup_table_insert_unhashed(ctx->wim->lookup_table, new_lte); - list_add_tail(&new_lte->staging_list, &ctx->staging_list); + lookup_table_insert_unhashed(ctx->wim->lookup_table, new_lte, my_ptr); *lte = new_lte; return 0; out_revert_fd_changes: @@ -777,34 +779,6 @@ delete_staging_dir(struct wimfs_context *ctx) return ret; } -static void -inode_update_lte_ptr(struct wim_inode *inode, - struct wim_lookup_table_entry *old_lte, - struct wim_lookup_table_entry *new_lte) -{ - if (inode->i_lte == old_lte) { - inode->i_lte = new_lte; - } else { - for (unsigned i = 0; i < inode->i_num_ads; i++) { - if (inode->i_ads_entries[i].lte == old_lte) { - inode->i_ads_entries[i].lte = new_lte; - break; - } - } - } -} - -static void -free_lte_if_unneeded(struct wim_lookup_table_entry *lte) -{ - - if (wim_resource_size(lte) == 0) { - /* Zero-length stream. No lookup table entry needed. */ - inode_update_lte_ptr(lte->lte_inode, lte, NULL); - free_lookup_table_entry(lte); - } -} - static int inode_close_fds(struct wim_inode *inode) { @@ -830,23 +804,26 @@ rebuild_wim(struct wimfs_context *ctx, int write_flags, int ret; struct wim_lookup_table_entry *lte, *tmp; WIMStruct *w = ctx->wim; + struct wim_image_metadata *imd = wim_get_current_image_metadata(ctx->wim); DEBUG("Closing all staging file descriptors."); - list_for_each_entry_safe(lte, tmp, &ctx->staging_list, staging_list) { + list_for_each_entry_safe(lte, tmp, &imd->unhashed_streams, staging_list) { ret = inode_close_fds(lte->lte_inode); - if (ret != 0) + if (ret) return ret; } DEBUG("Freeing entries for zero-length streams"); - list_for_each_entry_safe(lte, tmp, &ctx->staging_list, staging_list) - free_lte_if_unneeded(lte); + list_for_each_entry_safe(lte, tmp, &imd->unhashed_streams, staging_list) { + if (wim_resource_size(lte) == 0) { + *lte->my_ptr = NULL; + free_lookup_table_entry(lte); + } + } xml_update_image_info(w, w->current_image); - list_splice(&ctx->staging_list, - &w->image_metadata[w->current_image - 1].unhashed_streams); ret = wimlib_overwrite(w, write_flags, 0, progress_func); - if (ret != 0) + if (ret) ERROR("Failed to commit changes to mounted WIM image"); return ret; } @@ -2483,16 +2460,13 @@ wimlib_mount_image(WIMStruct *wim, int image, const char *dir, imd = wim_get_current_image_metadata(wim); - if (imd->root_dentry->refcnt != 1) { + if (imd->refcnt != 1) { ERROR("Cannot mount image that was just exported with " "wimlib_export_image()"); ret = WIMLIB_ERR_INVALID_PARAM; goto out; } - if (imd->inode_list.first) /* May be unneeded? */ - imd->inode_list.first->pprev = &imd->inode_list.first; - if (imd->modified) { ERROR("Cannot mount image that was added " "with wimlib_add_image()"); diff --git a/src/ntfs-apply.c b/src/ntfs-apply.c index 994bbd27..19c78458 100644 --- a/src/ntfs-apply.c +++ b/src/ntfs-apply.c @@ -92,7 +92,7 @@ extract_wim_resource_to_ntfs_attr(const struct wim_lookup_table_entry *lte, * Returns 0 on success, nonzero on failure. */ static int -write_ntfs_data_streams(ntfs_inode *ni, const struct wim_dentry *dentry, +write_ntfs_data_streams(ntfs_inode *ni, struct wim_dentry *dentry, union wimlib_progress_info *progress_info) { int ret = 0; @@ -105,7 +105,7 @@ write_ntfs_data_streams(ntfs_inode *ni, const struct wim_dentry *dentry, DEBUG("Writing %u NTFS data stream%s for `%s'", inode->i_num_ads + 1, (inode->i_num_ads == 0 ? "" : "s"), - dentry->full_path); + dentry_full_path(dentry)); lte = inode->i_lte; while (1) { @@ -125,7 +125,7 @@ write_ntfs_data_streams(ntfs_inode *ni, const struct wim_dentry *dentry, ERROR_WITH_ERRNO("Failed to create named data " "stream for extracted file " "`%s'", - dentry->full_path); + dentry_full_path(dentry)); ret = WIMLIB_ERR_NTFS_3G; break; @@ -142,7 +142,7 @@ write_ntfs_data_streams(ntfs_inode *ni, const struct wim_dentry *dentry, if (!na) { ERROR_WITH_ERRNO("Failed to open a data stream of " "extracted file `%s'", - dentry->full_path); + dentry_full_path(dentry)); ret = WIMLIB_ERR_NTFS_3G; break; } @@ -184,21 +184,24 @@ write_ntfs_data_streams(ntfs_inode *ni, const struct wim_dentry *dentry, /* Open the NTFS inode that corresponds to the parent of a WIM dentry. Returns * the opened inode, or NULL on failure. */ static ntfs_inode * -dentry_open_parent_ni(const struct wim_dentry *dentry, ntfs_volume *vol) +dentry_open_parent_ni(struct wim_dentry *dentry, ntfs_volume *vol) { char *p; const char *dir_name; ntfs_inode *dir_ni; char orig; - p = dentry->full_path + dentry->full_path_nbytes; + if (!dentry_full_path(dentry)) + return NULL; + + p = dentry->_full_path + dentry->full_path_nbytes; do { p--; } while (*p != '/'); orig = *p; *p = '\0'; - dir_name = dentry->full_path; + dir_name = dentry->_full_path; dir_ni = ntfs_pathname_to_inode(vol, NULL, dir_name); if (!dir_ni) { ERROR_WITH_ERRNO("Could not find NTFS inode for `%s'", @@ -224,7 +227,7 @@ dentry_open_parent_ni(const struct wim_dentry *dentry, ntfs_volume *vol) * Return 0 on success, nonzero on failure. dir_ni is closed either way. */ static int -apply_ntfs_hardlink(const struct wim_dentry *from_dentry, +apply_ntfs_hardlink(struct wim_dentry *from_dentry, const struct wim_inode *inode, ntfs_inode *dir_ni) { @@ -240,7 +243,7 @@ apply_ntfs_hardlink(const struct wim_dentry *from_dentry, } DEBUG("Extracting NTFS hard link `%s' => `%s'", - from_dentry->full_path, inode->i_extracted_file); + dentry_full_path(from_dentry), inode->i_extracted_file); to_ni = ntfs_pathname_to_inode(vol, NULL, inode->i_extracted_file); if (!to_ni) { @@ -262,7 +265,7 @@ apply_ntfs_hardlink(const struct wim_dentry *from_dentry, ret |= ntfs_inode_close(to_ni); if (ret) { ERROR_WITH_ERRNO("Could not create hard link `%s' => `%s'", - from_dentry->full_path, + dentry_full_path(from_dentry), inode->i_extracted_file); ret = WIMLIB_ERR_NTFS_3G; } @@ -283,7 +286,7 @@ apply_ntfs_hardlink(const struct wim_dentry *from_dentry, static int apply_file_attributes_and_security_data(ntfs_inode *ni, ntfs_inode *dir_ni, - const struct wim_dentry *dentry, + struct wim_dentry *dentry, const WIMStruct *w, int extract_flags) { @@ -295,7 +298,7 @@ apply_file_attributes_and_security_data(ntfs_inode *ni, inode = dentry->d_inode; DEBUG("Setting NTFS file attributes on `%s' to %#"PRIx32, - dentry->full_path, inode->i_attributes); + dentry_full_path(dentry), inode->i_attributes); attributes_le32 = cpu_to_le32(inode->i_attributes); memset(&ctx, 0, sizeof(ctx)); @@ -306,7 +309,7 @@ apply_file_attributes_and_security_data(ntfs_inode *ni, sizeof(u32), 0); if (ret != 0) { ERROR("Failed to set NTFS file attributes on `%s'", - dentry->full_path); + dentry_full_path(dentry)); return WIMLIB_ERR_NTFS_3G; } if (inode->i_security_id != -1 && @@ -319,7 +322,7 @@ apply_file_attributes_and_security_data(ntfs_inode *ni, wimlib_assert(inode->i_security_id < sd->num_entries); desc = (const char *)sd->descriptors[inode->i_security_id]; DEBUG("Applying security descriptor %d to `%s'", - inode->i_security_id, dentry->full_path); + inode->i_security_id, dentry_full_path(dentry)); ret = ntfs_xattr_system_setxattr(&ctx, XATTR_NTFS_ACL, ni, dir_ni, desc, @@ -327,7 +330,7 @@ apply_file_attributes_and_security_data(ntfs_inode *ni, if (ret != 0) { ERROR_WITH_ERRNO("Failed to set security data on `%s'", - dentry->full_path); + dentry_full_path(dentry)); return WIMLIB_ERR_NTFS_3G; } } @@ -339,7 +342,7 @@ apply_file_attributes_and_security_data(ntfs_inode *ni, * point) to a NTFS inode. */ static int -apply_reparse_data(ntfs_inode *ni, const struct wim_dentry *dentry, +apply_reparse_data(ntfs_inode *ni, struct wim_dentry *dentry, union wimlib_progress_info *progress_info) { struct wim_lookup_table_entry *lte; @@ -347,17 +350,17 @@ apply_reparse_data(ntfs_inode *ni, const struct wim_dentry *dentry, lte = inode_unnamed_lte_resolved(dentry->d_inode); - DEBUG("Applying reparse data to `%s'", dentry->full_path); + DEBUG("Applying reparse data to `%s'", dentry_full_path(dentry)); if (!lte) { ERROR("Could not find reparse data for `%s'", - dentry->full_path); + dentry_full_path(dentry)); return WIMLIB_ERR_INVALID_DENTRY; } if (wim_resource_size(lte) >= 0xffff) { ERROR("Reparse data of `%s' is too long (%"PRIu64" bytes)", - dentry->full_path, wim_resource_size(lte)); + dentry_full_path(dentry), wim_resource_size(lte)); return WIMLIB_ERR_INVALID_DENTRY; } @@ -376,7 +379,7 @@ apply_reparse_data(ntfs_inode *ni, const struct wim_dentry *dentry, wim_resource_size(lte) + 8, 0); if (ret) { ERROR_WITH_ERRNO("Failed to set NTFS reparse data on `%s'", - dentry->full_path); + dentry_full_path(dentry)); return WIMLIB_ERR_NTFS_3G; } progress_info->extract.completed_bytes += wim_resource_size(lte); @@ -419,8 +422,11 @@ do_apply_dentry_ntfs(struct wim_dentry *dentry, ntfs_inode *dir_ni, * extracted yet, so go ahead and extract the * first one. */ FREE(inode->i_extracted_file); - inode->i_extracted_file = STRDUP(dentry->full_path); - if (!inode->i_extracted_file) { + const tchar *full_path = dentry_full_path(dentry); + + if (!full_path || + !(inode->i_extracted_file = STRDUP(full_path))) + { ret = WIMLIB_ERR_NOMEM; goto out_close_dir_ni; } @@ -438,7 +444,7 @@ do_apply_dentry_ntfs(struct wim_dentry *dentry, ntfs_inode *dir_ni, if (!ni) { ERROR_WITH_ERRNO("Could not create NTFS inode for `%s'", - dentry->full_path); + dentry_full_path(dentry)); ret = WIMLIB_ERR_NTFS_3G; goto out_close_dir_ni; } @@ -476,14 +482,14 @@ do_apply_dentry_ntfs(struct wim_dentry *dentry, ntfs_inode *dir_ni, goto out_close_dir_ni; DEBUG("Setting short (DOS) name of `%s' to %s", - dentry->full_path, short_name_mbs); + dentry_full_path(dentry), short_name_mbs); ret = ntfs_set_ntfs_dos_name(ni, dir_ni, short_name_mbs, short_name_mbs_nbytes, 0); FREE(short_name_mbs); if (ret != 0) { ERROR_WITH_ERRNO("Could not set DOS (short) name for `%s'", - dentry->full_path); + dentry_full_path(dentry)); ret = WIMLIB_ERR_NTFS_3G; } /* inodes have been closed by ntfs_set_ntfs_dos_name(). */ @@ -496,14 +502,15 @@ out_close_dir_ni: if (ret == 0) ret = WIMLIB_ERR_NTFS_3G; ERROR_WITH_ERRNO("Failed to close inode for `%s'", - dentry->full_path); + dentry_full_path(dentry)); } } if (ntfs_inode_close(dir_ni)) { if (ret == 0) ret = WIMLIB_ERR_NTFS_3G; ERROR_WITH_ERRNO("Failed to close inode of directory " - "containing `%s'", dentry->full_path); + "containing `%s'", + dentry_full_path(dentry)); } } out: @@ -511,7 +518,7 @@ out: } static int -apply_root_dentry_ntfs(const struct wim_dentry *dentry, +apply_root_dentry_ntfs(struct wim_dentry *dentry, ntfs_volume *vol, const WIMStruct *w, int extract_flags) { @@ -635,12 +642,12 @@ apply_dentry_timestamps_ntfs(struct wim_dentry *dentry, void *arg) ntfs_inode *ni; int ret; - DEBUG("Setting timestamps on `%s'", dentry->full_path); + DEBUG("Setting timestamps on `%s'", dentry_full_path(dentry)); - ni = ntfs_pathname_to_inode(vol, NULL, dentry->full_path); + ni = ntfs_pathname_to_inode(vol, NULL, dentry_full_path(dentry)); if (!ni) { ERROR_WITH_ERRNO("Could not find NTFS inode for `%s'", - dentry->full_path); + dentry_full_path(dentry)); return WIMLIB_ERR_NTFS_3G; } @@ -651,7 +658,7 @@ apply_dentry_timestamps_ntfs(struct wim_dentry *dentry, void *arg) ret = ntfs_inode_set_times(ni, (const char*)buf, 3 * sizeof(u64), 0); if (ret != 0) { ERROR_WITH_ERRNO("Failed to set NTFS timestamps on `%s'", - dentry->full_path); + dentry_full_path(dentry)); ret = WIMLIB_ERR_NTFS_3G; } @@ -659,7 +666,7 @@ apply_dentry_timestamps_ntfs(struct wim_dentry *dentry, void *arg) if (ret == 0) ret = WIMLIB_ERR_NTFS_3G; ERROR_WITH_ERRNO("Failed to close NTFS inode for `%s'", - dentry->full_path); + dentry_full_path(dentry)); } return ret; } diff --git a/src/ntfs-capture.c b/src/ntfs-capture.c index 5c96c1b2..43d7ca2e 100644 --- a/src/ntfs-capture.c +++ b/src/ntfs-capture.c @@ -165,6 +165,7 @@ capture_ntfs_streams(struct wim_inode *inode, { u64 data_size = ntfs_get_attribute_value_length(actx->attr); u64 name_length = actx->attr->name_length; + struct wim_lookup_table_entry **my_ptr; if (data_size == 0) { if (errno != 0) { ERROR_WITH_ERRNO("Failed to get size of attribute of " @@ -218,12 +219,12 @@ capture_ntfs_streams(struct wim_inode *inode, /* Unnamed data stream. Put the reference to it in the * dentry's inode. */ if (inode->i_lte) { - WARNING("Found two un-named data streams for " - "`%s'", path); - free_lookup_table_entry(lte); - } else { - inode->i_lte = lte; + ERROR("Found two un-named data streams for `%s'", + path); + ret = WIMLIB_ERR_NTFS_3G; + goto out_free_lte; } + my_ptr = &inode->i_lte; } else { /* Named data stream. Put the reference to it in the * alternate data stream entries */ @@ -235,10 +236,9 @@ capture_ntfs_streams(struct wim_inode *inode, if (!new_ads_entry) goto out_free_lte; wimlib_assert(new_ads_entry->stream_name_nbytes == name_length * 2); - new_ads_entry->lte = lte; + my_ptr = &new_ads_entry->lte; } - if (lte) - lookup_table_insert_unhashed(lookup_table, lte); + lookup_table_insert_unhashed(lookup_table, lte, my_ptr); } ret = 0; goto out_put_actx; diff --git a/src/resource.c b/src/resource.c index 90e45ca1..344ba0d2 100644 --- a/src/resource.c +++ b/src/resource.c @@ -544,9 +544,10 @@ read_partial_wim_resource(const struct wim_lookup_table_entry *lte, goto out_release_fp; } if (cb) { - char buf[min(32768, size)]; + /* Send data to callback function */ + u8 buf[min(WIM_CHUNK_SIZE, size)]; while (size) { - size_t bytes_to_read = min(32768, size); + size_t bytes_to_read = min(WIM_CHUNK_SIZE, size); size_t bytes_read = fread(buf, 1, bytes_to_read, wim_fp); if (bytes_read != bytes_to_read) @@ -554,8 +555,10 @@ read_partial_wim_resource(const struct wim_lookup_table_entry *lte, ret = cb(buf, bytes_read, ctx_or_buf); if (ret) goto out_release_fp; + size -= bytes_read; } } else { + /* Send data directly to a buffer */ if (fread(ctx_or_buf, 1, size, wim_fp) != size) goto read_error; } @@ -572,7 +575,6 @@ out: if (ret) { if (errno == 0) errno = EIO; - ret = -1; } return ret; } @@ -618,16 +620,17 @@ read_file_on_disk_prefix(const struct wim_lookup_table_entry *lte, } if (cb) { /* Send data to callback function */ - char buf[min(32768, size)]; + u8 buf[min(WIM_CHUNK_SIZE, size)]; size_t bytes_to_read; while (size) { - bytes_to_read = min(32768, size); + bytes_to_read = min(WIM_CHUNK_SIZE, size); bytes_read = full_read(fd, buf, bytes_to_read); if (bytes_read != bytes_to_read) goto read_error; ret = cb(buf, bytes_read, ctx_or_buf); if (ret) goto out_close; + size -= bytes_read; } } else { /* Send data directly to a buffer */ diff --git a/src/security.c b/src/security.c index 94660de8..f4dfb8e8 100644 --- a/src/security.c +++ b/src/security.c @@ -186,7 +186,6 @@ read_security_data(const u8 metadata_resource[], u64 metadata_resource_len, } sd->sizes = NULL; sd->descriptors = NULL; - sd->refcnt = 1; p = metadata_resource; p = get_u32(p, &sd->total_length); @@ -437,17 +436,14 @@ void free_security_data(struct wim_security_data *sd) { if (sd) { - wimlib_assert(sd->refcnt != 0); - if (--sd->refcnt == 0) { - u8 **descriptors = sd->descriptors; - u32 num_entries = sd->num_entries; - if (descriptors) - while (num_entries--) - FREE(*descriptors++); - FREE(sd->sizes); - FREE(sd->descriptors); - FREE(sd); - } + u8 **descriptors = sd->descriptors; + u32 num_entries = sd->num_entries; + if (descriptors) + while (num_entries--) + FREE(*descriptors++); + FREE(sd->sizes); + FREE(sd->descriptors); + FREE(sd); } } diff --git a/src/split.c b/src/split.c index c0557837..660ffaf7 100644 --- a/src/split.c +++ b/src/split.c @@ -189,9 +189,9 @@ wimlib_split(WIMStruct *w, const tchar *swm_name, for (int i = 0; i < w->hdr.image_count; i++) { struct wim_lookup_table_entry *metadata_lte; - metadata_lte = w->image_metadata[i].metadata_lte; + metadata_lte = w->image_metadata[i]->metadata_lte; ret = copy_resource(metadata_lte, w); - if (ret != 0) + if (ret) goto out; args.size_remaining -= metadata_lte->resource_entry.size; args.progress.split.completed_bytes += metadata_lte->resource_entry.size; diff --git a/src/verify.c b/src/verify.c index 201132ae..f9808ada 100644 --- a/src/verify.c +++ b/src/verify.c @@ -35,8 +35,8 @@ verify_inode(struct wim_inode *inode, const WIMStruct *w) { const struct wim_lookup_table *table = w->lookup_table; const struct wim_security_data *sd = wim_const_security_data(w); - const struct wim_dentry *first_dentry = inode_first_dentry(inode); - const struct wim_dentry *dentry; + struct wim_dentry *first_dentry = inode_first_dentry(inode); + struct wim_dentry *dentry; int ret = WIMLIB_ERR_INVALID_DENTRY; /* Check the security ID. -1 is valid and means "no security @@ -44,14 +44,14 @@ verify_inode(struct wim_inode *inode, const WIMStruct *w) * image's security descriptors table. */ if (inode->i_security_id < -1) { ERROR("Dentry `%"TS"' has an invalid security ID (%d)", - first_dentry->full_path, inode->i_security_id); + dentry_full_path(first_dentry), inode->i_security_id); goto out; } if (inode->i_security_id >= sd->num_entries) { ERROR("Dentry `%"TS"' has an invalid security ID (%d) " "(there are only %u entries in the security table)", - first_dentry->full_path, inode->i_security_id, + dentry_full_path(first_dentry), inode->i_security_id, sd->num_entries); goto out; } @@ -70,7 +70,7 @@ verify_inode(struct wim_inode *inode, const WIMStruct *w) if (!lte && !is_zero_hash(hash)) { ERROR("Could not find lookup table entry for stream " "%u of dentry `%"TS"'", - i, first_dentry->full_path); + i, dentry_full_path(first_dentry)); goto out; } if (lte) @@ -88,21 +88,21 @@ verify_inode(struct wim_inode *inode, const WIMStruct *w) } if (num_unnamed_streams > 1) { ERROR("Dentry `%"TS"' has multiple (%u) un-named streams", - first_dentry->full_path, num_unnamed_streams); + dentry_full_path(first_dentry), num_unnamed_streams); goto out; } /* Files cannot have multiple DOS names, even if they have multiple * names in multiple directories (i.e. hard links). * Source: NTFS-3g authors. */ - const struct wim_dentry *dentry_with_dos_name = NULL; + struct wim_dentry *dentry_with_dos_name = NULL; inode_for_each_dentry(dentry, inode) { if (dentry_has_short_name(dentry)) { if (dentry_with_dos_name) { ERROR("Hard-linked file has a DOS name at " "both `%"TS"' and `%"TS"'", - dentry_with_dos_name->full_path, - dentry->full_path); + dentry_full_path(dentry_with_dos_name), + dentry_full_path(dentry)); goto out; } dentry_with_dos_name = dentry; @@ -112,7 +112,7 @@ verify_inode(struct wim_inode *inode, const WIMStruct *w) /* Directories with multiple links have not been tested. XXX */ if (inode->i_nlink > 1 && inode->i_attributes & FILE_ATTRIBUTE_DIRECTORY) { ERROR("Hard-linked directory `%"TS"' is unsupported", - first_dentry->full_path); + dentry_full_path(first_dentry)); goto out; } @@ -153,7 +153,7 @@ verify_dentry(struct wim_dentry *dentry, void *wim) } else { if (!dentry_has_long_name(dentry)) { ERROR("Dentry `%"TS"' has no long name!", - dentry->full_path); + dentry_full_path(dentry)); return WIMLIB_ERR_INVALID_DENTRY; } } @@ -220,8 +220,6 @@ wim_run_full_verifications(WIMStruct *w) ret = for_image(w, WIMLIB_ALL_IMAGES, image_run_full_verifications); if (ret == 0) { unsigned long num_ltes_with_bogus_refcnt = 0; - for (int i = 0; i < w->hdr.image_count; i++) - w->image_metadata[i].metadata_lte->real_refcnt++; for_lookup_table_entry(w->lookup_table, lte_fix_refcnt, &num_ltes_with_bogus_refcnt); if (num_ltes_with_bogus_refcnt != 0) { diff --git a/src/wim.c b/src/wim.c index 6859d5c7..ed20145e 100644 --- a/src/wim.c +++ b/src/wim.c @@ -200,14 +200,11 @@ select_wim_image(WIMStruct *w, int image) imd = wim_get_current_image_metadata(w); if (!imd->modified) { DEBUG("Freeing image %u", w->current_image); - destroy_image_metadata(imd, NULL); - imd->root_dentry = NULL; - imd->security_data = NULL; - INIT_HLIST_HEAD(&imd->inode_list); + destroy_image_metadata(imd, NULL, false); } } w->current_image = image; - imd = &w->image_metadata[image - 1]; + imd = wim_get_current_image_metadata(w); if (imd->root_dentry) { ret = 0; } else { @@ -375,17 +372,6 @@ wimlib_set_boot_idx(WIMStruct *w, int boot_idx) if (boot_idx < 0 || boot_idx > w->hdr.image_count) return WIMLIB_ERR_INVALID_IMAGE; w->hdr.boot_idx = boot_idx; - - if (boot_idx == 0) { - memset(&w->hdr.boot_metadata_res_entry, 0, - sizeof(struct resource_entry)); - } else { - memcpy(&w->hdr.boot_metadata_res_entry, - &w->image_metadata[ - boot_idx - 1].metadata_lte->resource_entry, - sizeof(struct resource_entry)); - } - return 0; } @@ -479,14 +465,9 @@ begin_read(WIMStruct *w, const tchar *in_wim_path, int open_flags, } if (w->hdr.image_count != 0 && w->hdr.part_number == 1) { - w->image_metadata = CALLOC(w->hdr.image_count, - sizeof(struct wim_image_metadata)); - - if (!w->image_metadata) { - ERROR("Failed to allocate memory for %u image metadata structures", - w->hdr.image_count); + w->image_metadata = new_image_metadata_array(w->hdr.image_count); + if (!w->image_metadata) return WIMLIB_ERR_NOMEM; - } } ret = read_lookup_table(w); @@ -539,12 +520,94 @@ wimlib_open_wim(const tchar *wim_file, int open_flags, void destroy_image_metadata(struct wim_image_metadata *imd, - struct wim_lookup_table *table) + struct wim_lookup_table *table, + bool free_metadata_lte) { free_dentry_tree(imd->root_dentry, table); + imd->root_dentry = NULL; free_security_data(imd->security_data); - if (table) + imd->security_data = NULL; + + if (free_metadata_lte) { free_lookup_table_entry(imd->metadata_lte); + imd->metadata_lte = NULL; + } + INIT_LIST_HEAD(&imd->unhashed_streams); + INIT_HLIST_HEAD(&imd->inode_list); +} + +void +put_image_metadata(struct wim_image_metadata *imd, + struct wim_lookup_table *table) +{ + if (imd && --imd->refcnt == 0) { + destroy_image_metadata(imd, table, true); + FREE(imd); + } +} + +/* Appends the specified image metadata structure to the array of image metadata + * for a WIM, and increments the image count. */ +int +append_image_metadata(WIMStruct *w, struct wim_image_metadata *imd) +{ + struct wim_image_metadata **imd_array; + + DEBUG("Reallocating image metadata array for image_count = %u", + w->hdr.image_count + 1); + imd_array = REALLOC(w->image_metadata, + sizeof(w->image_metadata[0]) * (w->hdr.image_count + 1)); + + if (!imd_array) + return WIMLIB_ERR_NOMEM; + w->image_metadata = imd_array; + imd_array[w->hdr.image_count++] = imd; + return 0; +} + + +struct wim_image_metadata * +new_image_metadata() +{ + struct wim_image_metadata *imd; + + imd = CALLOC(1, sizeof(*imd)); + if (imd) { + imd->refcnt = 1; + INIT_HLIST_HEAD(&imd->inode_list); + INIT_LIST_HEAD(&imd->unhashed_streams); + DEBUG("Created new image metadata (refcnt=1)"); + } else { + ERROR_WITH_ERRNO("Failed to allocate new image metadata structure"); + } + return imd; +} + +struct wim_image_metadata ** +new_image_metadata_array(unsigned num_images) +{ + struct wim_image_metadata **imd_array; + + DEBUG("Creating new image metadata array for %u images", + num_images); + + imd_array = CALLOC(num_images, sizeof(imd_array[0])); + + if (!imd_array) { + ERROR("Failed to allocate memory for %u image metadata structures", + num_images); + return NULL; + } + for (unsigned i = 0; i < num_images; i++) { + imd_array[i] = new_image_metadata(); + if (!imd_array[i]) { + for (unsigned j = 0; j < i; j++) + put_image_metadata(imd_array[j], NULL); + FREE(imd_array); + return NULL; + } + } + return imd_array; } /* Frees the memory for the WIMStruct, including all internal memory; also @@ -577,10 +640,8 @@ wimlib_free(WIMStruct *w) FREE(w->xml_data); free_wim_info(w->wim_info); if (w->image_metadata) { - for (unsigned i = 0; i < w->hdr.image_count; i++) { - destroy_image_metadata(&w->image_metadata[i], NULL); - free_lookup_table_entry(w->image_metadata[i].metadata_lte); - } + for (unsigned i = 0; i < w->hdr.image_count; i++) + put_image_metadata(w->image_metadata[i], NULL); FREE(w->image_metadata); } #ifdef WITH_NTFS_3G diff --git a/src/wimlib.h b/src/wimlib.h index f0be7acd..ea5966ce 100644 --- a/src/wimlib.h +++ b/src/wimlib.h @@ -899,7 +899,8 @@ struct wimlib_modify_command { * added at the end to maintain a compatible ABI, except when it's being broken * anyway. */ enum wimlib_error_code { - WIMLIB_ERR_ALREADY_LOCKED = 1, + WIMLIB_ERR_SUCCESS = 0, + WIMLIB_ERR_ALREADY_LOCKED, WIMLIB_ERR_COMPRESSED_LOOKUP_TABLE, WIMLIB_ERR_DECOMPRESSION, WIMLIB_ERR_DELETE_STAGING_DIR, @@ -949,7 +950,6 @@ enum wimlib_error_code { WIMLIB_ERR_SPLIT_INVALID, WIMLIB_ERR_SPLIT_UNSUPPORTED, WIMLIB_ERR_STAT, - WIMLIB_ERR_SUCCESS = 0, WIMLIB_ERR_TIMEOUT, WIMLIB_ERR_UNICODE_STRING_NOT_REPRESENTABLE, WIMLIB_ERR_UNKNOWN_VERSION, diff --git a/src/wimlib_internal.h b/src/wimlib_internal.h index 61a70f1c..18f02738 100644 --- a/src/wimlib_internal.h +++ b/src/wimlib_internal.h @@ -234,15 +234,13 @@ struct wim_security_data { /* Array of descriptors. */ u8 **descriptors; - - /* Keep track of how many WIMs reference this security data (used when - * exporting images between WIMs) */ - u32 refcnt; }; /* Metadata for a WIM image */ struct wim_image_metadata { + unsigned long refcnt; + /* Pointer to the root dentry of the image. */ struct wim_dentry *root_dentry; @@ -297,7 +295,7 @@ struct WIMStruct { struct wim_info *wim_info; /* Array of the image metadata, one for each image in the WIM. */ - struct wim_image_metadata *image_metadata; + struct wim_image_metadata **image_metadata; /* The header of the WIM file. */ struct wim_header hdr; @@ -321,27 +319,34 @@ struct WIMStruct { /* Inline utility functions for WIMStructs. */ +static inline struct wim_image_metadata * +wim_get_current_image_metadata(WIMStruct *w) +{ + return w->image_metadata[w->current_image - 1]; +} + +static inline const struct wim_image_metadata * +wim_get_const_current_image_metadata(const WIMStruct *w) +{ + return w->image_metadata[w->current_image - 1]; +} + static inline struct wim_dentry * wim_root_dentry(WIMStruct *w) { - return w->image_metadata[w->current_image - 1].root_dentry; + return wim_get_current_image_metadata(w)->root_dentry; } static inline struct wim_security_data * wim_security_data(WIMStruct *w) { - return w->image_metadata[w->current_image - 1].security_data; + return wim_get_current_image_metadata(w)->security_data; } + static inline const struct wim_security_data * wim_const_security_data(const WIMStruct *w) { - return w->image_metadata[w->current_image - 1].security_data; -} - -static inline struct wim_image_metadata * -wim_get_current_image_metadata(WIMStruct *w) -{ - return &w->image_metadata[w->current_image - 1]; + return wim_get_const_current_image_metadata(w)->security_data; } /* Nonzero if a struct resource_entry indicates a compressed resource. */ @@ -358,11 +363,6 @@ exclude_path(const tchar *path, size_t path_len, const struct wimlib_capture_config *config, bool exclude_prefix); -extern int -add_new_dentry_tree(WIMStruct *dest_wim, struct wim_dentry *root, - struct wim_security_data *sd); - - /* extract_image.c */ /* Internal use only */ @@ -612,7 +612,21 @@ for_image(WIMStruct *w, int image, int (*visitor)(WIMStruct *)); extern void destroy_image_metadata(struct wim_image_metadata *imd, - struct wim_lookup_table *lt); + struct wim_lookup_table *table, + bool free_metadata_lte); + +extern void +put_image_metadata(struct wim_image_metadata *imd, + struct wim_lookup_table *table); + +extern int +append_image_metadata(WIMStruct *w, struct wim_image_metadata *imd); + +extern struct wim_image_metadata * +new_image_metadata(); + +extern struct wim_image_metadata ** +new_image_metadata_array(unsigned num_images); /* write.c */ diff --git a/src/win32.c b/src/win32.c index b3aec1cb..8110d4fb 100644 --- a/src/win32.c +++ b/src/win32.c @@ -585,12 +585,14 @@ win32_capture_stream(const wchar_t *path, spath = NULL; lte->resource_location = RESOURCE_WIN32; lte->resource_entry.original_size = (u64)dat->StreamSize.QuadPart; - lookup_table_insert_unhashed(lookup_table, lte); + struct wim_lookup_table_entry **my_ptr; if (is_named_stream) - ads_entry->lte = lte; + my_ptr = &ads_entry->lte; else - inode->i_lte = lte; + my_ptr = &inode->i_lte; + + lookup_table_insert_unhashed(lookup_table, lte, my_ptr); out_free_spath: FREE(spath); out: diff --git a/src/write.c b/src/write.c index 9835dba0..55dbbc12 100644 --- a/src/write.c +++ b/src/write.c @@ -378,12 +378,12 @@ write_wim_resource(struct wim_lookup_table_entry *lte, ret = read_resource_prefix(lte, wim_resource_size(lte), write_resource_cb, &write_ctx, 0); - /* Verify SHA1 message digest of the resource, Or, if the hash we had - * before is all 0's, just re-set it to be the new hash. */ + /* Verify SHA1 message digest of the resource, or set the hash for the + * first time. */ if (write_ctx.doing_sha) { u8 md[SHA1_HASH_SIZE]; sha1_final(md, &write_ctx.sha_ctx); - if (is_zero_hash(lte->hash)) { + if (lte->unhashed) { copy_hash(lte->hash, md); } else if (!hashes_equal(md, lte->hash)) { ERROR("WIM resource has incorrect hash!"); @@ -599,8 +599,35 @@ do_write_streams_progress(union wimlib_progress_info *progress, } } +static int +sha1_chunk(const void *buf, size_t len, void *ctx) +{ + sha1_update(ctx, buf, len); + return 0; +} + +static int +sha1_resource(struct wim_lookup_table_entry *lte) +{ + int ret; + SHA_CTX sha_ctx; + + sha1_init(&sha_ctx); + ret = read_resource_prefix(lte, wim_resource_size(lte), + sha1_chunk, &sha_ctx, 0); + if (ret == 0) + sha1_final(lte->hash, &sha_ctx); + return ret; +} + +enum { + STREAMS_MERGED = 0, + STREAMS_NOT_MERGED = 1, +}; + static int do_write_stream_list(struct list_head *my_resources, + struct wim_lookup_table *lookup_table, FILE *out_fp, int out_ctype, wimlib_progress_func_t progress_func, @@ -608,18 +635,53 @@ do_write_stream_list(struct list_head *my_resources, int write_resource_flags) { int ret; - struct wim_lookup_table_entry *lte, *tmp; + struct wim_lookup_table_entry *lte; + + while (!list_empty(my_resources)) { + lte = container_of(my_resources->next, + struct wim_lookup_table_entry, + write_streams_list); + list_del(<e->write_streams_list); + if (lte->unhashed && !lte->unique_size) { + struct wim_lookup_table_entry *duplicate_lte; + + ret = sha1_resource(lte); + if (ret) + return ret; + list_del(<e->staging_list); + + duplicate_lte = __lookup_resource(lookup_table, lte->hash); + if (duplicate_lte) { + bool new_stream = (duplicate_lte->out_refcnt == 0); + duplicate_lte->refcnt += lte->refcnt; + duplicate_lte->out_refcnt += lte->refcnt; + free_lookup_table_entry(lte); + + if (new_stream) + lte = duplicate_lte; + else + continue; + } else { + lookup_table_insert(lookup_table, lte); + lte->out_refcnt = lte->refcnt; + lte->unhashed = 0; + } + } + + wimlib_assert(lte->out_refcnt != 0); - list_for_each_entry_safe(lte, tmp, my_resources, staging_list) { ret = write_wim_resource(lte, out_fp, out_ctype, <e->output_resource_entry, write_resource_flags); - if (ret != 0) + if (ret) return ret; - list_del(<e->staging_list); - + if (lte->unhashed) { + wimlib_assert(__lookup_resource(lookup_table, lte->hash) == NULL); + lookup_table_insert(lookup_table, lte); + lte->unhashed = 0; + } do_write_streams_progress(progress, progress_func, wim_resource_size(lte)); @@ -629,6 +691,7 @@ do_write_stream_list(struct list_head *my_resources, static int write_stream_list_serial(struct list_head *stream_list, + struct wim_lookup_table *lookup_table, FILE *out_fp, int out_ctype, int write_flags, @@ -644,7 +707,9 @@ write_stream_list_serial(struct list_head *stream_list, progress->write_streams.num_threads = 1; if (progress_func) progress_func(WIMLIB_PROGRESS_MSG_WRITE_STREAMS, progress); - return do_write_stream_list(stream_list, out_fp, + return do_write_stream_list(stream_list, + lookup_table, + out_fp, out_ctype, progress_func, progress, write_resource_flags); } @@ -1112,6 +1177,7 @@ get_default_num_threads() static int write_stream_list_parallel(struct list_head *stream_list, + struct wim_lookup_table *lookup_table, FILE *out_fp, int out_ctype, int write_flags, @@ -1206,6 +1272,7 @@ out_destroy_res_to_compress_queue: out_serial: WARNING("Falling back to single-threaded compression"); return write_stream_list_serial(stream_list, + lookup_table, out_fp, out_ctype, write_flags, @@ -1220,18 +1287,19 @@ out_serial: * @out_ctype and up to @num_threads compressor threads. */ static int -write_stream_list(struct list_head *stream_list, FILE *out_fp, - int out_ctype, int write_flags, - unsigned num_threads, - wimlib_progress_func_t progress_func) +write_stream_list(struct list_head *stream_list, + struct wim_lookup_table *lookup_table, + FILE *out_fp, int out_ctype, int write_flags, + unsigned num_threads, wimlib_progress_func_t progress_func) { struct wim_lookup_table_entry *lte; size_t num_streams = 0; u64 total_bytes = 0; u64 total_compression_bytes = 0; union wimlib_progress_info progress; + int ret; - list_for_each_entry(lte, stream_list, staging_list) { + list_for_each_entry(lte, stream_list, write_streams_list) { num_streams++; total_bytes += wim_resource_size(lte); if (out_ctype != WIMLIB_COMPRESSION_TYPE_NONE @@ -1251,21 +1319,24 @@ write_stream_list(struct list_head *stream_list, FILE *out_fp, #ifdef ENABLE_MULTITHREADED_COMPRESSION if (total_compression_bytes >= 1000000 && num_threads != 1) - return write_stream_list_parallel(stream_list, - out_fp, - out_ctype, - write_flags, - num_threads, - progress_func, - &progress); + ret = write_stream_list_parallel(stream_list, + lookup_table, + out_fp, + out_ctype, + write_flags, + num_threads, + progress_func, + &progress); else #endif - return write_stream_list_serial(stream_list, - out_fp, - out_ctype, - write_flags, - progress_func, - &progress); + ret = write_stream_list_serial(stream_list, + lookup_table, + out_fp, + out_ctype, + write_flags, + progress_func, + &progress); + return ret; } struct lte_overwrite_prepare_args { @@ -1311,7 +1382,7 @@ wim_prepare_streams(WIMStruct *wim, off_t end_offset, int ret; for (int i = 0; i < wim->hdr.image_count; i++) { - ret = lte_overwrite_prepare(wim->image_metadata[i].metadata_lte, + ret = lte_overwrite_prepare(wim->image_metadata[i]->metadata_lte, &args); if (ret) return ret; @@ -1320,18 +1391,72 @@ wim_prepare_streams(WIMStruct *wim, off_t end_offset, lte_overwrite_prepare, &args); } +struct stream_size_table { + struct hlist_head *array; + size_t num_entries; + size_t capacity; +}; + +static int +init_stream_size_table(struct stream_size_table *tab, size_t capacity) +{ + tab->array = CALLOC(capacity, sizeof(tab->array[0])); + if (!tab->array) + return WIMLIB_ERR_NOMEM; + tab->num_entries = 0; + tab->capacity = capacity; + return 0; +} + +static void +destroy_stream_size_table(struct stream_size_table *tab) +{ + FREE(tab->array); +} + +static int +stream_size_table_insert(struct wim_lookup_table_entry *lte, void *_tab) +{ + struct stream_size_table *tab = _tab; + size_t pos = hash_u64(wim_resource_size(lte)) % tab->capacity; + struct wim_lookup_table_entry *hashed_lte; + struct hlist_node *tmp; + + lte->unique_size = 1; + hlist_for_each_entry(hashed_lte, tmp, &tab->array[pos], hash_list_2) { + if (wim_resource_size(hashed_lte) == wim_resource_size(lte)) { + lte->unique_size = 0; + hashed_lte->unique_size = 0; + break; + } + } + + hlist_add_head(<e->hash_list_2, &tab->array[pos]); + tab->num_entries++; + return 0; +} + + +struct find_streams_ctx { + struct list_head stream_list; + struct stream_size_table stream_size_tab; +}; + static int inode_find_streams_to_write(struct wim_inode *inode, struct wim_lookup_table *table, - struct list_head *stream_list) + struct list_head *stream_list, + struct stream_size_table *tab) { struct wim_lookup_table_entry *lte; for (unsigned i = 0; i <= inode->i_num_ads; i++) { lte = inode_stream_lte(inode, i, table); if (lte) { if (lte->out_refcnt == 0) - list_add_tail(<e->staging_list, stream_list); + list_add_tail(<e->write_streams_list, stream_list); lte->out_refcnt += inode->i_nlink; + if (lte->unhashed) + stream_size_table_insert(lte, tab); } } return 0; @@ -1340,31 +1465,46 @@ inode_find_streams_to_write(struct wim_inode *inode, static int image_find_streams_to_write(WIMStruct *w) { + struct wim_image_metadata *imd; + struct find_streams_ctx *ctx; struct wim_inode *inode; struct hlist_node *cur; - struct hlist_head *inode_list; - inode_list = &wim_get_current_image_metadata(w)->inode_list; - hlist_for_each_entry(inode, cur, inode_list, i_hlist) { + ctx = w->private; + imd = wim_get_current_image_metadata(w); + hlist_for_each_entry(inode, cur, &imd->inode_list, i_hlist) { inode_find_streams_to_write(inode, w->lookup_table, - (struct list_head*)w->private); + &ctx->stream_list, + &ctx->stream_size_tab); } return 0; } static int write_wim_streams(WIMStruct *w, int image, int write_flags, - unsigned num_threads, - wimlib_progress_func_t progress_func) + unsigned num_threads, + wimlib_progress_func_t progress_func) { + struct find_streams_ctx ctx; + int ret; for_lookup_table_entry(w->lookup_table, lte_zero_out_refcnt, NULL); - LIST_HEAD(stream_list); - w->private = &stream_list; + ret = init_stream_size_table(&ctx.stream_size_tab, 9001); + if (ret) + return ret; + for_lookup_table_entry(w->lookup_table, stream_size_table_insert, + &ctx.stream_size_tab); + + INIT_LIST_HEAD(&ctx.stream_list); + w->private = &ctx; for_image(w, image, image_find_streams_to_write); - return write_stream_list(&stream_list, w->out_fp, - wimlib_get_compression_type(w), write_flags, - num_threads, progress_func); + destroy_stream_size_table(&ctx.stream_size_tab); + ret = write_stream_list(&ctx.stream_list, + w->lookup_table, + w->out_fp, + wimlib_get_compression_type(w), write_flags, + num_threads, progress_func); + return ret; } /* @@ -1497,7 +1637,7 @@ finish_write(WIMStruct *w, int image, int write_flags, } else { memcpy(&hdr.boot_metadata_res_entry, &w->image_metadata[ - hdr.boot_idx - 1].metadata_lte->output_resource_entry, + hdr.boot_idx - 1]->metadata_lte->output_resource_entry, sizeof(struct resource_entry)); } @@ -1509,7 +1649,7 @@ finish_write(WIMStruct *w, int image, int write_flags, } ret = write_header(&hdr, out); - if (ret != 0) + if (ret) goto out; if (write_flags & WIMLIB_WRITE_FLAG_FSYNC) { @@ -1656,7 +1796,7 @@ static bool any_images_modified(WIMStruct *w) { for (int i = 0; i < w->hdr.image_count; i++) - if (w->image_metadata[i].modified) + if (w->image_metadata[i]->modified) return true; return false; } @@ -1786,7 +1926,9 @@ overwrite_wim_inplace(WIMStruct *w, int write_flags, if (!list_empty(&stream_list)) { DEBUG("Writing newly added streams (offset = %"PRIu64")", old_wim_end); - ret = write_stream_list(&stream_list, w->out_fp, + ret = write_stream_list(&stream_list, + w->lookup_table, + w->out_fp, wimlib_get_compression_type(w), write_flags, num_threads, progress_func); @@ -1797,10 +1939,10 @@ overwrite_wim_inplace(WIMStruct *w, int write_flags, } for (int i = 0; i < w->hdr.image_count; i++) { - if (w->image_metadata[i].modified) { + if (w->image_metadata[i]->modified) { select_wim_image(w, i + 1); ret = write_metadata_resource(w); - if (ret != 0) + if (ret) goto out_ftruncate; } } diff --git a/src/xml.c b/src/xml.c index 4b1d708d..97fbfd50 100644 --- a/src/xml.c +++ b/src/xml.c @@ -1123,7 +1123,7 @@ xml_update_image_info(WIMStruct *w, int image) image_info->hard_link_bytes = 0; image_info->lookup_table = w->lookup_table; - for_dentry_in_tree(w->image_metadata[image - 1].root_dentry, + for_dentry_in_tree(w->image_metadata[image - 1]->root_dentry, calculate_dentry_statistics, image_info); image_info->last_modification_time = get_wim_timestamp(); -- 2.43.0