X-Git-Url: https://wimlib.net/git/?p=wimlib;a=blobdiff_plain;f=src%2Flookup_table.c;h=302b1750ece3335c0dfda989c036efeb552e7d38;hp=3b98553e9bb586d790df154f8de706351bc2ac0e;hb=acfc301115f69877a634f3141908747a54f678a0;hpb=61db93f82eca3fe9f7676355c709c58cc425a6ad diff --git a/src/lookup_table.c b/src/lookup_table.c index 3b98553e..302b1750 100644 --- a/src/lookup_table.c +++ b/src/lookup_table.c @@ -31,6 +31,7 @@ #include "wimlib/endianness.h" #include "wimlib/error.h" #include "wimlib/file_io.h" +#include "wimlib/glob.h" #include "wimlib/lookup_table.h" #include "wimlib/metadata.h" #include "wimlib/paths.h" @@ -74,8 +75,10 @@ new_lookup_table_entry(void) lte = CALLOC(1, sizeof(struct wim_lookup_table_entry)); if (lte) { - lte->part_number = 1; - lte->refcnt = 1; + lte->part_number = 1; + lte->refcnt = 1; + BUILD_BUG_ON(RESOURCE_NONEXISTENT != 0); + BUILD_BUG_ON(WIMLIB_COMPRESSION_TYPE_NONE != 0); } else { ERROR("Out of memory (tried to allocate %zu bytes for " "lookup table entry)", @@ -288,20 +291,110 @@ for_lookup_table_entry(struct wim_lookup_table *table, return 0; } -int -cmp_streams_by_wim_position(const void *p1, const void *p2) +/* qsort() callback that sorts streams (represented by `struct + * wim_lookup_table_entry's) into an order optimized for reading and writing. + * + * Sorting is done primarily by resource location, then secondarily by a + * per-resource location order. For example, resources in WIM files are sorted + * primarily by part number, then secondarily by offset, as to implement optimal + * reading of either a standalone or split WIM. */ +static int +cmp_streams_by_sequential_order(const void *p1, const void *p2) { const struct wim_lookup_table_entry *lte1, *lte2; + int v; + lte1 = *(const struct wim_lookup_table_entry**)p1; lte2 = *(const struct wim_lookup_table_entry**)p2; - if (lte1->resource_entry.offset < lte2->resource_entry.offset) - return -1; - else if (lte1->resource_entry.offset > lte2->resource_entry.offset) - return 1; - else + + v = (int)lte1->resource_location - (int)lte2->resource_location; + + /* Different resource locations? */ + if (v) + return v; + + switch (lte1->resource_location) { + case RESOURCE_IN_WIM: + + /* Different (possibly split) WIMs? */ + if (lte1->wim != lte2->wim) { + v = memcmp(lte1->wim->hdr.guid, lte2->wim->hdr.guid, + WIM_GID_LEN); + if (v) + return v; + } + + /* Different part numbers in the same WIM? */ + v = (int)lte1->wim->hdr.part_number - (int)lte2->wim->hdr.part_number; + if (v) + return v; + + /* Compare by offset. */ + if (lte1->resource_entry.offset < lte2->resource_entry.offset) + return -1; + else if (lte1->resource_entry.offset > lte2->resource_entry.offset) + return 1; return 0; + case RESOURCE_IN_FILE_ON_DISK: +#ifdef WITH_FUSE + case RESOURCE_IN_STAGING_FILE: +#endif +#ifdef __WIN32__ + case RESOURCE_WIN32_ENCRYPTED: +#endif + /* Compare files by path: just a heuristic that will place files + * in the same directory next to each other. */ + return tstrcmp(lte1->file_on_disk, lte2->file_on_disk); +#ifdef WITH_NTFS_3G + case RESOURCE_IN_NTFS_VOLUME: + return tstrcmp(lte1->ntfs_loc->path, lte2->ntfs_loc->path); +#endif + default: + /* No additional sorting order defined for this resource + * location (e.g. RESOURCE_IN_ATTACHED_BUFFER); simply compare + * everything equal to each other. */ + return 0; + } +} + +int +sort_stream_list_by_sequential_order(struct list_head *stream_list, + size_t list_head_offset) +{ + struct list_head *cur; + struct wim_lookup_table_entry **array; + size_t i; + size_t array_size; + size_t num_streams = 0; + + list_for_each(cur, stream_list) + num_streams++; + + array_size = num_streams * sizeof(array[0]); + array = MALLOC(array_size); + if (!array) + return WIMLIB_ERR_NOMEM; + cur = stream_list->next; + for (i = 0; i < num_streams; i++) { + array[i] = (struct wim_lookup_table_entry*)((u8*)cur - + list_head_offset); + cur = cur->next; + } + + qsort(array, num_streams, sizeof(array[0]), + cmp_streams_by_sequential_order); + + INIT_LIST_HEAD(stream_list); + for (i = 0; i < num_streams; i++) { + list_add_tail((struct list_head*) + ((u8*)array[i] + list_head_offset), + stream_list); + } + FREE(array); + return 0; } + static int add_lte_to_array(struct wim_lookup_table_entry *lte, void *_pp) @@ -333,7 +426,7 @@ for_lookup_table_entry_pos_sorted(struct wim_lookup_table *table, wimlib_assert(p == lte_array + num_streams); qsort(lte_array, num_streams, sizeof(lte_array[0]), - cmp_streams_by_wim_position); + cmp_streams_by_sequential_order); ret = 0; for (size_t i = 0; i < num_streams; i++) { ret = visitor(lte_array[i], arg); @@ -397,7 +490,7 @@ read_wim_lookup_table(WIMStruct *wim) size_t num_entries; struct wim_lookup_table *table; struct wim_lookup_table_entry *cur_entry, *duplicate_entry; - struct wim_lookup_table_entry_disk *buf; + void *buf; BUILD_BUG_ON(sizeof(struct wim_lookup_table_entry_disk) != WIM_LOOKUP_TABLE_ENTRY_DISK_SIZE); @@ -412,8 +505,7 @@ read_wim_lookup_table(WIMStruct *wim) /* Read the lookup table into a buffer. */ - ret = res_entry_to_data(&wim->hdr.lookup_table_res_entry, wim, - (void**)&buf); + ret = res_entry_to_data(&wim->hdr.lookup_table_res_entry, wim, &buf); if (ret) goto out; @@ -429,7 +521,8 @@ read_wim_lookup_table(WIMStruct *wim) * on-disk lookup table. */ wim->current_image = 0; for (i = 0; i < num_entries; i++) { - const struct wim_lookup_table_entry_disk *disk_entry = &buf[i]; + const struct wim_lookup_table_entry_disk *disk_entry = + &((const struct wim_lookup_table_entry_disk*)buf)[i]; cur_entry = new_lookup_table_entry(); if (!cur_entry) { @@ -486,6 +579,13 @@ read_wim_lookup_table(WIMStruct *wim) if (cur_entry->resource_entry.flags & WIM_RESHDR_FLAG_METADATA) { /* Lookup table entry for a metadata resource */ if (cur_entry->refcnt != 1) { + /* Metadata entries with no references must be + * ignored. See for example the WinPE WIMs from + * WAIK v2.1. */ + if (cur_entry->refcnt == 0) { + free_lookup_table_entry(cur_entry); + continue; + } if (wimlib_print_errors) { ERROR("Found metadata resource with refcnt != 1:"); print_lookup_table_entry(cur_entry, stderr); @@ -525,7 +625,7 @@ read_wim_lookup_table(WIMStruct *wim) } else { /* Lookup table entry for a stream that is not a * metadata resource */ - duplicate_entry = __lookup_resource(table, cur_entry->hash); + duplicate_entry = lookup_resource(table, cur_entry->hash); if (duplicate_entry) { if (wimlib_print_errors) { WARNING("The WIM lookup table contains two entries with the " @@ -583,7 +683,8 @@ static int write_wim_lookup_table_from_stream_list(struct list_head *stream_list, struct filedes *out_fd, struct resource_entry *out_res_entry, - int write_resource_flags) + int write_resource_flags, + struct wimlib_lzx_context **comp_ctx) { size_t table_size; struct wim_lookup_table_entry *lte; @@ -615,18 +716,42 @@ write_wim_lookup_table_from_stream_list(struct list_head *stream_list, WIM_RESHDR_FLAG_METADATA, out_fd, WIMLIB_COMPRESSION_TYPE_NONE, + 0, out_res_entry, NULL, - write_resource_flags); + write_resource_flags, + comp_ctx); FREE(table_buf); + DEBUG("ret=%d", ret); return ret; } static int append_lookup_table_entry(struct wim_lookup_table_entry *lte, void *_list) { - if (lte->out_refcnt != 0) + /* Lookup table entries with 'out_refcnt' == 0 correspond to streams not + * written and not present in the resulting WIM file, and should not be + * included in the lookup table. + * + * Lookup table entries marked as filtered (EXTERNAL_WIM) with + * 'out_refcnt != 0' were referenced as part of the logical write but + * correspond to streams that were not in fact written, and should not + * be included in the lookup table. + * + * Lookup table entries marked as filtered (SAME_WIM) with 'out_refcnt + * != 0' were referenced as part of the logical write but correspond to + * streams that were not in fact written, but nevertheless were already + * present in the WIM being overwritten in-place. These entries must be + * included in the lookup table, and the resource information to write + * needs to be copied from the resource information read originally. + */ + if (lte->out_refcnt != 0 && !(lte->filtered & FILTERED_EXTERNAL_WIM)) { + if (lte->filtered & FILTERED_SAME_WIM) { + copy_resource_entry(<e->output_resource_entry, + <e->resource_entry); + } list_add_tail(<e->lookup_table_list, (struct list_head*)_list); + } return 0; } @@ -673,7 +798,9 @@ write_wim_lookup_table(WIMStruct *wim, int image, int write_flags, } } - /* Append additional lookup table entries that have out_refcnt != 0. */ + /* Append additional lookup table entries that need to be written, with + * some special handling for streams that have been marked as filtered. + */ if (!stream_list_override) { for_lookup_table_entry(wim->lookup_table, append_lookup_table_entry, stream_list); @@ -685,7 +812,8 @@ write_wim_lookup_table(WIMStruct *wim, int image, int write_flags, return write_wim_lookup_table_from_stream_list(stream_list, &wim->out_fd, out_res_entry, - write_resource_flags); + write_resource_flags, + &wim->lzx_context); } @@ -831,28 +959,10 @@ wimlib_iterate_lookup_table(WIMStruct *wim, int flags, return for_lookup_table_entry(wim->lookup_table, do_iterate_lte, &ctx); } -static int -do_print_lookup_table_entry(struct wim_lookup_table_entry *lte, void *fp) -{ - print_lookup_table_entry(lte, (FILE*)fp); - return 0; -} - -/* API function documented in wimlib.h */ -WIMLIBAPI void -wimlib_print_lookup_table(WIMStruct *wim) -{ - for (int i = 0; i < wim->hdr.image_count; i++) - print_lookup_table_entry(wim->image_metadata[i]->metadata_lte, stdout); - for_lookup_table_entry(wim->lookup_table, - do_print_lookup_table_entry, - stdout); -} - /* Given a SHA1 message digest, return the corresponding entry in the WIM's * lookup table, or NULL if there is none. */ struct wim_lookup_table_entry * -__lookup_resource(const struct wim_lookup_table *table, const u8 hash[]) +lookup_resource(const struct wim_lookup_table *table, const u8 hash[]) { size_t i; struct wim_lookup_table_entry *lte; @@ -876,12 +986,12 @@ __lookup_resource(const struct wim_lookup_table *table, const u8 hash[]) * This is only for pre-resolved inodes. */ int -lookup_resource(WIMStruct *wim, - const tchar *path, - int lookup_flags, - struct wim_dentry **dentry_ret, - struct wim_lookup_table_entry **lte_ret, - u16 *stream_idx_ret) +wim_pathname_to_stream(WIMStruct *wim, + const tchar *path, + int lookup_flags, + struct wim_dentry **dentry_ret, + struct wim_lookup_table_entry **lte_ret, + u16 *stream_idx_ret) { struct wim_dentry *dentry; struct wim_lookup_table_entry *lte; @@ -927,8 +1037,7 @@ lookup_resource(WIMStruct *wim, return -ENOENT; } } else { - lte = inode->i_lte; - stream_idx = 0; + lte = inode_unnamed_stream_resolved(inode, &stream_idx); } out: if (dentry_ret) @@ -941,6 +1050,18 @@ out: } #endif +int +resource_not_found_error(const struct wim_inode *inode, const u8 *hash) +{ + if (wimlib_print_errors) { + ERROR("\"%"TS"\": resource not found", inode_first_full_path(inode)); + tfprintf(stderr, T(" SHA-1 message digest of missing resource:\n ")); + print_hash(hash, stderr); + tputc(T('\n'), stderr); + } + return WIMLIB_ERR_RESOURCE_NOT_FOUND; +} + /* * Resolve an inode's lookup table entries. * @@ -968,7 +1089,7 @@ inode_resolve_ltes(struct wim_inode *inode, struct wim_lookup_table *table, lte = NULL; hash = inode->i_hash; if (!is_zero_hash(hash)) { - lte = __lookup_resource(table, hash); + lte = lookup_resource(table, hash); if (!lte) { if (force) { lte = new_lookup_table_entry(); @@ -991,7 +1112,7 @@ inode_resolve_ltes(struct wim_inode *inode, struct wim_lookup_table *table, cur_entry = &inode->i_ads_entries[i]; hash = cur_entry->hash; if (!is_zero_hash(hash)) { - ads_lte = __lookup_resource(table, hash); + ads_lte = lookup_resource(table, hash); if (!ads_lte) { if (force) { ads_lte = new_lookup_table_entry(); @@ -1012,14 +1133,9 @@ inode_resolve_ltes(struct wim_inode *inode, struct wim_lookup_table *table, inode->i_resolved = 1; } return 0; + resource_not_found: - if (wimlib_print_errors) { - ERROR("\"%"TS"\": resource not found", inode_first_full_path(inode)); - tfprintf(stderr, T(" SHA-1 message digest of missing resource:\n ")); - print_hash(hash, stderr); - tputc(T('\n'), stderr); - } - return WIMLIB_ERR_RESOURCE_NOT_FOUND; + return resource_not_found_error(inode, hash); } void @@ -1060,19 +1176,28 @@ inode_stream_lte(const struct wim_inode *inode, unsigned stream_idx, } struct wim_lookup_table_entry * -inode_unnamed_lte_resolved(const struct wim_inode *inode) +inode_unnamed_stream_resolved(const struct wim_inode *inode, u16 *stream_idx_ret) { wimlib_assert(inode->i_resolved); for (unsigned i = 0; i <= inode->i_num_ads; i++) { if (inode_stream_name_nbytes(inode, i) == 0 && !is_zero_hash(inode_stream_hash_resolved(inode, i))) { + *stream_idx_ret = i; return inode_stream_lte_resolved(inode, i); } } + *stream_idx_ret = 0; return NULL; } +struct wim_lookup_table_entry * +inode_unnamed_lte_resolved(const struct wim_inode *inode) +{ + u16 stream_idx; + return inode_unnamed_stream_resolved(inode, &stream_idx); +} + struct wim_lookup_table_entry * inode_unnamed_lte_unresolved(const struct wim_inode *inode, const struct wim_lookup_table *table) @@ -1114,6 +1239,25 @@ inode_unnamed_lte(const struct wim_inode *inode, return inode_unnamed_lte_unresolved(inode, table); } +/* Returns the SHA1 message digest of the unnamed data stream of a WIM inode, or + * 'zero_hash' if the unnamed data stream is missing has all zeroes in its SHA1 + * message digest field. */ +const u8 * +inode_unnamed_stream_hash(const struct wim_inode *inode) +{ + const u8 *hash; + + for (unsigned i = 0; i <= inode->i_num_ads; i++) { + if (inode_stream_name_nbytes(inode, i) == 0) { + hash = inode_stream_hash(inode, i); + if (!is_zero_hash(hash)) + return hash; + } + } + return zero_hash; +} + + static int lte_add_stream_size(struct wim_lookup_table_entry *lte, void *total_bytes_p) { @@ -1179,7 +1323,7 @@ hash_unhashed_stream(struct wim_lookup_table_entry *lte, return ret; /* Look for a duplicate stream */ - duplicate_lte = __lookup_resource(lookup_table, lte->hash); + duplicate_lte = lookup_resource(lookup_table, lte->hash); list_del(<e->unhashed_list); if (duplicate_lte) { /* We have a duplicate stream. Transfer the reference counts @@ -1188,7 +1332,7 @@ hash_unhashed_stream(struct wim_lookup_table_entry *lte, * duplicate, then free this stream. */ wimlib_assert(!(duplicate_lte->unhashed)); duplicate_lte->refcnt += lte->refcnt; - duplicate_lte->out_refcnt += lte->refcnt; + duplicate_lte->out_refcnt += lte->out_refcnt; *back_ptr = duplicate_lte; free_lookup_table_entry(lte); lte = duplicate_lte; @@ -1203,3 +1347,183 @@ hash_unhashed_stream(struct wim_lookup_table_entry *lte, *lte_ret = lte; return 0; } + +static int +lte_clone_if_new(struct wim_lookup_table_entry *lte, void *_lookup_table) +{ + struct wim_lookup_table *lookup_table = _lookup_table; + + if (lookup_resource(lookup_table, lte->hash)) + return 0; /* Resource already present. */ + + lte = clone_lookup_table_entry(lte); + if (!lte) + return WIMLIB_ERR_NOMEM; + lte->out_refcnt = 1; + lookup_table_insert(lookup_table, lte); + return 0; +} + +static int +lte_delete_if_new(struct wim_lookup_table_entry *lte, void *_lookup_table) +{ + struct wim_lookup_table *lookup_table = _lookup_table; + + if (lte->out_refcnt) { + lookup_table_unlink(lookup_table, lte); + free_lookup_table_entry(lte); + } + return 0; +} + +/* API function documented in wimlib.h */ +WIMLIBAPI int +wimlib_reference_resources(WIMStruct *wim, + WIMStruct **resource_wims, unsigned num_resource_wims, + int ref_flags) +{ + int ret; + unsigned i; + + if (wim == NULL) + return WIMLIB_ERR_INVALID_PARAM; + + if (num_resource_wims != 0 && resource_wims == NULL) + return WIMLIB_ERR_INVALID_PARAM; + + for (i = 0; i < num_resource_wims; i++) + if (resource_wims[i] == NULL) + return WIMLIB_ERR_INVALID_PARAM; + + for_lookup_table_entry(wim->lookup_table, lte_zero_out_refcnt, NULL); + + for (i = 0; i < num_resource_wims; i++) { + ret = for_lookup_table_entry(resource_wims[i]->lookup_table, + lte_clone_if_new, + wim->lookup_table); + if (ret) + goto out_rollback; + } + return 0; + +out_rollback: + for_lookup_table_entry(wim->lookup_table, lte_delete_if_new, + wim->lookup_table); + return ret; +} + +static int +reference_resource_paths(WIMStruct *wim, + const tchar * const *resource_wimfiles, + unsigned num_resource_wimfiles, + int ref_flags, + int open_flags, + wimlib_progress_func_t progress_func) +{ + WIMStruct **resource_wims; + unsigned i; + int ret; + + resource_wims = CALLOC(num_resource_wimfiles, sizeof(resource_wims[0])); + if (!resource_wims) + return WIMLIB_ERR_NOMEM; + + for (i = 0; i < num_resource_wimfiles; i++) { + DEBUG("Referencing resources from path \"%"TS"\"", + resource_wimfiles[i]); + ret = wimlib_open_wim(resource_wimfiles[i], open_flags, + &resource_wims[i], progress_func); + if (ret) + goto out_free_resource_wims; + } + + ret = wimlib_reference_resources(wim, resource_wims, + num_resource_wimfiles, ref_flags); + if (ret) + goto out_free_resource_wims; + + for (i = 0; i < num_resource_wimfiles; i++) + list_add_tail(&resource_wims[i]->subwim_node, &wim->subwims); + + ret = 0; + goto out_free_array; + +out_free_resource_wims: + for (i = 0; i < num_resource_wimfiles; i++) + wimlib_free(resource_wims[i]); +out_free_array: + FREE(resource_wims); + return ret; +} + +static int +reference_resource_glob(WIMStruct *wim, const tchar *refglob, + int ref_flags, int open_flags, + wimlib_progress_func_t progress_func) +{ + glob_t globbuf; + int ret; + + /* Note: glob() is replaced in Windows native builds. */ + ret = tglob(refglob, GLOB_ERR | GLOB_NOSORT, NULL, &globbuf); + if (ret) { + if (ret == GLOB_NOMATCH) { + if (ref_flags & WIMLIB_REF_FLAG_GLOB_ERR_ON_NOMATCH) { + ERROR("Found no files for glob \"%"TS"\"", refglob); + return WIMLIB_ERR_GLOB_HAD_NO_MATCHES; + } else { + return reference_resource_paths(wim, + &refglob, + 1, + ref_flags, + open_flags, + progress_func); + } + } else { + ERROR_WITH_ERRNO("Failed to process glob \"%"TS"\"", refglob); + if (ret == GLOB_NOSPACE) + return WIMLIB_ERR_NOMEM; + else + return WIMLIB_ERR_READ; + } + } + + ret = reference_resource_paths(wim, + (const tchar * const *)globbuf.gl_pathv, + globbuf.gl_pathc, + ref_flags, + open_flags, + progress_func); + globfree(&globbuf); + return ret; +} + +/* API function documented in wimlib.h */ +WIMLIBAPI int +wimlib_reference_resource_files(WIMStruct *wim, + const tchar * const * resource_wimfiles_or_globs, + unsigned count, + int ref_flags, + int open_flags, + wimlib_progress_func_t progress_func) +{ + unsigned i; + int ret; + + if (ref_flags & WIMLIB_REF_FLAG_GLOB_ENABLE) { + for (i = 0; i < count; i++) { + ret = reference_resource_glob(wim, + resource_wimfiles_or_globs[i], + ref_flags, + open_flags, + progress_func); + if (ret) + return ret; + } + return 0; + } else { + return reference_resource_paths(wim, resource_wimfiles_or_globs, + count, ref_flags, + open_flags, progress_func); + } +}