X-Git-Url: https://wimlib.net/git/?p=wimlib;a=blobdiff_plain;f=src%2Flookup_table.c;h=815d576fe551177200b8b781c3a69621d7210530;hp=302b1750ece3335c0dfda989c036efeb552e7d38;hb=b6034a5dd44709341c46d553b1c0294ec91f13e4;hpb=acfc301115f69877a634f3141908747a54f678a0 diff --git a/src/lookup_table.c b/src/lookup_table.c index 302b1750..815d576f 100644 --- a/src/lookup_table.c +++ b/src/lookup_table.c @@ -74,16 +74,14 @@ new_lookup_table_entry(void) struct wim_lookup_table_entry *lte; lte = CALLOC(1, sizeof(struct wim_lookup_table_entry)); - if (lte) { - lte->part_number = 1; - lte->refcnt = 1; - BUILD_BUG_ON(RESOURCE_NONEXISTENT != 0); - BUILD_BUG_ON(WIMLIB_COMPRESSION_TYPE_NONE != 0); - } else { + if (lte == NULL) { ERROR("Out of memory (tried to allocate %zu bytes for " "lookup table entry)", sizeof(struct wim_lookup_table_entry)); + return NULL; } + lte->refcnt = 1; + BUILD_BUG_ON(RESOURCE_NONEXISTENT != 0); return lte; } @@ -93,11 +91,15 @@ clone_lookup_table_entry(const struct wim_lookup_table_entry *old) struct wim_lookup_table_entry *new; new = memdup(old, sizeof(struct wim_lookup_table_entry)); - if (!new) + if (new == NULL) return NULL; new->extracted_file = NULL; switch (new->resource_location) { + case RESOURCE_IN_WIM: + list_add(&new->rspec_node, &new->rspec->stream_list); + break; + case RESOURCE_IN_FILE_ON_DISK: #ifdef __WIN32__ case RESOURCE_WIN32_ENCRYPTED: @@ -108,13 +110,12 @@ clone_lookup_table_entry(const struct wim_lookup_table_entry *old) (void*)&old->staging_file_name); #endif new->file_on_disk = TSTRDUP(old->file_on_disk); - if (!new->file_on_disk) + if (new->file_on_disk == NULL) goto out_free; break; case RESOURCE_IN_ATTACHED_BUFFER: - new->attached_buffer = memdup(old->attached_buffer, - wim_resource_size(old)); - if (!new->attached_buffer) + new->attached_buffer = memdup(old->attached_buffer, old->size); + if (new->attached_buffer == NULL) goto out_free; break; #ifdef WITH_NTFS_3G @@ -122,18 +123,18 @@ clone_lookup_table_entry(const struct wim_lookup_table_entry *old) if (old->ntfs_loc) { struct ntfs_location *loc; loc = memdup(old->ntfs_loc, sizeof(struct ntfs_location)); - if (!loc) + if (loc == NULL) goto out_free; loc->path = NULL; loc->stream_name = NULL; new->ntfs_loc = loc; loc->path = STRDUP(old->ntfs_loc->path); - if (!loc->path) + if (loc->path == NULL) goto out_free; - if (loc->stream_name_nchars) { + if (loc->stream_name_nchars != 0) { loc->stream_name = memdup(old->ntfs_loc->stream_name, loc->stream_name_nchars * 2); - if (!loc->stream_name) + if (loc->stream_name == NULL) goto out_free; } } @@ -153,6 +154,11 @@ free_lookup_table_entry(struct wim_lookup_table_entry *lte) { if (lte) { switch (lte->resource_location) { + case RESOURCE_IN_WIM: + list_del(<e->rspec_node); + if (list_empty(<e->rspec->stream_list)) + FREE(lte->rspec); + break; case RESOURCE_IN_FILE_ON_DISK: #ifdef __WIN32__ case RESOURCE_WIN32_ENCRYPTED: @@ -194,7 +200,7 @@ do_free_lookup_table_entry(struct wim_lookup_table_entry *entry, void *ignore) void free_lookup_table(struct wim_lookup_table *table) { - DEBUG2("Freeing lookup table"); + DEBUG("Freeing lookup table."); if (table) { if (table->array) { for_lookup_table_entry(table, @@ -206,6 +212,43 @@ free_lookup_table(struct wim_lookup_table *table) } } +static void +lookup_table_insert_raw(struct wim_lookup_table *table, + struct wim_lookup_table_entry *lte) +{ + size_t i = lte->hash_short % table->capacity; + + hlist_add_head(<e->hash_list, &table->array[i]); +} + +static void +enlarge_lookup_table(struct wim_lookup_table *table) +{ + size_t old_capacity, new_capacity; + struct hlist_head *old_array, *new_array; + struct wim_lookup_table_entry *lte; + struct hlist_node *cur, *tmp; + size_t i; + + old_capacity = table->capacity; + new_capacity = old_capacity * 2; + new_array = CALLOC(new_capacity, sizeof(struct hlist_head)); + if (new_array == NULL) + return; + old_array = table->array; + table->array = new_array; + table->capacity = new_capacity; + + for (i = 0; i < old_capacity; i++) { + hlist_for_each_entry_safe(lte, cur, tmp, &old_array[i], hash_list) { + hlist_del(<e->hash_list); + lookup_table_insert_raw(table, lte); + } + } + FREE(old_array); +} + + /* * Inserts an entry into the lookup table. * @@ -216,11 +259,9 @@ void lookup_table_insert(struct wim_lookup_table *table, struct wim_lookup_table_entry *lte) { - size_t i = lte->hash_short % table->capacity; - hlist_add_head(<e->hash_list, &table->array[i]); - - /* XXX Make the table grow when too many entries have been inserted. */ - table->num_entries++; + lookup_table_insert_raw(table, lte); + if (++table->num_entries > table->capacity) + enlarge_lookup_table(table); } static void @@ -282,7 +323,6 @@ for_lookup_table_entry(struct wim_lookup_table *table, hlist_for_each_entry_safe(lte, pos, tmp, &table->array[i], hash_list) { - wimlib_assert2(!(lte->resource_entry.flags & WIM_RESHDR_FLAG_METADATA)); ret = visitor(lte, arg); if (ret) return ret; @@ -292,7 +332,7 @@ for_lookup_table_entry(struct wim_lookup_table *table, } /* qsort() callback that sorts streams (represented by `struct - * wim_lookup_table_entry's) into an order optimized for reading and writing. + * wim_lookup_table_entry's) into an order optimized for reading. * * Sorting is done primarily by resource location, then secondarily by a * per-resource location order. For example, resources in WIM files are sorted @@ -303,6 +343,7 @@ cmp_streams_by_sequential_order(const void *p1, const void *p2) { const struct wim_lookup_table_entry *lte1, *lte2; int v; + WIMStruct *wim1, *wim2; lte1 = *(const struct wim_lookup_table_entry**)p1; lte2 = *(const struct wim_lookup_table_entry**)p2; @@ -315,26 +356,27 @@ cmp_streams_by_sequential_order(const void *p1, const void *p2) switch (lte1->resource_location) { case RESOURCE_IN_WIM: + wim1 = lte1->rspec->wim; + wim2 = lte2->rspec->wim; /* Different (possibly split) WIMs? */ - if (lte1->wim != lte2->wim) { - v = memcmp(lte1->wim->hdr.guid, lte2->wim->hdr.guid, - WIM_GID_LEN); + if (wim1 != wim2) { + v = memcmp(wim1->hdr.guid, wim2->hdr.guid, WIM_GID_LEN); if (v) return v; } /* Different part numbers in the same WIM? */ - v = (int)lte1->wim->hdr.part_number - (int)lte2->wim->hdr.part_number; + v = (int)wim1->hdr.part_number - (int)wim2->hdr.part_number; if (v) return v; - /* Compare by offset. */ - if (lte1->resource_entry.offset < lte2->resource_entry.offset) - return -1; - else if (lte1->resource_entry.offset > lte2->resource_entry.offset) - return 1; - return 0; + if (lte1->rspec->offset_in_wim != lte2->rspec->offset_in_wim) + return cmp_u64(lte1->rspec->offset_in_wim, + lte2->rspec->offset_in_wim); + + return cmp_u64(lte1->offset_in_res, lte2->offset_in_res); + case RESOURCE_IN_FILE_ON_DISK: #ifdef WITH_FUSE case RESOURCE_IN_STAGING_FILE: @@ -358,8 +400,9 @@ cmp_streams_by_sequential_order(const void *p1, const void *p2) } int -sort_stream_list_by_sequential_order(struct list_head *stream_list, - size_t list_head_offset) +sort_stream_list(struct list_head *stream_list, + size_t list_head_offset, + int (*compar)(const void *, const void*)) { struct list_head *cur; struct wim_lookup_table_entry **array; @@ -370,10 +413,14 @@ sort_stream_list_by_sequential_order(struct list_head *stream_list, list_for_each(cur, stream_list) num_streams++; + if (num_streams <= 1) + return 0; + array_size = num_streams * sizeof(array[0]); array = MALLOC(array_size); - if (!array) + if (array == NULL) return WIMLIB_ERR_NOMEM; + cur = stream_list->next; for (i = 0; i < num_streams; i++) { array[i] = (struct wim_lookup_table_entry*)((u8*)cur - @@ -381,8 +428,7 @@ sort_stream_list_by_sequential_order(struct list_head *stream_list, cur = cur->next; } - qsort(array, num_streams, sizeof(array[0]), - cmp_streams_by_sequential_order); + qsort(array, num_streams, sizeof(array[0]), compar); INIT_LIST_HEAD(stream_list); for (i = 0; i < num_streams; i++) { @@ -394,6 +440,15 @@ sort_stream_list_by_sequential_order(struct list_head *stream_list, return 0; } +/* Sort the specified list of streams in an order optimized for reading. */ +int +sort_stream_list_by_sequential_order(struct list_head *stream_list, + size_t list_head_offset) +{ + return sort_stream_list(stream_list, list_head_offset, + cmp_streams_by_sequential_order); +} + static int add_lte_to_array(struct wim_lookup_table_entry *lte, @@ -439,9 +494,8 @@ for_lookup_table_entry_pos_sorted(struct wim_lookup_table *table, /* On-disk format of a WIM lookup table entry (stream entry). */ struct wim_lookup_table_entry_disk { - /* Location, offset, compression status, and metadata status of the - * stream. */ - struct resource_entry_disk resource_entry; + /* Size, offset, and flags of the stream. */ + struct wim_reshdr_disk reshdr; /* Which part of the split WIM this stream is in; indexed from 1. */ le16 part_number; @@ -456,22 +510,41 @@ struct wim_lookup_table_entry_disk { #define WIM_LOOKUP_TABLE_ENTRY_DISK_SIZE 50 -void -lte_init_wim(struct wim_lookup_table_entry *lte, WIMStruct *wim) +/* Validate the size and location of a WIM resource. */ +static int +validate_resource(const struct wim_resource_spec *rspec) { - lte->resource_location = RESOURCE_IN_WIM; - lte->wim = wim; - if (lte->resource_entry.flags & WIM_RESHDR_FLAG_COMPRESSED) - lte->compression_type = wim->compression_type; - else - lte->compression_type = WIMLIB_COMPRESSION_TYPE_NONE; + struct wim_lookup_table_entry *lte; + u64 cur_offset; + + /* Verify that calculating the offset of the end of the resource doesn't + * overflow. */ + if (rspec->offset_in_wim + rspec->size_in_wim < rspec->size_in_wim) + goto invalid; + + /* Verify that each stream in the resource has a valid offset and size, + * and that no streams overlap, and that the streams were added in order + * of increasing offset. */ + cur_offset = 0; + list_for_each_entry(lte, &rspec->stream_list, rspec_node) { + if (lte->offset_in_res + lte->size < lte->size || + lte->offset_in_res + lte->size > rspec->uncompressed_size || + lte->offset_in_res < cur_offset) + goto invalid; + + cur_offset = lte->offset_in_res + lte->size; + } + return 0; + +invalid: - if (wim_is_pipable(wim)) - lte->is_pipable = 1; + ERROR("Invalid resource entry!"); + return WIMLIB_ERR_INVALID_LOOKUP_TABLE_ENTRY; } /* - * Reads the lookup table from a WIM file. + * Reads the lookup table from a WIM file. Each entry specifies a stream that + * the WIM file contains, along with its location and SHA1 message digest. * * Saves lookup table entries for non-metadata streams in a hash table, and * saves the metadata entry for each image in a special per-image location (the @@ -481,6 +554,8 @@ lte_init_wim(struct wim_lookup_table_entry *lte, WIMStruct *wim) * WIMLIB_ERR_SUCCESS (0) * WIMLIB_ERR_INVALID_LOOKUP_TABLE_ENTRY * WIMLIB_ERR_RESOURCE_NOT_FOUND + * + * Or an error code caused by failure to read the lookup table into memory. */ int read_wim_lookup_table(WIMStruct *wim) @@ -490,108 +565,220 @@ read_wim_lookup_table(WIMStruct *wim) size_t num_entries; struct wim_lookup_table *table; struct wim_lookup_table_entry *cur_entry, *duplicate_entry; + struct wim_resource_spec *cur_rspec; void *buf; + bool back_to_back_pack; + DEBUG("Reading lookup table."); + + /* Sanity check: lookup table entries are 50 bytes each. */ BUILD_BUG_ON(sizeof(struct wim_lookup_table_entry_disk) != WIM_LOOKUP_TABLE_ENTRY_DISK_SIZE); - DEBUG("Reading lookup table: offset %"PRIu64", size %"PRIu64"", - wim->hdr.lookup_table_res_entry.offset, - wim->hdr.lookup_table_res_entry.size); - /* Calculate number of entries in the lookup table. */ - num_entries = wim->hdr.lookup_table_res_entry.size / + num_entries = wim->hdr.lookup_table_reshdr.uncompressed_size / sizeof(struct wim_lookup_table_entry_disk); - /* Read the lookup table into a buffer. */ - ret = res_entry_to_data(&wim->hdr.lookup_table_res_entry, wim, &buf); + ret = wim_reshdr_to_data(&wim->hdr.lookup_table_reshdr, wim, &buf); if (ret) goto out; - /* Allocate hash table. */ + /* Allocate a hash table to map SHA1 message digests into stream + * specifications. This is the in-memory "lookup table". */ table = new_lookup_table(num_entries * 2 + 1); - if (!table) { + if (table == NULL) { ERROR("Not enough memory to read lookup table."); ret = WIMLIB_ERR_NOMEM; goto out_free_buf; } - /* Allocate and initalize `struct wim_lookup_table_entry's from the - * on-disk lookup table. */ + /* Allocate and initalize stream entries from the raw lookup table + * buffer. */ wim->current_image = 0; + cur_rspec = NULL; for (i = 0; i < num_entries; i++) { const struct wim_lookup_table_entry_disk *disk_entry = &((const struct wim_lookup_table_entry_disk*)buf)[i]; + u16 part_number; + struct wim_reshdr reshdr; + + get_wim_reshdr(&disk_entry->reshdr, &reshdr); + + DEBUG("reshdr: size_in_wim=%"PRIu64", " + "uncompressed_size=%"PRIu64", " + "offset_in_wim=%"PRIu64", " + "flags=0x%02x", + reshdr.size_in_wim, reshdr.uncompressed_size, + reshdr.offset_in_wim, reshdr.flags); + + if (wim->hdr.wim_version == WIM_VERSION_DEFAULT) + reshdr.flags &= ~WIM_RESHDR_FLAG_PACKED_STREAMS; cur_entry = new_lookup_table_entry(); - if (!cur_entry) { - ERROR("Not enough memory to read lookup table."); + if (cur_entry == NULL) { + ERROR("Not enough memory to read lookup table!"); ret = WIMLIB_ERR_NOMEM; - goto out_free_lookup_table; + goto err; } - cur_entry->wim = wim; - cur_entry->resource_location = RESOURCE_IN_WIM; - get_resource_entry(&disk_entry->resource_entry, &cur_entry->resource_entry); - cur_entry->part_number = le16_to_cpu(disk_entry->part_number); + part_number = le16_to_cpu(disk_entry->part_number); cur_entry->refcnt = le32_to_cpu(disk_entry->refcnt); copy_hash(cur_entry->hash, disk_entry->hash); - lte_init_wim(cur_entry, wim); - if (cur_entry->part_number != wim->hdr.part_number) { + if (part_number != wim->hdr.part_number) { WARNING("A lookup table entry in part %hu of the WIM " "points to part %hu (ignoring it)", - wim->hdr.part_number, cur_entry->part_number); + wim->hdr.part_number, part_number); + free_lookup_table_entry(cur_entry); + continue; + } + + if (!(reshdr.flags & (WIM_RESHDR_FLAG_PACKED_STREAMS | + WIM_RESHDR_FLAG_COMPRESSED))) { + if (reshdr.uncompressed_size != reshdr.size_in_wim) { + ERROR("Invalid resource entry!"); + ret = WIMLIB_ERR_INVALID_LOOKUP_TABLE_ENTRY; + goto err; + } + } + + back_to_back_pack = false; + if (!(reshdr.flags & WIM_RESHDR_FLAG_PACKED_STREAMS) || + cur_rspec == NULL || + (back_to_back_pack = + ((reshdr.flags & WIM_RESHDR_FLAG_PACKED_STREAMS) && + reshdr.uncompressed_size == WIM_PACK_MAGIC_NUMBER && + cur_rspec != NULL && + cur_rspec->size_in_wim != 0))) + { + /* Starting new run of streams that share the same WIM + * resource. */ + struct wim_lookup_table_entry *prev_entry = NULL; + + if (back_to_back_pack && + !list_empty(&cur_rspec->stream_list)) + { + prev_entry = list_entry(cur_rspec->stream_list.prev, + struct wim_lookup_table_entry, + rspec_node); + lte_unbind_wim_resource_spec(prev_entry); + } + if (cur_rspec != NULL) { + ret = validate_resource(cur_rspec); + if (ret) + goto err; + } + + /* Allocate the resource specification and initialize it + * with values from the current stream entry. */ + cur_rspec = MALLOC(sizeof(*cur_rspec)); + if (cur_rspec == NULL) { + ERROR("Not enough memory to read lookup table!"); + ret = WIMLIB_ERR_NOMEM; + goto err; + } + wim_res_hdr_to_spec(&reshdr, wim, cur_rspec); + + if (prev_entry) + lte_bind_wim_resource_spec(prev_entry, cur_rspec); + } + + if ((reshdr.flags & WIM_RESHDR_FLAG_PACKED_STREAMS) && + reshdr.uncompressed_size == WIM_PACK_MAGIC_NUMBER) + { + /* Found the specification for the packed resource. + * Transfer the values to the `struct + * wim_resource_spec', and discard the current stream + * since this lookup table entry did not, in fact, + * correspond to a "stream". + */ + + /* Uncompressed size of the resource pack is actually + * stored in the header of the resource itself. Read + * it, and also grab the chunk size and compression type + * (which are not necessarily the defaults from the WIM + * header). */ + struct alt_chunk_table_header_disk hdr; + + ret = full_pread(&wim->in_fd, &hdr, + sizeof(hdr), reshdr.offset_in_wim); + if (ret) + goto err; + + cur_rspec->uncompressed_size = le64_to_cpu(hdr.res_usize); + cur_rspec->offset_in_wim = reshdr.offset_in_wim; + cur_rspec->size_in_wim = reshdr.size_in_wim; + cur_rspec->flags = reshdr.flags; + + /* Compression format numbers must be the same as in + * WIMGAPI to be compatible here. */ + BUILD_BUG_ON(WIMLIB_COMPRESSION_TYPE_NONE != 0); + BUILD_BUG_ON(WIMLIB_COMPRESSION_TYPE_LZX != 1); + BUILD_BUG_ON(WIMLIB_COMPRESSION_TYPE_XPRESS != 2); + BUILD_BUG_ON(WIMLIB_COMPRESSION_TYPE_LZMS != 3); + cur_rspec->compression_type = le32_to_cpu(hdr.compression_format); + + cur_rspec->chunk_size = le32_to_cpu(hdr.chunk_size); + + DEBUG("Full pack is %"PRIu64" compressed bytes " + "at file offset %"PRIu64" (flags 0x%02x)", + cur_rspec->size_in_wim, + cur_rspec->offset_in_wim, + cur_rspec->flags); free_lookup_table_entry(cur_entry); continue; } if (is_zero_hash(cur_entry->hash)) { - WARNING("The WIM lookup table contains an entry with a " - "SHA1 message digest of all 0's (ignoring it)"); free_lookup_table_entry(cur_entry); continue; } - if (!(cur_entry->resource_entry.flags & WIM_RESHDR_FLAG_COMPRESSED) - && (cur_entry->resource_entry.size != - cur_entry->resource_entry.original_size)) - { - if (wimlib_print_errors) { - WARNING("Found uncompressed resource with " - "original size (%"PRIu64") not the same " - "as compressed size (%"PRIu64")", - cur_entry->resource_entry.original_size, - cur_entry->resource_entry.size); - if (cur_entry->resource_entry.original_size) { - WARNING("Overriding compressed size with original size."); - cur_entry->resource_entry.size = - cur_entry->resource_entry.original_size; - } else { - WARNING("Overriding original size with compressed size"); - cur_entry->resource_entry.original_size = - cur_entry->resource_entry.size; - } - } + if (reshdr.flags & WIM_RESHDR_FLAG_PACKED_STREAMS) { + /* Continuing the pack with another stream. */ + DEBUG("Continuing pack with stream: " + "%"PRIu64" uncompressed bytes @ " + "resource offset %"PRIu64")", + reshdr.size_in_wim, reshdr.offset_in_wim); } - if (cur_entry->resource_entry.flags & WIM_RESHDR_FLAG_METADATA) { + lte_bind_wim_resource_spec(cur_entry, cur_rspec); + if (reshdr.flags & WIM_RESHDR_FLAG_PACKED_STREAMS) { + /* In packed runs, the offset field is used for + * in-resource offset, not the in-WIM offset, and the + * size field is used for the uncompressed size, not the + * compressed size. */ + cur_entry->offset_in_res = reshdr.offset_in_wim; + cur_entry->size = reshdr.size_in_wim; + cur_entry->flags = reshdr.flags; + } else { + /* Normal case: The stream corresponds one-to-one with + * the resource entry. */ + cur_entry->offset_in_res = 0; + cur_entry->size = reshdr.uncompressed_size; + cur_entry->flags = reshdr.flags; + cur_rspec = NULL; + } + + if (cur_entry->flags & WIM_RESHDR_FLAG_METADATA) { /* Lookup table entry for a metadata resource */ + + /* Metadata entries with no references must be ignored; + * see for example the WinPE WIMs from the WAIK v2.1. + * */ + if (cur_entry->refcnt == 0) { + free_lookup_table_entry(cur_entry); + continue; + } + if (cur_entry->refcnt != 1) { - /* Metadata entries with no references must be - * ignored. See for example the WinPE WIMs from - * WAIK v2.1. */ - if (cur_entry->refcnt == 0) { - free_lookup_table_entry(cur_entry); - continue; - } if (wimlib_print_errors) { ERROR("Found metadata resource with refcnt != 1:"); print_lookup_table_entry(cur_entry, stderr); } ret = WIMLIB_ERR_INVALID_LOOKUP_TABLE_ENTRY; - goto out_free_cur_entry; + goto err; } if (wim->hdr.part_number != 1) { @@ -619,28 +806,39 @@ read_wim_lookup_table(WIMStruct *wim) DEBUG("Found metadata resource for image %u at " "offset %"PRIu64".", wim->current_image + 1, - cur_entry->resource_entry.offset); + cur_entry->rspec->offset_in_wim); wim->image_metadata[ wim->current_image++]->metadata_lte = cur_entry; - } else { - /* Lookup table entry for a stream that is not a - * metadata resource */ - duplicate_entry = lookup_resource(table, cur_entry->hash); - if (duplicate_entry) { - if (wimlib_print_errors) { - WARNING("The WIM lookup table contains two entries with the " - "same SHA1 message digest!"); - WARNING("The first entry is:"); - print_lookup_table_entry(duplicate_entry, stderr); - WARNING("The second entry is:"); - print_lookup_table_entry(cur_entry, stderr); - } - free_lookup_table_entry(cur_entry); - continue; - } else { - lookup_table_insert(table, cur_entry); + continue; + } + + /* Lookup table entry for a stream that is not a metadata + * resource. */ + duplicate_entry = lookup_resource(table, cur_entry->hash); + if (duplicate_entry) { + if (wimlib_print_errors) { + WARNING("The WIM lookup table contains two entries with the " + "same SHA1 message digest!"); + WARNING("The first entry is:"); + print_lookup_table_entry(duplicate_entry, stderr); + WARNING("The second entry is:"); + print_lookup_table_entry(cur_entry, stderr); } + free_lookup_table_entry(cur_entry); + continue; } + + /* Finally, insert the stream into the lookup table, keyed by + * its SHA1 message digest. */ + lookup_table_insert(table, cur_entry); + } + cur_entry = NULL; + + /* Validate the last resource. */ + if (cur_rspec != NULL) { + ret = validate_resource(cur_rspec); + if (ret) + goto err; } if (wim->hdr.part_number == 1 && wim->current_image != wim->hdr.image_count) { @@ -657,9 +855,11 @@ read_wim_lookup_table(WIMStruct *wim) wim->lookup_table = table; ret = 0; goto out_free_buf; -out_free_cur_entry: - FREE(cur_entry); -out_free_lookup_table: + +err: + if (cur_rspec && list_empty(&cur_rspec->stream_list)) + FREE(cur_rspec); + free_lookup_table_entry(cur_entry); free_lookup_table(table); out_free_buf: FREE(buf); @@ -668,46 +868,91 @@ out: return ret; } - static void -write_wim_lookup_table_entry(const struct wim_lookup_table_entry *lte, - struct wim_lookup_table_entry_disk *disk_entry) +put_wim_lookup_table_entry(struct wim_lookup_table_entry_disk *disk_entry, + const struct wim_reshdr *out_reshdr, + u16 part_number, u32 refcnt, const u8 *hash) { - put_resource_entry(<e->output_resource_entry, &disk_entry->resource_entry); - disk_entry->part_number = cpu_to_le16(lte->part_number); - disk_entry->refcnt = cpu_to_le32(lte->out_refcnt); - copy_hash(disk_entry->hash, lte->hash); + put_wim_reshdr(out_reshdr, &disk_entry->reshdr); + disk_entry->part_number = cpu_to_le16(part_number); + disk_entry->refcnt = cpu_to_le32(refcnt); + copy_hash(disk_entry->hash, hash); } -static int +int write_wim_lookup_table_from_stream_list(struct list_head *stream_list, struct filedes *out_fd, - struct resource_entry *out_res_entry, - int write_resource_flags, - struct wimlib_lzx_context **comp_ctx) + u16 part_number, + struct wim_reshdr *out_reshdr, + int write_resource_flags) { size_t table_size; struct wim_lookup_table_entry *lte; struct wim_lookup_table_entry_disk *table_buf; struct wim_lookup_table_entry_disk *table_buf_ptr; int ret; + u64 prev_res_offset_in_wim = ~0ULL; table_size = 0; - list_for_each_entry(lte, stream_list, lookup_table_list) + list_for_each_entry(lte, stream_list, lookup_table_list) { table_size += sizeof(struct wim_lookup_table_entry_disk); + if (lte->out_reshdr.flags & WIM_RESHDR_FLAG_PACKED_STREAMS && + lte->out_res_offset_in_wim != prev_res_offset_in_wim) + { + table_size += sizeof(struct wim_lookup_table_entry_disk); + prev_res_offset_in_wim = lte->out_res_offset_in_wim; + } + } + DEBUG("Writing WIM lookup table (size=%zu, offset=%"PRIu64")", table_size, out_fd->offset); table_buf = MALLOC(table_size); - if (!table_buf) { + if (table_buf == NULL) { ERROR("Failed to allocate %zu bytes for temporary lookup table", table_size); return WIMLIB_ERR_NOMEM; } table_buf_ptr = table_buf; - list_for_each_entry(lte, stream_list, lookup_table_list) - write_wim_lookup_table_entry(lte, table_buf_ptr++); + + prev_res_offset_in_wim = ~0ULL; + list_for_each_entry(lte, stream_list, lookup_table_list) { + + put_wim_lookup_table_entry(table_buf_ptr++, + <e->out_reshdr, + part_number, + lte->out_refcnt, + lte->hash); + if (lte->out_reshdr.flags & WIM_RESHDR_FLAG_PACKED_STREAMS && + lte->out_res_offset_in_wim != prev_res_offset_in_wim) + { + /* Put the main resource entry for the pack. */ + + struct wim_reshdr reshdr; + + reshdr.offset_in_wim = lte->out_res_offset_in_wim; + reshdr.size_in_wim = lte->out_res_size_in_wim; + reshdr.uncompressed_size = WIM_PACK_MAGIC_NUMBER; + reshdr.flags = WIM_RESHDR_FLAG_PACKED_STREAMS; + + DEBUG("Putting main entry for pack: " + "size_in_wim=%"PRIu64", " + "offset_in_wim=%"PRIu64", " + "uncompressed_size=%"PRIu64, + reshdr.size_in_wim, + reshdr.offset_in_wim, + reshdr.uncompressed_size); + + put_wim_lookup_table_entry(table_buf_ptr++, + &reshdr, + part_number, + 1, zero_hash); + prev_res_offset_in_wim = lte->out_res_offset_in_wim; + } + + } + wimlib_assert((u8*)table_buf_ptr - (u8*)table_buf == table_size); /* Write the lookup table uncompressed. Although wimlib can handle a * compressed lookup table, MS software cannot. */ @@ -717,106 +962,14 @@ write_wim_lookup_table_from_stream_list(struct list_head *stream_list, out_fd, WIMLIB_COMPRESSION_TYPE_NONE, 0, - out_res_entry, + out_reshdr, NULL, - write_resource_flags, - comp_ctx); + write_resource_flags); FREE(table_buf); DEBUG("ret=%d", ret); return ret; } -static int -append_lookup_table_entry(struct wim_lookup_table_entry *lte, void *_list) -{ - /* Lookup table entries with 'out_refcnt' == 0 correspond to streams not - * written and not present in the resulting WIM file, and should not be - * included in the lookup table. - * - * Lookup table entries marked as filtered (EXTERNAL_WIM) with - * 'out_refcnt != 0' were referenced as part of the logical write but - * correspond to streams that were not in fact written, and should not - * be included in the lookup table. - * - * Lookup table entries marked as filtered (SAME_WIM) with 'out_refcnt - * != 0' were referenced as part of the logical write but correspond to - * streams that were not in fact written, but nevertheless were already - * present in the WIM being overwritten in-place. These entries must be - * included in the lookup table, and the resource information to write - * needs to be copied from the resource information read originally. - */ - if (lte->out_refcnt != 0 && !(lte->filtered & FILTERED_EXTERNAL_WIM)) { - if (lte->filtered & FILTERED_SAME_WIM) { - copy_resource_entry(<e->output_resource_entry, - <e->resource_entry); - } - list_add_tail(<e->lookup_table_list, (struct list_head*)_list); - } - return 0; -} - -int -write_wim_lookup_table(WIMStruct *wim, int image, int write_flags, - struct resource_entry *out_res_entry, - struct list_head *stream_list_override) -{ - int write_resource_flags; - struct list_head _stream_list; - struct list_head *stream_list; - - if (stream_list_override) { - stream_list = stream_list_override; - } else { - stream_list = &_stream_list; - INIT_LIST_HEAD(stream_list); - } - - if (!(write_flags & WIMLIB_WRITE_FLAG_NO_METADATA)) { - int start_image; - int end_image; - - if (image == WIMLIB_ALL_IMAGES) { - start_image = 1; - end_image = wim->hdr.image_count; - } else { - start_image = image; - end_image = image; - } - - /* Push metadata resource lookup table entries onto the front of - * the list in reverse order, so that they're written in order. - */ - for (int i = end_image; i >= start_image; i--) { - struct wim_lookup_table_entry *metadata_lte; - - metadata_lte = wim->image_metadata[i - 1]->metadata_lte; - metadata_lte->out_refcnt = 1; - metadata_lte->part_number = wim->hdr.part_number; - metadata_lte->output_resource_entry.flags |= WIM_RESHDR_FLAG_METADATA; - - list_add(&metadata_lte->lookup_table_list, stream_list); - } - } - - /* Append additional lookup table entries that need to be written, with - * some special handling for streams that have been marked as filtered. - */ - if (!stream_list_override) { - for_lookup_table_entry(wim->lookup_table, - append_lookup_table_entry, stream_list); - } - - write_resource_flags = 0; - if (write_flags & WIMLIB_WRITE_FLAG_PIPABLE) - write_resource_flags |= WIMLIB_WRITE_RESOURCE_FLAG_PIPABLE; - return write_wim_lookup_table_from_stream_list(stream_list, - &wim->out_fd, - out_res_entry, - write_resource_flags, - &wim->lzx_context); -} - - int lte_zero_real_refcnt(struct wim_lookup_table_entry *lte, void *_ignore) { @@ -844,33 +997,47 @@ lte_free_extracted_file(struct wim_lookup_table_entry *lte, void *_ignore) void print_lookup_table_entry(const struct wim_lookup_table_entry *lte, FILE *out) { - if (!lte) { + if (lte == NULL) { tputc(T('\n'), out); return; } - tfprintf(out, T("Offset = %"PRIu64" bytes\n"), - lte->resource_entry.offset); - tfprintf(out, T("Size = %"PRIu64" bytes\n"), - (u64)lte->resource_entry.size); - tfprintf(out, T("Original size = %"PRIu64" bytes\n"), - lte->resource_entry.original_size); + tprintf(T("Uncompressed size = %"PRIu64" bytes\n"), + lte->size); + if (lte->flags & WIM_RESHDR_FLAG_PACKED_STREAMS) { + tprintf(T("Offset = %"PRIu64" bytes\n"), + lte->offset_in_res); - tfprintf(out, T("Part Number = %hu\n"), lte->part_number); - tfprintf(out, T("Reference Count = %u\n"), lte->refcnt); + tprintf(T("Raw uncompressed size = %"PRIu64" bytes\n"), + lte->rspec->uncompressed_size); + + tprintf(T("Raw compressed size = %"PRIu64" bytes\n"), + lte->rspec->size_in_wim); + + tprintf(T("Raw offset = %"PRIu64" bytes\n"), + lte->rspec->offset_in_wim); + } else if (lte->resource_location == RESOURCE_IN_WIM) { + tprintf(T("Compressed size = %"PRIu64" bytes\n"), + lte->rspec->size_in_wim); + + tprintf(T("Offset = %"PRIu64" bytes\n"), + lte->rspec->offset_in_wim); + } + + tfprintf(out, T("Reference Count = %u\n"), lte->refcnt); if (lte->unhashed) { tfprintf(out, T("(Unhashed: inode %p, stream_id = %u)\n"), lte->back_inode, lte->back_stream_id); } else { - tfprintf(out, T("Hash = 0x")); + tfprintf(out, T("Hash = 0x")); print_hash(lte->hash, out); tputc(T('\n'), out); } - tfprintf(out, T("Flags = ")); - u8 flags = lte->resource_entry.flags; + tfprintf(out, T("Flags = ")); + u8 flags = lte->flags; if (flags & WIM_RESHDR_FLAG_COMPRESSED) tfputs(T("WIM_RESHDR_FLAG_COMPRESSED, "), out); if (flags & WIM_RESHDR_FLAG_FREE) @@ -879,24 +1046,26 @@ print_lookup_table_entry(const struct wim_lookup_table_entry *lte, FILE *out) tfputs(T("WIM_RESHDR_FLAG_METADATA, "), out); if (flags & WIM_RESHDR_FLAG_SPANNED) tfputs(T("WIM_RESHDR_FLAG_SPANNED, "), out); + if (flags & WIM_RESHDR_FLAG_PACKED_STREAMS) + tfputs(T("WIM_RESHDR_FLAG_PACKED_STREAMS, "), out); tputc(T('\n'), out); switch (lte->resource_location) { case RESOURCE_IN_WIM: - if (lte->wim->filename) { - tfprintf(out, T("WIM file = `%"TS"'\n"), - lte->wim->filename); + if (lte->rspec->wim->filename) { + tfprintf(out, T("WIM file = `%"TS"'\n"), + lte->rspec->wim->filename); } break; #ifdef __WIN32__ case RESOURCE_WIN32_ENCRYPTED: #endif case RESOURCE_IN_FILE_ON_DISK: - tfprintf(out, T("File on Disk = `%"TS"'\n"), + tfprintf(out, T("File on Disk = `%"TS"'\n"), lte->file_on_disk); break; #ifdef WITH_FUSE case RESOURCE_IN_STAGING_FILE: - tfprintf(out, T("Staging File = `%"TS"'\n"), + tfprintf(out, T("Staging File = `%"TS"'\n"), lte->staging_file_name); break; #endif @@ -910,16 +1079,29 @@ void lte_to_wimlib_resource_entry(const struct wim_lookup_table_entry *lte, struct wimlib_resource_entry *wentry) { - wentry->uncompressed_size = lte->resource_entry.original_size; - wentry->compressed_size = lte->resource_entry.size; - wentry->offset = lte->resource_entry.offset; + memset(wentry, 0, sizeof(*wentry)); + + wentry->uncompressed_size = lte->size; + if (lte->resource_location == RESOURCE_IN_WIM) { + wentry->part_number = lte->rspec->wim->hdr.part_number; + if (lte->flags & WIM_RESHDR_FLAG_PACKED_STREAMS) { + wentry->compressed_size = 0; + wentry->offset = lte->offset_in_res; + } else { + wentry->compressed_size = lte->rspec->size_in_wim; + wentry->offset = lte->rspec->offset_in_wim; + } + wentry->raw_resource_offset_in_wim = lte->rspec->offset_in_wim; + /*wentry->raw_resource_uncompressed_size = lte->rspec->uncompressed_size;*/ + wentry->raw_resource_compressed_size = lte->rspec->size_in_wim; + } copy_hash(wentry->sha1_hash, lte->hash); - wentry->part_number = lte->part_number; wentry->reference_count = lte->refcnt; - wentry->is_compressed = (lte->resource_entry.flags & WIM_RESHDR_FLAG_COMPRESSED) != 0; - wentry->is_metadata = (lte->resource_entry.flags & WIM_RESHDR_FLAG_METADATA) != 0; - wentry->is_free = (lte->resource_entry.flags & WIM_RESHDR_FLAG_FREE) != 0; - wentry->is_spanned = (lte->resource_entry.flags & WIM_RESHDR_FLAG_SPANNED) != 0; + wentry->is_compressed = (lte->flags & WIM_RESHDR_FLAG_COMPRESSED) != 0; + wentry->is_metadata = (lte->flags & WIM_RESHDR_FLAG_METADATA) != 0; + wentry->is_free = (lte->flags & WIM_RESHDR_FLAG_FREE) != 0; + wentry->is_spanned = (lte->flags & WIM_RESHDR_FLAG_SPANNED) != 0; + wentry->packed = (lte->flags & WIM_RESHDR_FLAG_PACKED_STREAMS) != 0; } struct iterate_lte_context { @@ -1008,7 +1190,7 @@ wim_pathname_to_stream(WIMStruct *wim, } } - dentry = get_dentry(wim, path); + dentry = get_dentry(wim, path, WIMLIB_CASE_SENSITIVE); if (p) *p = T(':'); if (!dentry) @@ -1257,22 +1439,6 @@ inode_unnamed_stream_hash(const struct wim_inode *inode) return zero_hash; } - -static int -lte_add_stream_size(struct wim_lookup_table_entry *lte, void *total_bytes_p) -{ - *(u64*)total_bytes_p += lte->resource_entry.size; - return 0; -} - -u64 -lookup_table_total_stream_size(struct wim_lookup_table *table) -{ - u64 total_size = 0; - for_lookup_table_entry(table, lte_add_stream_size, &total_size); - return total_size; -} - struct wim_lookup_table_entry ** retrieve_lte_pointer(struct wim_lookup_table_entry *lte) { @@ -1318,7 +1484,7 @@ hash_unhashed_stream(struct wim_lookup_table_entry *lte, * the SHA1 has been calculated. */ back_ptr = retrieve_lte_pointer(lte); - ret = sha1_resource(lte); + ret = sha1_stream(lte); if (ret) return ret; @@ -1327,24 +1493,23 @@ hash_unhashed_stream(struct wim_lookup_table_entry *lte, list_del(<e->unhashed_list); if (duplicate_lte) { /* We have a duplicate stream. Transfer the reference counts - * from this stream to the duplicate, update the reference to + * from this stream to the duplicate and update the reference to * this stream (in an inode or ads_entry) to point to the - * duplicate, then free this stream. */ + * duplicate. The caller is responsible for freeing @lte if + * needed. */ wimlib_assert(!(duplicate_lte->unhashed)); + wimlib_assert(duplicate_lte->size == lte->size); duplicate_lte->refcnt += lte->refcnt; - duplicate_lte->out_refcnt += lte->out_refcnt; + lte->refcnt = 0; *back_ptr = duplicate_lte; - free_lookup_table_entry(lte); lte = duplicate_lte; } else { - /* No duplicate stream, so we need to insert - * this stream into the lookup table and treat - * it as a hashed stream. */ + /* No duplicate stream, so we need to insert this stream into + * the lookup table and treat it as a hashed stream. */ lookup_table_insert(lookup_table, lte); lte->unhashed = 0; } - if (lte_ret) - *lte_ret = lte; + *lte_ret = lte; return 0; } @@ -1357,7 +1522,7 @@ lte_clone_if_new(struct wim_lookup_table_entry *lte, void *_lookup_table) return 0; /* Resource already present. */ lte = clone_lookup_table_entry(lte); - if (!lte) + if (lte == NULL) return WIMLIB_ERR_NOMEM; lte->out_refcnt = 1; lookup_table_insert(lookup_table, lte);