X-Git-Url: https://wimlib.net/git/?p=wimlib;a=blobdiff_plain;f=src%2Flookup_table.c;h=ec793205c46cce371e73efb0b933961d36e3de21;hp=01f7383571c5f00899b6c7814f9f0e8323d398df;hb=894f8dab7f174bf289e6b5e9ea54374d10d6e62f;hpb=882dc6e5dece7f521b9de6c4337aad0411ed00da diff --git a/src/lookup_table.c b/src/lookup_table.c index 01f73835..ec793205 100644 --- a/src/lookup_table.c +++ b/src/lookup_table.c @@ -6,28 +6,30 @@ */ /* - * Copyright (C) 2012, 2013 Eric Biggers + * Copyright (C) 2012, 2013, 2014 Eric Biggers * - * This file is part of wimlib, a library for working with WIM files. + * This file is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation; either version 3 of the License, or (at your option) any + * later version. * - * wimlib is free software; you can redistribute it and/or modify it under the - * terms of the GNU General Public License as published by the Free - * Software Foundation; either version 3 of the License, or (at your option) - * any later version. - * - * wimlib is distributed in the hope that it will be useful, but WITHOUT ANY - * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR - * A PARTICULAR PURPOSE. See the GNU General Public License for more + * This file is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more * details. * - * You should have received a copy of the GNU General Public License - * along with wimlib; if not, see http://www.gnu.org/licenses/. + * You should have received a copy of the GNU Lesser General Public License + * along with this file; if not, see http://www.gnu.org/licenses/. */ #ifdef HAVE_CONFIG_H # include "config.h" #endif +#include +#include +#include /* for unlink() */ + #include "wimlib/assert.h" #include "wimlib/endianness.h" #include "wimlib/error.h" @@ -35,13 +37,10 @@ #include "wimlib/metadata.h" #include "wimlib/ntfs_3g.h" #include "wimlib/resource.h" +#include "wimlib/unaligned.h" #include "wimlib/util.h" #include "wimlib/write.h" -#include -#include -#include /* for unlink() */ - /* WIM lookup table: * * This is a logical mapping from SHA1 message digests to the data streams @@ -94,17 +93,11 @@ do_free_lookup_table_entry(struct wim_lookup_table_entry *entry, void *ignore) void free_lookup_table(struct wim_lookup_table *table) { - DEBUG("Freeing lookup table."); - if (table == NULL) - return; - - if (table->array) { - for_lookup_table_entry(table, - do_free_lookup_table_entry, - NULL); + if (table) { + for_lookup_table_entry(table, do_free_lookup_table_entry, NULL); FREE(table->array); + FREE(table); } - FREE(table); } struct wim_lookup_table_entry * @@ -133,7 +126,6 @@ clone_lookup_table_entry(const struct wim_lookup_table_entry *old) if (new == NULL) return NULL; - new->extracted_file = NULL; switch (new->resource_location) { case RESOURCE_IN_WIM: list_add(&new->rspec_node, &new->rspec->stream_list); @@ -141,6 +133,7 @@ clone_lookup_table_entry(const struct wim_lookup_table_entry *old) case RESOURCE_IN_FILE_ON_DISK: #ifdef __WIN32__ + case RESOURCE_IN_WINNT_FILE_ON_DISK: case RESOURCE_WIN32_ENCRYPTED: #endif #ifdef WITH_FUSE @@ -200,6 +193,7 @@ lte_put_resource(struct wim_lookup_table_entry *lte) break; case RESOURCE_IN_FILE_ON_DISK: #ifdef __WIN32__ + case RESOURCE_IN_WINNT_FILE_ON_DISK: case RESOURCE_WIN32_ENCRYPTED: #endif #ifdef WITH_FUSE @@ -250,20 +244,34 @@ finalize_lte(struct wim_lookup_table_entry *lte) } /* - * Decrements the reference count for the lookup table entry @lte, which must be - * inserted in the stream lookup table @table. + * Decrements the reference count of the single-instance stream @lte, which must + * be inserted in the stream lookup table @table. + * + * If the stream's reference count reaches 0, we may unlink it from @table and + * free it. However, we retain streams with 0 reference count that originated + * from WIM files (RESOURCE_IN_WIM). We do this for two reasons: + * + * 1. This prevents information about valid streams in a WIM file --- streams + * which will continue to be present after appending to the WIM file --- from + * being lost merely because we dropped all references to them. + * + * 2. Stream reference counts we read from WIM files can't be trusted. It's + * possible that a WIM has reference counts that are too low; WIMGAPI + * sometimes creates WIMs where this is the case. It's also possible that + * streams have been referenced from an external WIM; those streams can + * potentially have any reference count at all, either lower or higher than + * would be expected for this WIM ("this WIM" meaning the owner of @table) if + * it were a standalone WIM. * - * If the reference count reaches 0, this may cause @lte to be destroyed. - * However, we may retain entries with 0 reference count. This does not affect - * correctness, but it prevents the entries for valid streams in a WIM archive, - * which will continue to be present after appending to the file, from being - * lost merely because we dropped all references to them. + * So we can't take the reference counts too seriously. But at least, we do + * recalculate by default when writing a new WIM file. */ void lte_decrement_refcnt(struct wim_lookup_table_entry *lte, struct wim_lookup_table *table) { - wimlib_assert(lte->refcnt != 0); + if (unlikely(lte->refcnt == 0)) /* See comment above */ + return; if (--lte->refcnt == 0) { if (lte->unhashed) { @@ -274,7 +282,8 @@ lte_decrement_refcnt(struct wim_lookup_table_entry *lte, * that there still may be open file descriptors to it.) * */ if (lte->resource_location == RESOURCE_IN_STAGING_FILE) - unlink(lte->staging_file_name); + unlinkat(lte->staging_dir_fd, + lte->staging_file_name, 0); #endif } else { if (!should_retain_lte(lte)) @@ -369,7 +378,7 @@ lookup_stream(const struct wim_lookup_table *table, const u8 hash[]) struct wim_lookup_table_entry *lte; struct hlist_node *pos; - i = *(size_t*)hash % table->capacity; + i = load_size_t_unaligned(hash) % table->capacity; hlist_for_each_entry(lte, pos, &table->array[i], hash_list) if (hashes_equal(hash, lte->hash)) return lte; @@ -429,7 +438,7 @@ cmp_streams_by_sequential_order(const void *p1, const void *p2) /* Different (possibly split) WIMs? */ if (wim1 != wim2) { - v = memcmp(wim1->hdr.guid, wim2->hdr.guid, WIM_GID_LEN); + v = memcmp(wim1->hdr.guid, wim2->hdr.guid, WIM_GUID_LEN); if (v) return v; } @@ -450,6 +459,7 @@ cmp_streams_by_sequential_order(const void *p1, const void *p2) case RESOURCE_IN_STAGING_FILE: #endif #ifdef __WIN32__ + case RESOURCE_IN_WINNT_FILE_ON_DISK: case RESOURCE_WIN32_ENCRYPTED: #endif /* Compare files by path: just a heuristic that will place files @@ -568,7 +578,8 @@ struct wim_lookup_table_entry_disk { /* Which part of the split WIM this stream is in; indexed from 1. */ le16 part_number; - /* Reference count of this stream over all WIM images. */ + /* Reference count of this stream over all WIM images. (But see comment + * above lte_decrement_refcnt().) */ le32 refcnt; /* SHA1 message digest of the uncompressed data of this stream, or @@ -578,6 +589,183 @@ struct wim_lookup_table_entry_disk { #define WIM_LOOKUP_TABLE_ENTRY_DISK_SIZE 50 +/* Given a nonempty run of consecutive lookup table entries with the SOLID flag + * set, count how many specify resources (as opposed to streams within those + * resources). + * + * Returns the resulting count. */ +static size_t +count_solid_resources(const struct wim_lookup_table_entry_disk *entries, size_t max) +{ + size_t count = 0; + do { + struct wim_reshdr reshdr; + + get_wim_reshdr(&(entries++)->reshdr, &reshdr); + + if (!(reshdr.flags & WIM_RESHDR_FLAG_SOLID)) { + /* Run was terminated by a stand-alone stream entry. */ + break; + } + + if (reshdr.uncompressed_size == SOLID_RESOURCE_MAGIC_NUMBER) { + /* This is a resource entry. */ + count++; + } + } while (--max); + return count; +} + +/* + * Given a run of consecutive lookup table entries with the SOLID flag set and + * having @num_rspecs resource entries, load resource information from them into + * the resource specifications in the @rspecs array. + * + * Returns 0 on success, or a nonzero error code on failure. + */ +static int +do_load_solid_info(WIMStruct *wim, struct wim_resource_spec **rspecs, + size_t num_rspecs, + const struct wim_lookup_table_entry_disk *entries) +{ + for (size_t i = 0; i < num_rspecs; i++) { + struct wim_reshdr reshdr; + struct alt_chunk_table_header_disk hdr; + struct wim_resource_spec *rspec; + int ret; + + /* Advance to next resource entry. */ + + do { + get_wim_reshdr(&(entries++)->reshdr, &reshdr); + } while (reshdr.uncompressed_size != SOLID_RESOURCE_MAGIC_NUMBER); + + rspec = rspecs[i]; + + wim_res_hdr_to_spec(&reshdr, wim, rspec); + + /* For solid resources, the uncompressed size, compression type, + * and chunk size are stored in the resource itself, not in the + * lookup table. */ + + ret = full_pread(&wim->in_fd, &hdr, + sizeof(hdr), reshdr.offset_in_wim); + if (ret) { + ERROR("Failed to read header of solid resource " + "(offset_in_wim=%"PRIu64")", + reshdr.offset_in_wim); + return ret; + } + + rspec->uncompressed_size = le64_to_cpu(hdr.res_usize); + + /* Compression format numbers must be the same as in + * WIMGAPI to be compatible here. */ + BUILD_BUG_ON(WIMLIB_COMPRESSION_TYPE_NONE != 0); + BUILD_BUG_ON(WIMLIB_COMPRESSION_TYPE_XPRESS != 1); + BUILD_BUG_ON(WIMLIB_COMPRESSION_TYPE_LZX != 2); + BUILD_BUG_ON(WIMLIB_COMPRESSION_TYPE_LZMS != 3); + rspec->compression_type = le32_to_cpu(hdr.compression_format); + + rspec->chunk_size = le32_to_cpu(hdr.chunk_size); + + DEBUG("Solid resource %zu/%zu: %"PRIu64" => %"PRIu64" " + "(%"TS"/%"PRIu32") @ +%"PRIu64"", + i + 1, num_rspecs, + rspec->uncompressed_size, + rspec->size_in_wim, + wimlib_get_compression_type_string(rspec->compression_type), + rspec->chunk_size, + rspec->offset_in_wim); + + } + return 0; +} + +/* + * Given a nonempty run of consecutive lookup table entries with the SOLID flag + * set, allocate a 'struct wim_resource_spec' for each resource within that run. + * + * Returns 0 on success, or a nonzero error code on failure. + * Returns the pointers and count in *rspecs_ret and *num_rspecs_ret. + */ +static int +load_solid_info(WIMStruct *wim, + const struct wim_lookup_table_entry_disk *entries, + size_t num_remaining_entries, + struct wim_resource_spec ***rspecs_ret, + size_t *num_rspecs_ret) +{ + size_t num_rspecs; + struct wim_resource_spec **rspecs; + size_t i; + int ret; + + num_rspecs = count_solid_resources(entries, num_remaining_entries); + rspecs = CALLOC(num_rspecs, sizeof(rspecs[0])); + if (!rspecs) + return WIMLIB_ERR_NOMEM; + + for (i = 0; i < num_rspecs; i++) { + rspecs[i] = MALLOC(sizeof(struct wim_resource_spec)); + if (!rspecs[i]) { + ret = WIMLIB_ERR_NOMEM; + goto out_free_rspecs; + } + } + + ret = do_load_solid_info(wim, rspecs, num_rspecs, entries); + if (ret) + goto out_free_rspecs; + + *rspecs_ret = rspecs; + *num_rspecs_ret = num_rspecs; + return 0; + +out_free_rspecs: + for (i = 0; i < num_rspecs; i++) + FREE(rspecs[i]); + FREE(rspecs); + return ret; +} + +/* Given a 'struct wim_lookup_table_entry' allocated for a stream entry with the + * SOLID flag set, try to bind it to resource in the current solid run. */ +static int +bind_stream_to_solid_resource(const struct wim_reshdr *reshdr, + struct wim_lookup_table_entry *stream, + struct wim_resource_spec **rspecs, + size_t num_rspecs) +{ + u64 offset = reshdr->offset_in_wim; + + /* XXX: This linear search will be slow in the degenerate case where the + * number of solid resources in the run is huge. */ + stream->size = reshdr->size_in_wim; + stream->flags = reshdr->flags; + for (size_t i = 0; i < num_rspecs; i++) { + if (offset + stream->size <= rspecs[i]->uncompressed_size) { + stream->offset_in_res = offset; + lte_bind_wim_resource_spec(stream, rspecs[i]); + return 0; + } + offset -= rspecs[i]->uncompressed_size; + } + ERROR("Stream could not be assigned to a solid resource"); + return WIMLIB_ERR_INVALID_LOOKUP_TABLE_ENTRY; +} + +static void +free_solid_rspecs(struct wim_resource_spec **rspecs, size_t num_rspecs) +{ + if (rspecs) { + for (size_t i = 0; i < num_rspecs; i++) + if (list_empty(&rspecs[i]->stream_list)) + FREE(rspecs[i]); + FREE(rspecs); + } +} + static int cmp_streams_by_offset_in_res(const void *p1, const void *p2) { @@ -644,22 +832,21 @@ invalid_due_to_overflow: return WIMLIB_ERR_INVALID_LOOKUP_TABLE_ENTRY; invalid_due_to_overlap: - ERROR("Invalid resource entry (streams in packed resource overlap)"); + ERROR("Invalid resource entry (streams in solid resource overlap)"); return WIMLIB_ERR_INVALID_LOOKUP_TABLE_ENTRY; } -/* Validate the resource, or free it if unused. */ static int -finish_resource(struct wim_resource_spec *rspec) +finish_solid_rspecs(struct wim_resource_spec **rspecs, size_t num_rspecs) { - if (!list_empty(&rspec->stream_list)) { - /* This resource contains at least one stream. */ - return validate_resource(rspec); - } else { - /* No streams are in this resource. Get rid of it. */ - FREE(rspec); - return 0; + int ret = 0; + for (size_t i = 0; i < num_rspecs; i++) { + ret = validate_resource(rspecs[i]); + if (ret) + break; } + free_solid_rspecs(rspecs, num_rspecs); + return ret; } /* @@ -672,7 +859,16 @@ finish_resource(struct wim_resource_spec *rspec) * per-image location (the wim->image_metadata array). * * This works for both version WIM_VERSION_DEFAULT (68864) and version - * WIM_VERSION_PACKED_STREAMS (3584) WIMs. + * WIM_VERSION_SOLID (3584) WIMs. In the latter, a consecutive run of lookup + * table entries that all have flag WIM_RESHDR_FLAG_SOLID (0x10) set is a "solid + * run". A solid run logically contains zero or more resources, each of which + * logically contains zero or more streams. Physically, in such a run, a + * "lookup table entry" with uncompressed size SOLID_RESOURCE_MAGIC_NUMBER + * (0x100000000) specifies a resource, whereas any other entry specifies a + * stream. Within such a run, stream entries and resource entries need not be + * in any particular order, except that the order of the resource entries is + * important, as it affects how streams are assigned to resources. See the code + * for details. * * Possible return values: * WIMLIB_ERR_SUCCESS (0) @@ -690,10 +886,11 @@ read_wim_lookup_table(WIMStruct *wim) void *buf = NULL; struct wim_lookup_table *table = NULL; struct wim_lookup_table_entry *cur_entry = NULL; - struct wim_resource_spec *cur_rspec = NULL; size_t num_duplicate_entries = 0; size_t num_wrong_part_entries = 0; u32 image_index = 0; + struct wim_resource_spec **cur_solid_rspecs = NULL; + size_t cur_num_solid_rspecs = 0; DEBUG("Reading lookup table."); @@ -720,14 +917,13 @@ read_wim_lookup_table(WIMStruct *wim) * wim_lookup_table_entry's) from the raw lookup table buffer. Each of * these entries will point to a 'struct wim_resource_spec' that * describes the underlying resource. In WIMs with version number - * WIM_VERSION_PACKED_STREAMS, a resource may contain multiple streams. + * WIM_VERSION_SOLID, a resource may contain multiple streams. */ for (size_t i = 0; i < num_entries; i++) { const struct wim_lookup_table_entry_disk *disk_entry = &((const struct wim_lookup_table_entry_disk*)buf)[i]; struct wim_reshdr reshdr; u16 part_number; - struct wim_lookup_table_entry *duplicate_entry; /* Get the resource header */ get_wim_reshdr(&disk_entry->reshdr, &reshdr); @@ -735,16 +931,16 @@ read_wim_lookup_table(WIMStruct *wim) DEBUG("reshdr: size_in_wim=%"PRIu64", " "uncompressed_size=%"PRIu64", " "offset_in_wim=%"PRIu64", " - "flags=0x%02x\n", + "flags=0x%02x", reshdr.size_in_wim, reshdr.uncompressed_size, reshdr.offset_in_wim, reshdr.flags); - /* Ignore PACKED_STREAMS flag if it isn't supposed to be used in - * this WIM version */ + /* Ignore SOLID flag if it isn't supposed to be used in this WIM + * version. */ if (wim->hdr.wim_version == WIM_VERSION_DEFAULT) - reshdr.flags &= ~WIM_RESHDR_FLAG_PACKED_STREAMS; + reshdr.flags &= ~WIM_RESHDR_FLAG_SOLID; - /* Allocate a 'struct wim_lookup_table_entry' */ + /* Allocate a new 'struct wim_lookup_table_entry'. */ cur_entry = new_lookup_table_entry(); if (!cur_entry) goto oom; @@ -754,147 +950,99 @@ read_wim_lookup_table(WIMStruct *wim) cur_entry->refcnt = le32_to_cpu(disk_entry->refcnt); copy_hash(cur_entry->hash, disk_entry->hash); - /* Verify that the part number matches that of the underlying - * WIM file. */ - if (part_number != wim->hdr.part_number) { - num_wrong_part_entries++; - goto free_cur_entry_and_continue; - } + if (reshdr.flags & WIM_RESHDR_FLAG_SOLID) { - /* If resource is uncompressed, check for (unexpected) size - * mismatch. */ - if (!(reshdr.flags & (WIM_RESHDR_FLAG_PACKED_STREAMS | - WIM_RESHDR_FLAG_COMPRESSED))) { - if (reshdr.uncompressed_size != reshdr.size_in_wim) { - /* So ... This is an uncompressed resource, but - * its uncompressed size is NOT the same as its - * "compressed" size (size_in_wim). What to do - * with it? - * - * Based on a simple test, WIMGAPI seems to - * handle this as follows: - * - * if (size_in_wim > uncompressed_size) { - * Ignore uncompressed_size; use - * size_in_wim instead. - * } else { - * Honor uncompressed_size, but treat the - * part of the file data above size_in_wim - * as all zeros. - * } - * - * So we will do the same. - */ - if (reshdr.size_in_wim > reshdr.uncompressed_size) - reshdr.uncompressed_size = reshdr.size_in_wim; - } - } + /* SOLID entry */ - /* - * Possibly start a new resource. - * - * We need to start a new resource if: - * - * - There is no previous resource (cur_rspec). - * - * OR - * - * - The resource header did not have PACKED_STREAMS set, so it - * specifies a new, single-stream resource. - * - * OR - * - * - The resource header had PACKED_STREAMS set, and it's a - * special entry that specifies the resource itself as opposed - * to a stream, and we already encountered one such entry in - * the current resource. We will interpret this as the - * beginning of a new packed resource. (However, note that - * wimlib does not currently allow create WIMs with multiple - * packed resources, as to remain compatible with WIMGAPI.) - */ - if (likely(!cur_rspec) || - !(reshdr.flags & WIM_RESHDR_FLAG_PACKED_STREAMS) || - (reshdr.uncompressed_size == WIM_PACK_MAGIC_NUMBER && - cur_rspec->size_in_wim != 0)) - { - /* Finish previous resource (if existent) */ - if (cur_rspec) { - ret = finish_resource(cur_rspec); - cur_rspec = NULL; + if (!cur_solid_rspecs) { + /* Starting new run */ + ret = load_solid_info(wim, disk_entry, + num_entries - i, + &cur_solid_rspecs, + &cur_num_solid_rspecs); if (ret) goto out; } - /* Allocate the resource specification and initialize it - * with values from the current stream entry. */ - cur_rspec = MALLOC(sizeof(*cur_rspec)); - if (!cur_rspec) - goto oom; - - wim_res_hdr_to_spec(&reshdr, wim, cur_rspec); - - /* If this is a packed run, the current stream entry may - * specify a stream within the resource, and not the - * resource itself. Zero possibly irrelevant data until - * it is read for certain. */ - if (reshdr.flags & WIM_RESHDR_FLAG_PACKED_STREAMS) { - cur_rspec->size_in_wim = 0; - cur_rspec->uncompressed_size = 0; - cur_rspec->offset_in_wim = 0; + if (reshdr.uncompressed_size == SOLID_RESOURCE_MAGIC_NUMBER) { + /* Resource entry, not stream entry */ + goto free_cur_entry_and_continue; } - } - /* Now cur_rspec != NULL. */ + /* Stream entry */ - /* Checked for packed resource specification. */ - if (unlikely((reshdr.flags & WIM_RESHDR_FLAG_PACKED_STREAMS) && - reshdr.uncompressed_size == WIM_PACK_MAGIC_NUMBER)) - { - /* Found the specification for the packed resource. - * Transfer the values to the `struct - * wim_resource_spec', and discard the current stream - * since this lookup table entry did not, in fact, - * correspond to a "stream". */ - - /* The uncompressed size of the packed resource is - * actually stored in the header of the resource itself. - * Read it, and also grab the chunk size and compression - * type (which are not necessarily the defaults from the - * WIM header). */ - struct alt_chunk_table_header_disk hdr; - - ret = full_pread(&wim->in_fd, &hdr, - sizeof(hdr), reshdr.offset_in_wim); + ret = bind_stream_to_solid_resource(&reshdr, + cur_entry, + cur_solid_rspecs, + cur_num_solid_rspecs); if (ret) goto out; - cur_rspec->uncompressed_size = le64_to_cpu(hdr.res_usize); - cur_rspec->offset_in_wim = reshdr.offset_in_wim; - cur_rspec->size_in_wim = reshdr.size_in_wim; - cur_rspec->flags = reshdr.flags; - - /* Compression format numbers must be the same as in - * WIMGAPI to be compatible here. */ - BUILD_BUG_ON(WIMLIB_COMPRESSION_TYPE_NONE != 0); - BUILD_BUG_ON(WIMLIB_COMPRESSION_TYPE_XPRESS != 1); - BUILD_BUG_ON(WIMLIB_COMPRESSION_TYPE_LZX != 2); - BUILD_BUG_ON(WIMLIB_COMPRESSION_TYPE_LZMS != 3); - cur_rspec->compression_type = le32_to_cpu(hdr.compression_format); - - cur_rspec->chunk_size = le32_to_cpu(hdr.chunk_size); - - DEBUG("Full pack is %"PRIu64" compressed bytes " - "at file offset %"PRIu64" (flags 0x%02x)", - cur_rspec->size_in_wim, - cur_rspec->offset_in_wim, - cur_rspec->flags); - goto free_cur_entry_and_continue; + } else { + /* Normal stream/resource entry; SOLID not set. */ + + struct wim_resource_spec *rspec; + + if (unlikely(cur_solid_rspecs)) { + /* This entry terminated a solid run. */ + ret = finish_solid_rspecs(cur_solid_rspecs, + cur_num_solid_rspecs); + cur_solid_rspecs = NULL; + if (ret) + goto out; + } + + /* How to handle an uncompressed resource with its + * uncompressed size different from its compressed size? + * + * Based on a simple test, WIMGAPI seems to handle this + * as follows: + * + * if (size_in_wim > uncompressed_size) { + * Ignore uncompressed_size; use size_in_wim + * instead. + * } else { + * Honor uncompressed_size, but treat the part of + * the file data above size_in_wim as all zeros. + * } + * + * So we will do the same. */ + if (unlikely(!(reshdr.flags & + WIM_RESHDR_FLAG_COMPRESSED) && + (reshdr.size_in_wim > + reshdr.uncompressed_size))) + { + reshdr.uncompressed_size = reshdr.size_in_wim; + } + + /* Set up a resource specification for this stream. */ + + rspec = MALLOC(sizeof(struct wim_resource_spec)); + if (!rspec) + goto oom; + + wim_res_hdr_to_spec(&reshdr, wim, rspec); + + cur_entry->offset_in_res = 0; + cur_entry->size = reshdr.uncompressed_size; + cur_entry->flags = reshdr.flags; + + lte_bind_wim_resource_spec(cur_entry, rspec); } + /* cur_entry is now a stream bound to a resource. */ + /* Ignore entries with all zeroes in the hash field. */ if (is_zero_hash(cur_entry->hash)) goto free_cur_entry_and_continue; + /* Verify that the part number matches that of the underlying + * WIM file. */ + if (part_number != wim->hdr.part_number) { + num_wrong_part_entries++; + goto free_cur_entry_and_continue; + } + if (reshdr.flags & WIM_RESHDR_FLAG_METADATA) { /* Lookup table entry for a metadata resource. */ @@ -909,8 +1057,7 @@ read_wim_lookup_table(WIMStruct *wim) /* We don't currently support this case due to * the complications of multiple images sharing * the same metadata resource or a metadata - * resource also being referenced by files. - */ + * resource also being referenced by files. */ ERROR("Found metadata resource with refcnt != 1"); ret = WIMLIB_ERR_INVALID_LOOKUP_TABLE_ENTRY; goto out; @@ -947,8 +1094,7 @@ read_wim_lookup_table(WIMStruct *wim) /* Lookup table entry for a non-metadata stream. */ /* Ignore this stream if it's a duplicate. */ - duplicate_entry = lookup_stream(table, cur_entry->hash); - if (duplicate_entry) { + if (lookup_stream(table, cur_entry->hash)) { num_duplicate_entries++; goto free_cur_entry_and_continue; } @@ -958,37 +1104,20 @@ read_wim_lookup_table(WIMStruct *wim) lookup_table_insert(table, cur_entry); } - /* Add the stream to the current resource specification. */ - lte_bind_wim_resource_spec(cur_entry, cur_rspec); - if (reshdr.flags & WIM_RESHDR_FLAG_PACKED_STREAMS) { - /* In packed runs, the offset field is used for - * in-resource offset, not the in-WIM offset, and the - * size field is used for the uncompressed size, not the - * compressed size. */ - cur_entry->offset_in_res = reshdr.offset_in_wim; - cur_entry->size = reshdr.size_in_wim; - cur_entry->flags = reshdr.flags; - /* cur_rspec stays the same */ - - } else { - /* Normal case: The stream corresponds one-to-one with - * the resource entry. */ - cur_entry->offset_in_res = 0; - cur_entry->size = reshdr.uncompressed_size; - cur_entry->flags = reshdr.flags; - cur_rspec = NULL; - } continue; free_cur_entry_and_continue: + if (cur_solid_rspecs && + cur_entry->resource_location == RESOURCE_IN_WIM) + lte_unbind_wim_resource_spec(cur_entry); free_lookup_table_entry(cur_entry); } cur_entry = NULL; - /* Validate the last resource. */ - if (cur_rspec) { - ret = finish_resource(cur_rspec); - cur_rspec = NULL; + if (cur_solid_rspecs) { + /* End of lookup table terminated a solid run. */ + ret = finish_solid_rspecs(cur_solid_rspecs, cur_num_solid_rspecs); + cur_solid_rspecs = NULL; if (ret) goto out; } @@ -1012,17 +1141,17 @@ read_wim_lookup_table(WIMStruct *wim) DEBUG("Done reading lookup table."); wim->lookup_table = table; - table = NULL; ret = 0; - goto out; + goto out_free_buf; + oom: ERROR("Not enough memory to read lookup table!"); ret = WIMLIB_ERR_NOMEM; out: - if (cur_rspec && list_empty(&cur_rspec->stream_list)) - FREE(cur_rspec); + free_solid_rspecs(cur_solid_rspecs, cur_num_solid_rspecs); free_lookup_table_entry(cur_entry); free_lookup_table(table); +out_free_buf: FREE(buf); return ret; } @@ -1038,6 +1167,10 @@ put_wim_lookup_table_entry(struct wim_lookup_table_entry_disk *disk_entry, copy_hash(disk_entry->hash, hash); } +/* Note: the list of stream entries must be sorted so that all entries for the + * same solid resource are consecutive. In addition, entries with + * WIM_RESHDR_FLAG_METADATA set must be in the same order as the indices of the + * underlying images. */ int write_wim_lookup_table_from_stream_list(struct list_head *stream_list, struct filedes *out_fd, @@ -1051,12 +1184,14 @@ write_wim_lookup_table_from_stream_list(struct list_head *stream_list, struct wim_lookup_table_entry_disk *table_buf_ptr; int ret; u64 prev_res_offset_in_wim = ~0ULL; + u64 prev_uncompressed_size; + u64 logical_offset; table_size = 0; list_for_each_entry(lte, stream_list, lookup_table_list) { table_size += sizeof(struct wim_lookup_table_entry_disk); - if (lte->out_reshdr.flags & WIM_RESHDR_FLAG_PACKED_STREAMS && + if (lte->out_reshdr.flags & WIM_RESHDR_FLAG_SOLID && lte->out_res_offset_in_wim != prev_res_offset_in_wim) { table_size += sizeof(struct wim_lookup_table_entry_disk); @@ -1076,38 +1211,46 @@ write_wim_lookup_table_from_stream_list(struct list_head *stream_list, table_buf_ptr = table_buf; prev_res_offset_in_wim = ~0ULL; + prev_uncompressed_size = 0; + logical_offset = 0; list_for_each_entry(lte, stream_list, lookup_table_list) { + if (lte->out_reshdr.flags & WIM_RESHDR_FLAG_SOLID) { + struct wim_reshdr tmp_reshdr; - put_wim_lookup_table_entry(table_buf_ptr++, - <e->out_reshdr, - part_number, - lte->out_refcnt, - lte->hash); - if (lte->out_reshdr.flags & WIM_RESHDR_FLAG_PACKED_STREAMS && - lte->out_res_offset_in_wim != prev_res_offset_in_wim) - { - /* Put the main resource entry for the pack. */ + /* Eww. When WIMGAPI sees multiple solid resources, it + * expects the offsets to be adjusted as if there were + * really only one solid resource. */ - struct wim_reshdr reshdr; + if (lte->out_res_offset_in_wim != prev_res_offset_in_wim) { + /* Put the resource entry for solid resource */ + tmp_reshdr.offset_in_wim = lte->out_res_offset_in_wim; + tmp_reshdr.size_in_wim = lte->out_res_size_in_wim; + tmp_reshdr.uncompressed_size = SOLID_RESOURCE_MAGIC_NUMBER; + tmp_reshdr.flags = WIM_RESHDR_FLAG_SOLID; - reshdr.offset_in_wim = lte->out_res_offset_in_wim; - reshdr.size_in_wim = lte->out_res_size_in_wim; - reshdr.uncompressed_size = WIM_PACK_MAGIC_NUMBER; - reshdr.flags = WIM_RESHDR_FLAG_PACKED_STREAMS; + put_wim_lookup_table_entry(table_buf_ptr++, + &tmp_reshdr, + part_number, + 1, zero_hash); - DEBUG("Putting main entry for pack: " - "size_in_wim=%"PRIu64", " - "offset_in_wim=%"PRIu64", " - "uncompressed_size=%"PRIu64, - reshdr.size_in_wim, - reshdr.offset_in_wim, - reshdr.uncompressed_size); + logical_offset += prev_uncompressed_size; + prev_res_offset_in_wim = lte->out_res_offset_in_wim; + prev_uncompressed_size = lte->out_res_uncompressed_size; + } + tmp_reshdr = lte->out_reshdr; + tmp_reshdr.offset_in_wim += logical_offset; put_wim_lookup_table_entry(table_buf_ptr++, - &reshdr, + &tmp_reshdr, part_number, - 1, zero_hash); - prev_res_offset_in_wim = lte->out_res_offset_in_wim; + lte->out_refcnt, + lte->hash); + } else { + put_wim_lookup_table_entry(table_buf_ptr++, + <e->out_reshdr, + part_number, + lte->out_refcnt, + lte->hash); } } @@ -1129,30 +1272,6 @@ write_wim_lookup_table_from_stream_list(struct list_head *stream_list, return ret; } -int -lte_zero_real_refcnt(struct wim_lookup_table_entry *lte, void *_ignore) -{ - lte->real_refcnt = 0; - return 0; -} - -int -lte_zero_out_refcnt(struct wim_lookup_table_entry *lte, void *_ignore) -{ - lte->out_refcnt = 0; - return 0; -} - -int -lte_free_extracted_file(struct wim_lookup_table_entry *lte, void *_ignore) -{ - if (lte->extracted_file != NULL) { - FREE(lte->extracted_file); - lte->extracted_file = NULL; - } - return 0; -} - /* Allocate a stream entry for the contents of the buffer, or re-use an existing * entry in @lookup_table for the same stream. */ struct wim_lookup_table_entry * @@ -1254,7 +1373,7 @@ lte_to_wimlib_resource_entry(const struct wim_lookup_table_entry *lte, wentry->uncompressed_size = lte->size; if (lte->resource_location == RESOURCE_IN_WIM) { wentry->part_number = lte->rspec->wim->hdr.part_number; - if (lte->flags & WIM_RESHDR_FLAG_PACKED_STREAMS) { + if (lte->flags & WIM_RESHDR_FLAG_SOLID) { wentry->compressed_size = 0; wentry->offset = lte->offset_in_res; } else { @@ -1271,7 +1390,7 @@ lte_to_wimlib_resource_entry(const struct wim_lookup_table_entry *lte, wentry->is_metadata = (lte->flags & WIM_RESHDR_FLAG_METADATA) != 0; wentry->is_free = (lte->flags & WIM_RESHDR_FLAG_FREE) != 0; wentry->is_spanned = (lte->flags & WIM_RESHDR_FLAG_SPANNED) != 0; - wentry->packed = (lte->flags & WIM_RESHDR_FLAG_PACKED_STREAMS) != 0; + wentry->packed = (lte->flags & WIM_RESHDR_FLAG_SOLID) != 0; } struct iterate_lte_context { @@ -1302,7 +1421,7 @@ wimlib_iterate_lookup_table(WIMStruct *wim, int flags, .cb = cb, .user_ctx = user_ctx, }; - if (wim->hdr.part_number == 1) { + if (wim_has_metadata(wim)) { int ret; for (int i = 0; i < wim->hdr.image_count; i++) { ret = do_iterate_lte(wim->image_metadata[i]->metadata_lte,