]> wimlib.net Git - wimlib/blobdiff - src/blob_table.c
wim.c: replace finalize_wim_struct() with wim_decrement_refcnt()
[wimlib] / src / blob_table.c
index 84835b5a7a03b6ca2d5a7774c55f98c025a2bad2..e1fb6adac77c3c46603468c9e1be058d450270f7 100644 (file)
@@ -34,6 +34,7 @@
 #include <unistd.h> /* for unlink()  */
 
 #include "wimlib/assert.h"
+#include "wimlib/bitops.h"
 #include "wimlib/blob_table.h"
 #include "wimlib/encoding.h"
 #include "wimlib/endianness.h"
 struct blob_table {
        struct hlist_head *array;
        size_t num_blobs;
-       size_t capacity;
+       size_t mask; /* capacity - 1; capacity is a power of 2  */
 };
 
+static size_t
+next_power_of_2(size_t n)
+{
+       if (n <= 1)
+               return 1;
+       return (size_t)1 << (1 + flsw(n - 1));
+}
+
 struct blob_table *
 new_blob_table(size_t capacity)
 {
        struct blob_table *table;
        struct hlist_head *array;
 
+       capacity = next_power_of_2(capacity);
+
        table = MALLOC(sizeof(struct blob_table));
        if (table == NULL)
                goto oom;
@@ -69,7 +80,7 @@ new_blob_table(size_t capacity)
        }
 
        table->num_blobs = 0;
-       table->capacity = capacity;
+       table->mask = capacity - 1;
        table->array = array;
        return table;
 
@@ -99,18 +110,8 @@ free_blob_table(struct blob_table *table)
 struct blob_descriptor *
 new_blob_descriptor(void)
 {
-       struct blob_descriptor *blob;
-
-       blob = CALLOC(1, sizeof(struct blob_descriptor));
-       if (blob == NULL)
-               return NULL;
-
-       blob->refcnt = 1;
-
-       /* blob->blob_location = BLOB_NONEXISTENT  */
-       BUILD_BUG_ON(BLOB_NONEXISTENT != 0);
-
-       return blob;
+       STATIC_ASSERT(BLOB_NONEXISTENT == 0);
+       return CALLOC(1, sizeof(struct blob_descriptor));
 }
 
 struct blob_descriptor *
@@ -134,8 +135,8 @@ clone_blob_descriptor(const struct blob_descriptor *old)
 #endif
 #ifdef WITH_FUSE
        case BLOB_IN_STAGING_FILE:
-               BUILD_BUG_ON((void*)&old->file_on_disk !=
-                            (void*)&old->staging_file_name);
+               STATIC_ASSERT((void*)&old->file_on_disk ==
+                             (void*)&old->staging_file_name);
 #endif
                new->file_on_disk = TSTRDUP(old->file_on_disk);
                if (new->file_on_disk == NULL)
@@ -148,26 +149,11 @@ clone_blob_descriptor(const struct blob_descriptor *old)
                break;
 #ifdef WITH_NTFS_3G
        case BLOB_IN_NTFS_VOLUME:
-               if (old->ntfs_loc) {
-                       new->ntfs_loc = memdup(old->ntfs_loc,
-                                              sizeof(struct ntfs_location));
-                       if (new->ntfs_loc == NULL)
-                               goto out_free;
-                       new->ntfs_loc->path = STRDUP(old->ntfs_loc->path);
-                       new->ntfs_loc->attr_name = NULL;
-                       if (new->ntfs_loc->path == NULL)
-                               goto out_free;
-                       if (new->ntfs_loc->attr_name_nchars != 0) {
-                               new->ntfs_loc->attr_name =
-                                       utf16le_dup(old->ntfs_loc->attr_name);
-                               if (new->ntfs_loc->attr_name == NULL)
-                                       goto out_free;
-                       }
-               }
+               new->ntfs_loc = clone_ntfs_location(old->ntfs_loc);
+               if (!new->ntfs_loc)
+                       goto out_free;
                break;
 #endif
-       default:
-               break;
        }
        return new;
 
@@ -176,15 +162,22 @@ out_free:
        return NULL;
 }
 
-static void
+/* Release a blob descriptor from its location, if any, and set its new location
+ * to BLOB_NONEXISTENT.  */
+void
 blob_release_location(struct blob_descriptor *blob)
 {
        switch (blob->blob_location) {
-       case BLOB_IN_WIM:
+       case BLOB_IN_WIM: {
+               struct wim_resource_descriptor *rdesc = blob->rdesc;
+
                list_del(&blob->rdesc_node);
-               if (list_empty(&blob->rdesc->blob_list))
-                       FREE(blob->rdesc);
+               if (list_empty(&rdesc->blob_list)) {
+                       wim_decrement_refcnt(rdesc->wim);
+                       FREE(rdesc);
+               }
                break;
+       }
        case BLOB_IN_FILE_ON_DISK:
 #ifdef __WIN32__
        case BLOB_IN_WINNT_FILE_ON_DISK:
@@ -192,26 +185,22 @@ blob_release_location(struct blob_descriptor *blob)
 #endif
 #ifdef WITH_FUSE
        case BLOB_IN_STAGING_FILE:
-               BUILD_BUG_ON((void*)&blob->file_on_disk !=
-                            (void*)&blob->staging_file_name);
+               STATIC_ASSERT((void*)&blob->file_on_disk ==
+                             (void*)&blob->staging_file_name);
 #endif
        case BLOB_IN_ATTACHED_BUFFER:
-               BUILD_BUG_ON((void*)&blob->file_on_disk !=
-                            (void*)&blob->attached_buffer);
+               STATIC_ASSERT((void*)&blob->file_on_disk ==
+                             (void*)&blob->attached_buffer);
                FREE(blob->file_on_disk);
                break;
 #ifdef WITH_NTFS_3G
        case BLOB_IN_NTFS_VOLUME:
-               if (blob->ntfs_loc) {
-                       FREE(blob->ntfs_loc->path);
-                       FREE(blob->ntfs_loc->attr_name);
-                       FREE(blob->ntfs_loc);
-               }
+               if (blob->ntfs_loc)
+                       free_ntfs_location(blob->ntfs_loc);
                break;
 #endif
-       default:
-               break;
        }
+       blob->blob_location = BLOB_NONEXISTENT;
 }
 
 void
@@ -263,10 +252,21 @@ finalize_blob(struct blob_descriptor *blob)
 void
 blob_decrement_refcnt(struct blob_descriptor *blob, struct blob_table *table)
 {
-       if (unlikely(blob->refcnt == 0))  /* See comment above  */
+       blob_subtract_refcnt(blob, table, 1);
+}
+
+void
+blob_subtract_refcnt(struct blob_descriptor *blob, struct blob_table *table,
+                    u32 count)
+{
+       if (unlikely(blob->refcnt < count)) {
+               blob->refcnt = 0; /* See comment above  */
                return;
+       }
+
+       blob->refcnt -= count;
 
-       if (--blob->refcnt != 0)
+       if (blob->refcnt != 0)
                return;
 
        if (blob->unhashed) {
@@ -306,7 +306,7 @@ blob_decrement_num_opened_fds(struct blob_descriptor *blob)
 static void
 blob_table_insert_raw(struct blob_table *table, struct blob_descriptor *blob)
 {
-       size_t i = blob->hash_short % table->capacity;
+       size_t i = blob->hash_short & table->mask;
 
        hlist_add_head(&blob->hash_list, &table->array[i]);
 }
@@ -317,24 +317,21 @@ enlarge_blob_table(struct blob_table *table)
        size_t old_capacity, new_capacity;
        struct hlist_head *old_array, *new_array;
        struct blob_descriptor *blob;
-       struct hlist_node *cur, *tmp;
+       struct hlist_node *tmp;
        size_t i;
 
-       old_capacity = table->capacity;
+       old_capacity = table->mask + 1;
        new_capacity = old_capacity * 2;
        new_array = CALLOC(new_capacity, sizeof(struct hlist_head));
        if (new_array == NULL)
                return;
        old_array = table->array;
        table->array = new_array;
-       table->capacity = new_capacity;
+       table->mask = new_capacity - 1;
 
-       for (i = 0; i < old_capacity; i++) {
-               hlist_for_each_entry_safe(blob, cur, tmp, &old_array[i], hash_list) {
-                       hlist_del(&blob->hash_list);
+       for (i = 0; i < old_capacity; i++)
+               hlist_for_each_entry_safe(blob, tmp, &old_array[i], hash_list)
                        blob_table_insert_raw(table, blob);
-               }
-       }
        FREE(old_array);
 }
 
@@ -343,7 +340,7 @@ void
 blob_table_insert(struct blob_table *table, struct blob_descriptor *blob)
 {
        blob_table_insert_raw(table, blob);
-       if (++table->num_blobs > table->capacity)
+       if (table->num_blobs++ > table->mask)
                enlarge_blob_table(table);
 }
 
@@ -365,10 +362,9 @@ lookup_blob(const struct blob_table *table, const u8 *hash)
 {
        size_t i;
        struct blob_descriptor *blob;
-       struct hlist_node *pos;
 
-       i = load_size_t_unaligned(hash) % table->capacity;
-       hlist_for_each_entry(blob, pos, &table->array[i], hash_list)
+       i = load_size_t_unaligned(hash) & table->mask;
+       hlist_for_each_entry(blob, &table->array[i], hash_list)
                if (hashes_equal(hash, blob->hash))
                        return blob;
        return NULL;
@@ -381,11 +377,11 @@ for_blob_in_table(struct blob_table *table,
                  int (*visitor)(struct blob_descriptor *, void *), void *arg)
 {
        struct blob_descriptor *blob;
-       struct hlist_node *pos, *tmp;
+       struct hlist_node *tmp;
        int ret;
 
-       for (size_t i = 0; i < table->capacity; i++) {
-               hlist_for_each_entry_safe(blob, pos, tmp, &table->array[i],
+       for (size_t i = 0; i <= table->mask; i++) {
+               hlist_for_each_entry_safe(blob, tmp, &table->array[i],
                                          hash_list)
                {
                        ret = visitor(blob, arg);
@@ -415,7 +411,12 @@ cmp_blobs_by_sequential_order(const void *p1, const void *p2)
 
        v = (int)blob1->blob_location - (int)blob2->blob_location;
 
-       /* Different resource locations?  */
+       /* Different locations?  Note: "unsafe compaction mode" requires that
+        * blobs in WIMs sort before all others.  For the logic here to ensure
+        * this, BLOB_IN_WIM must have the lowest value among all defined
+        * blob_locations.  Statically verify that the enum values haven't
+        * changed.  */
+       STATIC_ASSERT(BLOB_NONEXISTENT == 0 && BLOB_IN_WIM == 1);
        if (v)
                return v;
 
@@ -424,22 +425,39 @@ cmp_blobs_by_sequential_order(const void *p1, const void *p2)
                wim1 = blob1->rdesc->wim;
                wim2 = blob2->rdesc->wim;
 
-               /* Different (possibly split) WIMs?  */
+               /* Different WIM files?  */
                if (wim1 != wim2) {
-                       v = memcmp(wim1->hdr.guid, wim2->hdr.guid, WIM_GUID_LEN);
+
+                       /* Resources from the WIM file currently being compacted
+                        * (if any) must always sort first.  */
+                       v = (int)wim2->being_compacted - (int)wim1->being_compacted;
+                       if (v)
+                               return v;
+
+                       /* Different split WIMs?  */
+                       v = cmp_guids(wim1->hdr.guid, wim2->hdr.guid);
+                       if (v)
+                               return v;
+
+                       /* Different part numbers in the same split WIM?  */
+                       v = (int)wim1->hdr.part_number - (int)wim2->hdr.part_number;
                        if (v)
                                return v;
+
+                       /* Probably two WIMStructs for the same on-disk file.
+                        * Just sort by pointer.  */
+                       return wim1 < wim2 ? -1 : 1;
                }
 
-               /* Different part numbers in the same WIM?  */
-               v = (int)wim1->hdr.part_number - (int)wim2->hdr.part_number;
-               if (v)
-                       return v;
+               /* Same WIM file  */
 
+               /* Sort by increasing resource offset  */
                if (blob1->rdesc->offset_in_wim != blob2->rdesc->offset_in_wim)
                        return cmp_u64(blob1->rdesc->offset_in_wim,
                                       blob2->rdesc->offset_in_wim);
 
+               /* The blobs are in the same solid resource.  Sort by increasing
+                * offset in the resource.  */
                return cmp_u64(blob1->offset_in_res, blob2->offset_in_res);
 
        case BLOB_IN_FILE_ON_DISK:
@@ -449,13 +467,17 @@ cmp_blobs_by_sequential_order(const void *p1, const void *p2)
 #ifdef __WIN32__
        case BLOB_IN_WINNT_FILE_ON_DISK:
        case BLOB_WIN32_ENCRYPTED:
+               /* Windows: compare by starting LCN (logical cluster number)  */
+               v = cmp_u64(blob1->sort_key, blob2->sort_key);
+               if (v)
+                       return v;
 #endif
                /* Compare files by path: just a heuristic that will place files
                 * in the same directory next to each other.  */
                return tstrcmp(blob1->file_on_disk, blob2->file_on_disk);
 #ifdef WITH_NTFS_3G
        case BLOB_IN_NTFS_VOLUME:
-               return tstrcmp(blob1->ntfs_loc->path, blob2->ntfs_loc->path);
+               return cmp_ntfs_locations(blob1->ntfs_loc, blob2->ntfs_loc);
 #endif
        default:
                /* No additional sorting order defined for this resource
@@ -627,7 +649,7 @@ do_load_solid_info(WIMStruct *wim, struct wim_resource_descriptor **rdescs,
 
                rdesc = rdescs[i];
 
-               wim_res_hdr_to_desc(&reshdr, wim, rdesc);
+               wim_reshdr_to_desc(&reshdr, wim, rdesc);
 
                /* For solid resources, the uncompressed size, compression type,
                 * and chunk size are stored in the resource itself, not in the
@@ -646,22 +668,12 @@ do_load_solid_info(WIMStruct *wim, struct wim_resource_descriptor **rdescs,
 
                /* Compression format numbers must be the same as in
                 * WIMGAPI to be compatible here.  */
-               BUILD_BUG_ON(WIMLIB_COMPRESSION_TYPE_NONE != 0);
-               BUILD_BUG_ON(WIMLIB_COMPRESSION_TYPE_XPRESS != 1);
-               BUILD_BUG_ON(WIMLIB_COMPRESSION_TYPE_LZX != 2);
-               BUILD_BUG_ON(WIMLIB_COMPRESSION_TYPE_LZMS != 3);
+               STATIC_ASSERT(WIMLIB_COMPRESSION_TYPE_NONE == 0);
+               STATIC_ASSERT(WIMLIB_COMPRESSION_TYPE_XPRESS == 1);
+               STATIC_ASSERT(WIMLIB_COMPRESSION_TYPE_LZX == 2);
+               STATIC_ASSERT(WIMLIB_COMPRESSION_TYPE_LZMS == 3);
                rdesc->compression_type = le32_to_cpu(hdr.compression_format);
-
                rdesc->chunk_size = le32_to_cpu(hdr.chunk_size);
-
-               DEBUG("Solid resource %zu/%zu: %"PRIu64" => %"PRIu64" "
-                     "(%"TS"/%"PRIu32") @ +%"PRIu64"",
-                     i + 1, num_rdescs,
-                     rdesc->uncompressed_size,
-                     rdesc->size_in_wim,
-                     wimlib_get_compression_type_string(rdesc->compression_type),
-                     rdesc->chunk_size,
-                     rdesc->offset_in_wim);
        }
        return 0;
 }
@@ -703,6 +715,8 @@ load_solid_info(WIMStruct *wim,
        if (ret)
                goto out_free_rdescs;
 
+       wim->refcnt += num_rdescs;
+
        *rdescs_ret = rdescs;
        *num_rdescs_ret = num_rdescs;
        return 0;
@@ -728,11 +742,9 @@ assign_blob_to_solid_resource(const struct wim_reshdr *reshdr,
        /* XXX: This linear search will be slow in the degenerate case where the
         * number of solid resources in the run is huge.  */
        blob->size = reshdr->size_in_wim;
-       blob->flags = reshdr->flags;
        for (size_t i = 0; i < num_rdescs; i++) {
                if (offset + blob->size <= rdescs[i]->uncompressed_size) {
-                       blob->offset_in_res = offset;
-                       blob_set_is_located_in_wim_resource(blob, rdescs[i]);
+                       blob_set_is_located_in_wim_resource(blob, rdescs[i], offset);
                        return 0;
                }
                offset -= rdescs[i]->uncompressed_size;
@@ -745,9 +757,12 @@ static void
 free_solid_rdescs(struct wim_resource_descriptor **rdescs, size_t num_rdescs)
 {
        if (rdescs) {
-               for (size_t i = 0; i < num_rdescs; i++)
-                       if (list_empty(&rdescs[i]->blob_list))
+               for (size_t i = 0; i < num_rdescs; i++) {
+                       if (list_empty(&rdescs[i]->blob_list)) {
+                               rdescs[i]->wim->refcnt--;
                                FREE(rdescs[i]);
+                       }
+               }
                FREE(rdescs);
        }
 }
@@ -874,13 +889,12 @@ read_blob_table(WIMStruct *wim)
        struct blob_table *table = NULL;
        struct blob_descriptor *cur_blob = NULL;
        size_t num_duplicate_blobs = 0;
+       size_t num_empty_blobs = 0;
        size_t num_wrong_part_blobs = 0;
        u32 image_index = 0;
        struct wim_resource_descriptor **cur_solid_rdescs = NULL;
        size_t cur_num_solid_rdescs = 0;
 
-       DEBUG("Reading blob table.");
-
        /* Calculate the number of entries in the blob table.  */
        num_entries = wim->hdr.blob_table_reshdr.uncompressed_size /
                      sizeof(struct blob_descriptor_disk);
@@ -892,7 +906,7 @@ read_blob_table(WIMStruct *wim)
 
        /* Allocate a hash table to map SHA-1 message digests into blob
         * descriptors.  This is the in-memory "blob table".  */
-       table = new_blob_table(num_entries * 2 + 1);
+       table = new_blob_table(num_entries);
        if (!table)
                goto oom;
 
@@ -907,13 +921,6 @@ read_blob_table(WIMStruct *wim)
                /* Get the resource header  */
                get_wim_reshdr(&disk_entry->reshdr, &reshdr);
 
-               DEBUG("reshdr: size_in_wim=%"PRIu64", "
-                     "uncompressed_size=%"PRIu64", "
-                     "offset_in_wim=%"PRIu64", "
-                     "flags=0x%02x",
-                     reshdr.size_in_wim, reshdr.uncompressed_size,
-                     reshdr.offset_in_wim, reshdr.flags);
-
                /* Ignore SOLID flag if it isn't supposed to be used in this WIM
                 * version.  */
                if (wim->hdr.wim_version == WIM_VERSION_DEFAULT)
@@ -971,27 +978,13 @@ read_blob_table(WIMStruct *wim)
                                        goto out;
                        }
 
-                       /* How to handle an uncompressed resource with its
-                        * uncompressed size different from its compressed size?
-                        *
-                        * Based on a simple test, WIMGAPI seems to handle this
-                        * as follows:
-                        *
-                        * if (size_in_wim > uncompressed_size) {
-                        *      Ignore uncompressed_size; use size_in_wim
-                        *      instead.
-                        * } else {
-                        *      Honor uncompressed_size, but treat the part of
-                        *      the file data above size_in_wim as all zeros.
-                        * }
-                        *
-                        * So we will do the same.  */
-                       if (unlikely(!(reshdr.flags &
-                                      WIM_RESHDR_FLAG_COMPRESSED) &&
-                                    (reshdr.size_in_wim >
-                                     reshdr.uncompressed_size)))
+                       if (unlikely(!(reshdr.flags & WIM_RESHDR_FLAG_COMPRESSED) &&
+                                    (reshdr.size_in_wim != reshdr.uncompressed_size)))
                        {
-                               reshdr.uncompressed_size = reshdr.size_in_wim;
+                               ERROR("Uncompressed resource has "
+                                     "size_in_wim != uncompressed_size");
+                               ret = WIMLIB_ERR_INVALID_LOOKUP_TABLE_ENTRY;
+                               goto out;
                        }
 
                        /* Set up a resource descriptor for this blob.  */
@@ -1000,30 +993,33 @@ read_blob_table(WIMStruct *wim)
                        if (!rdesc)
                                goto oom;
 
-                       wim_res_hdr_to_desc(&reshdr, wim, rdesc);
-
-                       cur_blob->offset_in_res = 0;
-                       cur_blob->size = reshdr.uncompressed_size;
-                       cur_blob->flags = reshdr.flags;
-
-                       blob_set_is_located_in_wim_resource(cur_blob, rdesc);
+                       wim_reshdr_to_desc_and_blob(&reshdr, wim, rdesc, cur_blob);
+                       wim->refcnt++;
                }
 
                /* cur_blob is now a blob bound to a resource.  */
 
                /* Ignore entries with all zeroes in the hash field.  */
-               if (is_zero_hash(cur_blob->hash))
+               if (unlikely(is_zero_hash(cur_blob->hash)))
+                       goto free_cur_blob_and_continue;
+
+               /* Verify that the blob has nonzero size.  */
+               if (unlikely(cur_blob->size == 0)) {
+                       num_empty_blobs++;
                        goto free_cur_blob_and_continue;
+               }
 
                /* Verify that the part number matches that of the underlying
                 * WIM file.  */
-               if (part_number != wim->hdr.part_number) {
+               if (unlikely(part_number != wim->hdr.part_number)) {
                        num_wrong_part_blobs++;
                        goto free_cur_blob_and_continue;
                }
 
                if (reshdr.flags & WIM_RESHDR_FLAG_METADATA) {
 
+                       cur_blob->is_metadata = 1;
+
                        /* Blob table entry for a metadata resource.  */
 
                        /* Metadata entries with no references must be ignored.
@@ -1042,6 +1038,13 @@ read_blob_table(WIMStruct *wim)
                                goto out;
                        }
 
+                       if (reshdr.flags & WIM_RESHDR_FLAG_SOLID) {
+                               ERROR("Image metadata in solid resources "
+                                     "is unsupported.");
+                               ret = WIMLIB_ERR_INVALID_LOOKUP_TABLE_ENTRY;
+                               goto out;
+                       }
+
                        if (wim->hdr.part_number != 1) {
                                WARNING("Ignoring metadata resource found in a "
                                        "non-first part of the split WIM");
@@ -1063,11 +1066,6 @@ read_blob_table(WIMStruct *wim)
                         * this overrides the actual locations of the metadata
                         * resources themselves in the WIM file as well as any
                         * information written in the XML data.  */
-                       DEBUG("Found metadata resource for image %"PRIu32" at "
-                             "offset %"PRIu64".",
-                             image_index + 1,
-                             reshdr.offset_in_wim);
-
                        wim->image_metadata[image_index++]->metadata_blob = cur_blob;
                } else {
                        /* Blob table entry for a non-metadata blob.  */
@@ -1111,12 +1109,14 @@ read_blob_table(WIMStruct *wim)
        if (num_duplicate_blobs > 0)
                WARNING("Ignoring %zu duplicate blobs", num_duplicate_blobs);
 
+       if (num_empty_blobs > 0)
+               WARNING("Ignoring %zu empty blobs", num_empty_blobs);
+
        if (num_wrong_part_blobs > 0) {
                WARNING("Ignoring %zu blobs with wrong part number",
                        num_wrong_part_blobs);
        }
 
-       DEBUG("Done reading blob table.");
        wim->blob_table = table;
        ret = 0;
        goto out_free_buf;
@@ -1145,9 +1145,9 @@ write_blob_descriptor(struct blob_descriptor_disk *disk_entry,
 }
 
 /* Note: the list of blob descriptors must be sorted so that all entries for the
- * same solid resource are consecutive.  In addition, blob descriptors with
- * WIM_RESHDR_FLAG_METADATA set must be in the same order as the indices of the
- * underlying images.  */
+ * same solid resource are consecutive.  In addition, blob descriptors for
+ * metadata resources must be in the same order as the indices of the underlying
+ * images.  */
 int
 write_blob_table_from_blob_list(struct list_head *blob_list,
                                struct filedes *out_fd,
@@ -1176,9 +1176,6 @@ write_blob_table_from_blob_list(struct list_head *blob_list,
                }
        }
 
-       DEBUG("Writing WIM blob table (size=%zu, offset=%"PRIu64")",
-             table_size, out_fd->offset);
-
        table_buf = MALLOC(table_size);
        if (table_buf == NULL) {
                ERROR("Failed to allocate %zu bytes for temporary blob table",
@@ -1229,7 +1226,7 @@ write_blob_table_from_blob_list(struct list_head *blob_list,
         * compressed blob table, MS software cannot.  */
        ret = write_wim_resource_from_buffer(table_buf,
                                             table_size,
-                                            WIM_RESHDR_FLAG_METADATA,
+                                            true,
                                             out_fd,
                                             WIMLIB_COMPRESSION_TYPE_NONE,
                                             0,
@@ -1237,7 +1234,6 @@ write_blob_table_from_blob_list(struct list_head *blob_list,
                                             NULL,
                                             write_resource_flags);
        FREE(table_buf);
-       DEBUG("ret=%d", ret);
        return ret;
 }
 
@@ -1248,31 +1244,58 @@ new_blob_from_data_buffer(const void *buffer, size_t size,
                          struct blob_table *blob_table)
 {
        u8 hash[SHA1_HASH_SIZE];
-       struct blob_descriptor *blob, *existing_blob;
+       struct blob_descriptor *blob;
+       void *buffer_copy;
 
        sha1_buffer(buffer, size, hash);
-       existing_blob = lookup_blob(blob_table, hash);
-       if (existing_blob) {
-               wimlib_assert(existing_blob->size == size);
-               blob = existing_blob;
-               blob->refcnt++;
+
+       blob = lookup_blob(blob_table, hash);
+       if (blob)
+               return blob;
+
+       blob = new_blob_descriptor();
+       if (!blob)
+               return NULL;
+
+       buffer_copy = memdup(buffer, size);
+       if (!buffer_copy) {
+               free_blob_descriptor(blob);
+               return NULL;
+       }
+       blob_set_is_located_in_attached_buffer(blob, buffer_copy, size);
+       copy_hash(blob->hash, hash);
+       blob_table_insert(blob_table, blob);
+       return blob;
+}
+
+struct blob_descriptor *
+after_blob_hashed(struct blob_descriptor *blob,
+                 struct blob_descriptor **back_ptr,
+                 struct blob_table *blob_table)
+{
+       struct blob_descriptor *duplicate_blob;
+
+       list_del(&blob->unhashed_list);
+       blob->unhashed = 0;
+
+       /* Look for a duplicate blob  */
+       duplicate_blob = lookup_blob(blob_table, blob->hash);
+       if (duplicate_blob) {
+               /* We have a duplicate blob.  Transfer the reference counts from
+                * this blob to the duplicate and update the reference to this
+                * blob (from a stream) to point to the duplicate.  The caller
+                * is responsible for freeing @blob if needed.  */
+               wimlib_assert(duplicate_blob->size == blob->size);
+               duplicate_blob->refcnt += blob->refcnt;
+               blob->refcnt = 0;
+               *back_ptr = duplicate_blob;
+               return duplicate_blob;
        } else {
-               void *buffer_copy;
-               blob = new_blob_descriptor();
-               if (blob == NULL)
-                       return NULL;
-               buffer_copy = memdup(buffer, size);
-               if (buffer_copy == NULL) {
-                       free_blob_descriptor(blob);
-                       return NULL;
-               }
-               blob->blob_location = BLOB_IN_ATTACHED_BUFFER;
-               blob->attached_buffer = buffer_copy;
-               blob->size = size;
-               copy_hash(blob->hash, hash);
+               /* No duplicate blob, so we need to insert this blob into the
+                * blob table and treat it as a hashed blob.  */
                blob_table_insert(blob_table, blob);
+               return blob;
        }
-       return blob;
 }
 
 /*
@@ -1295,42 +1318,16 @@ int
 hash_unhashed_blob(struct blob_descriptor *blob, struct blob_table *blob_table,
                   struct blob_descriptor **blob_ret)
 {
-       int ret;
-       struct blob_descriptor *duplicate_blob;
        struct blob_descriptor **back_ptr;
+       int ret;
 
-       wimlib_assert(blob->unhashed);
-
-       /* back_ptr must be saved because @back_inode and @back_stream_id are in
-        * union with the SHA-1 message digest and will no longer be valid once
-        * the SHA-1 has been calculated. */
        back_ptr = retrieve_pointer_to_unhashed_blob(blob);
 
        ret = sha1_blob(blob);
        if (ret)
                return ret;
 
-       list_del(&blob->unhashed_list);
-       blob->unhashed = 0;
-
-       /* Look for a duplicate blob  */
-       duplicate_blob = lookup_blob(blob_table, blob->hash);
-       if (duplicate_blob) {
-               /* We have a duplicate blob.  Transfer the reference counts from
-                * this blob to the duplicate and update the reference to this
-                * blob (from an stream) to point to the duplicate.  The caller
-                * is responsible for freeing @blob if needed.  */
-               wimlib_assert(duplicate_blob->size == blob->size);
-               duplicate_blob->refcnt += blob->refcnt;
-               blob->refcnt = 0;
-               *back_ptr = duplicate_blob;
-               blob = duplicate_blob;
-       } else {
-               /* No duplicate blob, so we need to insert this blob into the
-                * blob table and treat it as a hashed blob. */
-               blob_table_insert(blob_table, blob);
-       }
-       *blob_ret = blob;
+       *blob_ret = after_blob_hashed(blob, back_ptr, blob_table);
        return 0;
 }
 
@@ -1342,8 +1339,10 @@ blob_to_wimlib_resource_entry(const struct blob_descriptor *blob,
 
        wentry->uncompressed_size = blob->size;
        if (blob->blob_location == BLOB_IN_WIM) {
+               unsigned res_flags = blob->rdesc->flags;
+
                wentry->part_number = blob->rdesc->wim->hdr.part_number;
-               if (blob->flags & WIM_RESHDR_FLAG_SOLID) {
+               if (res_flags & WIM_RESHDR_FLAG_SOLID) {
                        wentry->offset = blob->offset_in_res;
                } else {
                        wentry->compressed_size = blob->rdesc->size_in_wim;
@@ -1352,14 +1351,16 @@ blob_to_wimlib_resource_entry(const struct blob_descriptor *blob,
                wentry->raw_resource_offset_in_wim = blob->rdesc->offset_in_wim;
                wentry->raw_resource_compressed_size = blob->rdesc->size_in_wim;
                wentry->raw_resource_uncompressed_size = blob->rdesc->uncompressed_size;
+
+               wentry->is_compressed = (res_flags & WIM_RESHDR_FLAG_COMPRESSED) != 0;
+               wentry->is_free = (res_flags & WIM_RESHDR_FLAG_FREE) != 0;
+               wentry->is_spanned = (res_flags & WIM_RESHDR_FLAG_SPANNED) != 0;
+               wentry->packed = (res_flags & WIM_RESHDR_FLAG_SOLID) != 0;
        }
-       copy_hash(wentry->sha1_hash, blob->hash);
+       if (!blob->unhashed)
+               copy_hash(wentry->sha1_hash, blob->hash);
        wentry->reference_count = blob->refcnt;
-       wentry->is_compressed = (blob->flags & WIM_RESHDR_FLAG_COMPRESSED) != 0;
-       wentry->is_metadata = (blob->flags & WIM_RESHDR_FLAG_METADATA) != 0;
-       wentry->is_free = (blob->flags & WIM_RESHDR_FLAG_FREE) != 0;
-       wentry->is_spanned = (blob->flags & WIM_RESHDR_FLAG_SPANNED) != 0;
-       wentry->packed = (blob->flags & WIM_RESHDR_FLAG_SOLID) != 0;
+       wentry->is_metadata = blob->is_metadata;
 }
 
 struct iterate_blob_context {
@@ -1393,10 +1394,17 @@ wimlib_iterate_lookup_table(WIMStruct *wim, int flags,
        if (wim_has_metadata(wim)) {
                int ret;
                for (int i = 0; i < wim->hdr.image_count; i++) {
-                       ret = do_iterate_blob(wim->image_metadata[i]->metadata_blob,
-                                             &ctx);
+                       struct blob_descriptor *blob;
+                       struct wim_image_metadata *imd = wim->image_metadata[i];
+
+                       ret = do_iterate_blob(imd->metadata_blob, &ctx);
                        if (ret)
                                return ret;
+                       image_for_each_unhashed_blob(blob, imd) {
+                               ret = do_iterate_blob(blob, &ctx);
+                               if (ret)
+                                       return ret;
+                       }
                }
        }
        return for_blob_in_table(wim->blob_table, do_iterate_blob, &ctx);