]> wimlib.net Git - wimlib/blobdiff - src/lookup_table.c
Make padded structures work properly with MinGW
[wimlib] / src / lookup_table.c
index 9502db4770b110b873a7f574ecca6564a2021132..a24cf68887d40a4246d3a29ad7639e29f67fb506 100644 (file)
  * along with wimlib; if not, see http://www.gnu.org/licenses/.
  */
 
-#include "wimlib_internal.h"
-#include "lookup_table.h"
-#include "buffer_io.h"
+#ifdef HAVE_CONFIG_H
+#  include "config.h"
+#endif
+
+#include "wimlib/endianness.h"
+#include "wimlib/error.h"
+#include "wimlib/file_io.h"
+#include "wimlib/lookup_table.h"
+#include "wimlib/metadata.h"
+#include "wimlib/paths.h"
+#include "wimlib/resource.h"
+#include "wimlib/util.h"
+
 #include <errno.h>
 #include <stdlib.h>
-
 #ifdef WITH_FUSE
-#include <unistd.h>
+#  include <unistd.h> /* for unlink() */
 #endif
 
 struct wim_lookup_table *
@@ -50,15 +59,15 @@ new_lookup_table(size_t capacity)
                } else {
                        FREE(table);
                        table = NULL;
-                       ERROR("Failed to allocate memory for lookup table with capacity %zu",
-                             capacity);
+                       ERROR("Failed to allocate memory for lookup table "
+                             "with capacity %zu", capacity);
                }
        }
        return table;
 }
 
 struct wim_lookup_table_entry *
-new_lookup_table_entry()
+new_lookup_table_entry(void)
 {
        struct wim_lookup_table_entry *lte;
 
@@ -79,51 +88,52 @@ clone_lookup_table_entry(const struct wim_lookup_table_entry *old)
 {
        struct wim_lookup_table_entry *new;
 
-       new = MALLOC(sizeof(*new));
+       new = memdup(old, sizeof(struct wim_lookup_table_entry));
        if (!new)
                return NULL;
 
-       memcpy(new, old, sizeof(*old));
        new->extracted_file = NULL;
        switch (new->resource_location) {
 #ifdef __WIN32__
        case RESOURCE_WIN32:
+       case RESOURCE_WIN32_ENCRYPTED:
+#else
+       case RESOURCE_IN_FILE_ON_DISK:
 #endif
+#ifdef WITH_FUSE
        case RESOURCE_IN_STAGING_FILE:
-       case RESOURCE_IN_FILE_ON_DISK:
                BUILD_BUG_ON((void*)&old->file_on_disk !=
                             (void*)&old->staging_file_name);
-               new->staging_file_name = TSTRDUP(old->staging_file_name);
-               if (!new->staging_file_name)
+#endif
+               new->file_on_disk = TSTRDUP(old->file_on_disk);
+               if (!new->file_on_disk)
                        goto out_free;
                break;
        case RESOURCE_IN_ATTACHED_BUFFER:
-               new->attached_buffer = MALLOC(wim_resource_size(old));
+               new->attached_buffer = memdup(old->attached_buffer,
+                                             wim_resource_size(old));
                if (!new->attached_buffer)
                        goto out_free;
-               memcpy(new->attached_buffer, old->attached_buffer,
-                      wim_resource_size(old));
                break;
 #ifdef WITH_NTFS_3G
        case RESOURCE_IN_NTFS_VOLUME:
                if (old->ntfs_loc) {
                        struct ntfs_location *loc;
-                       loc = MALLOC(sizeof(*loc));
+                       loc = memdup(old->ntfs_loc, sizeof(struct ntfs_location));
                        if (!loc)
                                goto out_free;
-                       memcpy(loc, old->ntfs_loc, sizeof(*loc));
                        loc->path = NULL;
                        loc->stream_name = NULL;
                        new->ntfs_loc = loc;
                        loc->path = STRDUP(old->ntfs_loc->path);
                        if (!loc->path)
                                goto out_free;
-                       loc->stream_name = MALLOC((loc->stream_name_nchars + 1) * 2);
-                       if (!loc->stream_name)
-                               goto out_free;
-                       memcpy(loc->stream_name,
-                              old->ntfs_loc->stream_name,
-                              (loc->stream_name_nchars + 1) * 2);
+                       if (loc->stream_name_nchars) {
+                               loc->stream_name = memdup(old->ntfs_loc->stream_name,
+                                                         loc->stream_name_nchars * 2);
+                               if (!loc->stream_name)
+                                       goto out_free;
+                       }
                }
                break;
 #endif
@@ -141,14 +151,18 @@ free_lookup_table_entry(struct wim_lookup_table_entry *lte)
 {
        if (lte) {
                switch (lte->resource_location) {
-               case RESOURCE_IN_STAGING_FILE:
-               case RESOURCE_IN_ATTACHED_BUFFER:
-               case RESOURCE_IN_FILE_ON_DISK:
-#ifdef __WIN32__
+       #ifdef __WIN32__
                case RESOURCE_WIN32:
-#endif
+               case RESOURCE_WIN32_ENCRYPTED:
+       #else
+               case RESOURCE_IN_FILE_ON_DISK:
+       #endif
+       #ifdef WITH_FUSE
+               case RESOURCE_IN_STAGING_FILE:
                        BUILD_BUG_ON((void*)&lte->file_on_disk !=
                                     (void*)&lte->staging_file_name);
+       #endif
+               case RESOURCE_IN_ATTACHED_BUFFER:
                        BUILD_BUG_ON((void*)&lte->file_on_disk !=
                                     (void*)&lte->attached_buffer);
                        FREE(lte->file_on_disk);
@@ -215,7 +229,7 @@ finalize_lte(struct wim_lookup_table_entry *lte)
        #ifdef WITH_FUSE
        if (lte->resource_location == RESOURCE_IN_STAGING_FILE) {
                unlink(lte->staging_file_name);
-               list_del(&lte->staging_list);
+               list_del(&lte->unhashed_list);
        }
        #endif
        free_lookup_table_entry(lte);
@@ -232,7 +246,10 @@ lte_decrement_refcnt(struct wim_lookup_table_entry *lte,
        wimlib_assert(lte != NULL);
        wimlib_assert(lte->refcnt != 0);
        if (--lte->refcnt == 0) {
-               lookup_table_unlink(table, lte);
+               if (lte->unhashed)
+                       list_del(&lte->unhashed_list);
+               else
+                       lookup_table_unlink(table, lte);
        #ifdef WITH_FUSE
                if (lte->num_opened_fds == 0)
        #endif
@@ -267,14 +284,14 @@ for_lookup_table_entry(struct wim_lookup_table *table,
                {
                        wimlib_assert2(!(lte->resource_entry.flags & WIM_RESHDR_FLAG_METADATA));
                        ret = visitor(lte, arg);
-                       if (ret != 0)
+                       if (ret)
                                return ret;
                }
        }
        return 0;
 }
 
-static int
+int
 cmp_streams_by_wim_position(const void *p1, const void *p2)
 {
        const struct wim_lookup_table_entry *lte1, *lte2;
@@ -288,40 +305,6 @@ cmp_streams_by_wim_position(const void *p1, const void *p2)
                return 0;
 }
 
-int
-sort_stream_list_by_wim_position(struct list_head *stream_list)
-{
-       struct list_head *cur;
-       size_t num_streams;
-       struct wim_lookup_table_entry **array;
-       size_t i;
-       size_t array_size;
-
-       num_streams = 0;
-       list_for_each(cur, stream_list)
-               num_streams++;
-       array_size = num_streams * sizeof(array[0]);
-       array = MALLOC(array_size);
-       if (!array) {
-               ERROR("Failed to allocate %zu bytes to sort stream entries",
-                     array_size);
-               return WIMLIB_ERR_NOMEM;
-       }
-       cur = stream_list->next;
-       for (i = 0; i < num_streams; i++) {
-               array[i] = container_of(cur, struct wim_lookup_table_entry, staging_list);
-               cur = cur->next;
-       }
-
-       qsort(array, num_streams, sizeof(array[0]), cmp_streams_by_wim_position);
-
-       INIT_LIST_HEAD(stream_list);
-       for (i = 0; i < num_streams; i++)
-               list_add_tail(&array[i]->staging_list, stream_list);
-       FREE(array);
-       return 0;
-}
-
 
 static int
 add_lte_to_array(struct wim_lookup_table_entry *lte,
@@ -356,12 +339,34 @@ for_lookup_table_entry_pos_sorted(struct wim_lookup_table *table,
        qsort(lte_array, num_streams, sizeof(lte_array[0]),
              cmp_streams_by_wim_position);
        ret = 0;
-       for (size_t i = 0; i < num_streams && ret == 0; i++)
+       for (size_t i = 0; i < num_streams; i++) {
                ret = visitor(lte_array[i], arg);
+               if (ret)
+                       break;
+       }
        FREE(lte_array);
        return ret;
 }
 
+/* On-disk format of a WIM lookup table entry (stream entry). */
+struct wim_lookup_table_entry_disk {
+       /* Location, offset, compression status, and metadata status of the
+        * stream. */
+       struct resource_entry_disk resource_entry;
+
+       /* Which part of the split WIM this stream is in; indexed from 1. */
+       le16 part_number;
+
+       /* Reference count of this stream over all WIM images. */
+       le32 refcnt;
+
+       /* SHA1 message digest of the uncompressed data of this stream, or
+        * optionally all zeroes if this stream is of zero length. */
+       u8 hash[SHA1_HASH_SIZE];
+} _packed_attribute;
+
+#define WIM_LOOKUP_TABLE_ENTRY_DISK_SIZE 50
+
 /*
  * Reads the lookup table from a WIM file.
  *
@@ -372,49 +377,59 @@ for_lookup_table_entry_pos_sorted(struct wim_lookup_table *table,
 int
 read_lookup_table(WIMStruct *w)
 {
-       u64 num_entries;
-       u8 buf[WIM_LOOKUP_TABLE_ENTRY_DISK_SIZE];
        int ret;
+       size_t num_entries;
        struct wim_lookup_table *table;
        struct wim_lookup_table_entry *cur_entry, *duplicate_entry;
+       struct wim_lookup_table_entry_disk
+                       table_buf[BUFFER_SIZE / sizeof(struct wim_lookup_table_entry_disk)]
+                               _aligned_attribute(8);
 
-       if (resource_is_compressed(&w->hdr.lookup_table_res_entry)) {
-               ERROR("Didn't expect a compressed lookup table!");
-               ERROR("Ask the author to implement support for this.");
-               return WIMLIB_ERR_COMPRESSED_LOOKUP_TABLE;
-       }
+       BUILD_BUG_ON(sizeof(struct wim_lookup_table_entry_disk) !=
+                    WIM_LOOKUP_TABLE_ENTRY_DISK_SIZE);
+
+       off_t offset;
+       size_t buf_entries_remaining;
+       const struct wim_lookup_table_entry_disk *disk_entry;
 
        DEBUG("Reading lookup table: offset %"PRIu64", size %"PRIu64"",
              w->hdr.lookup_table_res_entry.offset,
              w->hdr.lookup_table_res_entry.original_size);
 
-       if (fseeko(w->fp, w->hdr.lookup_table_res_entry.offset, SEEK_SET) != 0)
-       {
-               ERROR_WITH_ERRNO("Failed to seek to byte %"PRIu64" to read "
-                                "lookup table",
-                                w->hdr.lookup_table_res_entry.offset);
-               return WIMLIB_ERR_READ;
+       if (resource_is_compressed(&w->hdr.lookup_table_res_entry)) {
+               ERROR("Didn't expect a compressed lookup table!");
+               ERROR("Ask the author to implement support for this.");
+               return WIMLIB_ERR_COMPRESSED_LOOKUP_TABLE;
        }
 
-       num_entries = w->hdr.lookup_table_res_entry.original_size /
-                     WIM_LOOKUP_TABLE_ENTRY_DISK_SIZE;
+       num_entries = w->hdr.lookup_table_res_entry.size /
+                     sizeof(struct wim_lookup_table_entry_disk);
        table = new_lookup_table(num_entries * 2 + 1);
        if (!table)
                return WIMLIB_ERR_NOMEM;
 
        w->current_image = 0;
-       while (num_entries--) {
-               const u8 *p;
-
-               if (fread(buf, 1, sizeof(buf), w->fp) != sizeof(buf)) {
-                       if (feof(w->fp)) {
-                               ERROR("Unexpected EOF in WIM lookup table!");
-                       } else {
-                               ERROR_WITH_ERRNO("Error reading WIM lookup "
-                                                "table");
+       offset = w->hdr.lookup_table_res_entry.offset;
+       buf_entries_remaining = 0;
+       for (; num_entries != 0;
+            num_entries--, buf_entries_remaining--, disk_entry++)
+       {
+               if (buf_entries_remaining == 0) {
+                       size_t entries_to_read, bytes_to_read;
+
+                       entries_to_read = min(ARRAY_LEN(table_buf), num_entries);
+                       bytes_to_read = entries_to_read * sizeof(struct wim_lookup_table_entry_disk);
+                       if (full_pread(w->in_fd, table_buf,
+                                      bytes_to_read, offset) != bytes_to_read)
+                       {
+                               ERROR_WITH_ERRNO("Error reading lookup table "
+                                                "(offset=%"PRIu64")", offset);
+                               ret = WIMLIB_ERR_READ;
+                               goto out_free_lookup_table;
                        }
-                       ret = WIMLIB_ERR_READ;
-                       goto out_free_lookup_table;
+                       offset += bytes_to_read;
+                       disk_entry = table_buf;
+                       buf_entries_remaining = entries_to_read;
                }
                cur_entry = new_lookup_table_entry();
                if (!cur_entry) {
@@ -424,10 +439,10 @@ read_lookup_table(WIMStruct *w)
 
                cur_entry->wim = w;
                cur_entry->resource_location = RESOURCE_IN_WIM;
-               p = get_resource_entry(buf, &cur_entry->resource_entry);
-               p = get_u16(p, &cur_entry->part_number);
-               p = get_u32(p, &cur_entry->refcnt);
-               p = get_bytes(p, SHA1_HASH_SIZE, cur_entry->hash);
+               get_resource_entry(&disk_entry->resource_entry, &cur_entry->resource_entry);
+               cur_entry->part_number = le16_to_cpu(disk_entry->part_number);
+               cur_entry->refcnt = le32_to_cpu(disk_entry->refcnt);
+               copy_hash(cur_entry->hash, disk_entry->hash);
 
                if (cur_entry->part_number != w->hdr.part_number) {
                        ERROR("A lookup table entry in part %hu of the WIM "
@@ -495,7 +510,7 @@ read_lookup_table(WIMStruct *w)
                              w->current_image + 1,
                              cur_entry->resource_entry.offset);
                        w->image_metadata[
-                               w->current_image++].metadata_lte = cur_entry;
+                               w->current_image++]->metadata_lte = cur_entry;
                } else {
                        /* Lookup table entry for a stream that is not a
                         * metadata resource */
@@ -516,8 +531,7 @@ read_lookup_table(WIMStruct *w)
                }
        }
 
-       if (w->hdr.part_number == 1 &&
-           w->current_image != w->hdr.image_count)
+       if (w->hdr.part_number == 1 && w->current_image != w->hdr.image_count)
        {
                ERROR("The WIM header says there are %u images "
                      "in the WIM, but we only found %d metadata "
@@ -539,37 +553,74 @@ out:
 }
 
 
-/*
- * Writes a lookup table entry to the output file.
- */
-int
-write_lookup_table_entry(struct wim_lookup_table_entry *lte, void *_out)
+static void
+write_lookup_table_entry(const struct wim_lookup_table_entry *lte,
+                        struct wim_lookup_table_entry_disk *disk_entry)
 {
-       FILE *out;
-       u8 buf[WIM_LOOKUP_TABLE_ENTRY_DISK_SIZE];
-       u8 *p;
-
-       out = _out;
+       put_resource_entry(&lte->output_resource_entry, &disk_entry->resource_entry);
+       disk_entry->part_number = cpu_to_le16(lte->part_number);
+       disk_entry->refcnt = cpu_to_le32(lte->out_refcnt);
+       copy_hash(disk_entry->hash, lte->hash);
+}
 
-       /* Don't write entries that have not had file resources or metadata
-        * resources written for them. */
-       if (lte->out_refcnt == 0)
-               return 0;
+int
+write_lookup_table_from_stream_list(struct list_head *stream_list,
+                                   int out_fd,
+                                   struct resource_entry *out_res_entry)
+{
+       int ret;
+       off_t start_offset;
+       struct wim_lookup_table_entry_disk
+                       table_buf[BUFFER_SIZE / sizeof(struct wim_lookup_table_entry_disk)]
+                               _aligned_attribute(8);
+       size_t table_size;
+       size_t bytes_to_write;
+       struct wim_lookup_table_entry *lte;
+       size_t cur_idx;
 
-       if (lte->output_resource_entry.flags & WIM_RESHDR_FLAG_METADATA) {
-               DEBUG("Writing metadata entry at %"PRIu64" "
-                     "(orig size = %"PRIu64")",
-                     ftello(out), lte->output_resource_entry.original_size);
+       start_offset = filedes_offset(out_fd);
+       if (start_offset == -1)
+               goto write_error;
+
+       table_size = 0;
+       cur_idx = 0;
+       list_for_each_entry(lte, stream_list, lookup_table_list) {
+               if (cur_idx == ARRAY_LEN(table_buf)) {
+                       bytes_to_write = sizeof(table_buf);
+                       if (full_write(out_fd, table_buf,
+                                      bytes_to_write) != bytes_to_write)
+                               goto write_error;
+                       table_size += bytes_to_write;
+                       cur_idx = 0;
+               }
+               write_lookup_table_entry(lte, &table_buf[cur_idx]);
+               cur_idx++;
        }
-
-       p = put_resource_entry(buf, &lte->output_resource_entry);
-       p = put_u16(p, lte->part_number);
-       p = put_u32(p, lte->out_refcnt);
-       p = put_bytes(p, SHA1_HASH_SIZE, lte->hash);
-       if (fwrite(buf, 1, sizeof(buf), out) != sizeof(buf)) {
-               ERROR_WITH_ERRNO("Failed to write lookup table entry");
-               return WIMLIB_ERR_WRITE;
+       if (cur_idx != 0) {
+               bytes_to_write = cur_idx * sizeof(struct wim_lookup_table_entry_disk);
+               if (full_write(out_fd, table_buf,
+                              bytes_to_write) != bytes_to_write)
+                       goto write_error;
+               table_size += bytes_to_write;
        }
+       out_res_entry->offset        = start_offset;
+       out_res_entry->size          = table_size;
+       out_res_entry->original_size = table_size;
+       out_res_entry->flags         = WIM_RESHDR_FLAG_METADATA;
+       ret = 0;
+out:
+       return ret;
+write_error:
+       ERROR_WITH_ERRNO("Failed to write lookup table");
+       ret = WIMLIB_ERR_WRITE;
+       goto out;
+}
+
+static int
+append_lookup_table_entry(struct wim_lookup_table_entry *lte, void *_list)
+{
+       if (lte->out_refcnt != 0)
+               list_add_tail(&lte->lookup_table_list, (struct list_head*)_list);
        return 0;
 }
 
@@ -577,16 +628,10 @@ write_lookup_table_entry(struct wim_lookup_table_entry *lte, void *_out)
 int
 write_lookup_table(WIMStruct *w, int image, struct resource_entry *out_res_entry)
 {
-       FILE *out = w->out_fp;
-       off_t start_offset, end_offset;
-       int ret;
-       int start_image, end_image;
+       LIST_HEAD(stream_list);
+       int start_image;
+       int end_image;
 
-       start_offset = ftello(out);
-       if (start_offset == -1)
-               return WIMLIB_ERR_WRITE;
-
-       /* Write lookup table entries for metadata resources */
        if (image == WIMLIB_ALL_IMAGES) {
                start_image = 1;
                end_image = w->hdr.image_count;
@@ -594,51 +639,39 @@ write_lookup_table(WIMStruct *w, int image, struct resource_entry *out_res_entry
                start_image = image;
                end_image = image;
        }
+
        for (int i = start_image; i <= end_image; i++) {
                struct wim_lookup_table_entry *metadata_lte;
 
-               metadata_lte = w->image_metadata[i - 1].metadata_lte;
+               metadata_lte = w->image_metadata[i - 1]->metadata_lte;
                metadata_lte->out_refcnt = 1;
                metadata_lte->output_resource_entry.flags |= WIM_RESHDR_FLAG_METADATA;
-               ret = write_lookup_table_entry(metadata_lte, out);
-               if (ret)
-                       return ret;
+               append_lookup_table_entry(metadata_lte, &stream_list);
        }
-
-       /* Write lookup table entries for other resources */
-       ret = for_lookup_table_entry(w->lookup_table, write_lookup_table_entry, out);
-       if (ret)
-               return ret;
-
-       /* Fill in the resource entry for the lookup table itself */
-       end_offset = ftello(out);
-       if (end_offset == -1)
-               return WIMLIB_ERR_WRITE;
-
-       out_res_entry->offset        = start_offset;
-       out_res_entry->size          = end_offset - start_offset;
-       out_res_entry->original_size = end_offset - start_offset;
-       out_res_entry->flags         = WIM_RESHDR_FLAG_METADATA;
-       return 0;
+       for_lookup_table_entry(w->lookup_table,
+                              append_lookup_table_entry,
+                              &stream_list);
+       return write_lookup_table_from_stream_list(&stream_list,
+                                                  w->out_fd,
+                                                  out_res_entry);
 }
 
-
 int
-lte_zero_real_refcnt(struct wim_lookup_table_entry *lte, void *ignore)
+lte_zero_real_refcnt(struct wim_lookup_table_entry *lte, void *_ignore)
 {
        lte->real_refcnt = 0;
        return 0;
 }
 
 int
-lte_zero_out_refcnt(struct wim_lookup_table_entry *lte, void *ignore)
+lte_zero_out_refcnt(struct wim_lookup_table_entry *lte, void *_ignore)
 {
        lte->out_refcnt = 0;
        return 0;
 }
 
 int
-lte_free_extracted_file(struct wim_lookup_table_entry *lte, void *ignore)
+lte_free_extracted_file(struct wim_lookup_table_entry *lte, void *_ignore)
 {
        if (lte->extracted_file != NULL) {
                FREE(lte->extracted_file);
@@ -666,9 +699,14 @@ print_lookup_table_entry(const struct wim_lookup_table_entry *lte, FILE *out)
        tfprintf(out, T("Part Number       = %hu\n"), lte->part_number);
        tfprintf(out, T("Reference Count   = %u\n"), lte->refcnt);
 
-       tfprintf(out, T("Hash              = 0x"));
-       print_hash(lte->hash, out);
-       tputc(T('\n'), out);
+       if (lte->unhashed) {
+               tfprintf(out, T("(Unhashed: inode %p, stream_id = %u)\n"),
+                        lte->back_inode, lte->back_stream_id);
+       } else {
+               tfprintf(out, T("Hash              = 0x"));
+               print_hash(lte->hash, out);
+               tputc(T('\n'), out);
+       }
 
        tfprintf(out, T("Flags             = "));
        u8 flags = lte->resource_entry.flags;
@@ -690,15 +728,19 @@ print_lookup_table_entry(const struct wim_lookup_table_entry *lte, FILE *out)
                break;
 #ifdef __WIN32__
        case RESOURCE_WIN32:
-#endif
+       case RESOURCE_WIN32_ENCRYPTED:
+#else
        case RESOURCE_IN_FILE_ON_DISK:
+#endif
                tfprintf(out, T("File on Disk      = `%"TS"'\n"),
                         lte->file_on_disk);
                break;
+#ifdef WITH_FUSE
        case RESOURCE_IN_STAGING_FILE:
                tfprintf(out, T("Staging File      = `%"TS"'\n"),
                                lte->staging_file_name);
                break;
+#endif
        default:
                break;
        }
@@ -813,6 +855,19 @@ out:
 }
 #endif
 
+/*
+ * XXX Probably should store the compression type directly in the lookup table
+ * entry
+ */
+int
+wim_resource_compression_type(const struct wim_lookup_table_entry *lte)
+{
+       if (!(lte->resource_entry.flags & WIM_RESHDR_FLAG_COMPRESSED)
+           || lte->resource_location != RESOURCE_IN_WIM)
+               return WIMLIB_COMPRESSION_TYPE_NONE;
+       return wimlib_get_compression_type(lte->wim);
+}
+
 /* Resolve an inode's lookup table entries
  *
  * This replaces the SHA1 hash fields (which are used to lookup an entry in the
@@ -879,6 +934,34 @@ inode_stream_lte(const struct wim_inode *inode, unsigned stream_idx,
                return inode_stream_lte_unresolved(inode, stream_idx, table);
 }
 
+struct wim_lookup_table_entry *
+inode_unnamed_lte_resolved(const struct wim_inode *inode)
+{
+       wimlib_assert(inode->i_resolved);
+       for (unsigned i = 0; i <= inode->i_num_ads; i++) {
+               if (inode_stream_name_nbytes(inode, i) == 0 &&
+                   !is_zero_hash(inode_stream_hash_resolved(inode, i)))
+               {
+                       return inode_stream_lte_resolved(inode, i);
+               }
+       }
+       return NULL;
+}
+
+struct wim_lookup_table_entry *
+inode_unnamed_lte_unresolved(const struct wim_inode *inode,
+                            const struct wim_lookup_table *table)
+{
+       wimlib_assert(!inode->i_resolved);
+       for (unsigned i = 0; i <= inode->i_num_ads; i++) {
+               if (inode_stream_name_nbytes(inode, i) == 0 &&
+                   !is_zero_hash(inode_stream_hash_unresolved(inode, i)))
+               {
+                       return inode_stream_lte_unresolved(inode, i, table);
+               }
+       }
+       return NULL;
+}
 
 /* Return the lookup table entry for the unnamed data stream of an inode, or
  * NULL if there is none.
@@ -921,12 +1004,78 @@ lookup_table_total_stream_size(struct wim_lookup_table *table)
        return total_size;
 }
 
-void
-lookup_table_free_unhashed_streams(struct wim_lookup_table *table)
+struct wim_lookup_table_entry **
+retrieve_lte_pointer(struct wim_lookup_table_entry *lte)
+{
+       wimlib_assert(lte->unhashed);
+       struct wim_inode *inode = lte->back_inode;
+       u32 stream_id = lte->back_stream_id;
+       if (stream_id == 0)
+               return &inode->i_lte;
+       else
+               for (u16 i = 0; i < inode->i_num_ads; i++)
+                       if (inode->i_ads_entries[i].stream_id == stream_id)
+                               return &inode->i_ads_entries[i].lte;
+       wimlib_assert(0);
+       return NULL;
+}
+
+/* Calculate the SHA1 message digest of a stream and move it from the list of
+ * unhashed streams to the stream lookup table, possibly joining it with an
+ * existing lookup table entry for an identical stream.
+ *
+ * @lte:  An unhashed lookup table entry.
+ * @lookup_table:  Lookup table for the WIM.
+ * @lte_ret:  On success, write a pointer to the resulting lookup table
+ *            entry to this location.  This will be the same as @lte
+ *            if it was inserted into the lookup table, or different if
+ *            a duplicate stream was found.
+ *
+ * Returns 0 on success; nonzero if there is an error reading the stream.
+ */
+int
+hash_unhashed_stream(struct wim_lookup_table_entry *lte,
+                    struct wim_lookup_table *lookup_table,
+                    struct wim_lookup_table_entry **lte_ret)
 {
-       struct wim_lookup_table_entry *lte, *tmp;
+       int ret;
+       struct wim_lookup_table_entry *duplicate_lte;
+       struct wim_lookup_table_entry **back_ptr;
+
+       wimlib_assert(lte->unhashed);
+
+       /* back_ptr must be saved because @back_inode and @back_stream_id are in
+        * union with the SHA1 message digest and will no longer be valid once
+        * the SHA1 has been calculated. */
+       back_ptr = retrieve_lte_pointer(lte);
 
-       list_for_each_entry_safe(lte, tmp, table->unhashed_streams, staging_list)
+       ret = sha1_resource(lte);
+       if (ret)
+               return ret;
+
+       /* Look for a duplicate stream */
+       duplicate_lte = __lookup_resource(lookup_table, lte->hash);
+       list_del(&lte->unhashed_list);
+       if (duplicate_lte) {
+               /* We have a duplicate stream.  Transfer the reference counts
+                * from this stream to the duplicate, update the reference to
+                * this stream (in an inode or ads_entry) to point to the
+                * duplicate, then free this stream. */
+               wimlib_assert(!(duplicate_lte->unhashed));
+               duplicate_lte->refcnt += lte->refcnt;
+               duplicate_lte->out_refcnt += lte->refcnt;
+               *back_ptr = duplicate_lte;
                free_lookup_table_entry(lte);
-       INIT_LIST_HEAD(table->unhashed_streams);
+               lte = duplicate_lte;
+       } else {
+               /* No duplicate stream, so we need to insert
+                * this stream into the lookup table and treat
+                * it as a hashed stream. */
+               lookup_table_insert(lookup_table, lte);
+               lte->unhashed = 0;
+       }
+       if (lte_ret)
+               *lte_ret = lte;
+       return 0;
 }
+