Fixes, comments
authorEric Biggers <ebiggers3@gmail.com>
Sat, 30 Mar 2013 21:25:04 +0000 (16:25 -0500)
committerEric Biggers <ebiggers3@gmail.com>
Sat, 30 Mar 2013 21:25:04 +0000 (16:25 -0500)
14 files changed:
src/dentry.h
src/export_image.c
src/extract_image.c
src/hardlink.c
src/lookup_table.c
src/lookup_table.h
src/metadata_resource.c
src/mount_image.c
src/resource.c
src/symlink.c
src/verify.c
src/wim.c
src/wimlib_internal.h
src/write.c

index 28fdeaf..299abc9 100644 (file)
@@ -265,7 +265,10 @@ struct wim_inode {
         * link_count of them) */
        struct list_head i_dentry;
 
-       struct hlist_node i_hlist;
+       union {
+               struct hlist_node i_hlist;
+               struct list_head i_list;
+       };
 
        union {
                /* Used during image extraction to build a list of inodes that
index 7e6d278..784938e 100644 (file)
@@ -104,7 +104,6 @@ wimlib_export_image(WIMStruct *src_wim,
        int ret;
        struct wim_lookup_table *joined_tab, *src_wim_tab_save;
        struct wim_image_metadata *src_imd;
-       struct hlist_node *cur_node;
        struct list_head lte_list_head;
        struct wim_inode *inode;
 
@@ -209,7 +208,7 @@ wimlib_export_image(WIMStruct *src_wim,
        for_lookup_table_entry(src_wim->lookup_table, lte_zero_out_refcnt, NULL);
        src_imd = wim_get_current_image_metadata(src_wim);
        INIT_LIST_HEAD(&lte_list_head);
-       hlist_for_each_entry(inode, cur_node, &src_imd->inode_list, i_hlist) {
+       image_for_each_inode(inode, src_imd) {
                ret = inode_allocate_needed_ltes(inode,
                                                 src_wim->lookup_table,
                                                 dest_wim->lookup_table,
@@ -235,7 +234,7 @@ wimlib_export_image(WIMStruct *src_wim,
        /* All memory allocations have been taken care of, so it's no longer
         * possible for this function to fail.  Go ahead and update the lookup
         * table of the destination WIM and the boot index, if needed. */
-       hlist_for_each_entry(inode, cur_node, &src_imd->inode_list, i_hlist) {
+       image_for_each_inode(inode, src_imd) {
                inode_move_ltes_to_table(inode,
                                         src_wim->lookup_table,
                                         dest_wim->lookup_table,
index 85b5af8..dff7dd4 100644 (file)
@@ -268,7 +268,7 @@ extract_regular_file_unlinked(struct wim_dentry *dentry,
        }
 
        ret = extract_wim_resource_to_fd(lte, out_fd, wim_resource_size(lte));
-       if (ret != 0) {
+       if (ret) {
                ERROR("Failed to extract resource to `%s'", output_path);
                goto out;
        }
@@ -586,7 +586,7 @@ calculate_bytes_to_extract(struct list_head *stream_list,
        u64 num_streams = 0;
 
        /* For each stream to be extracted... */
-       list_for_each_entry(lte, stream_list, staging_list) {
+       list_for_each_entry(lte, stream_list, extraction_list) {
                if (extract_flags &
                    (WIMLIB_EXTRACT_FLAG_SYMLINK | WIMLIB_EXTRACT_FLAG_HARDLINK))
                {
@@ -614,7 +614,7 @@ maybe_add_stream_for_extraction(struct wim_lookup_table_entry *lte,
 {
        if (++lte->out_refcnt == 1) {
                INIT_LIST_HEAD(&lte->inode_list);
-               list_add_tail(&lte->staging_list, stream_list);
+               list_add_tail(&lte->extraction_list, stream_list);
        }
 }
 
@@ -653,18 +653,17 @@ inode_find_streams_for_extraction(struct wim_inode *inode,
 }
 
 static void
-find_streams_for_extraction(struct hlist_head *inode_list,
+find_streams_for_extraction(struct wim_image_metadata *imd,
                            struct list_head *stream_list,
                            struct wim_lookup_table *lookup_table,
                            int extract_flags)
 {
        struct wim_inode *inode;
-       struct hlist_node *cur;
        struct wim_dentry *dentry;
 
        for_lookup_table_entry(lookup_table, lte_zero_out_refcnt, NULL);
        INIT_LIST_HEAD(stream_list);
-       hlist_for_each_entry(inode, cur, inode_list, i_hlist) {
+       image_for_each_inode(inode, imd) {
                if (!inode->i_resolved)
                        inode_resolve_ltes(inode, lookup_table);
                inode_for_each_dentry(dentry, inode)
@@ -713,7 +712,7 @@ apply_stream_list(struct list_head *stream_list,
         * sequential reading of the WIM can be implemented. */
 
        /* For each distinct stream to be extracted */
-       list_for_each_entry(lte, stream_list, staging_list) {
+       list_for_each_entry(lte, stream_list, extraction_list) {
                /* For each inode that contains the stream */
                list_for_each_entry(inode, &lte->inode_list, i_lte_inode_list) {
                        /* For each dentry that points to the inode */
@@ -745,6 +744,41 @@ apply_stream_list(struct list_head *stream_list,
        return 0;
 }
 
+static int
+sort_stream_list_by_wim_position(struct list_head *stream_list)
+{
+       struct list_head *cur;
+       size_t num_streams;
+       struct wim_lookup_table_entry **array;
+       size_t i;
+       size_t array_size;
+
+       num_streams = 0;
+       list_for_each(cur, stream_list)
+               num_streams++;
+       array_size = num_streams * sizeof(array[0]);
+       array = MALLOC(array_size);
+       if (!array) {
+               ERROR("Failed to allocate %zu bytes to sort stream entries",
+                     array_size);
+               return WIMLIB_ERR_NOMEM;
+       }
+       cur = stream_list->next;
+       for (i = 0; i < num_streams; i++) {
+               array[i] = container_of(cur, struct wim_lookup_table_entry, extraction_list);
+               cur = cur->next;
+       }
+
+       qsort(array, num_streams, sizeof(array[0]), cmp_streams_by_wim_position);
+
+       INIT_LIST_HEAD(stream_list);
+       for (i = 0; i < num_streams; i++)
+               list_add_tail(&array[i]->extraction_list, stream_list);
+       FREE(array);
+       return 0;
+}
+
+
 /* Extracts the image @image from the WIM @w to the directory or NTFS volume
  * @target. */
 static int
@@ -754,7 +788,6 @@ extract_single_image(WIMStruct *w, int image,
 {
        int ret;
        struct list_head stream_list;
-       struct hlist_head *inode_list;
 
        struct apply_args args;
        const struct apply_operations *ops;
@@ -792,10 +825,9 @@ extract_single_image(WIMStruct *w, int image,
        if (ret)
                goto out;
 
-       inode_list = &wim_get_current_image_metadata(w)->inode_list;
-
        /* Build a list of the streams that need to be extracted */
-       find_streams_for_extraction(inode_list, &stream_list,
+       find_streams_for_extraction(wim_get_current_image_metadata(w),
+                                   &stream_list,
                                    w->lookup_table, extract_flags);
 
        /* Calculate the number of bytes of data that will be extracted */
index 08f6225..c7c5a2f 100644 (file)
@@ -60,7 +60,7 @@ init_inode_table(struct wim_inode_table *table, size_t capacity)
        }
        table->num_entries  = 0;
        table->capacity     = capacity;
-       INIT_HLIST_HEAD(&table->extra_inodes);
+       INIT_LIST_HEAD(&table->extra_inodes);
        return 0;
 }
 
@@ -87,7 +87,7 @@ inode_table_insert(struct wim_dentry *dentry, void *_table)
                /* A dentry with a hard link group ID of 0 indicates that it's
                 * in a hard link group by itself.  Add it to the list of extra
                 * inodes rather than inserting it into the hash lists. */
-               hlist_add_head(&d_inode->i_hlist, &table->extra_inodes);
+               list_add_tail(&d_inode->i_list, &table->extra_inodes);
 
                wimlib_assert(d_inode->i_dentry.next == &dentry->d_alias);
                wimlib_assert(d_inode->i_dentry.prev == &dentry->d_alias);
@@ -148,6 +148,22 @@ inode_table_get_inode(struct wim_inode_table *table, u64 ino, u64 devno)
        return inode;
 }
 
+void
+inode_ref_streams(struct wim_inode *inode)
+{
+       for (unsigned i = 0; i <= inode->i_num_ads; i++) {
+               struct wim_lookup_table_entry *lte;
+               lte = inode_stream_lte_resolved(inode, i);
+               if (lte)
+                       lte->refcnt++;
+       }
+}
+
+void
+inode_add_link(struct wim_inode *inode, struct wim_dentry *dentry)
+{
+}
+
 /* Given a directory entry with the name @name for the file with the inode
  * number @ino and device number @devno, create a new WIM dentry with an
  * associated inode, where the inode is shared if an inode with the same @ino
@@ -172,6 +188,8 @@ inode_table_new_dentry(struct wim_inode_table *table, const tchar *name,
                free_dentry(dentry);
                return WIMLIB_ERR_NOMEM;
        }
+       if (inode->i_nlink > 1)
+               inode_ref_streams(inode);
        dentry->d_inode = inode;
        inode_add_dentry(dentry, inode);
        *dentry_ret = dentry;
@@ -251,7 +269,7 @@ inodes_consistent(const struct wim_inode * restrict ref_inode,
 
 /* Fix up a "true" inode and check for inconsistencies */
 static int
-fix_true_inode(struct wim_inode *inode, struct hlist_head *inode_list)
+fix_true_inode(struct wim_inode *inode, struct list_head *inode_list)
 {
        struct wim_dentry *dentry;
        struct wim_dentry *ref_dentry = NULL;
@@ -273,7 +291,7 @@ fix_true_inode(struct wim_inode *inode, struct hlist_head *inode_list)
 
        ref_inode = ref_dentry->d_inode;
        ref_inode->i_nlink = 1;
-       hlist_add_head(&ref_inode->i_hlist, inode_list);
+       list_add_tail(&ref_inode->i_list, inode_list);
 
        list_del(&inode->i_dentry);
        list_add(&ref_inode->i_dentry, &ref_dentry->d_alias);
@@ -314,7 +332,7 @@ fix_true_inode(struct wim_inode *inode, struct hlist_head *inode_list)
  * group remaining.
  */
 static int
-fix_nominal_inode(struct wim_inode *inode, struct hlist_head *inode_list,
+fix_nominal_inode(struct wim_inode *inode, struct list_head *inode_list,
                  bool *ino_changes_needed)
 {
        struct wim_dentry *dentry;
@@ -429,29 +447,28 @@ next_dentry_2:
 
        hlist_for_each_entry_safe(inode, cur, tmp, &true_inodes, i_hlist) {
                ret = fix_true_inode(inode, inode_list);
-               if (ret != 0)
+               if (ret)
                        return ret;
        }
        return 0;
 }
 
 static int
-fix_inodes(struct wim_inode_table *table, struct hlist_head *inode_list,
+fix_inodes(struct wim_inode_table *table, struct list_head *inode_list,
           bool *ino_changes_needed)
 {
        struct wim_inode *inode;
        struct hlist_node *cur, *tmp;
        int ret;
-       INIT_HLIST_HEAD(inode_list);
+       INIT_LIST_HEAD(inode_list);
        for (u64 i = 0; i < table->capacity; i++) {
                hlist_for_each_entry_safe(inode, cur, tmp, &table->array[i], i_hlist) {
                        ret = fix_nominal_inode(inode, inode_list, ino_changes_needed);
-                       if (ret != 0)
+                       if (ret)
                                return ret;
                }
        }
-       hlist_for_each_safe(cur, tmp, &table->extra_inodes)
-               hlist_add_head(cur, inode_list);
+       list_splice_tail(&table->extra_inodes, inode_list);
        return 0;
 }
 
@@ -484,7 +501,7 @@ fix_inodes(struct wim_inode_table *table, struct hlist_head *inode_list,
  * is returned in the hlist @inode_list.
  */
 int
-dentry_tree_fix_inodes(struct wim_dentry *root, struct hlist_head *inode_list)
+dentry_tree_fix_inodes(struct wim_dentry *root, struct list_head *inode_list)
 {
        struct wim_inode_table inode_tab;
        int ret;
@@ -504,11 +521,10 @@ dentry_tree_fix_inodes(struct wim_dentry *root, struct hlist_head *inode_list)
 
        if (ret == 0 && ino_changes_needed) {
                u64 cur_ino = 1;
-               struct hlist_node *tmp;
                struct wim_inode *inode;
 
                WARNING("Re-assigning inode numbers due to inode inconsistencies");
-               hlist_for_each_entry(inode, tmp, inode_list, i_hlist) {
+               list_for_each_entry(inode, inode_list, i_list) {
                        if (inode->i_nlink > 1)
                                inode->i_ino = cur_ino++;
                        else
@@ -522,13 +538,13 @@ dentry_tree_fix_inodes(struct wim_dentry *root, struct hlist_head *inode_list)
  * the inodes to a single list @head. */
 void
 inode_table_prepare_inode_list(struct wim_inode_table *table,
-                              struct hlist_head *head)
+                              struct list_head *head)
 {
        struct wim_inode *inode;
        struct hlist_node *cur, *tmp;
        u64 cur_ino = 1;
 
-       INIT_HLIST_HEAD(head);
+       INIT_LIST_HEAD(head);
        for (size_t i = 0; i < table->capacity; i++) {
                hlist_for_each_entry_safe(inode, cur, tmp, &table->array[i], i_hlist)
                {
@@ -536,7 +552,7 @@ inode_table_prepare_inode_list(struct wim_inode_table *table,
                                inode->i_ino = cur_ino++;
                        else
                                inode->i_ino = 0;
-                       hlist_add_head(&inode->i_hlist, head);
+                       list_add_tail(&inode->i_list, head);
                }
                INIT_HLIST_HEAD(&table->array[i]);
        }
index 5b57e3a..4d62ca5 100644 (file)
@@ -215,7 +215,7 @@ finalize_lte(struct wim_lookup_table_entry *lte)
        #ifdef WITH_FUSE
        if (lte->resource_location == RESOURCE_IN_STAGING_FILE) {
                unlink(lte->staging_file_name);
-               list_del(&lte->staging_list);
+               list_del(&lte->unhashed_list);
        }
        #endif
        free_lookup_table_entry(lte);
@@ -275,7 +275,7 @@ for_lookup_table_entry(struct wim_lookup_table *table,
        return 0;
 }
 
-static int
+int
 cmp_streams_by_wim_position(const void *p1, const void *p2)
 {
        const struct wim_lookup_table_entry *lte1, *lte2;
@@ -289,40 +289,6 @@ cmp_streams_by_wim_position(const void *p1, const void *p2)
                return 0;
 }
 
-int
-sort_stream_list_by_wim_position(struct list_head *stream_list)
-{
-       struct list_head *cur;
-       size_t num_streams;
-       struct wim_lookup_table_entry **array;
-       size_t i;
-       size_t array_size;
-
-       num_streams = 0;
-       list_for_each(cur, stream_list)
-               num_streams++;
-       array_size = num_streams * sizeof(array[0]);
-       array = MALLOC(array_size);
-       if (!array) {
-               ERROR("Failed to allocate %zu bytes to sort stream entries",
-                     array_size);
-               return WIMLIB_ERR_NOMEM;
-       }
-       cur = stream_list->next;
-       for (i = 0; i < num_streams; i++) {
-               array[i] = container_of(cur, struct wim_lookup_table_entry, staging_list);
-               cur = cur->next;
-       }
-
-       qsort(array, num_streams, sizeof(array[0]), cmp_streams_by_wim_position);
-
-       INIT_LIST_HEAD(stream_list);
-       for (i = 0; i < num_streams; i++)
-               list_add_tail(&array[i]->staging_list, stream_list);
-       FREE(array);
-       return 0;
-}
-
 
 static int
 add_lte_to_array(struct wim_lookup_table_entry *lte,
@@ -667,9 +633,13 @@ print_lookup_table_entry(const struct wim_lookup_table_entry *lte, FILE *out)
        tfprintf(out, T("Part Number       = %hu\n"), lte->part_number);
        tfprintf(out, T("Reference Count   = %u\n"), lte->refcnt);
 
-       tfprintf(out, T("Hash              = 0x"));
-       print_hash(lte->hash, out);
-       tputc(T('\n'), out);
+       if (lte->unhashed) {
+               tfprintf(out, T("(Unhashed, back ptr at %p)\n"), lte->my_ptr);
+       } else {
+               tfprintf(out, T("Hash              = 0x"));
+               print_hash(lte->hash, out);
+               tputc(T('\n'), out);
+       }
 
        tfprintf(out, T("Flags             = "));
        u8 flags = lte->resource_entry.flags;
index 62d02d6..633c874 100644 (file)
@@ -139,7 +139,6 @@ struct wim_lookup_table_entry {
        u16 resource_location : 5;
        u8 unique_size : 1;
        u8 unhashed : 1;
-       u8 is_ads : 1;
 
        /* (On-disk field)
         * Number of times this lookup table entry is referenced by dentries.
@@ -160,6 +159,10 @@ struct wim_lookup_table_entry {
                 * table. */
                size_t hash_short;
 
+               /* Unhashed entries only (unhashed == 1): this points directly
+                * to the pointer to this 'struct wim_lookup_table_entry'
+                * contained in a 'struct wim_ads_entry' or 'struct wim_inode'.
+                * */
                struct wim_lookup_table_entry **my_ptr;
        };
 
@@ -226,7 +229,11 @@ struct wim_lookup_table_entry {
         *
         * This field is also used to make other lists of lookup table entries.
         * */
-       struct list_head staging_list;
+       union {
+               struct list_head unhashed_list;
+               struct list_head staging_list;
+               struct list_head extraction_list;
+       };
 };
 
 static inline u64
@@ -317,7 +324,7 @@ for_lookup_table_entry(struct wim_lookup_table *table,
                       void *arg);
 
 extern int
-sort_stream_list_by_wim_position(struct list_head *stream_list);
+cmp_streams_by_wim_position(const void *p1, const void *p2);
 
 extern int
 for_lookup_table_entry_pos_sorted(struct wim_lookup_table *table,
@@ -481,7 +488,7 @@ lookup_table_insert_unhashed(struct wim_lookup_table *table,
                             struct wim_lookup_table_entry **my_ptr)
 {
        lte->unhashed = 1;
-       list_add_tail(&lte->staging_list, table->unhashed_streams);
+       list_add_tail(&lte->unhashed_list, table->unhashed_streams);
        lte->my_ptr = my_ptr;
        *my_ptr = lte;
 }
index a9e2e38..91b1121 100644 (file)
@@ -51,7 +51,6 @@ read_metadata_resource(WIMStruct *w, struct wim_image_metadata *imd)
        struct wim_dentry *dentry;
        const struct wim_lookup_table_entry *metadata_lte;
        u64 metadata_len;
-       struct hlist_head inode_list;
 
        metadata_lte = imd->metadata_lte;
        metadata_len = wim_resource_size(metadata_lte);
@@ -151,7 +150,7 @@ read_metadata_resource(WIMStruct *w, struct wim_image_metadata *imd)
                goto out_free_dentry_tree;
 
        /* Build hash table that maps hard link group IDs to dentry sets */
-       ret = dentry_tree_fix_inodes(dentry, &inode_list);
+       ret = dentry_tree_fix_inodes(dentry, &imd->inode_list);
        if (ret)
                goto out_free_dentry_tree;
 
@@ -159,16 +158,13 @@ read_metadata_resource(WIMStruct *w, struct wim_image_metadata *imd)
                DEBUG("Running miscellaneous verifications on the dentry tree");
                for_lookup_table_entry(w->lookup_table, lte_zero_real_refcnt, NULL);
                ret = for_dentry_in_tree(dentry, verify_dentry, w);
-               if (ret != 0)
+               if (ret)
                        goto out_free_dentry_tree;
        }
 
        DEBUG("Done reading image metadata");
 
        imd->root_dentry = dentry;
-       imd->inode_list  = inode_list;
-       if (imd->inode_list.first)
-               imd->inode_list.first->pprev = &imd->inode_list.first;
        INIT_LIST_HEAD(&imd->unhashed_streams);
        goto out_free_buf;
 out_free_dentry_tree:
@@ -203,16 +199,15 @@ write_wim_resource_from_buffer(const void *buf, u64 buf_size,
         * write_wim_resource(). */
        struct wim_lookup_table_entry lte;
        int ret;
-       lte.resource_entry.original_size = buf_size;
        lte.resource_location            = RESOURCE_IN_ATTACHED_BUFFER;
        lte.attached_buffer              = (void*)buf;
+       lte.resource_entry.original_size = buf_size;
+       lte.resource_entry.flags         = 0;
        lte.unhashed                     = 1;
-       zero_out_hash(lte.hash);
        ret = write_wim_resource(&lte, out_fp, out_ctype, out_res_entry, 0);
-       if (ret)
-               return ret;
-       copy_hash(hash, lte.hash);
-       return 0;
+       if (ret == 0)
+               copy_hash(hash, lte.hash);
+       return ret;
 }
 
 /* Write the metadata resource for the current WIM image. */
index 4700cb5..2d0d56c 100644 (file)
@@ -99,7 +99,7 @@ struct wimfs_context {
        u64 next_ino;
 
        /* List of inodes in the mounted image */
-       struct hlist_head *image_inode_list;
+       struct list_head *image_inode_list;
 
        /* Name and message queue descriptors for message queues between the
         * filesystem daemon process and the unmount process.  These are used
@@ -346,7 +346,7 @@ create_dentry(struct fuse_context *fuse_ctx, const char *path,
                }
        }
        dentry_add_child(parent, new);
-       hlist_add_head(&new->d_inode->i_hlist, wimfs_ctx->image_inode_list);
+       list_add_tail(&new->d_inode->i_list, wimfs_ctx->image_inode_list);
        if (dentry_ret)
                *dentry_ret = new;
        return 0;
@@ -802,19 +802,19 @@ rebuild_wim(struct wimfs_context *ctx, int write_flags,
            wimlib_progress_func_t progress_func)
 {
        int ret;
-       struct wim_lookup_table_entry *lte, *tmp;
+       struct wim_lookup_table_entry *lte;
        WIMStruct *w = ctx->wim;
        struct wim_image_metadata *imd = wim_get_current_image_metadata(ctx->wim);
 
        DEBUG("Closing all staging file descriptors.");
-       list_for_each_entry_safe(lte, tmp, &imd->unhashed_streams, staging_list) {
+       image_for_each_unhashed_stream(lte, imd) {
                ret = inode_close_fds(lte->lte_inode);
                if (ret)
                        return ret;
        }
 
        DEBUG("Freeing entries for zero-length streams");
-       list_for_each_entry_safe(lte, tmp, &imd->unhashed_streams, staging_list) {
+       image_for_each_unhashed_stream(lte, imd) {
                if (wim_resource_size(lte) == 0) {
                        *lte->my_ptr = NULL;
                        free_lookup_table_entry(lte);
@@ -1671,9 +1671,7 @@ wimfs_link(const char *to, const char *from)
        struct wim_dentry *from_dentry, *from_dentry_parent;
        const char *link_name;
        struct wim_inode *inode;
-       struct wim_lookup_table_entry *lte;
        WIMStruct *w = wimfs_get_WIMStruct();
-       u16 i;
        int ret;
 
        inode = wim_pathname_to_inode(w, to);
@@ -1698,15 +1696,10 @@ wimfs_link(const char *to, const char *from)
        if (ret)
                return -ENOMEM;
 
-       inode_add_dentry(from_dentry, inode);
-       from_dentry->d_inode = inode;
        inode->i_nlink++;
-
-       for (i = 0; i <= inode->i_num_ads; i++) {
-               lte = inode_stream_lte_resolved(inode, i);
-               if (lte)
-                       lte->refcnt++;
-       }
+       inode_ref_streams(inode);
+       from_dentry->d_inode = inode;
+       inode_add_dentry(from_dentry, inode);
        dentry_add_child(from_dentry_parent, from_dentry);
        return 0;
 }
@@ -2418,7 +2411,6 @@ wimlib_mount_image(WIMStruct *wim, int image, const char *dir,
        struct wim_lookup_table *joined_tab, *wim_tab_save;
        struct wim_image_metadata *imd;
        struct wimfs_context ctx;
-       struct hlist_node *cur_node;
        struct wim_inode *inode;
 
        DEBUG("Mount: wim = %p, image = %d, dir = %s, flags = %d, ",
@@ -2428,7 +2420,7 @@ wimlib_mount_image(WIMStruct *wim, int image, const char *dir,
                return WIMLIB_ERR_INVALID_PARAM;
 
        ret = verify_swm_set(wim, additional_swms, num_additional_swms);
-       if (ret != 0)
+       if (ret)
                return ret;
 
        if ((mount_flags & WIMLIB_MOUNT_FLAG_READWRITE) && (wim->hdr.total_parts != 1)) {
@@ -2440,7 +2432,7 @@ wimlib_mount_image(WIMStruct *wim, int image, const char *dir,
                ret = new_joined_lookup_table(wim, additional_swms,
                                              num_additional_swms,
                                              &joined_tab);
-               if (ret != 0)
+               if (ret)
                        return ret;
                wim_tab_save = wim->lookup_table;
                wim->lookup_table = joined_tab;
@@ -2448,12 +2440,12 @@ wimlib_mount_image(WIMStruct *wim, int image, const char *dir,
 
        if (mount_flags & WIMLIB_MOUNT_FLAG_READWRITE) {
                ret = wim_run_full_verifications(wim);
-               if (ret != 0)
+               if (ret)
                        goto out;
        }
 
        ret = select_wim_image(wim, image);
-       if (ret != 0)
+       if (ret)
                goto out;
 
        DEBUG("Selected image %d", image);
@@ -2476,7 +2468,7 @@ wimlib_mount_image(WIMStruct *wim, int image, const char *dir,
 
        if (mount_flags & WIMLIB_MOUNT_FLAG_READWRITE) {
                ret = lock_wim(wim, wim->fp);
-               if (ret != 0)
+               if (ret)
                        goto out;
        }
 
@@ -2494,12 +2486,13 @@ wimlib_mount_image(WIMStruct *wim, int image, const char *dir,
        ctx.image_inode_list = &imd->inode_list;
        ctx.default_uid = getuid();
        ctx.default_gid = getgid();
+       ctx.wim->lookup_table->unhashed_streams = &imd->unhashed_streams;
        if (mount_flags & WIMLIB_MOUNT_FLAG_STREAM_INTERFACE_WINDOWS)
                ctx.default_lookup_flags = LOOKUP_FLAG_ADS_OK;
 
        DEBUG("Unlinking message queues in case they already exist");
        ret = set_message_queue_names(&ctx, dir);
-       if (ret != 0)
+       if (ret)
                goto out_unlock;
        unlink_message_queues(&ctx);
 
@@ -2537,7 +2530,7 @@ wimlib_mount_image(WIMStruct *wim, int image, const char *dir,
        if ((mount_flags & WIMLIB_MOUNT_FLAG_READWRITE)) {
                /* Read-write mount.  Make the staging directory */
                ret = make_staging_dir(&ctx, staging_dir);
-               if (ret != 0)
+               if (ret)
                        goto out_free_dir_copy;
        } else {
                /* Read-only mount */
@@ -2570,7 +2563,7 @@ wimlib_mount_image(WIMStruct *wim, int image, const char *dir,
         * assign inode numbers */
        DEBUG("Resolving lookup table entries and assigning inode numbers");
        ctx.next_ino = 1;
-       hlist_for_each_entry(inode, cur_node, &imd->inode_list, i_hlist) {
+       image_for_each_inode(inode, imd) {
                inode_resolve_ltes(inode, wim->lookup_table);
                inode->i_ino = ctx.next_ino++;
        }
index 344ba0d..7a5d171 100644 (file)
@@ -537,6 +537,8 @@ read_partial_wim_resource(const struct wim_lookup_table_entry *lte,
                                               cb,
                                               ctx_or_buf);
        } else {
+               offset += lte->resource_entry.offset;
+
                if (fseeko(wim_fp, offset, SEEK_SET)) {
                        ERROR_WITH_ERRNO("Failed to seek to offset %"PRIu64
                                         " in WIM", offset);
@@ -654,12 +656,21 @@ read_buffer_prefix(const struct wim_lookup_table_entry *lte,
                   void *ctx_or_buf, int _ignored_flags)
 {
        const void *inbuf = lte->attached_buffer;
+       int ret;
+
        if (cb) {
-               return cb(inbuf, size, ctx_or_buf);
+               while (size) {
+                       size_t chunk_size = min(WIM_CHUNK_SIZE, size);
+                       ret = cb(inbuf, chunk_size, ctx_or_buf);
+                       if (ret)
+                               return ret;
+                       size -= chunk_size;
+                       inbuf += chunk_size;
+               }
        } else {
                memcpy(ctx_or_buf, inbuf, size);
-               return 0;
        }
+       return 0;
 }
 
 typedef int (*read_resource_prefix_handler_t)(const struct wim_lookup_table_entry *lte,
@@ -668,6 +679,23 @@ typedef int (*read_resource_prefix_handler_t)(const struct wim_lookup_table_entr
                                              void *ctx_or_buf,
                                              int flags);
 
+/*
+ * Read the first @size bytes from a generic "resource", which may be located in
+ * the WIM (compressed or uncompressed), in an external file, or directly in an
+ * in-memory buffer.
+ *
+ * Feed the data either to a callback function (cb != NULL, passing it
+ * ctx_or_buf), or write it directly into a buffer (cb == NULL, ctx_or_buf
+ * specifies the buffer, which must have room for @size bytes).
+ *
+ * When using a callback function, it is called with chunks up to 32768 bytes in
+ * size until the resource is exhausted.
+ *
+ * If the resource is located in a WIM file, @flags can be
+ * WIMLIB_RESOURCE_FLAG_MULTITHREADED if it must be safe to access the resource
+ * concurrently by multiple threads, or WIMLIB_RESOURCE_FLAG_RAW if the raw
+ * compressed data is to be supplied instead of the uncompressed data.
+ */
 int
 read_resource_prefix(const struct wim_lookup_table_entry *lte,
                     u64 size, consume_data_callback_t cb, void *ctx_or_buf,
@@ -703,6 +731,22 @@ read_full_resource_into_buf(const struct wim_lookup_table_entry *lte,
                                    thread_safe ? WIMLIB_RESOURCE_FLAG_MULTITHREADED : 0);
 }
 
+struct extract_ctx {
+       SHA_CTX sha_ctx;
+       consume_data_callback_t extract_chunk;
+       void *extract_chunk_arg;
+};
+
+static int
+extract_chunk_sha1_wrapper(const void *chunk, size_t chunk_size,
+                          void *_ctx)
+{
+       struct extract_ctx *ctx = _ctx;
+
+       sha1_update(&ctx->sha_ctx, chunk, chunk_size);
+       return ctx->extract_chunk(chunk, chunk_size, ctx->extract_chunk_arg);
+}
+
 /* Extracts the first @size bytes of a WIM resource to somewhere.  In the
  * process, the SHA1 message digest of the resource is checked if the full
  * resource is being extracted.
@@ -715,8 +759,36 @@ extract_wim_resource(const struct wim_lookup_table_entry *lte,
                     consume_data_callback_t extract_chunk,
                     void *extract_chunk_arg)
 {
-       return read_resource_prefix(lte, size, extract_chunk,
-                                   extract_chunk_arg, 0);
+       int ret;
+       if (size == wim_resource_size(lte)) {
+               /* Do SHA1 */
+               struct extract_ctx ctx;
+               ctx.extract_chunk = extract_chunk;
+               ctx.extract_chunk_arg = extract_chunk_arg;
+               sha1_init(&ctx.sha_ctx);
+               ret = read_resource_prefix(lte, size,
+                                          extract_chunk_sha1_wrapper,
+                                          &ctx, 0);
+               if (ret == 0) {
+                       u8 hash[SHA1_HASH_SIZE];
+                       sha1_final(hash, &ctx.sha_ctx);
+                       if (!hashes_equal(hash, lte->hash)) {
+                       #ifdef ENABLE_ERROR_MESSAGES
+                               ERROR_WITH_ERRNO("Invalid SHA1 message digest "
+                                                "on the following WIM resource:");
+                               print_lookup_table_entry(lte, stderr);
+                               if (lte->resource_location == RESOURCE_IN_WIM)
+                                       ERROR("The WIM file appears to be corrupt!");
+                               ret = WIMLIB_ERR_INVALID_RESOURCE_HASH;
+                       #endif
+                       }
+               }
+       } else {
+               /* Don't do SHA1 */
+               ret = read_resource_prefix(lte, size, extract_chunk,
+                                          extract_chunk_arg, 0);
+       }
+       return ret;
 }
 
 static int
index bd370e3..538b3e8 100644 (file)
@@ -219,7 +219,7 @@ inode_set_symlink(struct wim_inode *inode,
 
        ret = make_symlink_reparse_data_buf(target, &symlink_buf_len,
                                            &symlink_buf);
-       if (ret != 0)
+       if (ret)
                return ret;
 
        DEBUG("Made symlink reparse data buf (len = %zu, name len = %zu)",
@@ -243,7 +243,6 @@ inode_set_symlink(struct wim_inode *inode,
                lte->resource_location            = RESOURCE_IN_ATTACHED_BUFFER;
                lte->attached_buffer              = symlink_buf;
                lte->resource_entry.original_size = symlink_buf_len;
-               lte->resource_entry.size          = symlink_buf_len;
                copy_hash(lte->hash, symlink_buf_hash);
        }
 
index f9808ad..6da6b6a 100644 (file)
@@ -175,10 +175,9 @@ image_run_full_verifications(WIMStruct *w)
 {
        struct wim_image_metadata *imd;
        struct wim_inode *inode;
-       struct hlist_node *cur;
 
        imd = wim_get_current_image_metadata(w);
-       hlist_for_each_entry(inode, cur, &imd->inode_list, i_hlist)
+       image_for_each_inode(inode, imd)
                inode->i_verified = 0;
        return for_dentry_in_tree(imd->root_dentry, verify_dentry, w);
 }
index ed20145..96fba59 100644 (file)
--- a/src/wim.c
+++ b/src/wim.c
@@ -533,7 +533,7 @@ destroy_image_metadata(struct wim_image_metadata *imd,
                imd->metadata_lte = NULL;
        }
        INIT_LIST_HEAD(&imd->unhashed_streams);
-       INIT_HLIST_HEAD(&imd->inode_list);
+       INIT_LIST_HEAD(&imd->inode_list);
 }
 
 void
@@ -574,7 +574,7 @@ new_image_metadata()
        imd = CALLOC(1, sizeof(*imd));
        if (imd) {
                imd->refcnt = 1;
-               INIT_HLIST_HEAD(&imd->inode_list);
+               INIT_LIST_HEAD(&imd->inode_list);
                INIT_LIST_HEAD(&imd->unhashed_streams);
                DEBUG("Created new image metadata (refcnt=1)");
        } else {
index 18f0273..91f99eb 100644 (file)
@@ -239,10 +239,12 @@ struct wim_security_data {
 /* Metadata for a WIM image */
 struct wim_image_metadata {
 
+       /* Number of WIMStruct's that are sharing this image metadata (from
+        * calls to wimlib_export_image().) */
        unsigned long refcnt;
 
        /* Pointer to the root dentry of the image. */
-       struct wim_dentry    *root_dentry;
+       struct wim_dentry *root_dentry;
 
        /* Pointer to the security data of the image. */
        struct wim_security_data *security_data;
@@ -251,9 +253,14 @@ struct wim_image_metadata {
         */
        struct wim_lookup_table_entry *metadata_lte;
 
-       /* Linked list of inodes of this image */
-       struct hlist_head inode_list;
+       /* Linked list of 'struct wim_inode's for this image. */
+       struct list_head inode_list;
 
+       /* Linked list of 'struct wim_lookup_table_entry's for this image that
+        * are referred to in the dentry tree, but have not had a SHA1 message
+        * digest calculated yet and therefore have not been inserted into the
+        * WIM's lookup table.  This list is added to during wimlib_add_image()
+        * and wimlib_mount_image() (read-write only). */
        struct list_head unhashed_streams;
 
        /* 1 iff the dentry tree has been modified.  If this is the case, the
@@ -356,6 +363,12 @@ resource_is_compressed(const struct resource_entry *entry)
        return (entry->flags & WIM_RESHDR_FLAG_COMPRESSED);
 }
 
+#define image_for_each_inode(inode, imd) \
+       list_for_each_entry(inode, &imd->inode_list, i_list)
+
+#define image_for_each_unhashed_stream(lte, imd) \
+       list_for_each_entry(lte, &imd->unhashed_streams, unhashed_list)
+
 /* add_image.c */
 
 extern bool
@@ -392,7 +405,7 @@ struct wim_inode_table {
         *   number with some other inode until assign_inode_numbers() is
         *   called.
         */
-       struct hlist_head extra_inodes;
+       struct list_head extra_inodes;
 };
 
 extern int
@@ -403,8 +416,11 @@ inode_table_new_dentry(struct wim_inode_table *table, const tchar *name,
                       u64 ino, u64 devno, struct wim_dentry **dentry_ret);
 
 extern void
+inode_ref_streams(struct wim_inode *inode);
+
+extern void
 inode_table_prepare_inode_list(struct wim_inode_table *table,
-                              struct hlist_head *head);
+                              struct list_head *head);
 
 static inline void
 destroy_inode_table(struct wim_inode_table *table)
@@ -414,7 +430,7 @@ destroy_inode_table(struct wim_inode_table *table)
 
 
 extern int
-dentry_tree_fix_inodes(struct wim_dentry *root, struct hlist_head *inode_list);
+dentry_tree_fix_inodes(struct wim_dentry *root, struct list_head *inode_list);
 
 /* header.c */
 
index 55dbbc1..ccad7f0 100644 (file)
@@ -187,9 +187,9 @@ get_compress_func(int out_ctype)
  *
  * @chunk:       Uncompressed data of the chunk.
  * @chunk_size:          Size of the chunk (<= WIM_CHUNK_SIZE)
- * @out_fp:      FILE * to write tho chunk to.
- * @out_ctype:   Compression type to use when writing the chunk (ignored if no
- *                     chunk table provided)
+ * @out_fp:      FILE * to write the chunk to.
+ * @compress:     Compression function to use (NULL if writing uncompressed
+ *                     data).
  * @chunk_tab:   Pointer to chunk table being created.  It is updated with the
  *                     offset of the chunk we write.
  *
@@ -202,7 +202,7 @@ write_wim_resource_chunk(const void *chunk, unsigned chunk_size,
 {
        const u8 *out_chunk;
        unsigned out_chunk_size;
-       if (chunk_tab) {
+       if (compress) {
                u8 *compressed_chunk = alloca(chunk_size);
 
                out_chunk_size = compress(chunk, chunk_size, compressed_chunk);
@@ -404,7 +404,6 @@ write_wim_resource(struct wim_lookup_table_entry *lte,
                /* Doing a raw write:  The new compressed size is the same as
                 * the compressed size in the other WIM. */
                new_size = lte->resource_entry.size;
-               out_res_entry->flags = lte->resource_entry.flags;
        } else if (out_ctype == WIMLIB_COMPRESSION_TYPE_NONE) {
                /* Using WIMLIB_COMPRESSION_TYPE_NONE:  The new compressed size
                 * is the original size. */
@@ -644,23 +643,31 @@ do_write_stream_list(struct list_head *my_resources,
                list_del(&lte->write_streams_list);
                if (lte->unhashed && !lte->unique_size) {
                        struct wim_lookup_table_entry *duplicate_lte;
+                       struct wim_lookup_table_entry **my_ptr;
 
+                       my_ptr = lte->my_ptr;
                        ret = sha1_resource(lte);
                        if (ret)
                                return ret;
-                       list_del(&lte->staging_list);
-
                        duplicate_lte = __lookup_resource(lookup_table, lte->hash);
                        if (duplicate_lte) {
                                bool new_stream = (duplicate_lte->out_refcnt == 0);
                                duplicate_lte->refcnt += lte->refcnt;
                                duplicate_lte->out_refcnt += lte->refcnt;
+                               *my_ptr = duplicate_lte;
                                free_lookup_table_entry(lte);
 
-                               if (new_stream)
+                               if (new_stream) {
                                        lte = duplicate_lte;
-                               else
-                                       continue;
+                                       DEBUG("Stream of length %"PRIu64" is duplicate "
+                                             "with one already in WIM",
+                                             wim_resource_size(duplicate_lte));
+                               } else {
+                                       DEBUG("Discarding duplicate stream of length %"PRIu64,
+                                             wim_resource_size(duplicate_lte));
+                                       goto skip_to_progress;
+                               }
+
                        } else {
                                lookup_table_insert(lookup_table, lte);
                                lte->out_refcnt = lte->refcnt;
@@ -678,10 +685,10 @@ do_write_stream_list(struct list_head *my_resources,
                if (ret)
                        return ret;
                if (lte->unhashed) {
-                       wimlib_assert(__lookup_resource(lookup_table, lte->hash) == NULL);
                        lookup_table_insert(lookup_table, lte);
                        lte->unhashed = 0;
                }
+       skip_to_progress:
                do_write_streams_progress(progress,
                                          progress_func,
                                          wim_resource_size(lte));
@@ -1418,10 +1425,11 @@ static int
 stream_size_table_insert(struct wim_lookup_table_entry *lte, void *_tab)
 {
        struct stream_size_table *tab = _tab;
-       size_t pos = hash_u64(wim_resource_size(lte)) % tab->capacity;
+       size_t pos;
        struct wim_lookup_table_entry *hashed_lte;
        struct hlist_node *tmp;
 
+       pos = hash_u64(wim_resource_size(lte)) % tab->capacity;
        lte->unique_size = 1;
        hlist_for_each_entry(hashed_lte, tmp, &tab->array[pos], hash_list_2) {
                if (wim_resource_size(hashed_lte) == wim_resource_size(lte)) {
@@ -1452,11 +1460,12 @@ inode_find_streams_to_write(struct wim_inode *inode,
        for (unsigned i = 0; i <= inode->i_num_ads; i++) {
                lte = inode_stream_lte(inode, i, table);
                if (lte) {
-                       if (lte->out_refcnt == 0)
+                       if (lte->out_refcnt == 0) {
+                               if (lte->unhashed)
+                                       stream_size_table_insert(lte, tab);
                                list_add_tail(&lte->write_streams_list, stream_list);
+                       }
                        lte->out_refcnt += inode->i_nlink;
-                       if (lte->unhashed)
-                               stream_size_table_insert(lte, tab);
                }
        }
        return 0;
@@ -1468,11 +1477,20 @@ image_find_streams_to_write(WIMStruct *w)
        struct wim_image_metadata *imd;
        struct find_streams_ctx *ctx;
        struct wim_inode *inode;
-       struct hlist_node *cur;
+       struct wim_lookup_table_entry *lte;
 
        ctx = w->private;
        imd = wim_get_current_image_metadata(w);
-       hlist_for_each_entry(inode, cur, &imd->inode_list, i_hlist) {
+
+       image_for_each_unhashed_stream(lte, imd) {
+               lte->out_refcnt = 0;
+               wimlib_assert(lte->unhashed);
+               wimlib_assert(lte->my_ptr != NULL);
+       }
+
+       /* Go through this image's inodes to find any streams that have not been
+        * found yet. */
+       image_for_each_inode(inode, imd) {
                inode_find_streams_to_write(inode, w->lookup_table,
                                            &ctx->stream_list,
                                            &ctx->stream_size_tab);
@@ -1480,31 +1498,61 @@ image_find_streams_to_write(WIMStruct *w)
        return 0;
 }
 
+/* Given a WIM that from which one or all of the images is being written, build
+ * the list of unique streams ('struct wim_lookup_table_entry's) that must be
+ * written, plus any unhashed streams that need to be written but may be
+ * identical to other hashed or unhashed streams being written.  These unhashed
+ * streams are checksummed while the streams are being written.  To aid this
+ * process, the member @unique_size is set to 1 on streams that have a unique
+ * size and therefore must be written.
+ *
+ * The out_refcnt member of each 'struct wim_lookup_table_entry' is set to
+ * indicate the number of times the stream is referenced in only the streams
+ * that are being written; this may still be adjusted later when unhashed
+ * streams are being resolved.
+ */
 static int
-write_wim_streams(WIMStruct *w, int image, int write_flags,
-                 unsigned num_threads,
-                 wimlib_progress_func_t progress_func)
+prepare_stream_list(WIMStruct *wim, int image, struct list_head *stream_list)
 {
-       struct find_streams_ctx ctx;
        int ret;
+       struct find_streams_ctx ctx;
 
-       for_lookup_table_entry(w->lookup_table, lte_zero_out_refcnt, NULL);
+       for_lookup_table_entry(wim->lookup_table, lte_zero_out_refcnt, NULL);
        ret = init_stream_size_table(&ctx.stream_size_tab, 9001);
        if (ret)
                return ret;
-       for_lookup_table_entry(w->lookup_table, stream_size_table_insert,
+       for_lookup_table_entry(wim->lookup_table, stream_size_table_insert,
                               &ctx.stream_size_tab);
-
        INIT_LIST_HEAD(&ctx.stream_list);
-       w->private = &ctx;
-       for_image(w, image, image_find_streams_to_write);
+       wim->private = &ctx;
+       for_image(wim, image, image_find_streams_to_write);
        destroy_stream_size_table(&ctx.stream_size_tab);
-       ret = write_stream_list(&ctx.stream_list,
-                               w->lookup_table,
-                               w->out_fp,
-                               wimlib_get_compression_type(w), write_flags,
-                               num_threads, progress_func);
-       return ret;
+
+       INIT_LIST_HEAD(stream_list);
+       list_splice(&ctx.stream_list, stream_list);
+       return 0;
+}
+
+/* Writes the streams for the specified @image in @wim to @wim->out_fp.
+ */
+static int
+write_wim_streams(WIMStruct *wim, int image, int write_flags,
+                 unsigned num_threads,
+                 wimlib_progress_func_t progress_func)
+{
+       int ret;
+       struct list_head stream_list;
+
+       ret = prepare_stream_list(wim, image, &stream_list);
+       if (ret)
+               return ret;
+       return write_stream_list(&stream_list,
+                                wim->lookup_table,
+                                wim->out_fp,
+                                wimlib_get_compression_type(wim),
+                                write_flags,
+                                num_threads,
+                                progress_func);
 }
 
 /*
@@ -1767,19 +1815,19 @@ wimlib_write(WIMStruct *w, const tchar *path,
        }
 
        ret = begin_write(w, path, write_flags);
-       if (ret != 0)
+       if (ret)
                goto out;
 
        ret = write_wim_streams(w, image, write_flags, num_threads,
                                progress_func);
-       if (ret != 0)
+       if (ret)
                goto out;
 
        if (progress_func)
                progress_func(WIMLIB_PROGRESS_MSG_WRITE_METADATA_BEGIN, NULL);
 
        ret = for_image(w, image, write_metadata_resource);
-       if (ret != 0)
+       if (ret)
                goto out;
 
        if (progress_func)