Initial rewrite of resource code

diff --git a/src/add_image.c b/src/add_image.c
index de9a465f15d012f82065fc0ce700471e21065472..02ca2f2584734808dde746e85b8c0cdb009e75ec 100644 (file)
--- a/src/add_image.c
+++ b/src/add_image.c
@@ -104,40 +104,20 @@ err:
 
 static int
 unix_capture_regular_file(const char *path,
 
 static int
 unix_capture_regular_file(const char *path,

-                         uint64_t size,
+                         u64 size,

                          struct wim_inode *inode,
                          struct wim_lookup_table *lookup_table)
 {
                          struct wim_inode *inode,
                          struct wim_lookup_table *lookup_table)
 {

-       struct wim_lookup_table_entry *lte;
-       u8 hash[SHA1_HASH_SIZE];
-       int ret;
-

        inode->i_attributes = FILE_ATTRIBUTE_NORMAL;
 
        /* Empty files do not have to have a lookup table entry. */
        inode->i_attributes = FILE_ATTRIBUTE_NORMAL;
 
        /* Empty files do not have to have a lookup table entry. */

-       if (size == 0)
-               return 0;
-
-       /* For each regular file, we must check to see if the file is in
-        * the lookup table already; if it is, we increment its refcnt;
-        * otherwise, we create a new lookup table entry and insert it.
-        * */
-
-       ret = sha1sum(path, hash);
-       if (ret)
-               return ret;
+       if (size != 0) {
+               struct wim_lookup_table_entry *lte;
+               char *file_on_disk;

 
 

-       lte = __lookup_resource(lookup_table, hash);
-       if (lte) {
-               lte->refcnt++;
-               DEBUG("Add lte reference %u for `%s'", lte->refcnt,
-                     path);
-       } else {
-               char *file_on_disk = STRDUP(path);
-               if (!file_on_disk) {
-                       ERROR("Failed to allocate memory for file path");
+               file_on_disk = STRDUP(path);
+               if (!file_on_disk)

                        return WIMLIB_ERR_NOMEM;
                        return WIMLIB_ERR_NOMEM;

-               }

                lte = new_lookup_table_entry();
                if (!lte) {
                        FREE(file_on_disk);
                lte = new_lookup_table_entry();
                if (!lte) {
                        FREE(file_on_disk);


@@ -146,11 +126,9 @@ unix_capture_regular_file(const char *path,
                lte->file_on_disk = file_on_disk;
                lte->resource_location = RESOURCE_IN_FILE_ON_DISK;
                lte->resource_entry.original_size = size;
                lte->file_on_disk = file_on_disk;
                lte->resource_location = RESOURCE_IN_FILE_ON_DISK;
                lte->resource_entry.original_size = size;

-               lte->resource_entry.size = size;
-               copy_hash(lte->hash, hash);
-               lookup_table_insert(lookup_table, lte);
+               lookup_table_insert_unhashed(lookup_table, lte);
+               inode->i_lte = lte;

        }
        }

-       inode->i_lte = lte;

        return 0;
 }
 
        return 0;
 }
 


@@ -393,10 +371,12 @@ unix_build_dentry_tree_recursive(struct wim_dentry **root_ret,
        else
                ret = unix_capture_symlink(path, inode, lookup_table);
 out:
        else
                ret = unix_capture_symlink(path, inode, lookup_table);
 out:

-       if (ret == 0)
+       if (ret == 0) {

                *root_ret = root;
                *root_ret = root;

-       else
+       } else {

                free_dentry_tree(root, lookup_table);
                free_dentry_tree(root, lookup_table);

+               lookup_table_free_unhashed_streams(lookup_table);
+       }

        return ret;
 }
 
        return ret;
 }
 

diff --git a/src/dentry.c b/src/dentry.c
index 49d5d8cb1569d1289e54f230f344d6b42b7d9a45..2ed77da7ffaaed52f256a376049203b043de0a92 100644 (file)
--- a/src/dentry.c
+++ b/src/dentry.c
@@ -1158,7 +1158,7 @@ inode_get_unix_data(const struct wim_inode *inode,
        if (size != sizeof(struct wimlib_unix_data))
                return BAD_UNIX_DATA;
 
        if (size != sizeof(struct wimlib_unix_data))
                return BAD_UNIX_DATA;
 

-       ret = read_full_wim_resource(lte, unix_data, 0);
+       ret = read_full_resource_into_buf(lte, unix_data, true);

        if (ret)
                return ret;
 
        if (ret)
                return ret;
 

diff --git a/src/extract_image.c b/src/extract_image.c
index 5148ddcc20da430468ca3e088a99a98944f59446..a24f0ab551358192d01f9286e8f219de1a373205 100644 (file)
--- a/src/extract_image.c
+++ b/src/extract_image.c
@@ -328,7 +328,7 @@ extract_symlink(struct wim_dentry *dentry,
 {
        char target[4096];
        ssize_t ret = inode_readlink(dentry->d_inode, target,
 {
        char target[4096];
        ssize_t ret = inode_readlink(dentry->d_inode, target,

-                                    sizeof(target), args->w, 0);
+                                    sizeof(target), args->w, false);

        struct wim_lookup_table_entry *lte;
 
        if (ret <= 0) {
        struct wim_lookup_table_entry *lte;
 
        if (ret <= 0) {

diff --git a/src/list.h b/src/list.h
index 82f08e5ff47e9e192777edfebc9358a590dcd119..889097b5acf36521157178f9d1e0995fce306bcc 100644 (file)
--- a/src/list.h
+++ b/src/list.h
@@ -119,6 +119,44 @@ static inline int list_empty(const struct list_head *head)
        return head->next == head;
 }
 
        return head->next == head;
 }
 

+static inline void __list_splice(const struct list_head *list,
+                                struct list_head *prev,
+                                struct list_head *next)
+{
+       struct list_head *first = list->next;
+       struct list_head *last = list->prev;
+
+       first->prev = prev;
+       prev->next = first;
+
+       last->next = next;
+       next->prev = last;
+}
+
+/**
+ * list_splice - join two lists, this is designed for stacks
+ * @list: the new list to add.
+ * @head: the place to add it in the first list.
+ */
+static inline void list_splice(const struct list_head *list,
+                               struct list_head *head)
+{
+       if (!list_empty(list))
+               __list_splice(list, head, head->next);
+}
+
+/**
+ * list_splice_tail - join two lists, each list being a queue
+ * @list: the new list to add.
+ * @head: the place to add it in the first list.
+ */
+static inline void list_splice_tail(struct list_head *list,
+                               struct list_head *head)
+{
+       if (!list_empty(list))
+               __list_splice(list, head->prev, head);
+}
+

 /**
  * list_entry - get the struct for this entry
  * @ptr:       the &struct list_head pointer.
 /**
  * list_entry - get the struct for this entry
  * @ptr:       the &struct list_head pointer.

diff --git a/src/lookup_table.c b/src/lookup_table.c
index 064215ec0cf75529de50fe677ea758dbdd2fff5b..9502db4770b110b873a7f574ecca6564a2021132 100644 (file)
--- a/src/lookup_table.c
+++ b/src/lookup_table.c
@@ -40,7 +40,7 @@ new_lookup_table(size_t capacity)
        struct wim_lookup_table *table;
        struct hlist_head *array;
 
        struct wim_lookup_table *table;
        struct hlist_head *array;
 

-       table = MALLOC(sizeof(struct wim_lookup_table));
+       table = CALLOC(1, sizeof(struct wim_lookup_table));

        if (table) {
                array = CALLOC(capacity, sizeof(array[0]));
                if (array) {
        if (table) {
                array = CALLOC(capacity, sizeof(array[0]));
                if (array) {


@@ -920,3 +920,13 @@ lookup_table_total_stream_size(struct wim_lookup_table *table)
        for_lookup_table_entry(table, lte_add_stream_size, &total_size);
        return total_size;
 }
        for_lookup_table_entry(table, lte_add_stream_size, &total_size);
        return total_size;
 }

+
+void
+lookup_table_free_unhashed_streams(struct wim_lookup_table *table)
+{
+       struct wim_lookup_table_entry *lte, *tmp;
+
+       list_for_each_entry_safe(lte, tmp, table->unhashed_streams, staging_list)
+               free_lookup_table_entry(lte);
+       INIT_LIST_HEAD(table->unhashed_streams);
+}

diff --git a/src/lookup_table.h b/src/lookup_table.h
index eacd9aa4596dfae4441de5ec9fb338f2973f1048..ab545f2d3ea94cd38b139af34ab7fee4bdbcaaa9 100644 (file)
--- a/src/lookup_table.h
+++ b/src/lookup_table.h
@@ -32,6 +32,7 @@ struct wim_lookup_table {
        struct hlist_head *array;
        u64 num_entries;
        u64 capacity;
        struct hlist_head *array;
        u64 num_entries;
        u64 capacity;

+       struct list_head *unhashed_streams;

 };
 
 #ifdef WITH_NTFS_3G
 };
 
 #ifdef WITH_NTFS_3G


@@ -68,23 +69,35 @@ enum resource_location {
         * */
        RESOURCE_IN_FILE_ON_DISK,
 
         * */
        RESOURCE_IN_FILE_ON_DISK,
 

-       /* The stream resource is located in an external file in the staging
-        * directory for a read-write mount.  */
-       RESOURCE_IN_STAGING_FILE,
-

        /* The stream resource is directly attached in an in-memory buffer
         * pointed to by @attached_buffer. */
        RESOURCE_IN_ATTACHED_BUFFER,
 
        /* The stream resource is directly attached in an in-memory buffer
         * pointed to by @attached_buffer. */
        RESOURCE_IN_ATTACHED_BUFFER,
 

+#ifdef WITH_FUSE
+       /* The stream resource is located in an external file in the staging
+        * directory for a read-write mount.  */
+       RESOURCE_IN_STAGING_FILE,
+#endif
+
+#ifdef WITH_NTFS_3G

        /* The stream resource is located in an NTFS volume.  It is identified
         * by volume, filename, data stream name, and by whether it is a reparse
         * point or not. @ntfs_loc points to a structure containing this
         * information. */
        RESOURCE_IN_NTFS_VOLUME,
        /* The stream resource is located in an NTFS volume.  It is identified
         * by volume, filename, data stream name, and by whether it is a reparse
         * point or not. @ntfs_loc points to a structure containing this
         * information. */
        RESOURCE_IN_NTFS_VOLUME,

+#endif

 
 

+#ifdef __WIN32__

        /* Resource must be accessed using Win32 API (may be a named data
         * stream) */
        RESOURCE_WIN32,
        /* Resource must be accessed using Win32 API (may be a named data
         * stream) */
        RESOURCE_WIN32,

+
+       /* Windows only: the file is on disk in the file named @file_on_disk,
+        * but the file is encrypted and must be read using special functions.
+        * */
+       RESOURCE_WIN32_ENCRYPTED,
+#endif
+

 };
 
 /*
 };
 
 /*


@@ -168,27 +181,11 @@ struct wim_lookup_table_entry {
                struct ntfs_location *ntfs_loc;
        #endif
        };
                struct ntfs_location *ntfs_loc;
        #endif
        };

-       union {
-               /* @file_on_disk_fp and @attr are both used to cache file/stream
-                * handles so we don't have re-open them on every read */
-

 
 

-               /* Valid iff resource_location == RESOURCE_IN_FILE_ON_DISK */
-               FILE *file_on_disk_fp;
-       #ifdef WITH_NTFS_3G
-               /* Valid iff resource_location == RESOURCE_IN_NTFS_VOLUME */
-               struct _ntfs_attr *attr;
-       #endif
-
-       #ifdef __WIN32__
-               HANDLE win32_file_on_disk_fp;
-       #endif
-
-               /* Pointer to inode that contains the opened file descriptors to
-                * this stream (valid iff resource_location ==
-                * RESOURCE_IN_STAGING_FILE) */
-               struct wim_inode *lte_inode;
-       };
+       /* Pointer to inode that contains the opened file descriptors to
+        * this stream (valid iff resource_location ==
+        * RESOURCE_IN_STAGING_FILE) */
+       struct wim_inode *lte_inode;

 
        /* When a WIM file is written, out_refcnt starts at 0 and is incremented
         * whenever the file resource pointed to by this lookup table entry
 
        /* When a WIM file is written, out_refcnt starts at 0 and is incremented
         * whenever the file resource pointed to by this lookup table entry


@@ -253,6 +250,19 @@ wim_resource_compression_type(const struct wim_lookup_table_entry *lte)
        return wimlib_get_compression_type(lte->wim);
 }
 
        return wimlib_get_compression_type(lte->wim);
 }
 

+static inline bool
+lte_filename_valid(const struct wim_lookup_table_entry *lte)
+{
+       return lte->resource_location == RESOURCE_IN_FILE_ON_DISK
+       #ifdef __WIN32__
+               || lte->resource_location == RESOURCE_WIN32
+               || lte->resource_location == RESOURCE_WIN32_ENCRYPTED
+       #endif
+       #ifdef WITH_FUSE
+               || lte->resource_location == RESOURCE_IN_STAGING_FILE
+       #endif
+               ;
+}

 
 extern struct wim_lookup_table *
 new_lookup_table(size_t capacity);
 
 extern struct wim_lookup_table *
 new_lookup_table(size_t capacity);


@@ -461,4 +471,15 @@ inode_unnamed_lte(const struct wim_inode *inode, const struct wim_lookup_table *
 extern u64
 lookup_table_total_stream_size(struct wim_lookup_table *table);
 
 extern u64
 lookup_table_total_stream_size(struct wim_lookup_table *table);
 

+
+static inline void
+lookup_table_insert_unhashed(struct wim_lookup_table *table,
+                            struct wim_lookup_table_entry *lte)
+{
+       list_add_tail(&lte->staging_list, table->unhashed_streams);
+}
+
+extern void
+lookup_table_free_unhashed_streams(struct wim_lookup_table *table);
+

 #endif
 #endif

diff --git a/src/metadata_resource.c b/src/metadata_resource.c
index db35ba649cfc15581fef8c9de2489f7e13607822..9b5a83050abd51a95a6d47ebc791ce63d2c5d5fd 100644 (file)
--- a/src/metadata_resource.c
+++ b/src/metadata_resource.c
@@ -86,8 +86,8 @@ read_metadata_resource(WIMStruct *w, struct wim_image_metadata *imd)
        }
 
        /* Read the metadata resource into memory.  (It may be compressed.) */
        }
 
        /* Read the metadata resource into memory.  (It may be compressed.) */

-       ret = read_full_wim_resource(metadata_lte, buf, 0);
-       if (ret != 0)
+       ret = read_full_resource_into_buf(metadata_lte, buf, false);
+       if (ret)

                goto out_free_buf;
 
        DEBUG("Finished reading metadata resource into memory.");
                goto out_free_buf;
 
        DEBUG("Finished reading metadata resource into memory.");


@@ -208,16 +208,12 @@ write_wim_resource_from_buffer(const u8 *buf, u64 buf_size,
         * write_wim_resource(). */
        struct wim_lookup_table_entry lte;
        int ret;
         * write_wim_resource(). */
        struct wim_lookup_table_entry lte;
        int ret;

-       lte.resource_entry.flags         = 0;

        lte.resource_entry.original_size = buf_size;
        lte.resource_entry.original_size = buf_size;

-       lte.resource_entry.size          = buf_size;
-       lte.resource_entry.offset        = 0;

        lte.resource_location            = RESOURCE_IN_ATTACHED_BUFFER;
        lte.attached_buffer              = (void*)buf;
        lte.resource_location            = RESOURCE_IN_ATTACHED_BUFFER;
        lte.attached_buffer              = (void*)buf;

-

        zero_out_hash(lte.hash);
        ret = write_wim_resource(&lte, out_fp, out_ctype, out_res_entry, 0);
        zero_out_hash(lte.hash);
        ret = write_wim_resource(&lte, out_fp, out_ctype, out_res_entry, 0);

-       if (ret != 0)
+       if (ret)

                return ret;
        copy_hash(hash, lte.hash);
        return 0;
                return ret;
        copy_hash(hash, lte.hash);
        return 0;


@@ -293,8 +289,8 @@ write_metadata_resource(WIMStruct *w)
                                             wimlib_get_compression_type(w),
                                             &lte->output_resource_entry,
                                             lte->hash);
                                             wimlib_get_compression_type(w),
                                             &lte->output_resource_entry,
                                             lte->hash);

-       if (ret != 0)
-               goto out;
+       if (ret)
+               goto out_free_buf;

 
        /* Note that although the SHA1 message digest of the metadata resource
         * is very likely to have changed, the corresponding lookup table entry
 
        /* Note that although the SHA1 message digest of the metadata resource
         * is very likely to have changed, the corresponding lookup table entry


@@ -302,7 +298,7 @@ write_metadata_resource(WIMStruct *w)
         * re-inserted in the hash table. */
        lte->out_refcnt = 1;
        lte->output_resource_entry.flags |= WIM_RESHDR_FLAG_METADATA;
         * re-inserted in the hash table. */
        lte->out_refcnt = 1;
        lte->output_resource_entry.flags |= WIM_RESHDR_FLAG_METADATA;

-out:
+out_free_buf:

        /* All the data has been written to the new WIM; no need for the buffer
         * anymore */
        FREE(buf);
        /* All the data has been written to the new WIM; no need for the buffer
         * anymore */
        FREE(buf);

diff --git a/src/mount_image.c b/src/mount_image.c
index 024c82b7253a2d21758b8f9942f07e803d8c6b6d..128e1736dcb21d3b157a6c7854a486618424f971 100644 (file)
--- a/src/mount_image.c
+++ b/src/mount_image.c
@@ -422,22 +422,10 @@ inode_to_stbuf(const struct wim_inode *inode,
        }
        stbuf->st_ino = (ino_t)inode->i_ino;
        stbuf->st_nlink = inode->i_nlink;
        }
        stbuf->st_ino = (ino_t)inode->i_ino;
        stbuf->st_nlink = inode->i_nlink;

-       if (lte) {
-               if (lte->resource_location == RESOURCE_IN_STAGING_FILE) {
-                       struct stat native_stat;
-                       if (stat(lte->staging_file_name, &native_stat) != 0) {
-                               DEBUG("Failed to stat `%s': %m",
-                                     lte->staging_file_name);
-                               return -errno;
-                       }
-                       stbuf->st_size = native_stat.st_size;
-               } else {
-                       stbuf->st_size = wim_resource_size(lte);
-               }
-       } else {
+       if (lte)
+               stbuf->st_size = wim_resource_size(lte);
+       else

                stbuf->st_size = 0;
                stbuf->st_size = 0;

-       }
-

 #ifdef HAVE_STAT_NANOSECOND_PRECISION
        stbuf->st_atim = wim_timestamp_to_timespec(inode->i_last_access_time);
        stbuf->st_mtim = wim_timestamp_to_timespec(inode->i_last_write_time);
 #ifdef HAVE_STAT_NANOSECOND_PRECISION
        stbuf->st_atim = wim_timestamp_to_timespec(inode->i_last_access_time);
        stbuf->st_mtim = wim_timestamp_to_timespec(inode->i_last_write_time);


@@ -658,11 +646,10 @@ extract_resource_to_staging_dir(struct wim_inode *inode,
                }
        }
 
                }
        }
 

-       new_lte->refcnt                       = inode->i_nlink;
-       new_lte->resource_location            = RESOURCE_IN_STAGING_FILE;
-       new_lte->staging_file_name            = staging_file_name;
-       new_lte->lte_inode                    = inode;
-       random_hash(new_lte->hash);
+       new_lte->refcnt            = inode->i_nlink;
+       new_lte->resource_location = RESOURCE_IN_STAGING_FILE;
+       new_lte->staging_file_name = staging_file_name;
+       new_lte->lte_inode         = inode;

 
        if (stream_id == 0)
                inode->i_lte = new_lte;
 
        if (stream_id == 0)
                inode->i_lte = new_lte;


@@ -671,8 +658,8 @@ extract_resource_to_staging_dir(struct wim_inode *inode,
                        if (inode->i_ads_entries[i].stream_id == stream_id)
                                inode->i_ads_entries[i].lte = new_lte;
 
                        if (inode->i_ads_entries[i].stream_id == stream_id)
                                inode->i_ads_entries[i].lte = new_lte;
 

-       lookup_table_insert(ctx->wim->lookup_table, new_lte);
-       list_add(&new_lte->staging_list, &ctx->staging_list);
+       lookup_table_insert_unhashed(ctx->wim->lookup_table, new_lte);
+       list_add_tail(&new_lte->staging_list, &ctx->staging_list);

        *lte = new_lte;
        return 0;
 out_revert_fd_changes:
        *lte = new_lte;
        return 0;
 out_revert_fd_changes:


@@ -807,45 +794,15 @@ inode_update_lte_ptr(struct wim_inode *inode,
        }
 }
 
        }
 }
 

-static int
-update_lte_of_staging_file(struct wim_lookup_table_entry *lte,
-                          struct wim_lookup_table *table)
+static void
+free_lte_if_unneeded(struct wim_lookup_table_entry *lte)

 {
 {

-       struct wim_lookup_table_entry *duplicate_lte;
-       int ret;
-       u8 hash[SHA1_HASH_SIZE];
-       struct stat stbuf;

 
 

-       ret = sha1sum(lte->staging_file_name, hash);
-       if (ret != 0)
-               return ret;
-       lookup_table_unlink(table, lte);
-       duplicate_lte = __lookup_resource(table, hash);
-       if (duplicate_lte) {
-               /* Merge duplicate lookup table entries */
-               duplicate_lte->refcnt += lte->refcnt;
-               inode_update_lte_ptr(lte->lte_inode, lte, duplicate_lte);
+       if (wim_resource_size(lte) == 0) {
+               /* Zero-length stream.  No lookup table entry needed. */
+               inode_update_lte_ptr(lte->lte_inode, lte, NULL);

                free_lookup_table_entry(lte);
                free_lookup_table_entry(lte);

-       } else {
-               if (stat(lte->staging_file_name, &stbuf) != 0) {
-                       ERROR_WITH_ERRNO("Failed to stat `%s'", lte->staging_file_name);
-                       return WIMLIB_ERR_STAT;
-               }
-               if (stbuf.st_size == 0) {
-                       /* Zero-length stream.  No lookup table entry needed. */
-                       inode_update_lte_ptr(lte->lte_inode, lte, NULL);
-                       free_lookup_table_entry(lte);
-               } else {
-                       BUILD_BUG_ON(&lte->file_on_disk != &lte->staging_file_name);
-                       lte->resource_entry.original_size = stbuf.st_size;
-                       lte->resource_entry.size = stbuf.st_size;
-                       lte->resource_location = RESOURCE_IN_FILE_ON_DISK;
-                       lte->file_on_disk_fp = NULL;
-                       copy_hash(lte->hash, hash);
-                       lookup_table_insert(table, lte);
-               }

        }
        }

-       return 0;

 }
 
 static int
 }
 
 static int


@@ -881,14 +838,13 @@ rebuild_wim(struct wimfs_context *ctx, int write_flags,
                        return ret;
        }
 
                        return ret;
        }
 

-       DEBUG("Calculating SHA1 checksums for all new staging files.");
-       list_for_each_entry_safe(lte, tmp, &ctx->staging_list, staging_list) {
-               ret = update_lte_of_staging_file(lte, w->lookup_table);
-               if (ret != 0)
-                       return ret;
-       }
+       DEBUG("Freeing entries for zero-length streams");
+       list_for_each_entry_safe(lte, tmp, &ctx->staging_list, staging_list)
+               free_lte_if_unneeded(lte);

 
        xml_update_image_info(w, w->current_image);
 
        xml_update_image_info(w, w->current_image);

+       list_splice(&ctx->staging_list,
+                   &w->image_metadata[w->current_image - 1].unhashed_streams);

        ret = wimlib_overwrite(w, write_flags, 0, progress_func);
        if (ret != 0)
                ERROR("Failed to commit changes to mounted WIM image");
        ret = wimlib_overwrite(w, write_flags, 0, progress_func);
        if (ret != 0)
                ERROR("Failed to commit changes to mounted WIM image");


@@ -1660,10 +1616,10 @@ wimfs_ftruncate(const char *path, off_t size, struct fuse_file_info *fi)
 {
        struct wimfs_fd *fd = (struct wimfs_fd*)(uintptr_t)fi->fh;
        int ret = ftruncate(fd->staging_fd, size);
 {
        struct wimfs_fd *fd = (struct wimfs_fd*)(uintptr_t)fi->fh;
        int ret = ftruncate(fd->staging_fd, size);

-       if (ret != 0)
+       if (ret)

                return -errno;
                return -errno;

-       if (fd->f_lte && size < fd->f_lte->resource_entry.original_size)
-               fd->f_lte->resource_entry.original_size = size;
+       touch_inode(fd->f_inode);
+       fd->f_lte->resource_entry.original_size = size;

        return 0;
 }
 
        return 0;
 }
 


@@ -1723,8 +1679,7 @@ wimfs_getxattr(const char *path, const char *name, char *value,
        if (res_size > size)
                return -ERANGE;
 
        if (res_size > size)
                return -ERANGE;
 

-       ret = read_full_wim_resource(lte, (u8*)value,
-                                    WIMLIB_RESOURCE_FLAG_MULTITHREADED);
+       ret = read_full_resource_into_buf(lte, value, true);

        if (ret != 0)
                return -EIO;
 
        if (ret != 0)
                return -EIO;
 


@@ -1861,8 +1816,7 @@ wimfs_mknod(const char *path, mode_t mode, dev_t rdev)
                inode = wim_pathname_to_inode(wimfs_ctx->wim, path);
                if (!inode)
                        return -errno;
                inode = wim_pathname_to_inode(wimfs_ctx->wim, path);
                if (!inode)
                        return -errno;

-               if (inode->i_attributes &
-                   (FILE_ATTRIBUTE_REPARSE_POINT | FILE_ATTRIBUTE_DIRECTORY))
+               if (inode->i_attributes & FILE_ATTRIBUTE_REPARSE_POINT)

                        return -ENOENT;
                if (inode_get_ads_entry(inode, stream_name, NULL))
                        return -EEXIST;
                        return -ENOENT;
                if (inode_get_ads_entry(inode, stream_name, NULL))
                        return -EEXIST;


@@ -1965,6 +1919,7 @@ wimfs_read(const char *path, char *buf, size_t size,
 {
        struct wimfs_fd *fd = (struct wimfs_fd*)(uintptr_t)fi->fh;
        ssize_t ret;
 {
        struct wimfs_fd *fd = (struct wimfs_fd*)(uintptr_t)fi->fh;
        ssize_t ret;

+       u64 res_size;

 
        if (!fd)
                return -EBADF;
 
        if (!fd)
                return -EBADF;


@@ -1972,32 +1927,34 @@ wimfs_read(const char *path, char *buf, size_t size,
        if (!fd->f_lte) /* Empty stream with no lookup table entry */
                return 0;
 
        if (!fd->f_lte) /* Empty stream with no lookup table entry */
                return 0;
 

-       if (fd->f_lte->resource_location == RESOURCE_IN_STAGING_FILE) {
-               /* Read from staging file */
-
-               wimlib_assert(fd->f_lte->staging_file_name);
-               wimlib_assert(fd->staging_fd != -1);
-
-               DEBUG("Seek to offset %"PRIu64, offset);
+       res_size = wim_resource_size(fd->f_lte);
+       if (offset > res_size)
+               return -EOVERFLOW;
+       size = min(size, INT_MAX);
+       size = min(size, res_size - offset);

 
 

-               if (lseek(fd->staging_fd, offset, SEEK_SET) == -1)
-                       return -errno;
-               ret = read(fd->staging_fd, buf, size);
-               if (ret == -1)
-                       return -errno;
-               return ret;
-       } else {
-               /* Read from WIM */
-               u64 res_size = wim_resource_size(fd->f_lte);
-               if (offset > res_size)
-                       return -EOVERFLOW;
-               size = min(size, res_size - offset);
-               if (read_wim_resource(fd->f_lte, buf,
-                                     size, offset,
-                                     WIMLIB_RESOURCE_FLAG_MULTITHREADED) != 0)
-                       return -EIO;
-               return size;
+       switch (fd->f_lte->resource_location) {
+       case RESOURCE_IN_STAGING_FILE:
+               ret = pread(fd->staging_fd, buf, size, offset);
+               if (ret < 0)
+                       ret = -errno;
+               break;
+       case RESOURCE_IN_WIM:
+               if (read_partial_wim_resource_into_buf(fd->f_lte, size,
+                                                      offset, buf, true))
+                       ret = -errno;
+               ret = size;
+               break;
+       case RESOURCE_IN_ATTACHED_BUFFER:
+               memcpy(buf, fd->f_lte->attached_buffer + offset, size);
+               ret = size;
+               break;
+       default:
+               ERROR("Invalid resource location");
+               ret = -EIO;
+               break;

        }
        }

+       return ret;

 }
 
 struct fill_params {
 }
 
 struct fill_params {


@@ -2064,8 +2021,7 @@ wimfs_readlink(const char *path, char *buf, size_t buf_len)
        if (!inode_is_symlink(inode))
                return -EINVAL;
 
        if (!inode_is_symlink(inode))
                return -EINVAL;
 

-       ret = inode_readlink(inode, buf, buf_len, ctx->wim,
-                            WIMLIB_RESOURCE_FLAG_MULTITHREADED);
+       ret = inode_readlink(inode, buf, buf_len, ctx->wim, true);

        if (ret > 0)
                ret = 0;
        return ret;
        if (ret > 0)
                ret = 0;
        return ret;


@@ -2299,6 +2255,8 @@ wimfs_truncate(const char *path, off_t size)
                ret = extract_resource_to_staging_dir(inode, stream_id,
                                                      &lte, size, ctx);
        }
                ret = extract_resource_to_staging_dir(inode, stream_id,
                                                      &lte, size, ctx);
        }

+       if (ret == 0)
+               lte->resource_entry.original_size = size;

        return ret;
 }
 
        return ret;
 }
 


@@ -2390,20 +2348,20 @@ wimfs_write(const char *path, const char *buf, size_t size,
        if (!fd)
                return -EBADF;
 
        if (!fd)
                return -EBADF;
 

-       wimlib_assert(fd->f_lte);
-       wimlib_assert(fd->f_lte->staging_file_name);
+       wimlib_assert(fd->f_lte != NULL);
+       wimlib_assert(fd->f_lte->staging_file_name != NULL);

        wimlib_assert(fd->staging_fd != -1);
        wimlib_assert(fd->staging_fd != -1);

-       wimlib_assert(fd->f_inode);
-
-       /* Seek to the requested position */
-       if (lseek(fd->staging_fd, offset, SEEK_SET) == -1)
-               return -errno;
+       wimlib_assert(fd->f_inode != NULL);

 
        /* Write the data. */
 
        /* Write the data. */

-       ret = write(fd->staging_fd, buf, size);
+       ret = pwrite(fd->staging_fd, buf, size, offset);

        if (ret == -1)
                return -errno;
 
        if (ret == -1)
                return -errno;
 

+       /* Update file size */
+       if (offset + size > fd->f_lte->resource_entry.original_size)
+               fd->f_lte->resource_entry.original_size = offset + size;
+

        /* Update timestamps */
        touch_inode(fd->f_inode);
        return ret;
        /* Update timestamps */
        touch_inode(fd->f_inode);
        return ret;

diff --git a/src/ntfs-apply.c b/src/ntfs-apply.c
index ca73db2ffc9ac83593d5d8ed59ef7e67a291be27..994bbd275979deeb2bfef1c15c9e110586c668c6 100644 (file)
--- a/src/ntfs-apply.c
+++ b/src/ntfs-apply.c
@@ -43,12 +43,17 @@
 #include <string.h>
 #include <locale.h>
 
 #include <string.h>
 #include <locale.h>
 

+struct ntfs_attr_extract_ctx {
+       u64 offset;
+       ntfs_attr *na;
+};
+

 static int
 static int

-extract_wim_chunk_to_ntfs_attr(const void *buf, size_t len,
-                              u64 offset, void *arg)
+extract_wim_chunk_to_ntfs_attr(const void *buf, size_t len, void *_ctx)

 {
 {

-       ntfs_attr *na = arg;
-       if (ntfs_attr_pwrite(na, offset, len, buf) == len) {
+       struct ntfs_attr_extract_ctx *ctx = _ctx;
+       if (ntfs_attr_pwrite(ctx->na, ctx->offset, len, buf) == len) {
+               ctx->offset += len;

                return 0;
        } else {
                ERROR_WITH_ERRNO("Error extracting WIM resource to NTFS attribute");
                return 0;
        } else {
                ERROR_WITH_ERRNO("Error extracting WIM resource to NTFS attribute");


@@ -63,8 +68,11 @@ static int
 extract_wim_resource_to_ntfs_attr(const struct wim_lookup_table_entry *lte,
                                  ntfs_attr *na)
 {
 extract_wim_resource_to_ntfs_attr(const struct wim_lookup_table_entry *lte,
                                  ntfs_attr *na)
 {

+       struct ntfs_attr_extract_ctx ctx;
+       ctx.na = na;
+       ctx.offset = 0;

        return extract_wim_resource(lte, wim_resource_size(lte),
        return extract_wim_resource(lte, wim_resource_size(lte),

-                                   extract_wim_chunk_to_ntfs_attr, na);
+                                   extract_wim_chunk_to_ntfs_attr, &ctx);

 }
 
 /* Writes the data streams of a WIM inode to the data attributes of a NTFS
 }
 
 /* Writes the data streams of a WIM inode to the data attributes of a NTFS


@@ -360,13 +368,13 @@ apply_reparse_data(ntfs_inode *ni, const struct wim_dentry *dentry,
        p = put_u16(p, wim_resource_size(lte)); /* ReparseDataLength */
        p = put_u16(p, 0); /* Reserved */
 
        p = put_u16(p, wim_resource_size(lte)); /* ReparseDataLength */
        p = put_u16(p, 0); /* Reserved */
 

-       ret = read_full_wim_resource(lte, p, 0);
-       if (ret != 0)
+       ret = read_full_resource_into_buf(lte, p, false);
+       if (ret)

                return ret;
 
        ret = ntfs_set_ntfs_reparse_data(ni, (char*)reparse_data_buf,
                                         wim_resource_size(lte) + 8, 0);
                return ret;
 
        ret = ntfs_set_ntfs_reparse_data(ni, (char*)reparse_data_buf,
                                         wim_resource_size(lte) + 8, 0);

-       if (ret != 0) {
+       if (ret) {

                ERROR_WITH_ERRNO("Failed to set NTFS reparse data on `%s'",
                                 dentry->full_path);
                return WIMLIB_ERR_NTFS_3G;
                ERROR_WITH_ERRNO("Failed to set NTFS reparse data on `%s'",
                                 dentry->full_path);
                return WIMLIB_ERR_NTFS_3G;

diff --git a/src/ntfs-capture.c b/src/ntfs-capture.c
index 94e74cee4f360cb68a2f3c78e47642d7edf7b533..5c96c1b2e437fef8c5f3f42566a181ac84bf8ee0 100644 (file)
--- a/src/ntfs-capture.c
+++ b/src/ntfs-capture.c
@@ -58,62 +58,79 @@ attr_record_name(ATTR_RECORD *ar)
        return (ntfschar*)((u8*)ar + le16_to_cpu(ar->name_offset));
 }
 
        return (ntfschar*)((u8*)ar + le16_to_cpu(ar->name_offset));
 }
 

-/* Calculates the SHA1 message digest of a NTFS attribute.
- *
- * @ni:  The NTFS inode containing the attribute.
- * @ar:         The ATTR_RECORD describing the attribute.
- * @md:  If successful, the returned SHA1 message digest.
- * @reparse_tag_ret:   Optional pointer into which the first 4 bytes of the
- *                             attribute will be written (to get the reparse
- *                             point ID)
- *
- * Return 0 on success or nonzero on error.
- */
-static int
-ntfs_attr_sha1sum(ntfs_inode *ni, ATTR_RECORD *ar,
-                 u8 md[SHA1_HASH_SIZE],
-                 bool is_reparse_point,
-                 u32 *reparse_tag_ret)
+int
+read_ntfs_file_prefix(const struct wim_lookup_table_entry *lte,
+                     u64 size,
+                     consume_data_callback_t cb,
+                     void *ctx_or_buf,
+                     int _ignored_flags)

 {
 {

-       s64 pos = 0;
-       s64 bytes_remaining;
-       char buf[BUFFER_SIZE];
+       struct ntfs_location *loc = lte->ntfs_loc;
+       ntfs_volume *vol = *loc->ntfs_vol_p;
+       ntfs_inode *ni;

        ntfs_attr *na;
        ntfs_attr *na;

-       SHA_CTX ctx;
+       s64 pos;
+       s64 bytes_remaining;
+       void *out_buf;
+       int ret;

 
 

-       na = ntfs_attr_open(ni, ar->type, attr_record_name(ar),
-                           ar->name_length);
-       if (!na) {
-               ERROR_WITH_ERRNO("Failed to open NTFS attribute");
-               return WIMLIB_ERR_NTFS_3G;
+       ni = ntfs_pathname_to_inode(vol, NULL, loc->path);
+       if (!ni) {
+               ERROR_WITH_ERRNO("Can't find NTFS inode for \"%"TS"\"", loc->path);
+               ret = WIMLIB_ERR_NTFS_3G;
+               goto out;

        }
 
        }
 

-       bytes_remaining = na->data_size;
-
-       if (is_reparse_point) {
-               if (ntfs_attr_pread(na, 0, 8, buf) != 8)
-                       goto out_error;
-               *reparse_tag_ret = le32_to_cpu(*(u32*)buf);
-               DEBUG("ReparseTag = %#x", *reparse_tag_ret);
-               pos = 8;
-               bytes_remaining -= 8;
+       na = ntfs_attr_open(ni,
+                           loc->is_reparse_point ? AT_REPARSE_POINT : AT_DATA,
+                           loc->stream_name,
+                           loc->stream_name_nchars);
+       if (!na) {
+               ERROR_WITH_ERRNO("Failed to open attribute of \"%"TS"\" in "
+                                "NTFS volume", loc->path);
+               ret = WIMLIB_ERR_NTFS_3G;
+               goto out_close_ntfs_inode;

        }
 
        }
 

-       sha1_init(&ctx);
+       /*if (is_reparse_point) {*/
+               /*if (ntfs_attr_pread(na, 0, 8, buf) != 8)*/
+                       /*goto out_error;*/
+               /**reparse_tag_ret = le32_to_cpu(*(u32*)buf);*/
+               /*DEBUG("ReparseTag = %#x", *reparse_tag_ret);*/
+               /*pos = 8;*/
+               /*bytes_remaining -= 8;*/
+       /*}*/
+
+       if (cb)
+               out_buf = alloca(WIM_CHUNK_SIZE);
+       else
+               out_buf = ctx_or_buf;
+       pos = 0;
+       bytes_remaining = na->data_size;

        while (bytes_remaining) {
        while (bytes_remaining) {

-               s64 to_read = min(bytes_remaining, sizeof(buf));
-               if (ntfs_attr_pread(na, pos, to_read, buf) != to_read)
-                       goto out_error;
-               sha1_update(&ctx, buf, to_read);
+               s64 to_read = min(bytes_remaining, WIM_CHUNK_SIZE);
+               if (ntfs_attr_pread(na, pos, to_read, out_buf) != to_read) {
+                       ERROR_WITH_ERRNO("Error reading \"%"TS"\"", loc->path);
+                       ret = WIMLIB_ERR_NTFS_3G;
+                       goto out_close_ntfs_attr;
+               }

                pos += to_read;
                bytes_remaining -= to_read;
                pos += to_read;
                bytes_remaining -= to_read;

+               if (cb) {
+                       ret = cb(out_buf, to_read, ctx_or_buf);
+                       if (ret)
+                               goto out_close_ntfs_attr;
+               } else {
+                       out_buf += to_read;
+               }

        }
        }

-       sha1_final(md, &ctx);
+       ret = 0;
+out_close_ntfs_attr:

        ntfs_attr_close(na);
        ntfs_attr_close(na);

-       return 0;
-out_error:
-       ERROR_WITH_ERRNO("Error reading NTFS attribute");
-       return WIMLIB_ERR_NTFS_3G;
+out_close_ntfs_inode:
+       ntfs_inode_close(ni);
+out:
+       return ret;

 }
 
 /* Load the streams from a file or reparse point in the NTFS volume into the WIM
 }
 
 /* Load the streams from a file or reparse point in the NTFS volume into the WIM


@@ -128,7 +145,6 @@ capture_ntfs_streams(struct wim_inode *inode,
                     ATTR_TYPES type)
 {
        ntfs_attr_search_ctx *actx;
                     ATTR_TYPES type)
 {
        ntfs_attr_search_ctx *actx;

-       u8 attr_hash[SHA1_HASH_SIZE];

        struct ntfs_location *ntfs_loc = NULL;
        int ret = 0;
        struct wim_lookup_table_entry *lte;
        struct ntfs_location *ntfs_loc = NULL;
        int ret = 0;
        struct wim_lookup_table_entry *lte;


@@ -147,7 +163,6 @@ capture_ntfs_streams(struct wim_inode *inode,
        while (!ntfs_attr_lookup(type, NULL, 0,
                                 CASE_SENSITIVE, 0, NULL, 0, actx))
        {
        while (!ntfs_attr_lookup(type, NULL, 0,
                                 CASE_SENSITIVE, 0, NULL, 0, actx))
        {

-               u32 reparse_tag;

                u64 data_size = ntfs_get_attribute_value_length(actx->attr);
                u64 name_length = actx->attr->name_length;
                if (data_size == 0) {
                u64 data_size = ntfs_get_attribute_value_length(actx->attr);
                u64 name_length = actx->attr->name_length;
                if (data_size == 0) {


@@ -160,64 +175,44 @@ capture_ntfs_streams(struct wim_inode *inode,
                        /* Empty stream.  No lookup table entry is needed. */
                        lte = NULL;
                } else {
                        /* Empty stream.  No lookup table entry is needed. */
                        lte = NULL;
                } else {

-                       if (type == AT_REPARSE_POINT && data_size < 8) {
-                               ERROR("`%s': reparse point buffer too small",
-                                     path);
-                               ret = WIMLIB_ERR_NTFS_3G;
+                       ntfs_loc = CALLOC(1, sizeof(*ntfs_loc));
+                       if (!ntfs_loc)

                                goto out_put_actx;
                                goto out_put_actx;

+                       ntfs_loc->ntfs_vol_p = ntfs_vol_p;
+                       ntfs_loc->path = MALLOC(path_len + 1);
+                       if (!ntfs_loc->path)
+                               goto out_free_ntfs_loc;
+                       memcpy(ntfs_loc->path, path, path_len + 1);
+                       if (name_length) {
+                               ntfs_loc->stream_name = MALLOC(name_length * 2);
+                               if (!ntfs_loc->stream_name)
+                                       goto out_free_ntfs_loc;
+                               memcpy(ntfs_loc->stream_name,
+                                      attr_record_name(actx->attr),
+                                      actx->attr->name_length * 2);
+                               ntfs_loc->stream_name_nchars = name_length;

                        }
                        }

-                       /* Checksum the stream. */
-                       ret = ntfs_attr_sha1sum(ni, actx->attr, attr_hash,
-                                               type == AT_REPARSE_POINT, &reparse_tag);
-                       if (ret != 0)
-                               goto out_put_actx;
-
-                       if (type == AT_REPARSE_POINT)
-                               inode->i_reparse_tag = reparse_tag;

 
 

-                       /* Make a lookup table entry for the stream, or use an existing
-                        * one if there's already an identical stream. */
-                       lte = __lookup_resource(lookup_table, attr_hash);
-                       ret = WIMLIB_ERR_NOMEM;
-                       if (lte) {
-                               lte->refcnt++;
+                       lte = new_lookup_table_entry();
+                       if (!lte)
+                               goto out_free_ntfs_loc;
+                       lte->ntfs_loc = ntfs_loc;
+                       lte->resource_location = RESOURCE_IN_NTFS_VOLUME;
+               #if 0
+                       if (type == AT_REPARSE_POINT) {
+                               ntfs_loc->is_reparse_point = true;
+                               lte->resource_entry.original_size = data_size - 8;
+                               lte->resource_entry.size = data_size - 8;

                        } else {
                        } else {

-                               ntfs_loc = CALLOC(1, sizeof(*ntfs_loc));
-                               if (!ntfs_loc)
-                                       goto out_put_actx;
-                               ntfs_loc->ntfs_vol_p = ntfs_vol_p;
-                               ntfs_loc->path = MALLOC(path_len + 1);
-                               if (!ntfs_loc->path)
-                                       goto out_free_ntfs_loc;
-                               memcpy(ntfs_loc->path, path, path_len + 1);
-                               if (name_length) {
-                                       ntfs_loc->stream_name = MALLOC(name_length * 2);
-                                       if (!ntfs_loc->stream_name)
-                                               goto out_free_ntfs_loc;
-                                       memcpy(ntfs_loc->stream_name,
-                                              attr_record_name(actx->attr),
-                                              actx->attr->name_length * 2);
-                                       ntfs_loc->stream_name_nchars = name_length;
-                               }
-
-                               lte = new_lookup_table_entry();
-                               if (!lte)
-                                       goto out_free_ntfs_loc;
-                               lte->ntfs_loc = ntfs_loc;
-                               lte->resource_location = RESOURCE_IN_NTFS_VOLUME;
-                               if (type == AT_REPARSE_POINT) {
-                                       ntfs_loc->is_reparse_point = true;
-                                       lte->resource_entry.original_size = data_size - 8;
-                                       lte->resource_entry.size = data_size - 8;
-                               } else {
-                                       ntfs_loc->is_reparse_point = false;
-                                       lte->resource_entry.original_size = data_size;
-                                       lte->resource_entry.size = data_size;
-                               }
-                               ntfs_loc = NULL;
-                               copy_hash(lte->hash, attr_hash);
-                               lookup_table_insert(lookup_table, lte);
+                               ntfs_loc->is_reparse_point = false;
+                               lte->resource_entry.original_size = data_size;
+                               lte->resource_entry.size = data_size;

                        }
                        }

+               #else
+                       ntfs_loc->is_reparse_point = (type == AT_REPARSE_POINT);
+                       lte->resource_entry.original_size = data_size;
+               #endif
+                       ntfs_loc = NULL;

                }
                if (name_length == 0) {
                        /* Unnamed data stream.  Put the reference to it in the
                }
                if (name_length == 0) {
                        /* Unnamed data stream.  Put the reference to it in the


@@ -242,6 +237,8 @@ capture_ntfs_streams(struct wim_inode *inode,
                        wimlib_assert(new_ads_entry->stream_name_nbytes == name_length * 2);
                        new_ads_entry->lte = lte;
                }
                        wimlib_assert(new_ads_entry->stream_name_nbytes == name_length * 2);
                        new_ads_entry->lte = lte;
                }

+               if (lte)
+                       lookup_table_insert_unhashed(lookup_table, lte);

        }
        ret = 0;
        goto out_put_actx;
        }
        ret = 0;
        goto out_put_actx;

diff --git a/src/resource.c b/src/resource.c
index a8e98ca41d2c88786795b724b17bd4da523415c7..90e45ca1092db0eb3f0a65617734613beceab1c0 100644 (file)
--- a/src/resource.c
+++ b/src/resource.c
@@ -36,33 +36,56 @@
 #include <stdarg.h>
 #include <stdlib.h>
 #include <unistd.h>
 #include <stdarg.h>
 #include <stdlib.h>
 #include <unistd.h>

+#include <fcntl.h>

 
 

-#ifdef WITH_NTFS_3G
-#  include <time.h>
-#  include <ntfs-3g/attrib.h>
-#  include <ntfs-3g/inode.h>
-#  include <ntfs-3g/dir.h>
-#endif
+/* Write @n bytes from @buf to the file descriptor @fd, retrying on internupt
+ * and on short writes.
+ *
+ * Returns short count and set errno on failure. */
+static ssize_t
+full_write(int fd, const void *buf, size_t n)
+{
+       const void *p = buf;
+       ssize_t ret;
+       ssize_t total = 0;

 
 

-#if defined(__WIN32__) && !defined(INVALID_HANDLE_VALUE)
-#  define INVALID_HANDLE_VALUE ((HANDLE)(-1))
-#endif
+       while (total != n) {
+               ret = write(fd, p, n);
+               if (ret < 0) {
+                       if (errno == EINTR)
+                               continue;
+                       else
+                               break;
+               }
+               total += ret;
+               p += ret;
+       }
+       return total;
+}

 
 

-/*
- * Reads all or part of a compressed resource into an in-memory buffer.
+/* Read @n bytes from the file descriptor @fd to the buffer @buf, retrying on
+ * internupt and on short reads.

  *
  *

- * @fp:                The FILE* for the WIM file.
- * @resource_compressed_size:           The compressed size of the resource.
- * @resource_uncompressed_size:  The uncompressed size of the resource.
- * @resource_offset:            The offset of the start of the resource from
- *                                     the start of the stream @fp.
- * @resource_ctype:    The compression type of the resource.
- * @len:               The number of bytes of uncompressed data to read from
- *                             the resource.
- * @offset:            The offset of the bytes to read within the uncompressed
- *                             resource.
- * @contents_len:      An array into which the uncompressed data is written.
- *                             It must be at least @len bytes long.
+ * Returns short count and set errno on failure. */
+static size_t
+full_read(int fd, void *buf, size_t n)
+{
+       size_t bytes_remaining = n;
+       while (bytes_remaining) {
+               ssize_t bytes_read = read(fd, buf, bytes_remaining);
+               if (bytes_read < 0) {
+                       if (errno == EINTR)
+                               continue;
+                       break;
+               }
+               bytes_remaining -= bytes_read;
+               buf += bytes_read;
+       }
+       return n - bytes_remaining;
+}
+
+/*
+ * Reads all or part of a compressed WIM resource.

  *
  * Returns zero on success, nonzero on failure.
  */
  *
  * Returns zero on success, nonzero on failure.
  */


@@ -70,16 +93,12 @@ static int
 read_compressed_resource(FILE *fp, u64 resource_compressed_size,
                         u64 resource_uncompressed_size,
                         u64 resource_offset, int resource_ctype,
 read_compressed_resource(FILE *fp, u64 resource_compressed_size,
                         u64 resource_uncompressed_size,
                         u64 resource_offset, int resource_ctype,

-                        u64 len, u64 offset, void *contents_ret)
+                        u64 len, u64 offset,
+                        consume_data_callback_t cb,
+                        void *ctx_or_buf)

 {
 {

+       int ret;

 
 

-       DEBUG2("comp size = %"PRIu64", uncomp size = %"PRIu64", "
-              "res offset = %"PRIu64"",
-              resource_compressed_size,
-              resource_uncompressed_size,
-              resource_offset);
-       DEBUG2("resource_ctype = %"TS", len = %"PRIu64", offset = %"PRIu64"",
-              wimlib_get_compression_type_string(resource_ctype), len, offset);

        /* Trivial case */
        if (len == 0)
                return 0;
        /* Trivial case */
        if (len == 0)
                return 0;


@@ -168,36 +187,34 @@ read_compressed_resource(FILE *fp, u64 resource_compressed_size,
        /* Skip over unneeded chunk table entries. */
        u64 file_offset_of_needed_chunk_entries = resource_offset +
                                start_table_idx * chunk_entry_size;
        /* Skip over unneeded chunk table entries. */
        u64 file_offset_of_needed_chunk_entries = resource_offset +
                                start_table_idx * chunk_entry_size;

-       if (fseeko(fp, file_offset_of_needed_chunk_entries, SEEK_SET) != 0) {
-               ERROR_WITH_ERRNO("Failed to seek to byte %"PRIu64" to read "
-                                "chunk table of compressed resource",
-                                file_offset_of_needed_chunk_entries);
-               return WIMLIB_ERR_READ;
-       }
+       if (fseeko(fp, file_offset_of_needed_chunk_entries, SEEK_SET))
+               goto read_error;

 
        /* Number of bytes we need to read from the chunk table. */
        size_t size = num_needed_chunk_entries * chunk_entry_size;
 
 
        /* Number of bytes we need to read from the chunk table. */
        size_t size = num_needed_chunk_entries * chunk_entry_size;
 

-       u8 chunk_tab_buf[size];
+       {
+               u8 chunk_tab_buf[size];

 
 

-       if (fread(chunk_tab_buf, 1, size, fp) != size)
-               goto err;
+               if (fread(chunk_tab_buf, 1, size, fp) != size)
+                       goto read_error;

 
 

-       /* Now fill in chunk_offsets from the entries we have read in
-        * chunk_tab_buf. */
+               /* Now fill in chunk_offsets from the entries we have read in
+                * chunk_tab_buf. */

 
 

-       u64 *chunk_tab_p = chunk_offsets;
-       if (start_chunk == 0)
-               chunk_tab_p++;
+               u64 *chunk_tab_p = chunk_offsets;
+               if (start_chunk == 0)
+                       chunk_tab_p++;

 
 

-       if (chunk_entry_size == 4) {
-               u32 *entries = (u32*)chunk_tab_buf;
-               while (num_needed_chunk_entries--)
-                       *chunk_tab_p++ = le32_to_cpu(*entries++);
-       } else {
-               u64 *entries = (u64*)chunk_tab_buf;
-               while (num_needed_chunk_entries--)
-                       *chunk_tab_p++ = le64_to_cpu(*entries++);
+               if (chunk_entry_size == 4) {
+                       u32 *entries = (u32*)chunk_tab_buf;
+                       while (num_needed_chunk_entries--)
+                               *chunk_tab_p++ = le32_to_cpu(*entries++);
+               } else {
+                       u64 *entries = (u64*)chunk_tab_buf;
+                       while (num_needed_chunk_entries--)
+                               *chunk_tab_p++ = le64_to_cpu(*entries++);
+               }

        }
 
        /* Done with the chunk table now.  We must now seek to the first chunk
        }
 
        /* Done with the chunk table now.  We must now seek to the first chunk


@@ -205,16 +222,16 @@ read_compressed_resource(FILE *fp, u64 resource_compressed_size,
 
        u64 file_offset_of_first_needed_chunk = resource_offset +
                                chunk_table_size + chunk_offsets[0];
 
        u64 file_offset_of_first_needed_chunk = resource_offset +
                                chunk_table_size + chunk_offsets[0];

-       if (fseeko(fp, file_offset_of_first_needed_chunk, SEEK_SET) != 0) {
-               ERROR_WITH_ERRNO("Failed to seek to byte %"PRIu64" to read "
-                                "first chunk of compressed resource",
-                                file_offset_of_first_needed_chunk);
-               return WIMLIB_ERR_READ;
-       }
+       if (fseeko(fp, file_offset_of_first_needed_chunk, SEEK_SET))
+               goto read_error;

 
        /* Pointer to current position in the output buffer for uncompressed
         * data. */
 
        /* Pointer to current position in the output buffer for uncompressed
         * data. */

-       u8 *out_p = contents_ret;
+       u8 *out_p;
+       if (cb)
+               out_p = alloca(32768);
+       else
+               out_p = ctx_or_buf;

 
        /* Buffer for compressed data.  While most compressed chunks will have a
         * size much less than WIM_CHUNK_SIZE, WIM_CHUNK_SIZE - 1 is the maximum
 
        /* Buffer for compressed data.  While most compressed chunks will have a
         * size much less than WIM_CHUNK_SIZE, WIM_CHUNK_SIZE - 1 is the maximum


@@ -222,15 +239,11 @@ read_compressed_resource(FILE *fp, u64 resource_compressed_size,
         * happen to compress to more than the uncompressed size (i.e. a
         * sequence of random bytes) are always stored uncompressed. But this seems
         * to be the case in M$'s WIM files, even though it is undocumented. */
         * happen to compress to more than the uncompressed size (i.e. a
         * sequence of random bytes) are always stored uncompressed. But this seems
         * to be the case in M$'s WIM files, even though it is undocumented. */

-       u8 compressed_buf[WIM_CHUNK_SIZE - 1];
-
+       void *compressed_buf = alloca(WIM_CHUNK_SIZE - 1);

 
        /* Decompress all the chunks. */
        for (u64 i = start_chunk; i <= end_chunk; i++) {
 
 
        /* Decompress all the chunks. */
        for (u64 i = start_chunk; i <= end_chunk; i++) {
 

-               DEBUG2("Chunk %"PRIu64" (start %"PRIu64", end %"PRIu64").",
-                      i, start_chunk, end_chunk);
-

                /* Calculate the sizes of the compressed chunk and of the
                 * uncompressed chunk. */
                unsigned compressed_chunk_size;
                /* Calculate the sizes of the compressed chunk and of the
                 * uncompressed chunk. */
                unsigned compressed_chunk_size;


@@ -265,11 +278,6 @@ read_compressed_resource(FILE *fp, u64 resource_compressed_size,
                                uncompressed_chunk_size = WIM_CHUNK_SIZE;
                }
 
                                uncompressed_chunk_size = WIM_CHUNK_SIZE;
                }
 

-               DEBUG2("compressed_chunk_size = %u, "
-                      "uncompressed_chunk_size = %u",
-                      compressed_chunk_size, uncompressed_chunk_size);
-
-

                /* Figure out how much of this chunk we actually need to read */
                u64 start_offset;
                if (i == start_chunk)
                /* Figure out how much of this chunk we actually need to read */
                u64 start_offset;
                if (i == start_chunk)


@@ -282,75 +290,80 @@ read_compressed_resource(FILE *fp, u64 resource_compressed_size,
                else
                        end_offset = WIM_CHUNK_SIZE - 1;
 
                else
                        end_offset = WIM_CHUNK_SIZE - 1;
 

-               u64 partial_chunk_size = end_offset + 1 - start_offset;
-               bool is_partial_chunk = (partial_chunk_size !=
-                                               uncompressed_chunk_size);
-
-               DEBUG2("start_offset = %"PRIu64", end_offset = %"PRIu64"",
-                      start_offset, end_offset);
-               DEBUG2("partial_chunk_size = %"PRIu64"", partial_chunk_size);
+               unsigned partial_chunk_size = end_offset + 1 - start_offset;
+               bool is_partial_chunk = (partial_chunk_size != uncompressed_chunk_size);

 
                /* This is undocumented, but chunks can be uncompressed.  This
                 * appears to always be the case when the compressed chunk size
                 * is equal to the uncompressed chunk size. */
                if (compressed_chunk_size == uncompressed_chunk_size) {
 
                /* This is undocumented, but chunks can be uncompressed.  This
                 * appears to always be the case when the compressed chunk size
                 * is equal to the uncompressed chunk size. */
                if (compressed_chunk_size == uncompressed_chunk_size) {

-                       /* Probably an uncompressed chunk */
+                       /* Uncompressed chunk */

 
 

-                       if (start_offset != 0) {
-                               if (fseeko(fp, start_offset, SEEK_CUR) != 0) {
-                                       ERROR_WITH_ERRNO("Uncompressed partial "
-                                                        "chunk fseek() error");
-                                       return WIMLIB_ERR_READ;
-                               }
-                       }
-                       if (fread(out_p, 1, partial_chunk_size, fp) !=
-                                       partial_chunk_size)
-                               goto err;
+                       if (start_offset != 0)
+                               if (fseeko(fp, start_offset, SEEK_CUR))
+                                       goto read_error;
+                       if (fread(out_p, 1, partial_chunk_size, fp) != partial_chunk_size)
+                               goto read_error;

                } else {
                        /* Compressed chunk */
                } else {
                        /* Compressed chunk */

-                       int ret;

 
                        /* Read the compressed data into compressed_buf. */
                        if (fread(compressed_buf, 1, compressed_chunk_size,
                                                fp) != compressed_chunk_size)
 
                        /* Read the compressed data into compressed_buf. */
                        if (fread(compressed_buf, 1, compressed_chunk_size,
                                                fp) != compressed_chunk_size)

-                               goto err;
+                               goto read_error;

 
 

-                       /* For partial chunks we must buffer the uncompressed
-                        * data because we don't need all of it. */
-                       if (is_partial_chunk) {
+                       /* For partial chunks and when writing directly to a
+                        * buffer, we must buffer the uncompressed data because
+                        * we don't need all of it. */
+                       if (is_partial_chunk && !cb) {

                                u8 uncompressed_buf[uncompressed_chunk_size];
 
                                ret = decompress(compressed_buf,
                                u8 uncompressed_buf[uncompressed_chunk_size];
 
                                ret = decompress(compressed_buf,

-                                               compressed_chunk_size,
-                                               uncompressed_buf,
-                                               uncompressed_chunk_size);
-                               if (ret != 0)
-                                       return WIMLIB_ERR_DECOMPRESSION;
+                                                compressed_chunk_size,
+                                                uncompressed_buf,
+                                                uncompressed_chunk_size);
+                               if (ret) {
+                                       ret = WIMLIB_ERR_DECOMPRESSION;
+                                       goto out;
+                               }

                                memcpy(out_p, uncompressed_buf + start_offset,
                                memcpy(out_p, uncompressed_buf + start_offset,

-                                               partial_chunk_size);
+                                      partial_chunk_size);

                        } else {
                                ret = decompress(compressed_buf,
                        } else {
                                ret = decompress(compressed_buf,

-                                               compressed_chunk_size,
-                                               out_p,
-                                               uncompressed_chunk_size);
-                               if (ret != 0)
-                                       return WIMLIB_ERR_DECOMPRESSION;
+                                                compressed_chunk_size,
+                                                out_p,
+                                                uncompressed_chunk_size);
+                               if (ret) {
+                                       ret = WIMLIB_ERR_DECOMPRESSION;
+                                       goto out;
+                               }

                        }
                }
                        }
                }

-
-               /* Advance the pointer into the uncompressed output data by the
-                * number of uncompressed bytes that were written.  */
-               out_p += partial_chunk_size;
+               if (cb) {
+                       /* Feed the data to the callback function */
+                       ret = cb(out_p, partial_chunk_size, ctx_or_buf);
+                       if (ret)
+                               goto out;
+               } else {
+                       /* No callback function provided; we are writing
+                        * directly to a buffer.  Advance the pointer into this
+                        * buffer by the number of uncompressed bytes that were
+                        * written.  */
+                       out_p += partial_chunk_size;
+               }

        }
 
        }
 

-       return 0;
+       ret = 0;
+out:
+       return ret;

 
 

-err:
+read_error:

        if (feof(fp))
                ERROR("Unexpected EOF in compressed file resource");
        else
                ERROR_WITH_ERRNO("Error reading compressed file resource");
        if (feof(fp))
                ERROR("Unexpected EOF in compressed file resource");
        else
                ERROR_WITH_ERRNO("Error reading compressed file resource");

-       return WIMLIB_ERR_READ;
+       ret = WIMLIB_ERR_READ;
+       goto out;

 }
 
 /*
 }
 
 /*


@@ -381,8 +394,8 @@ read_uncompressed_resource(FILE *fp, u64 offset, u64 len, void *contents_ret)
 /* Reads the contents of a struct resource_entry, as represented in the on-disk
  * format, from the memory pointed to by @p, and fills in the fields of @entry.
  * A pointer to the byte after the memory read at @p is returned. */
 /* Reads the contents of a struct resource_entry, as represented in the on-disk
  * format, from the memory pointed to by @p, and fills in the fields of @entry.
  * A pointer to the byte after the memory read at @p is returned. */

-const u8 *
-get_resource_entry(const u8 *p, struct resource_entry *entry)
+const void *
+get_resource_entry(const void *p, struct resource_entry *entry)

 {
        u64 size;
        u8 flags;
 {
        u64 size;
        u8 flags;


@@ -412,8 +425,8 @@ get_resource_entry(const u8 *p, struct resource_entry *entry)
 /* Copies the struct resource_entry @entry to the memory pointed to by @p in the
  * on-disk format.  A pointer to the byte after the memory written at @p is
  * returned. */
 /* Copies the struct resource_entry @entry to the memory pointed to by @p in the
  * on-disk format.  A pointer to the byte after the memory written at @p is
  * returned. */

-u8 *
-put_resource_entry(u8 *p, const struct resource_entry *entry)
+void *
+put_resource_entry(void *p, const struct resource_entry *entry)

 {
        p = put_u56(p, entry->size);
        p = put_u8(p, entry->flags);
 {
        p = put_u56(p, entry->size);
        p = put_u8(p, entry->flags);


@@ -422,10 +435,10 @@ put_resource_entry(u8 *p, const struct resource_entry *entry)
        return p;
 }
 
        return p;
 }
 

-#ifdef WITH_FUSE

 static FILE *
 wim_get_fp(WIMStruct *w)
 {
 static FILE *
 wim_get_fp(WIMStruct *w)
 {

+#ifdef WITH_FUSE

        pthread_mutex_lock(&w->fp_tab_mutex);
        FILE *fp;
 
        pthread_mutex_lock(&w->fp_tab_mutex);
        FILE *fp;
 


@@ -435,15 +448,18 @@ wim_get_fp(WIMStruct *w)
                if (w->fp_tab[i]) {
                        fp = w->fp_tab[i];
                        w->fp_tab[i] = NULL;
                if (w->fp_tab[i]) {
                        fp = w->fp_tab[i];
                        w->fp_tab[i] = NULL;

-                       goto out;
+                       goto out_unlock;

                }
        }
        DEBUG("Opening extra file descriptor to `%"TS"'", w->filename);
        fp = tfopen(w->filename, T("rb"));
        if (!fp)
                ERROR_WITH_ERRNO("Failed to open `%"TS"'", w->filename);
                }
        }
        DEBUG("Opening extra file descriptor to `%"TS"'", w->filename);
        fp = tfopen(w->filename, T("rb"));
        if (!fp)
                ERROR_WITH_ERRNO("Failed to open `%"TS"'", w->filename);

-out:
+out_unlock:

        pthread_mutex_unlock(&w->fp_tab_mutex);
        pthread_mutex_unlock(&w->fp_tab_mutex);

+#else /* WITH_FUSE */
+       fp = w->fp;
+#endif /* !WITH_FUSE */

        return fp;
 }
 
        return fp;
 }
 


@@ -451,6 +467,7 @@ static int
 wim_release_fp(WIMStruct *w, FILE *fp)
 {
        int ret = 0;
 wim_release_fp(WIMStruct *w, FILE *fp)
 {
        int ret = 0;

+#ifdef WITH_FUSE

        FILE **fp_tab;
 
        pthread_mutex_lock(&w->fp_tab_mutex);
        FILE **fp_tab;
 
        pthread_mutex_lock(&w->fp_tab_mutex);


@@ -458,251 +475,251 @@ wim_release_fp(WIMStruct *w, FILE *fp)
        for (size_t i = 0; i < w->num_allocated_fps; i++) {
                if (w->fp_tab[i] == NULL) {
                        w->fp_tab[i] = fp;
        for (size_t i = 0; i < w->num_allocated_fps; i++) {
                if (w->fp_tab[i] == NULL) {
                        w->fp_tab[i] = fp;

-                       goto out;
+                       goto out_unlock;

                }
        }
 
        fp_tab = REALLOC(w->fp_tab, sizeof(FILE*) * (w->num_allocated_fps + 4));
        if (!fp_tab) {
                ret = WIMLIB_ERR_NOMEM;
                }
        }
 
        fp_tab = REALLOC(w->fp_tab, sizeof(FILE*) * (w->num_allocated_fps + 4));
        if (!fp_tab) {
                ret = WIMLIB_ERR_NOMEM;

-               goto out;
+               fclose(fp);
+               goto out_unlock;

        }
        w->fp_tab = fp_tab;
        memset(&w->fp_tab[w->num_allocated_fps], 0, 4 * sizeof(FILE*));
        w->fp_tab[w->num_allocated_fps] = fp;
        w->num_allocated_fps += 4;
        }
        w->fp_tab = fp_tab;
        memset(&w->fp_tab[w->num_allocated_fps], 0, 4 * sizeof(FILE*));
        w->fp_tab[w->num_allocated_fps] = fp;
        w->num_allocated_fps += 4;

-out:
+out_unlock:

        pthread_mutex_unlock(&w->fp_tab_mutex);
        pthread_mutex_unlock(&w->fp_tab_mutex);

+#endif /* WITH_FUSE */

        return ret;
 }
        return ret;
 }

-#endif /* !WITH_FUSE */

 
 

-/*
- * Reads some data from the resource corresponding to a WIM lookup table entry.
- *
- * @lte:       The WIM lookup table entry for the resource.
- * @buf:       Buffer into which to write the data.
- * @size:      Number of bytes to read.
- * @offset:    Offset at which to start reading the resource.
- *
- * Returns zero on success, nonzero on failure.
- */
-int
-read_wim_resource(const struct wim_lookup_table_entry *lte, void *buf,
-                 size_t size, u64 offset, int flags)
+static int
+read_partial_wim_resource(const struct wim_lookup_table_entry *lte,
+                         u64 size,
+                         consume_data_callback_t cb,
+                         void *ctx_or_buf,
+                         int flags,
+                         u64 offset)

 {
 {

-       int ctype;
-       int ret = 0;
-       FILE *fp;
+       FILE *wim_fp;
+       WIMStruct *wim;
+       int ret;

 
 

-       /* We shouldn't be allowing read over-runs in any part of the library.
-        * */
-       if (flags & WIMLIB_RESOURCE_FLAG_RAW)
-               wimlib_assert(offset + size <= lte->resource_entry.size);
-       else
-               wimlib_assert(offset + size <= lte->resource_entry.original_size);
-
-       switch (lte->resource_location) {
-       case RESOURCE_IN_WIM:
-               /* The resource is in a WIM file, and its WIMStruct is given by
-                * the lte->wim member.  The resource may be either compressed
-                * or uncompressed. */
-               wimlib_assert(lte->wim != NULL);
-
-               #ifdef WITH_FUSE
-               if (flags & WIMLIB_RESOURCE_FLAG_MULTITHREADED) {
-                       fp = wim_get_fp(lte->wim);
-                       if (!fp)
-                               return WIMLIB_ERR_OPEN;
-               } else
-               #endif
-               {
-                       wimlib_assert(!(flags & WIMLIB_RESOURCE_FLAG_MULTITHREADED));
-                       wimlib_assert(lte->wim->fp != NULL);
-                       fp = lte->wim->fp;
-               }
+       wimlib_assert(lte->resource_location == RESOURCE_IN_WIM);
+       wimlib_assert(offset + size <= lte->resource_entry.original_size);

 
 

-               ctype = wim_resource_compression_type(lte);
+       wim = lte->wim;

 
 

-               wimlib_assert(ctype != WIMLIB_COMPRESSION_TYPE_NONE ||
-                             (lte->resource_entry.original_size ==
-                              lte->resource_entry.size));
+       if (flags & WIMLIB_RESOURCE_FLAG_MULTITHREADED) {
+               wim_fp = wim_get_fp(wim);
+               if (!wim_fp) {
+                       ret = -1;
+                       goto out;
+               }
+       } else {
+               wim_fp = lte->wim->fp;
+       }

 
 

-               if ((flags & WIMLIB_RESOURCE_FLAG_RAW)
-                   || ctype == WIMLIB_COMPRESSION_TYPE_NONE)
-                       ret = read_uncompressed_resource(fp,
-                                                        lte->resource_entry.offset + offset,
-                                                        size, buf);
-               else
-                       ret = read_compressed_resource(fp,
-                                                      lte->resource_entry.size,
-                                                      lte->resource_entry.original_size,
-                                                      lte->resource_entry.offset,
-                                                      ctype, size, offset, buf);
-       #ifdef WITH_FUSE
-               if (flags & WIMLIB_RESOURCE_FLAG_MULTITHREADED) {
-                       int ret2 = wim_release_fp(lte->wim, fp);
-                       if (ret == 0)
-                               ret = ret2;
+       wimlib_assert(wim_fp != NULL);
+
+       if (lte->resource_entry.flags & WIM_RESHDR_FLAG_COMPRESSED &&
+           !(flags & WIMLIB_RESOURCE_FLAG_RAW))
+       {
+               ret = read_compressed_resource(wim_fp,
+                                              lte->resource_entry.size,
+                                              lte->resource_entry.original_size,
+                                              lte->resource_entry.offset,
+                                              wimlib_get_compression_type(wim),
+                                              size,
+                                              offset,
+                                              cb,
+                                              ctx_or_buf);
+       } else {
+               if (fseeko(wim_fp, offset, SEEK_SET)) {
+                       ERROR_WITH_ERRNO("Failed to seek to offset %"PRIu64
+                                        " in WIM", offset);
+                       ret = WIMLIB_ERR_READ;
+                       goto out_release_fp;

                }
                }

-       #endif
-               break;
-       case RESOURCE_IN_STAGING_FILE:
-       case RESOURCE_IN_FILE_ON_DISK:
-               /* The resource is in some file on the external filesystem and
-                * needs to be read uncompressed */
-               wimlib_assert(lte->file_on_disk != NULL);
-               BUILD_BUG_ON(&lte->file_on_disk != &lte->staging_file_name);
-               /* Use existing file pointer if available; otherwise open one
-                * temporarily */
-               if (lte->file_on_disk_fp) {
-                       fp = lte->file_on_disk_fp;
-               } else {
-                       fp = tfopen(lte->file_on_disk, T("rb"));
-                       if (!fp) {
-                               ERROR_WITH_ERRNO("Failed to open the file "
-                                                "`%"TS"'", lte->file_on_disk);
-                               ret = WIMLIB_ERR_OPEN;
-                               break;
+               if (cb) {
+                       char buf[min(32768, size)];
+                       while (size) {
+                               size_t bytes_to_read = min(32768, size);
+                               size_t bytes_read = fread(buf, 1, bytes_to_read, wim_fp);
+                               
+                               if (bytes_read != bytes_to_read)
+                                       goto read_error;
+                               ret = cb(buf, bytes_read, ctx_or_buf);
+                               if (ret)
+                                       goto out_release_fp;

                        }
                        }

+               } else {
+                       if (fread(ctx_or_buf, 1, size, wim_fp) != size)
+                               goto read_error;

                }
                }

-               ret = read_uncompressed_resource(fp, offset, size, buf);
-               if (fp != lte->file_on_disk_fp)
-                       fclose(fp);
-               break;
-#ifdef __WIN32__
-       case RESOURCE_WIN32:
-               wimlib_assert(lte->win32_file_on_disk_fp != INVALID_HANDLE_VALUE);
-               ret = win32_read_file(lte->file_on_disk,
-                                     lte->win32_file_on_disk_fp, offset,
-                                     size, buf);
-               break;
-#endif
-       case RESOURCE_IN_ATTACHED_BUFFER:
-               /* The resource is directly attached uncompressed in an
-                * in-memory buffer. */
-               wimlib_assert(lte->attached_buffer != NULL);
-               memcpy(buf, lte->attached_buffer + offset, size);
-               break;
-#ifdef WITH_NTFS_3G
-       case RESOURCE_IN_NTFS_VOLUME:
-               wimlib_assert(lte->ntfs_loc != NULL);
-               wimlib_assert(lte->attr != NULL);
-               if (lte->ntfs_loc->is_reparse_point)
-                       offset += 8;
-               if (ntfs_attr_pread(lte->attr, offset, size, buf) != size) {
-                       ERROR_WITH_ERRNO("Error reading NTFS attribute "
-                                        "at `%"TS"'",
-                                        lte->ntfs_loc->path);
-                       ret = WIMLIB_ERR_NTFS_3G;
-               }
-               break;
-#endif
-       default:
-               wimlib_assert(0);
+               ret = 0;
+       }
+       goto out_release_fp;
+read_error:
+       ERROR_WITH_ERRNO("Error reading data from WIM");
+       ret = WIMLIB_ERR_READ;
+out_release_fp:
+       if (flags & WIMLIB_RESOURCE_FLAG_MULTITHREADED)
+               ret |= wim_release_fp(wim, wim_fp);
+out:
+       if (ret) {
+               if (errno == 0)
+                       errno = EIO;

                ret = -1;
                ret = -1;

-               break;

        }
        return ret;
 }
 
        }
        return ret;
 }
 

-/*
- * Reads all the data from the resource corresponding to a WIM lookup table
- * entry.
- *
- * @lte:       The WIM lookup table entry for the resource.
- * @buf:       Buffer into which to write the data.  It must be at least
- *             wim_resource_size(lte) bytes long.
- *
- * Returns 0 on success; nonzero on failure.
- */
+

 int
 int

-read_full_wim_resource(const struct wim_lookup_table_entry *lte,
-                      void *buf, int flags)
+read_partial_wim_resource_into_buf(const struct wim_lookup_table_entry *lte,
+                                  size_t size, u64 offset, void *buf,
+                                  bool threadsafe)

 {
 {

-       return read_wim_resource(lte, buf, wim_resource_size(lte), 0, flags);
+       return read_partial_wim_resource(lte, size, NULL, buf,
+                                        threadsafe ? WIMLIB_RESOURCE_FLAG_MULTITHREADED : 0,
+                                        offset);

 }
 
 }
 

-/* Extracts the first @size bytes of a WIM resource to somewhere.  In the
- * process, the SHA1 message digest of the resource is checked if the full
- * resource is being extracted.
- *
- * @extract_chunk is a function that is called to extract each chunk of the
- * resource. */
-int
-extract_wim_resource(const struct wim_lookup_table_entry *lte,
-                    u64 size,
-                    extract_chunk_func_t extract_chunk,
-                    void *extract_chunk_arg)
+static int
+read_wim_resource_prefix(const struct wim_lookup_table_entry *lte,
+                        u64 size,
+                        consume_data_callback_t cb,
+                        void *ctx_or_buf,
+                        int flags)

 {
 {

-       u64 bytes_remaining = size;
-       u8 buf[min(WIM_CHUNK_SIZE, bytes_remaining)];
-       u64 offset = 0;
-       int ret = 0;
-       u8 hash[SHA1_HASH_SIZE];
-       bool check_hash = (size == wim_resource_size(lte));
-       SHA_CTX ctx;
+       return read_partial_wim_resource(lte, size, cb, ctx_or_buf, flags, 0);
+}

 
 

-       if (check_hash)
-               sha1_init(&ctx);

 
 

-       while (bytes_remaining) {
-               u64 to_read = min(bytes_remaining, sizeof(buf));
-               ret = read_wim_resource(lte, buf, to_read, offset, 0);
-               if (ret != 0)
-                       return ret;
-               if (check_hash)
-                       sha1_update(&ctx, buf, to_read);
-               ret = extract_chunk(buf, to_read, offset, extract_chunk_arg);
-               if (ret != 0) {
-                       ERROR_WITH_ERRNO("Error extracting WIM resource");
-                       return ret;
-               }
-               bytes_remaining -= to_read;
-               offset += to_read;
+static int
+read_file_on_disk_prefix(const struct wim_lookup_table_entry *lte,
+                        u64 size,
+                        consume_data_callback_t cb,
+                        void *ctx_or_buf,
+                        int _ignored_flags)
+{
+       const tchar *filename = lte->file_on_disk;
+       int ret;
+       int fd;
+       size_t bytes_read;
+
+       fd = open(filename, O_RDONLY);
+       if (fd < 0) {
+               ERROR_WITH_ERRNO("Can't open \"%"TS"\"", filename);
+               return WIMLIB_ERR_OPEN;

        }
        }

-       if (check_hash) {
-               sha1_final(hash, &ctx);
-               if (!hashes_equal(hash, lte->hash)) {
-               #ifdef ENABLE_ERROR_MESSAGES
-                       ERROR("Invalid checksum on the following WIM resource:");
-                       print_lookup_table_entry(lte, stderr);
-               #endif
-                       return WIMLIB_ERR_INVALID_RESOURCE_HASH;
+       if (cb) {
+               /* Send data to callback function */
+               char buf[min(32768, size)];
+               size_t bytes_to_read;
+               while (size) {
+                       bytes_to_read = min(32768, size);
+                       bytes_read = full_read(fd, buf, bytes_to_read);
+                       if (bytes_read != bytes_to_read)
+                               goto read_error;
+                       ret = cb(buf, bytes_read, ctx_or_buf);
+                       if (ret)
+                               goto out_close;

                }
                }

+       } else {
+               /* Send data directly to a buffer */
+               bytes_read = full_read(fd, ctx_or_buf, size);
+               if (bytes_read != size)
+                       goto read_error;

        }
        }

-       return 0;
+       ret = 0;
+       goto out_close;
+read_error:
+       ERROR_WITH_ERRNO("Error reading \"%"TS"\"", filename);
+       ret = WIMLIB_ERR_READ;
+out_close:
+       close(fd);
+       return ret;

 }
 
 }
 

-/* Write @n bytes from @buf to the file descriptor @fd, retrying on internupt
- * and on short writes.
- *
- * Returns short count and set errno on failure. */
-static ssize_t
-full_write(int fd, const void *buf, size_t n)
+static int
+read_buffer_prefix(const struct wim_lookup_table_entry *lte,
+                  u64 size, consume_data_callback_t cb,
+                  void *ctx_or_buf, int _ignored_flags)

 {
 {

-       const void *p = buf;
-       ssize_t ret;
-       ssize_t total = 0;
-
-       while (total != n) {
-               ret = write(fd, p, n);
-               if (ret < 0) {
-                       if (errno == EINTR)
-                               continue;
-                       else
-                               break;
-               }
-               total += ret;
-               p += ret;
+       const void *inbuf = lte->attached_buffer;
+       if (cb) {
+               return cb(inbuf, size, ctx_or_buf);
+       } else {
+               memcpy(ctx_or_buf, inbuf, size);
+               return 0;

        }
        }

-       return total;

 }
 
 }
 

+typedef int (*read_resource_prefix_handler_t)(const struct wim_lookup_table_entry *lte,
+                                             u64 size,
+                                             consume_data_callback_t cb,
+                                             void *ctx_or_buf,
+                                             int flags);
+

 int
 int

-extract_wim_chunk_to_fd(const void *buf, size_t len, u64 offset, void *arg)
+read_resource_prefix(const struct wim_lookup_table_entry *lte,
+                    u64 size, consume_data_callback_t cb, void *ctx_or_buf,
+                    int flags)

 {
 {

-       int fd = *(int*)arg;
+       static const read_resource_prefix_handler_t handlers[] = {
+               [RESOURCE_IN_WIM]             = read_wim_resource_prefix,
+               [RESOURCE_IN_FILE_ON_DISK]    = read_file_on_disk_prefix,
+               [RESOURCE_IN_ATTACHED_BUFFER] = read_buffer_prefix,
+       #ifdef WITH_FUSE
+               [RESOURCE_IN_STAGING_FILE]    = read_file_on_disk_prefix,
+       #endif
+       #ifdef WITH_NTFS_3G
+               [RESOURCE_IN_NTFS_VOLUME]     = read_ntfs_file_prefix,
+       #endif
+       #ifdef __WIN32__
+               [RESOURCE_WIN32]              = read_win32_file_prefix,
+               [RESOURCE_WIN32_ENCRYPTED]    = read_win32_encrypted_file_prefix,
+       #endif
+       };
+       wimlib_assert(lte->resource_location < ARRAY_LEN(handlers)
+                     && handlers[lte->resource_location] != NULL);
+       return handlers[lte->resource_location](lte, size, cb, ctx_or_buf, flags);
+}
+
+int
+read_full_resource_into_buf(const struct wim_lookup_table_entry *lte,
+                           void *buf, bool thread_safe)
+{
+       return read_resource_prefix(lte,
+                                   wim_resource_size(lte),
+                                   NULL, buf,
+                                   thread_safe ? WIMLIB_RESOURCE_FLAG_MULTITHREADED : 0);
+}
+
+/* Extracts the first @size bytes of a WIM resource to somewhere.  In the
+ * process, the SHA1 message digest of the resource is checked if the full
+ * resource is being extracted.
+ *
+ * @extract_chunk is a function that is called to extract each chunk of the
+ * resource. */
+int
+extract_wim_resource(const struct wim_lookup_table_entry *lte,
+                    u64 size,
+                    consume_data_callback_t extract_chunk,
+                    void *extract_chunk_arg)
+{
+       return read_resource_prefix(lte, size, extract_chunk,
+                                   extract_chunk_arg, 0);
+}
+
+static int
+extract_wim_chunk_to_fd(const void *buf, size_t len, void *_fd_p)
+{
+       int fd = *(int*)_fd_p;

        ssize_t ret = full_write(fd, buf, len);
        if (ret < len) {
                ERROR_WITH_ERRNO("Error writing to file descriptor");
        ssize_t ret = full_write(fd, buf, len);
        if (ret < len) {
                ERROR_WITH_ERRNO("Error writing to file descriptor");


@@ -712,6 +729,13 @@ extract_wim_chunk_to_fd(const void *buf, size_t len, u64 offset, void *arg)
        }
 }
 
        }
 }
 

+int
+extract_wim_resource_to_fd(const struct wim_lookup_table_entry *lte,
+                          int fd, u64 size)
+{
+       return extract_wim_resource(lte, size, extract_wim_chunk_to_fd, &fd);
+}
+

 /*
  * Copies the file resource specified by the lookup table entry @lte from the
  * input WIM to the output WIM that has its FILE * given by
 /*
  * Copies the file resource specified by the lookup table entry @lte from the
  * input WIM to the output WIM that has its FILE * given by

diff --git a/src/symlink.c b/src/symlink.c
index 70301b35269d266b93f0c15b507939de9654648f..bd370e30bedca28fe10a52185a8ac48b85f172f1 100644 (file)
--- a/src/symlink.c
+++ b/src/symlink.c
@@ -171,7 +171,7 @@ make_symlink_reparse_data_buf(const char *symlink_target,
  */
 ssize_t
 inode_readlink(const struct wim_inode *inode, char *buf, size_t buf_len,
  */
 ssize_t
 inode_readlink(const struct wim_inode *inode, char *buf, size_t buf_len,

-              const WIMStruct *w, int read_resource_flags)
+              const WIMStruct *w, bool threadsafe)

 {
        const struct wim_lookup_table_entry *lte;
        int ret;
 {
        const struct wim_lookup_table_entry *lte;
        int ret;


@@ -186,7 +186,7 @@ inode_readlink(const struct wim_inode *inode, char *buf, size_t buf_len,
                return -EIO;
 
        u8 res_buf[wim_resource_size(lte)];
                return -EIO;
 
        u8 res_buf[wim_resource_size(lte)];

-       ret = read_full_wim_resource(lte, res_buf, read_resource_flags);
+       ret = read_full_resource_into_buf(lte, res_buf, threadsafe);

        if (ret != 0)
                return -EIO;
        return get_symlink_name(res_buf, wim_resource_size(lte), buf,
        if (ret != 0)
                return -EIO;
        return get_symlink_name(res_buf, wim_resource_size(lte), buf,

diff --git a/src/wimlib_internal.h b/src/wimlib_internal.h
index 3d7fb488f519fdc55cc4f91ba224337886d80447..61a70f1c354db7b77f50faa93f244636c5634a32 100644 (file)
--- a/src/wimlib_internal.h
+++ b/src/wimlib_internal.h
@@ -256,6 +256,8 @@ struct wim_image_metadata {
        /* Linked list of inodes of this image */
        struct hlist_head inode_list;
 
        /* Linked list of inodes of this image */
        struct hlist_head inode_list;
 

+       struct list_head unhashed_streams;
+

        /* 1 iff the dentry tree has been modified.  If this is the case, the
         * memory for the dentry tree should not be freed when switching to a
         * different WIM image. */
        /* 1 iff the dentry tree has been modified.  If this is the case, the
         * memory for the dentry tree should not be freed when switching to a
         * different WIM image. */


@@ -495,6 +497,16 @@ extern void
 libntfs3g_global_init();
 
 /* ntfs-capture.c */
 libntfs3g_global_init();
 
 /* ntfs-capture.c */

+
+/* The types of these two callbacks are intentionally the same. */
+typedef int (*consume_data_callback_t)(const void *buf, size_t len, void *ctx);
+
+extern int
+read_ntfs_file_prefix(const struct wim_lookup_table_entry *lte,
+                     u64 size,
+                     consume_data_callback_t cb,
+                     void *ctx_or_buf,
+                     int _ignored_flags);

 extern int
 build_dentry_tree_ntfs(struct wim_dentry **root_p,
                       const tchar *device,
 extern int
 build_dentry_tree_ntfs(struct wim_dentry **root_p,
                       const tchar *device,


@@ -512,61 +524,46 @@ build_dentry_tree_ntfs(struct wim_dentry **root_p,
 #define WIMLIB_RESOURCE_FLAG_MULTITHREADED     0x2
 #define WIMLIB_RESOURCE_FLAG_RECOMPRESS                0x4
 
 #define WIMLIB_RESOURCE_FLAG_MULTITHREADED     0x2
 #define WIMLIB_RESOURCE_FLAG_RECOMPRESS                0x4
 

-extern const u8 *
-get_resource_entry(const u8 *p, struct resource_entry *entry);
+extern int
+read_resource_prefix(const struct wim_lookup_table_entry *lte,
+                    u64 size, consume_data_callback_t cb, void *ctx_or_buf,
+                    int flags);

 
 

-extern u8 *
-put_resource_entry(u8 *p, const struct resource_entry *entry);
+extern const void *
+get_resource_entry(const void *p, struct resource_entry *entry);
+
+extern void *
+put_resource_entry(void *p, const struct resource_entry *entry);

 
 extern int
 read_uncompressed_resource(FILE *fp, u64 offset, u64 size, void *buf);
 
 extern int
 
 extern int
 read_uncompressed_resource(FILE *fp, u64 offset, u64 size, void *buf);
 
 extern int

-read_wim_resource(const struct wim_lookup_table_entry *lte, void *buf,
-                 size_t size, u64 offset, int flags);
-
+read_partial_wim_resource_into_buf(const struct wim_lookup_table_entry *lte,
+                                  size_t size, u64 offset, void *buf,
+                                  bool threadsafe);

 extern int
 extern int

-read_full_wim_resource(const struct wim_lookup_table_entry *lte,
-                      void *buf, int flags);
+read_full_resource_into_buf(const struct wim_lookup_table_entry *lte,
+                           void *buf, bool thread_safe);

 
 extern int
 write_wim_resource(struct wim_lookup_table_entry *lte, FILE *out_fp,
                   int out_ctype, struct resource_entry *out_res_entry,
                   int flags);
 
 
 extern int
 write_wim_resource(struct wim_lookup_table_entry *lte, FILE *out_fp,
                   int out_ctype, struct resource_entry *out_res_entry,
                   int flags);
 

-
-typedef int (*extract_chunk_func_t)(const void *, size_t, u64, void *);
-
-extern int
-extract_wim_chunk_to_fd(const void *buf, size_t len, u64 offset, void *arg);
-

 extern int
 extract_wim_resource(const struct wim_lookup_table_entry *lte,
 extern int
 extract_wim_resource(const struct wim_lookup_table_entry *lte,

-                    u64 size, extract_chunk_func_t extract_chunk,
+                    u64 size,
+                    consume_data_callback_t extract_chunk,

                     void *extract_chunk_arg);
 
                     void *extract_chunk_arg);
 

-/*
- * Extracts the first @size bytes of the WIM resource specified by @lte to the
- * open file descriptor @fd.
- *
- * Returns 0 on success; nonzero on failure.
- */
-static inline int
-extract_wim_resource_to_fd(const struct wim_lookup_table_entry *lte,
-                          int fd, u64 size)
-{
-       return extract_wim_resource(lte, size,
-                                   extract_wim_chunk_to_fd, &fd);
-}
-
-

 extern int
 extern int

-write_dentry_resources(struct wim_dentry *dentry, void *wim_p);
+extract_wim_resource_to_fd(const struct wim_lookup_table_entry *lte,
+                          int fd, u64 size);

 
 extern int
 copy_resource(struct wim_lookup_table_entry *lte, void *w);
 
 
 extern int
 copy_resource(struct wim_lookup_table_entry *lte, void *w);
 

-

 /* security.c */
 extern int
 read_security_data(const u8 metadata_resource[],
 /* security.c */
 extern int
 read_security_data(const u8 metadata_resource[],


@@ -585,7 +582,7 @@ free_security_data(struct wim_security_data *sd);
 #ifndef __WIN32__
 ssize_t
 inode_readlink(const struct wim_inode *inode, char *buf, size_t buf_len,
 #ifndef __WIN32__
 ssize_t
 inode_readlink(const struct wim_inode *inode, char *buf, size_t buf_len,

-              const WIMStruct *w, int read_resource_flags);
+              const WIMStruct *w, bool threadsafe);

 
 extern int
 inode_set_symlink(struct wim_inode *inode, const char *target,
 
 extern int
 inode_set_symlink(struct wim_inode *inode, const char *target,

diff --git a/src/win32.c b/src/win32.c
index 05f457a87df3e28c9e692b2d35f2ae266d5ec5d4..b3aec1cbfc2b97e3d06a0ae642feea851504baf3 100644 (file)
--- a/src/win32.c
+++ b/src/win32.c
@@ -53,6 +53,28 @@ struct win32_capture_state {
 #define MAX_SET_SD_ACCESS_DENIED_WARNINGS 1
 #define MAX_SET_SACL_PRIV_NOTHELD_WARNINGS 1
 
 #define MAX_SET_SD_ACCESS_DENIED_WARNINGS 1
 #define MAX_SET_SACL_PRIV_NOTHELD_WARNINGS 1
 

+#ifdef ENABLE_ERROR_MESSAGES
+static void
+win32_error(u32 err_code)
+{
+       wchar_t *buffer;
+       DWORD nchars;
+       nchars = FormatMessageW(FORMAT_MESSAGE_FROM_SYSTEM |
+                                   FORMAT_MESSAGE_ALLOCATE_BUFFER,
+                               NULL, err_code, 0,
+                               (wchar_t*)&buffer, 0, NULL);
+       if (nchars == 0) {
+               ERROR("Error printing error message! "
+                     "Computer will self-destruct in 3 seconds.");
+       } else {
+               ERROR("Win32 error: %ls", buffer);
+               LocalFree(buffer);
+       }
+}
+#else /* ENABLE_ERROR_MESSAGES */
+#  define win32_error(err_code)
+#endif /* !ENABLE_ERROR_MESSAGES */
+

 /* Pointers to functions that are not available on all targetted versions of
  * Windows (XP and later).  NOTE: The WINAPI annotations seem to be important; I
  * assume it specifies a certain calling convention. */
 /* Pointers to functions that are not available on all targetted versions of
  * Windows (XP and later).  NOTE: The WINAPI annotations seem to be important; I
  * assume it specifies a certain calling convention. */


@@ -128,32 +150,6 @@ L"If you are not running this program as the administrator, you may\n"
  "          `wimlib-imagex apply' with the --no-acls flag instead.\n"
  ;
 
  "          `wimlib-imagex apply' with the --no-acls flag instead.\n"
  ;
 

-#ifdef ENABLE_ERROR_MESSAGES
-void
-win32_error(u32 err_code)
-{
-       wchar_t *buffer;
-       DWORD nchars;
-       nchars = FormatMessageW(FORMAT_MESSAGE_FROM_SYSTEM |
-                                   FORMAT_MESSAGE_ALLOCATE_BUFFER,
-                               NULL, err_code, 0,
-                               (wchar_t*)&buffer, 0, NULL);
-       if (nchars == 0) {
-               ERROR("Error printing error message! "
-                     "Computer will self-destruct in 3 seconds.");
-       } else {
-               ERROR("Win32 error: %ls", buffer);
-               LocalFree(buffer);
-       }
-}
-
-void
-win32_error_last()
-{
-       win32_error(GetLastError());
-}
-#endif
-

 static HANDLE
 win32_open_existing_file(const wchar_t *path, DWORD dwDesiredAccess)
 {
 static HANDLE
 win32_open_existing_file(const wchar_t *path, DWORD dwDesiredAccess)
 {


@@ -174,29 +170,58 @@ win32_open_file_data_only(const wchar_t *path)
 }
 
 int
 }
 
 int

-win32_read_file(const wchar_t *filename,
-               void *handle, u64 offset, size_t size, void *buf)
+read_win32_file_prefix(const struct lookup_table_entry *lte,
+                      u64 size,
+                      consume_data_callback_t cb,
+                      void *ctx_or_buf,
+                      int _ignored_flags)

 {
 {

-       HANDLE h = handle;
+       int ret;
+       void *out_buf;

        DWORD err;
        DWORD err;

-       DWORD bytesRead;
-       LARGE_INTEGER liOffset = {.QuadPart = offset};
+       u64 bytes_remaining;

 
 

-       wimlib_assert(size <= 0xffffffff);
+       HANDLE hFile = win32_open_file_data_only(lte->file_on_disk);
+       if (hFile == INVALID_HANDLE_VALUE) {
+               err = GetLastError();
+               ERROR("Failed to open \"%ls\"", lte->file_on_disk);
+               win32_error(err);
+               ret = WIMLIB_ERR_OPEN;
+               goto out;
+       }

 
 

-       if (SetFilePointerEx(h, liOffset, NULL, FILE_BEGIN))
-               if (ReadFile(h, buf, size, &bytesRead, NULL) && bytesRead == size)
-                       return 0;
-       err = GetLastError();
-       ERROR("Error reading \"%ls\"", filename);
-       win32_error(err);
-       return WIMLIB_ERR_READ;
-}
+       if (cb)
+               out_buf = alloca(WIM_CHUNK_SIZE);
+       else
+               out_buf = ctx_or_buf;

 
 

-void
-win32_close_file(void *handle)
-{
-       CloseHandle((HANDLE)handle);
+       bytes_remaining = size;
+       while (bytes_remaining) {
+               DWORD bytesToRead, bytesRead;
+
+               bytesToRead = min(WIM_CHUNK_SIZE, bytes_remaining);
+               if (!ReadFile(h, out_buf, bytesToRead, &bytesRead, NULL) ||
+                   bytesRead != bytesToRead)
+               {
+                       err = GetLastError();
+                       ERROR("Failed to read data from \"%ls\"", lte->file_on_disk);
+                       win32_error(err);
+                       ret = WIMLIB_ERR_READ;
+                       goto out_close_handle;
+               }
+               bytes_remaining -= bytesRead;
+               if (cb) {
+                       ret = cb(out_buf, bytesRead, ctx_or_buf);
+                       if (ret)
+                               goto out_close_handle;
+               } else {
+                       out_buf += bytesRead;
+               }
+       }
+out_close_handle:
+       CloseHandle(hFile);
+out:
+       return ret;

 }
 
 static u64
 }
 
 static u64


@@ -451,47 +476,6 @@ win32_capture_reparse_point(HANDLE hFile,
                                       bytesReturned - 8, lookup_table);
 }
 
                                       bytesReturned - 8, lookup_table);
 }
 

-/* Calculate the SHA1 message digest of a Win32 data stream, which may be either
- * an unnamed or named data stream.
- *
- * @path:      Path to the file, with the stream noted at the end for named
- *              streams.  UTF-16LE encoding.
- *
- * @hash:       On success, the SHA1 message digest of the stream is written to
- *              this location.
- *
- * Returns 0 on success; nonzero on failure.
- */
-static int
-win32_sha1sum(const wchar_t *path, u8 hash[SHA1_HASH_SIZE])
-{
-       HANDLE hFile;
-       SHA_CTX ctx;
-       u8 buf[32768];
-       DWORD bytesRead;
-       int ret;
-
-       hFile = win32_open_file_data_only(path);
-       if (hFile == INVALID_HANDLE_VALUE)
-               return WIMLIB_ERR_OPEN;
-
-       sha1_init(&ctx);
-       for (;;) {
-               if (!ReadFile(hFile, buf, sizeof(buf), &bytesRead, NULL)) {
-                       ret = WIMLIB_ERR_READ;
-                       goto out_close_handle;
-               }
-               if (bytesRead == 0)
-                       break;
-               sha1_update(&ctx, buf, bytesRead);
-       }
-       ret = 0;
-       sha1_final(hash, &ctx);
-out_close_handle:
-       CloseHandle(hFile);
-       return ret;
-}
-

 /* Scans an unnamed or named stream of a Win32 file (not a reparse point
  * stream); calculates its SHA1 message digest and either creates a `struct
  * wim_lookup_table_entry' in memory for it, or uses an existing 'struct
 /* Scans an unnamed or named stream of a Win32 file (not a reparse point
  * stream); calculates its SHA1 message digest and either creates a `struct
  * wim_lookup_table_entry' in memory for it, or uses an existing 'struct


@@ -591,35 +575,18 @@ win32_capture_stream(const wchar_t *path,
        swprintf(spath, L"%ls%ls%ls%ls",
                 relpath_prefix, path, colonchar, stream_name);
 
        swprintf(spath, L"%ls%ls%ls%ls",
                 relpath_prefix, path, colonchar, stream_name);
 

-       ret = win32_sha1sum(spath, hash);
-       if (ret) {
-               err = GetLastError();
-               ERROR("Failed to read \"%ls\" to calculate SHA1sum", spath);
-               win32_error(err);
+       /* Make a new wim_lookup_table_entry */
+       lte = new_lookup_table_entry();
+       if (!lte) {
+               ret = WIMLIB_ERR_NOMEM;

                goto out_free_spath;
        }
                goto out_free_spath;
        }

+       lte->file_on_disk = spath;
+       spath = NULL;
+       lte->resource_location = RESOURCE_WIN32;
+       lte->resource_entry.original_size = (u64)dat->StreamSize.QuadPart;
+       lookup_table_insert_unhashed(lookup_table, lte);

 
 

-       lte = __lookup_resource(lookup_table, hash);
-       if (lte) {
-               /* Use existing wim_lookup_table_entry that has the same SHA1
-                * message digest */
-               lte->refcnt++;
-       } else {
-               /* Make a new wim_lookup_table_entry */
-               lte = new_lookup_table_entry();
-               if (!lte) {
-                       ret = WIMLIB_ERR_NOMEM;
-                       goto out_free_spath;
-               }
-               lte->file_on_disk = spath;
-               lte->win32_file_on_disk_fp = INVALID_HANDLE_VALUE;
-               spath = NULL;
-               lte->resource_location = RESOURCE_WIN32;
-               lte->resource_entry.original_size = (uint64_t)dat->StreamSize.QuadPart;
-               lte->resource_entry.size = (uint64_t)dat->StreamSize.QuadPart;
-               copy_hash(lte->hash, hash);
-               lookup_table_insert(lookup_table, lte);
-       }

        if (is_named_stream)
                ads_entry->lte = lte;
        else
        if (is_named_stream)
                ads_entry->lte = lte;
        else

diff --git a/src/win32.h b/src/win32.h
index 1df793f1936c431412376a2e428bfb5e7ae9d823..0ef410a8d3aaaea8fe29cbc1713429327d511ff8 100644 (file)
--- a/src/win32.h
+++ b/src/win32.h
@@ -20,19 +20,14 @@ extern int
 win32_read_file(const tchar *filename, HANDLE handle,
                u64 offset, size_t size, void *buf);
 
 win32_read_file(const tchar *filename, HANDLE handle,
                u64 offset, size_t size, void *buf);
 

-extern HANDLE
-win32_open_file_data_only(const wchar_t *path_utf16);
-
-extern void
-win32_close_file(void *handle);
-
-#ifdef ENABLE_ERROR_MESSAGES
-extern void win32_error(u32 err);
-extern void win32_error_last();
-#else
-#  define win32_error(err)
-#  define win32_error_last()
-#endif
+extern int
+read_win32_file_prefix(const struct lookup_table_entry *lte,
+                      u64 size,
+                      consume_data_callback_t cb,
+                      void *ctx_or_buf,
+                      int _ignored_flags)
+{
+}

 
 #define FNM_PATHNAME 0x1
 #define FNM_NOESCAPE 0x2
 
 #define FNM_PATHNAME 0x1
 #define FNM_NOESCAPE 0x2

diff --git a/src/write.c b/src/write.c
index e5dbbdf95c600f377a67f8af4bc96b70f61dc711..9835dba03f67b70e15e5e9a812532774bc702cdd 100644 (file)
--- a/src/write.c
+++ b/src/write.c
@@ -196,7 +196,7 @@ get_compress_func(int out_ctype)
  * Returns 0 on success; nonzero on failure.
  */
 static int
  * Returns 0 on success; nonzero on failure.
  */
 static int

-write_wim_resource_chunk(const u8 chunk[], unsigned chunk_size,
+write_wim_resource_chunk(const void *chunk, unsigned chunk_size,

                         FILE *out_fp, compress_func_t compress,
                         struct chunk_table *chunk_tab)
 {
                         FILE *out_fp, compress_func_t compress,
                         struct chunk_table *chunk_tab)
 {


@@ -269,107 +269,6 @@ finish_wim_resource_chunk_tab(struct chunk_table *chunk_tab,
        return 0;
 }
 
        return 0;
 }
 

-/* Prepare for multiple reads to a resource by caching a FILE * or NTFS
- * attribute pointer in the lookup table entry. */
-static int
-prepare_resource_for_read(struct wim_lookup_table_entry *lte
-
-                               #ifdef WITH_NTFS_3G
-                               , ntfs_inode **ni_ret
-                               #endif
-                       )
-{
-       switch (lte->resource_location) {
-       case RESOURCE_IN_FILE_ON_DISK:
-               if (!lte->file_on_disk_fp) {
-                       lte->file_on_disk_fp = tfopen(lte->file_on_disk, T("rb"));
-                       if (!lte->file_on_disk_fp) {
-                               ERROR_WITH_ERRNO("Failed to open the file "
-                                                "`%"TS"'", lte->file_on_disk);
-                               return WIMLIB_ERR_OPEN;
-                       }
-               }
-               break;
-#ifdef WITH_NTFS_3G
-       case RESOURCE_IN_NTFS_VOLUME:
-               if (!lte->attr) {
-                       struct ntfs_location *loc = lte->ntfs_loc;
-                       ntfs_inode *ni;
-                       wimlib_assert(loc);
-                       ni = ntfs_pathname_to_inode(*loc->ntfs_vol_p, NULL, loc->path);
-                       if (!ni) {
-                               ERROR_WITH_ERRNO("Failed to open inode `%"TS"' in NTFS "
-                                                "volume", loc->path);
-                               return WIMLIB_ERR_NTFS_3G;
-                       }
-                       lte->attr = ntfs_attr_open(ni,
-                                                  loc->is_reparse_point ? AT_REPARSE_POINT : AT_DATA,
-                                                  loc->stream_name,
-                                                  loc->stream_name_nchars);
-                       if (!lte->attr) {
-                               ERROR_WITH_ERRNO("Failed to open attribute of `%"TS"' in "
-                                                "NTFS volume", loc->path);
-                               ntfs_inode_close(ni);
-                               return WIMLIB_ERR_NTFS_3G;
-                       }
-                       *ni_ret = ni;
-               }
-               break;
-#endif
-#ifdef __WIN32__
-       case RESOURCE_WIN32:
-               if (lte->win32_file_on_disk_fp == INVALID_HANDLE_VALUE) {
-                       lte->win32_file_on_disk_fp =
-                               win32_open_file_data_only(lte->file_on_disk);
-                       if (lte->win32_file_on_disk_fp == INVALID_HANDLE_VALUE) {
-                               ERROR("Win32 API: Can't open %"TS, lte->file_on_disk);
-                               win32_error_last();
-                               return WIMLIB_ERR_OPEN;
-                       }
-               }
-               break;
-#endif
-       default:
-               break;
-       }
-       return 0;
-}
-
-/* Undo prepare_resource_for_read() by closing the cached FILE * or NTFS
- * attribute. */
-static void
-end_wim_resource_read(struct wim_lookup_table_entry *lte
-                       #ifdef WITH_NTFS_3G
-                               , ntfs_inode *ni
-                       #endif
-                       )
-{
-       if (lte->resource_location == RESOURCE_IN_FILE_ON_DISK
-           && lte->file_on_disk_fp)
-       {
-               fclose(lte->file_on_disk_fp);
-               lte->file_on_disk_fp = NULL;
-       }
-#ifdef WITH_NTFS_3G
-       else if (lte->resource_location == RESOURCE_IN_NTFS_VOLUME) {
-               if (lte->attr) {
-                       ntfs_attr_close(lte->attr);
-                       lte->attr = NULL;
-               }
-               if (ni)
-                       ntfs_inode_close(ni);
-       }
-#endif
-#ifdef __WIN32__
-       else if (lte->resource_location == RESOURCE_WIN32
-                && lte->win32_file_on_disk_fp != INVALID_HANDLE_VALUE)
-       {
-               win32_close_file(lte->win32_file_on_disk_fp);
-               lte->win32_file_on_disk_fp = INVALID_HANDLE_VALUE;
-       }
-#endif
-}
-

 static int
 write_uncompressed_resource_and_truncate(struct wim_lookup_table_entry *lte,
                                         FILE *out_fp,
 static int
 write_uncompressed_resource_and_truncate(struct wim_lookup_table_entry *lte,
                                         FILE *out_fp,


@@ -382,217 +281,158 @@ write_uncompressed_resource_and_truncate(struct wim_lookup_table_entry *lte,
                                 "output WIM file", file_offset);
                return WIMLIB_ERR_WRITE;
        }
                                 "output WIM file", file_offset);
                return WIMLIB_ERR_WRITE;
        }

-       ret = write_wim_resource(lte, out_fp, WIMLIB_COMPRESSION_TYPE_NONE,
-                                out_res_entry, 0);
-       if (ret != 0)
+       ret = write_wim_resource(lte, out_fp,
+                                WIMLIB_COMPRESSION_TYPE_NONE,
+                                out_res_entry,
+                                0);
+       if (ret)

                return ret;
 
        return fflush_and_ftruncate(out_fp,
                                    file_offset + wim_resource_size(lte));
 }
 
                return ret;
 
        return fflush_and_ftruncate(out_fp,
                                    file_offset + wim_resource_size(lte));
 }
 

-/*
- * Writes a WIM resource to a FILE * opened for writing.  The resource may be
- * written uncompressed or compressed depending on the @out_ctype parameter.
- *
- * If by chance the resource compresses to more than the original size (this may
- * happen with random data or files than are pre-compressed), the resource is
- * instead written uncompressed (and this is reflected in the @out_res_entry by
- * removing the WIM_RESHDR_FLAG_COMPRESSED flag).
- *
- * @lte:       The lookup table entry for the WIM resource.
- * @out_fp:    The FILE * to write the resource to.
- * @out_ctype:  The compression type of the resource to write.  Note: if this is
- *                     the same as the compression type of the WIM resource we
- *                     need to read, we simply copy the data (i.e. we do not
- *                     uncompress it, then compress it again).
- * @out_res_entry:  If non-NULL, a resource entry that is filled in with the
- *                 offset, original size, compressed size, and compression flag
- *                 of the output resource.
- *
- * Returns 0 on success; nonzero on failure.
- */
+struct write_resource_ctx {
+       compress_func_t compress;
+       struct chunk_table *chunk_tab;
+       FILE *out_fp;
+       SHA_CTX sha_ctx;
+       bool doing_sha;
+};
+
+static int
+write_resource_cb(const void *chunk, size_t chunk_size, void *_ctx)
+{
+       struct write_resource_ctx *ctx = _ctx;
+
+       if (ctx->doing_sha)
+               sha1_update(&ctx->sha_ctx, chunk, chunk_size);
+
+       if (ctx->compress) {
+               return write_wim_resource_chunk(chunk, chunk_size,
+                                               ctx->out_fp, ctx->compress,
+                                               ctx->chunk_tab);
+       } else {
+               if (fwrite(chunk, 1, chunk_size, ctx->out_fp) != chunk_size) {
+                       ERROR_WITH_ERRNO("Error writing to output WIM");
+                       return WIMLIB_ERR_WRITE;
+               } else {
+                       return 0;
+               }
+       }
+}
+

 int
 write_wim_resource(struct wim_lookup_table_entry *lte,
                   FILE *out_fp, int out_ctype,
                   struct resource_entry *out_res_entry,
                   int flags)
 {
 int
 write_wim_resource(struct wim_lookup_table_entry *lte,
                   FILE *out_fp, int out_ctype,
                   struct resource_entry *out_res_entry,
                   int flags)
 {

-       u64 bytes_remaining;
-       u64 original_size;
-       u64 old_compressed_size;
-       u64 new_compressed_size;
-       u64 offset;
+       struct write_resource_ctx write_ctx;
+       u64 new_size;
+       off_t offset;

        int ret;
        int ret;

-       struct chunk_table *chunk_tab = NULL;
-       bool raw;
-       off_t file_offset;
-       compress_func_t compress = NULL;
-#ifdef WITH_NTFS_3G
-       ntfs_inode *ni = NULL;
-#endif
-
-       wimlib_assert(lte);
-
-       /* Original size of the resource */
-       original_size = wim_resource_size(lte);

 
 

-       /* Compressed size of the resource (as it exists now) */
-       old_compressed_size = wim_resource_compressed_size(lte);
+       if (wim_resource_size(lte) == 0) {
+               /* Empty resource; nothing needs to be done, so just return
+                * success. */
+               return 0;
+       }

 
 

-       /* Current offset in output file */
-       file_offset = ftello(out_fp);
-       if (file_offset == -1) {
-               ERROR_WITH_ERRNO("Failed to get offset in output "
-                                "stream");
+       offset = ftello(out_fp);
+       if (offset == -1) {
+               ERROR_WITH_ERRNO("Can't get position in output WIM");

                return WIMLIB_ERR_WRITE;
        }
 
                return WIMLIB_ERR_WRITE;
        }
 

-       /* Are the compression types the same?  If so, do a raw copy (copy
-        * without decompressing and recompressing the data). */
-       raw = (wim_resource_compression_type(lte) == out_ctype
-              && out_ctype != WIMLIB_COMPRESSION_TYPE_NONE
-              && !(flags & WIMLIB_RESOURCE_FLAG_RECOMPRESS));
+       /* Can we simply copy the compressed data without recompressing it? */

 
 

-       if (raw) {
+       if (!(flags & WIMLIB_RESOURCE_FLAG_RECOMPRESS) &&
+           lte->resource_location == RESOURCE_IN_WIM &&
+           wimlib_get_compression_type(lte->wim) == out_ctype)
+       {

                flags |= WIMLIB_RESOURCE_FLAG_RAW;
                flags |= WIMLIB_RESOURCE_FLAG_RAW;

-               bytes_remaining = old_compressed_size;
+               write_ctx.doing_sha = false;

        } else {
        } else {

-               flags &= ~WIMLIB_RESOURCE_FLAG_RAW;
-               bytes_remaining = original_size;
-       }
-
-       /* Empty resource; nothing needs to be done, so just return success. */
-       if (bytes_remaining == 0)
-               return 0;
-
-       /* Buffer for reading chunks for the resource */
-       u8 buf[min(WIM_CHUNK_SIZE, bytes_remaining)];
-
-       /* If we are writing a compressed resource and not doing a raw copy, we
-        * need to initialize the chunk table */
-       if (out_ctype != WIMLIB_COMPRESSION_TYPE_NONE && !raw) {
-               ret = begin_wim_resource_chunk_tab(lte, out_fp, file_offset,
-                                                  &chunk_tab);
-               if (ret != 0)
-                       goto out;
+               write_ctx.doing_sha = true;
+               sha1_init(&write_ctx.sha_ctx);

        }
 
        }
 

-       /* If the WIM resource is in an external file, open a FILE * to it so we
-        * don't have to open a temporary one in read_wim_resource() for each
-        * chunk. */
-#ifdef WITH_NTFS_3G
-       ret = prepare_resource_for_read(lte, &ni);
-#else
-       ret = prepare_resource_for_read(lte);
-#endif
-       if (ret != 0)
-               goto out;
-
-       /* If we aren't doing a raw copy, we will compute the SHA1 message
-        * digest of the resource as we read it, and verify it's the same as the
-        * hash given in the lookup table entry once we've finished reading the
-        * resource. */
-       SHA_CTX ctx;
-       if (!raw) {
-               sha1_init(&ctx);
-               compress = get_compress_func(out_ctype);
-       }
-       offset = 0;
-
-       /* While there are still bytes remaining in the WIM resource, read a
-        * chunk of the resource, update SHA1, then write that chunk using the
-        * desired compression type. */
-       do {
-               u64 to_read = min(bytes_remaining, WIM_CHUNK_SIZE);
-               ret = read_wim_resource(lte, buf, to_read, offset, flags);
-               if (ret != 0)
-                       goto out_fclose;
-               if (!raw)
-                       sha1_update(&ctx, buf, to_read);
-               ret = write_wim_resource_chunk(buf, to_read, out_fp,
-                                              compress, chunk_tab);
-               if (ret != 0)
-                       goto out_fclose;
-               bytes_remaining -= to_read;
-               offset += to_read;
-       } while (bytes_remaining);
-
-       /* Raw copy:  The new compressed size is the same as the old compressed
-        * size
-        *
-        * Using WIMLIB_COMPRESSION_TYPE_NONE:  The new compressed size is the
-        * original size
-        *
-        * Using a different compression type:  Call
-        * finish_wim_resource_chunk_tab() and it will provide the new
-        * compressed size.
-        */
-       if (raw) {
-               new_compressed_size = old_compressed_size;
+       /* Initialize the chunk table and set the compression function if
+        * compressing the resource. */
+       if (out_ctype == WIMLIB_COMPRESSION_TYPE_NONE ||
+           (flags & WIMLIB_RESOURCE_FLAG_RAW)) {
+               write_ctx.compress = NULL;
+               write_ctx.chunk_tab = NULL;

        } else {
        } else {

-               if (out_ctype == WIMLIB_COMPRESSION_TYPE_NONE)
-                       new_compressed_size = original_size;
-               else {
-                       ret = finish_wim_resource_chunk_tab(chunk_tab, out_fp,
-                                                           &new_compressed_size);
-                       if (ret != 0)
-                               goto out_fclose;
-               }
+               write_ctx.compress = get_compress_func(out_ctype);
+               ret = begin_wim_resource_chunk_tab(lte, out_fp,
+                                                  offset,
+                                                  &write_ctx.chunk_tab);
+               if (ret)
+                       return ret;

        }
 
        }
 

-       /* Verify SHA1 message digest of the resource, unless we are doing a raw
-        * write (in which case we never even saw the uncompressed data).  Or,
-        * if the hash we had before is all 0's, just re-set it to be the new
-        * hash. */
-       if (!raw) {
+       /* Write the data */
+       write_ctx.out_fp = out_fp;
+       ret = read_resource_prefix(lte, wim_resource_size(lte),
+                                  write_resource_cb, &write_ctx, 0);
+
+       /* Verify SHA1 message digest of the resource,  Or, if the hash we had
+        * before is all 0's, just re-set it to be the new hash. */
+       if (write_ctx.doing_sha) {

                u8 md[SHA1_HASH_SIZE];
                u8 md[SHA1_HASH_SIZE];

-               sha1_final(md, &ctx);
+               sha1_final(md, &write_ctx.sha_ctx);

                if (is_zero_hash(lte->hash)) {
                        copy_hash(lte->hash, md);
                } else if (!hashes_equal(md, lte->hash)) {
                        ERROR("WIM resource has incorrect hash!");
                if (is_zero_hash(lte->hash)) {
                        copy_hash(lte->hash, md);
                } else if (!hashes_equal(md, lte->hash)) {
                        ERROR("WIM resource has incorrect hash!");

-                       if (lte->resource_location == RESOURCE_IN_FILE_ON_DISK) {
-                               ERROR("We were reading it from `%"TS"'; maybe "
+                       if (lte_filename_valid(lte)) {
+                               ERROR("We were reading it from \"%"TS"\"; maybe "

                                      "it changed while we were reading it.",
                                      lte->file_on_disk);
                        }
                        ret = WIMLIB_ERR_INVALID_RESOURCE_HASH;
                                      "it changed while we were reading it.",
                                      lte->file_on_disk);
                        }
                        ret = WIMLIB_ERR_INVALID_RESOURCE_HASH;

-                       goto out_fclose;
+                       goto out_free_chunk_tab;

                }
        }
 
                }
        }
 

-       if (!raw && new_compressed_size >= original_size &&
-           out_ctype != WIMLIB_COMPRESSION_TYPE_NONE)
-       {
-               /* Oops!  We compressed the resource to larger than the original
-                * size.  Write the resource uncompressed instead. */
-               ret = write_uncompressed_resource_and_truncate(lte,
-                                                              out_fp,
-                                                              file_offset,
-                                                              out_res_entry);
-               if (ret != 0)
-                       goto out_fclose;
+       out_res_entry->flags = lte->resource_entry.flags;
+       out_res_entry->original_size = wim_resource_size(lte);
+       out_res_entry->offset = offset;
+       if (flags & WIMLIB_RESOURCE_FLAG_RAW) {
+               /* Doing a raw write:  The new compressed size is the same as
+                * the compressed size in the other WIM. */
+               new_size = lte->resource_entry.size;
+               out_res_entry->flags = lte->resource_entry.flags;
+       } else if (out_ctype == WIMLIB_COMPRESSION_TYPE_NONE) {
+               /* Using WIMLIB_COMPRESSION_TYPE_NONE:  The new compressed size
+                * is the original size. */
+               new_size = lte->resource_entry.original_size;
+               out_res_entry->flags &= ~WIM_RESHDR_FLAG_COMPRESSED;

        } else {
        } else {

-               if (out_res_entry) {
-                       out_res_entry->size          = new_compressed_size;
-                       out_res_entry->original_size = original_size;
-                       out_res_entry->offset        = file_offset;
-                       out_res_entry->flags         = lte->resource_entry.flags
-                                                       & ~WIM_RESHDR_FLAG_COMPRESSED;
-                       if (out_ctype != WIMLIB_COMPRESSION_TYPE_NONE)
-                               out_res_entry->flags |= WIM_RESHDR_FLAG_COMPRESSED;
+               /* Using a different compression type:  Call
+                * finish_wim_resource_chunk_tab() and it will provide the new
+                * compressed size. */
+               ret = finish_wim_resource_chunk_tab(write_ctx.chunk_tab, out_fp,
+                                                   &new_size);
+               if (ret)
+                       goto out_free_chunk_tab;
+               if (new_size >= wim_resource_size(lte)) {
+                       /* Oops!  We compressed the resource to larger than the original
+                        * size.  Write the resource uncompressed instead. */
+                       ret = write_uncompressed_resource_and_truncate(lte,
+                                                                      out_fp,
+                                                                      offset,
+                                                                      out_res_entry);
+                       goto out_free_chunk_tab;

                }
                }

+               out_res_entry->flags |= WIM_RESHDR_FLAG_COMPRESSED;

        }
        }

+       out_res_entry->size = new_size;

        ret = 0;
        ret = 0;

-out_fclose:
-#ifdef WITH_NTFS_3G
-       end_wim_resource_read(lte, ni);
-#else
-       end_wim_resource_read(lte);
-#endif
-out:
-       FREE(chunk_tab);
+out_free_chunk_tab:
+       FREE(write_ctx.chunk_tab);

        return ret;
 }
 
        return ret;
 }