Rewrite of write_stream_list(), and writing packed resources

diff --git a/Makefile.am b/Makefile.am
index 223f884ed53db043a08afa6a54291c060f0f26a2..7b20b58a9bda9790b66bdc04c827e28f81207c43 100644 (file)
--- a/Makefile.am
+++ b/Makefile.am
@@ -21,6 +21,9 @@ libwim_la_SOURCES =           \
        src/add_image.c         \
        src/capture_common.c    \
        src/compress.c          \
        src/add_image.c         \
        src/capture_common.c    \
        src/compress.c          \

+       src/compress_chunk.c    \
+       src/compress_parallel.c \
+       src/compress_serial.c   \

        src/decompress.c        \
        src/delete_image.c      \
        src/dentry.c            \
        src/decompress.c        \
        src/delete_image.c      \
        src/dentry.c            \


@@ -67,6 +70,7 @@ libwim_la_SOURCES =           \
        include/wimlib/capture.h        \
        include/wimlib/compiler.h       \
        include/wimlib/compress.h       \
        include/wimlib/capture.h        \
        include/wimlib/compiler.h       \
        include/wimlib/compress.h       \

+       include/wimlib/compress_chunks.h \

        include/wimlib/decompress.h     \
        include/wimlib/dentry.h         \
        include/wimlib/encoding.h       \
        include/wimlib/decompress.h     \
        include/wimlib/dentry.h         \
        include/wimlib/encoding.h       \

diff --git a/include/wimlib.h b/include/wimlib.h
index 930422a87075e853fe582a9429aaf1d9cf186e26..efc770cfdf536bfb3b8f5a9317e1b9ca6a04182a 100644 (file)
--- a/include/wimlib.h
+++ b/include/wimlib.h
@@ -987,7 +987,7 @@ struct wimlib_resource_entry {
         */
        uint32_t is_missing : 1;
 
         */
        uint32_t is_missing : 1;
 

-       uint32_t is_partial : 1;
+       uint32_t is_packed_streams : 1;

 
        uint32_t reserved_flags : 26;
 
 
        uint32_t reserved_flags : 26;
 


@@ -1513,6 +1513,12 @@ typedef int (*wimlib_iterate_lookup_table_callback_t)(const struct wimlib_resour
 
 #define WIMLIB_WRITE_FLAG_RESERVED                     0x00000800
 
 
 #define WIMLIB_WRITE_FLAG_RESERVED                     0x00000800
 

+/** TODO */
+#define WIMLIB_WRITE_FLAG_PACK_STREAMS                 0x00001000
+
+/** TODO */
+#define WIMLIB_WRITE_FLAG_NO_PACK_STREAMS              0x00002000
+

 /** @} */
 /** @ingroup G_general
  * @{ */
 /** @} */
 /** @ingroup G_general
  * @{ */

diff --git a/include/wimlib/header.h b/include/wimlib/header.h
index 5f26f7c6acd1841ccca51fbb6d60a063a403baea..d67811c121a0034b28d1edbb40273fa01bbb92c5 100644 (file)
--- a/include/wimlib/header.h
+++ b/include/wimlib/header.h
@@ -49,63 +49,65 @@
 /* On-disk format of the WIM header. */
 struct wim_header_disk {
 
 /* On-disk format of the WIM header. */
 struct wim_header_disk {
 

-       /* Magic characters "MSWIM\0\0\0" */
+       /* +0x00: Magic characters "MSWIM\0\0\0" */

        le64 magic;
 
        le64 magic;
 

-       /* Size of the WIM header, in bytes; WIM_HEADER_DISK_SIZE expected
-        * (currently the only supported value). */
+       /* +0x08: Size of the WIM header, in bytes; WIM_HEADER_DISK_SIZE
+        * expected (currently the only supported value). */

        u32 hdr_size;
 
        u32 hdr_size;
 

-       /* Version of the WIM file
+       /* +0x0c: Version of the WIM file

         * TODO  */
        u32 wim_version;
 
         * TODO  */
        u32 wim_version;
 

-       /* Flags for the WIM file (WIM_HDR_FLAG_*) */
+       /* +0x10: Flags for the WIM file (WIM_HDR_FLAG_*) */

        u32 wim_flags;
 
        u32 wim_flags;
 

-       /* Chunk size for compressed resources in the WIM, or 0 if the WIM is
-        * uncompressed.  */
+       /* +0x14: Chunk size for compressed resources in the WIM, or 0 if the
+        * WIM is uncompressed.  */

        u32 chunk_size;
 
        u32 chunk_size;
 

-       /* Globally unique identifier for the WIM file.  Basically a bunch of
-        * random bytes. */
+       /* +0x18: Globally unique identifier for the WIM file.  Basically a
+        * bunch of random bytes. */

        u8 guid[WIM_GID_LEN];
 
        u8 guid[WIM_GID_LEN];
 

-       /* Number of this WIM part in the split WIM file, indexed from 1, or 1
-        * if the WIM is not split. */
+       /* +0x28: Number of this WIM part in the split WIM file, indexed from 1,
+        * or 1 if the WIM is not split. */

        u16 part_number;
 
        u16 part_number;
 

-       /* Total number of parts of the split WIM file, or 1 if the WIM is not
-        * split. */
+       /* +0x2a: Total number of parts of the split WIM file, or 1 if the WIM
+        * is not split. */

        u16 total_parts;
 
        u16 total_parts;
 

-       /* Number of images in the WIM. */
+       /* +0x2c: Number of images in the WIM. */

        u32 image_count;
 
        u32 image_count;
 

-       /* Location and size of the WIM's lookup table. */
+       /* +0x30: Location and size of the WIM's lookup table. */

        struct wim_reshdr_disk lookup_table_reshdr;
 
        struct wim_reshdr_disk lookup_table_reshdr;
 

-       /* Location and size of the WIM's XML data. */
+       /* +0x48: Location and size of the WIM's XML data. */

        struct wim_reshdr_disk xml_data_reshdr;
 
        struct wim_reshdr_disk xml_data_reshdr;
 

-       /* Location and size of metadata resource for the bootable image of the
-        * WIM, or all zeroes if no image is bootable. */
+       /* +0x60: Location and size of metadata resource for the bootable image
+        * of the WIM, or all zeroes if no image is bootable. */

        struct wim_reshdr_disk boot_metadata_reshdr;
 
        struct wim_reshdr_disk boot_metadata_reshdr;
 

-       /* 1-based index of the bootable image of the WIM, or 0 if no image is
-        * bootable. */
+       /* +0x78: 1-based index of the bootable image of the WIM, or 0 if no
+        * image is bootable. */

        u32 boot_idx;
 
        u32 boot_idx;
 

-       /* Location and size of the WIM's integrity table, or all zeroes if the
-        * WIM has no integrity table.
+       /* +0x7c: Location and size of the WIM's integrity table, or all zeroes
+        * if the WIM has no integrity table.

         *
         * Note the integrity_table_reshdr here is 4-byte aligned even though
         * it would ordinarily be 8-byte aligned--- hence, the _packed_attribute
         * on the `struct wim_header_disk' is essential. */
        struct wim_reshdr_disk integrity_table_reshdr;
 
         *
         * Note the integrity_table_reshdr here is 4-byte aligned even though
         * it would ordinarily be 8-byte aligned--- hence, the _packed_attribute
         * on the `struct wim_header_disk' is essential. */
        struct wim_reshdr_disk integrity_table_reshdr;
 

-       /* Unused bytes. */
+       /* +0x94: Unused bytes. */

        u8 unused[60];
        u8 unused[60];

+
+       /* +0xd0 (208)  */

 } _packed_attribute;
 
 
 } _packed_attribute;
 
 

diff --git a/include/wimlib/list.h b/include/wimlib/list.h
index d6bd55c11013b54dadb2d0e615882058ace32e64..4ac2d0b414f38e8c99d9161e9a144513256ec7c6 100644 (file)
--- a/include/wimlib/list.h
+++ b/include/wimlib/list.h
@@ -86,6 +86,29 @@ static inline void list_add_tail(struct list_head *new, struct list_head *head)
        __list_add(new, head->prev, head);
 }
 
        __list_add(new, head->prev, head);
 }
 

+/**
+ * list_replace - replace old entry by new one
+ * @old : the element to be replaced
+ * @new : the new element to insert
+ *
+ * If @old was empty, it will be overwritten.
+ */
+static inline void list_replace(struct list_head *old,
+                               struct list_head *new)
+{
+       new->next = old->next;
+       new->next->prev = new;
+       new->prev = old->prev;
+       new->prev->next = new;
+}
+
+static inline void list_replace_init(struct list_head *old,
+                                       struct list_head *new)
+{
+       list_replace(old, new);
+       INIT_LIST_HEAD(old);
+}
+

 /*
  * Delete a list entry by making the prev/next entries
  * point to each other.
 /*
  * Delete a list entry by making the prev/next entries
  * point to each other.

diff --git a/include/wimlib/lookup_table.h b/include/wimlib/lookup_table.h
index f0cfb768f1e92410f022180c1c3393f9d1d04e62..6a934e4931e3f0a9440a7164a1411284c023fd2d 100644 (file)
--- a/include/wimlib/lookup_table.h
+++ b/include/wimlib/lookup_table.h
@@ -118,39 +118,24 @@ struct wim_lookup_table_entry {
        u64 size;
 
        /* Stream flags (WIM_RESHDR_FLAG_*).  */
        u64 size;
 
        /* Stream flags (WIM_RESHDR_FLAG_*).  */

-       u16 flags : 8;
+       u32 flags : 8;

 
        /* One of the `enum resource_location' values documented above. */
 
        /* One of the `enum resource_location' values documented above. */

-       u16 resource_location : 5;
-
-       /* 1 if this stream is a unique size (only set while writing streams). */
-       u16 unique_size : 1;
+       u32 resource_location : 5;

 
        /* 1 if this stream has not had a SHA1 message digest calculated for it
         * yet */
 
        /* 1 if this stream has not had a SHA1 message digest calculated for it
         * yet */

-       u16 unhashed : 1;
-
-       u16 deferred : 1;
+       u32 unhashed : 1;

 
 

-       u16 no_progress : 1;
+       /* Temoorary files used for writing; set as documented for
+        * prepare_stream_list_for_write().  */
+       u32 unique_size : 1;
+       u32 will_be_in_output_wim : 1;

 
        /* Set to 1 when a metadata entry has its checksum changed; in such
         * cases the hash is no longer valid to verify the data if the metadata
         * resource is read again.  */
 
        /* Set to 1 when a metadata entry has its checksum changed; in such
         * cases the hash is no longer valid to verify the data if the metadata
         * resource is read again.  */

-       u16 dont_check_metadata_hash : 1;
-
-       /* Only used during WIM write.  Normal value is 0 (resource not
-        * filtered).  */
-       u16 filtered : 2;
-#define FILTERED_SAME_WIM      0x1  /* Resource already in same WIM  */
-#define FILTERED_EXTERNAL_WIM  0x2  /* Resource already in external WIM  */
-
-       /* (On-disk field)
-        * Number of times this lookup table entry is referenced by dentries.
-        * Unfortunately, this field is not always set correctly in Microsoft's
-        * WIMs, so we have no choice but to fix it if more references to the
-        * lookup table entry are found than stated here.  */
-       u32 refcnt;
+       u32 dont_check_metadata_hash : 1;

 
        union {
                /* (On-disk field) SHA1 message digest of the stream referenced
 
        union {
                /* (On-disk field) SHA1 message digest of the stream referenced


@@ -176,12 +161,26 @@ struct wim_lookup_table_entry {
                };
        };
 
                };
        };
 

+       /* Number of times this lookup table entry is referenced by dentries in
+        * the WIM.  */
+       u32 refcnt;
+
+       /* Actual reference count to this stream (only used while verifying an
+        * image). */
+       u32 real_refcnt;
+

        /* When a WIM file is written, out_refcnt starts at 0 and is incremented
         * whenever the stream pointed to by this lookup table entry needs to be
         * written.  The stream only need to be written when out_refcnt is
         * nonzero, since otherwise it is not referenced by any dentries. */
        u32 out_refcnt;
 
        /* When a WIM file is written, out_refcnt starts at 0 and is incremented
         * whenever the stream pointed to by this lookup table entry needs to be
         * written.  The stream only need to be written when out_refcnt is
         * nonzero, since otherwise it is not referenced by any dentries. */
        u32 out_refcnt;
 

+#ifdef WITH_FUSE
+       /* Number of times this stream has been opened (used only during
+        * mounting) */
+       u16 num_opened_fds;
+#endif
+

        /* Pointers to somewhere where the stream is actually located.  See the
         * comments for the @resource_location field above. */
        union {
        /* Pointers to somewhere where the stream is actually located.  See the
         * comments for the @resource_location field above. */
        union {


@@ -199,48 +198,38 @@ struct wim_lookup_table_entry {
        #endif
        };
 
        #endif
        };
 

-       /* Actual reference count to this stream (only used while
-        * verifying an image). */
-       u32 real_refcnt;
-
-       union {
-       #ifdef WITH_FUSE
-               /* Number of times this stream has been opened (used only during
-                * mounting) */
-               u16 num_opened_fds;
-       #endif
+       /* Links together streams that share the same underlying WIM resource.
+        * The head is wim_resource_spec.stream_list.  */
+       struct list_head rspec_node;

 
 

-               /* This field is used for the special hardlink or symlink image
-                * extraction mode.   In these mode, all identical files are linked
-                * together, and @extracted_file will be set to the filename of the
-                * first extracted file containing this stream.  */
-               tchar *extracted_file;
-       };
+       /* This field is used for the special hardlink or symlink image
+        * extraction mode.   In these mode, all identical files are linked
+        * together, and @extracted_file will be set to the filename of the
+        * first extracted file containing this stream.  */
+       tchar *extracted_file;

 
        /* Temporary fields  */
        union {
                /* Used temporarily during WIM file writing  */
                struct {
 
        /* Temporary fields  */
        union {
                /* Used temporarily during WIM file writing  */
                struct {

-                       struct hlist_node hash_list_2;
-
+                       union {
+                               struct hlist_node hash_list_2;
+                               struct {
+                                       u64 out_res_offset_in_wim;
+                                       u64 out_res_size_in_wim;
+                                       u64 out_res_uncompressed_size;
+                               };
+                       };

                        /* Links streams being written to the WIM.  */
                        struct list_head write_streams_list;
                        /* Links streams being written to the WIM.  */
                        struct list_head write_streams_list;

-               };

 
 

-               /* Used temporarily during WIM file writing (after above)  */
-               struct {
-                       struct list_head msg_list;
-                       struct list_head being_compressed_list;
+                       struct wim_reshdr out_reshdr;

                };
 
                };
 

-               /* When a WIM file is written, @output_reshdr is filled in with
-                * the resource header for the output WIM.  */
-               struct wim_reshdr out_reshdr;
-

                /* Used temporarily during extraction  */
                union {
                        /* out_refcnt tracks number of slots filled */
                /* Used temporarily during extraction  */
                union {
                        /* out_refcnt tracks number of slots filled */

-                       struct wim_dentry *inline_lte_dentries[4];
+                       struct wim_dentry *inline_lte_dentries[8];

                        struct {
                                struct wim_dentry **lte_dentries;
                                unsigned long alloc_lte_dentries;
                        struct {
                                struct wim_dentry **lte_dentries;
                                unsigned long alloc_lte_dentries;


@@ -250,7 +239,7 @@ struct wim_lookup_table_entry {
 
        /* Temporary list fields */
        union {
 
        /* Temporary list fields */
        union {

-               /* Links streams when writing lookup table.  */
+               /* Links streams for writing lookup table.  */

                struct list_head lookup_table_list;
 
                /* Links streams being extracted.  */
                struct list_head lookup_table_list;
 
                /* Links streams being extracted.  */


@@ -260,11 +249,9 @@ struct wim_lookup_table_entry {
                struct list_head export_stream_list;
        };
 
                struct list_head export_stream_list;
        };
 

-       /* Links streams that are still unhashed after being been added
-        * to a WIM.  */
+       /* Links streams that are still unhashed after being been added to a
+        * WIM.  */

        struct list_head unhashed_list;
        struct list_head unhashed_list;

-
-       struct list_head wim_resource_list;

 };
 
 static inline bool
 };
 
 static inline bool


@@ -294,9 +281,12 @@ extern int
 read_wim_lookup_table(WIMStruct *wim);
 
 extern int
 read_wim_lookup_table(WIMStruct *wim);
 
 extern int

-write_wim_lookup_table(WIMStruct *wim, int image, int write_flags,
-                      struct wim_reshdr *out_reshdr,
-                      struct list_head *stream_list_override);
+write_wim_lookup_table_from_stream_list(struct list_head *stream_list,
+                                       struct filedes *out_fd,
+                                       u16 part_number,
+                                       struct wim_reshdr *out_reshdr,
+                                       int write_resource_flags,
+                                       struct wimlib_lzx_context **comp_ctx);

 
 extern void
 free_lookup_table(struct wim_lookup_table *table);
 
 extern void
 free_lookup_table(struct wim_lookup_table *table);


@@ -336,6 +326,11 @@ for_lookup_table_entry(struct wim_lookup_table *table,
                       int (*visitor)(struct wim_lookup_table_entry *, void *),
                       void *arg);
 
                       int (*visitor)(struct wim_lookup_table_entry *, void *),
                       void *arg);
 

+extern int
+sort_stream_list(struct list_head *stream_list,
+                size_t list_head_offset,
+                int (*compar)(const void *, const void*));
+

 extern int
 sort_stream_list_by_sequential_order(struct list_head *stream_list,
                                     size_t list_head_offset);
 extern int
 sort_stream_list_by_sequential_order(struct list_head *stream_list,
                                     size_t list_head_offset);


@@ -379,13 +374,13 @@ lte_bind_wim_resource_spec(struct wim_lookup_table_entry *lte,
 {
        lte->resource_location = RESOURCE_IN_WIM;
        lte->rspec = rspec;
 {
        lte->resource_location = RESOURCE_IN_WIM;
        lte->rspec = rspec;

-       list_add_tail(&lte->wim_resource_list, &rspec->stream_list);
+       list_add_tail(&lte->rspec_node, &rspec->stream_list);

 }
 
 static inline void
 lte_unbind_wim_resource_spec(struct wim_lookup_table_entry *lte)
 {
 }
 
 static inline void
 lte_unbind_wim_resource_spec(struct wim_lookup_table_entry *lte)
 {

-       list_del(&lte->wim_resource_list);
+       list_del(&lte->rspec_node);

        lte->rspec = NULL;
        lte->resource_location = RESOURCE_NONEXISTENT;
 }
        lte->rspec = NULL;
        lte->resource_location = RESOURCE_NONEXISTENT;
 }

diff --git a/include/wimlib/ntfs_3g.h b/include/wimlib/ntfs_3g.h
index b8ff37232bdfd1aec8f28d7788242fb56f136216..00b9fbdd21cd47a214d7284bb14b90749fa84bd6 100644 (file)
--- a/include/wimlib/ntfs_3g.h
+++ b/include/wimlib/ntfs_3g.h
@@ -14,8 +14,7 @@ extern int
 read_ntfs_file_prefix(const struct wim_lookup_table_entry *lte,
                      u64 size,
                      consume_data_callback_t cb,
 read_ntfs_file_prefix(const struct wim_lookup_table_entry *lte,
                      u64 size,
                      consume_data_callback_t cb,

-                     void *cb_ctx,
-                     int _ignored_flags);
+                     void *cb_ctx);

 
 
 extern int
 
 
 extern int

diff --git a/include/wimlib/resource.h b/include/wimlib/resource.h
index 9b176879b10af7f4d99707166c0fbd156b29007f..3d58bd24524fe6081499bfbf486de54a75df206d 100644 (file)
--- a/include/wimlib/resource.h
+++ b/include/wimlib/resource.h
@@ -48,6 +48,9 @@ struct wim_resource_spec {
         * resource will be in a slightly different format if it is compressed.
         * This is a wimlib extension.  */
        u32 is_pipable : 1;
         * resource will be in a slightly different format if it is compressed.
         * This is a wimlib extension.  */
        u32 is_pipable : 1;

+
+       /* Temporary flag.  */
+       u32 raw_copy_ok : 1;

 };
 
 /* On-disk version of a WIM resource header.  */
 };
 
 /* On-disk version of a WIM resource header.  */


@@ -99,6 +102,10 @@ struct wim_reshdr {
  * should be WIM_VERSION_PACKED_STREAMS.  */
 #define WIM_RESHDR_FLAG_PACKED_STREAMS 0x10
 
  * should be WIM_VERSION_PACKED_STREAMS.  */
 #define WIM_RESHDR_FLAG_PACKED_STREAMS 0x10
 

+/* Magic number in the 'uncompressed_size' field of the resource header that
+ * identifies the main entry for a pack.  */
+#define WIM_PACK_MAGIC_NUMBER          0x100000000ULL
+

 /* Returns true if the specified WIM resource is compressed, using either the
  * original chunk table layout or the alternate layout for resources that may
  * contain multiple packed streams.  */
 /* Returns true if the specified WIM resource is compressed, using either the
  * original chunk table layout or the alternate layout for resources that may
  * contain multiple packed streams.  */


@@ -137,17 +144,31 @@ void
 put_wim_reshdr(const struct wim_reshdr *reshdr,
               struct wim_reshdr_disk *disk_reshdr);
 
 put_wim_reshdr(const struct wim_reshdr *reshdr,
               struct wim_reshdr_disk *disk_reshdr);
 

-/* wimlib internal flags used when reading or writing resources.  */
+/* Alternate chunk table format for resources with
+ * WIM_RESHDR_FLAG_PACKED_STREAMS set.  */
+struct alt_chunk_table_header_disk {
+       /* Uncompressed size of the resource in bytes.  */
+       le64 res_usize;
+
+       /* Number of bytes each compressed chunk decompresses into, except
+        * possibly the last which decompresses into the remainder.  */
+       le32 chunk_size;
+
+       /* Compression format used for compressed chunks:
+        * 0 = None
+        * 1 = LZX
+        * 2 = XPRESS
+        * 3 = LZMS  */
+       le32 compression_format;
+
+       /* This header is directly followed by a table of compressed sizes of
+        * the chunks.  */
+} _packed_attribute;
+
+/* wimlib internal flags used when writing resources.  */

 #define WIMLIB_WRITE_RESOURCE_FLAG_RECOMPRESS          0x00000001
 #define WIMLIB_WRITE_RESOURCE_FLAG_PIPABLE             0x00000002
 #define WIMLIB_WRITE_RESOURCE_FLAG_RECOMPRESS          0x00000001
 #define WIMLIB_WRITE_RESOURCE_FLAG_PIPABLE             0x00000002

-#define WIMLIB_WRITE_RESOURCE_MASK                     0x0000ffff
-
-#define WIMLIB_READ_RESOURCE_FLAG_RAW_FULL             0x80000000
-#define WIMLIB_READ_RESOURCE_FLAG_RAW_CHUNKS           0x40000000
-#define WIMLIB_READ_RESOURCE_FLAG_RAW          (WIMLIB_READ_RESOURCE_FLAG_RAW_FULL |  \
-                                                WIMLIB_READ_RESOURCE_FLAG_RAW_CHUNKS)
-#define WIMLIB_READ_RESOURCE_MASK                      0xffff0000
-
+#define WIMLIB_WRITE_RESOURCE_FLAG_PACK_STREAMS                0x00000004

 
 /* Functions to read streams  */
 
 
 /* Functions to read streams  */
 


@@ -172,7 +193,7 @@ skip_wim_stream(struct wim_lookup_table_entry *lte);
 extern int
 read_stream_prefix(const struct wim_lookup_table_entry *lte,
                   u64 size, consume_data_callback_t cb,
 extern int
 read_stream_prefix(const struct wim_lookup_table_entry *lte,
                   u64 size, consume_data_callback_t cb,

-                  void *cb_ctx, int flags);
+                  void *cb_ctx);

 
 typedef int (*read_stream_list_begin_stream_t)(struct wim_lookup_table_entry *lte,
                                               bool is_partial_res,
 
 typedef int (*read_stream_list_begin_stream_t)(struct wim_lookup_table_entry *lte,
                                               bool is_partial_res,


@@ -181,6 +202,11 @@ typedef int (*read_stream_list_end_stream_t)(struct wim_lookup_table_entry *lte,
                                             int status,
                                             void *ctx);
 
                                             int status,
                                             void *ctx);
 

+#define VERIFY_STREAM_HASHES           0x1
+#define COMPUTE_MISSING_STREAM_HASHES  0x2
+#define STREAM_LIST_ALREADY_SORTED     0x4
+#define BEGIN_STREAM_STATUS_SKIP_STREAM                -1
+

 /* Callback functions and contexts for read_stream_list().  */
 struct read_stream_list_callbacks {
 
 /* Callback functions and contexts for read_stream_list().  */
 struct read_stream_list_callbacks {
 


@@ -206,8 +232,8 @@ struct read_stream_list_callbacks {
 extern int
 read_stream_list(struct list_head *stream_list,
                 size_t list_head_offset,
 extern int
 read_stream_list(struct list_head *stream_list,
                 size_t list_head_offset,

-                u32 cb_chunk_size,
-                const struct read_stream_list_callbacks *cbs);
+                const struct read_stream_list_callbacks *cbs,
+                int flags);

 
 /* Functions to extract streams.  */
 
 
 /* Functions to extract streams.  */
 

diff --git a/include/wimlib/util.h b/include/wimlib/util.h
index cd0d599f28875596f059b517da55bfd67e37dbc2..8e9d695ed090a1cbd35ad75e37da425be7663a7b 100644 (file)
--- a/include/wimlib/util.h
+++ b/include/wimlib/util.h
@@ -138,6 +138,17 @@ hash_u64(u64 n)
        return n * 0x9e37fffffffc0001ULL;
 }
 
        return n * 0x9e37fffffffc0001ULL;
 }
 

+static inline int
+cmp_u64(u64 n1, u64 n2)
+{
+       if (n1 < n2)
+               return -1;
+       else if (n1 > n2)
+               return 1;
+       else
+               return 0;
+}
+

 /* is_any_path_separator() - characters treated as path separators in WIM path
  * specifications and capture configuration files (the former will be translated
  * to WIM_PATH_SEPARATOR; the latter will be translated to
 /* is_any_path_separator() - characters treated as path separators in WIM path
  * specifications and capture configuration files (the former will be translated
  * to WIM_PATH_SEPARATOR; the latter will be translated to

diff --git a/include/wimlib/write.h b/include/wimlib/write.h
index 1240c89f35b96f00e4d8470ad7b6d3a53821e24a..9857fc03d7bfdb13f009ab714a94335fc24b5c53 100644 (file)
--- a/include/wimlib/write.h
+++ b/include/wimlib/write.h
@@ -46,7 +46,8 @@ write_wim_resource_from_buffer(const void *buf, size_t buf_size,
                               int out_ctype,
                               u32 out_chunk_size,
                               struct wim_reshdr *out_reshdr,
                               int out_ctype,
                               u32 out_chunk_size,
                               struct wim_reshdr *out_reshdr,

-                              u8 *hash_ret, int write_resource_flags,
+                              u8 *hash,
+                              int write_resource_flags,

                               struct wimlib_lzx_context **comp_ctx);
 
 #endif /* _WIMLIB_WRITE_H */
                               struct wimlib_lzx_context **comp_ctx);
 
 #endif /* _WIMLIB_WRITE_H */

diff --git a/programs/imagex.c b/programs/imagex.c
index c3187f74f94341a4b3bf089c134cde7c31a0c00c..b983c7f317b36bce77ebbb8c0cb5245a7f4dd6f4 100644 (file)
--- a/programs/imagex.c
+++ b/programs/imagex.c
@@ -142,7 +142,9 @@ enum {
        IMAGEX_NORPFIX_OPTION,
        IMAGEX_NOCHECK_OPTION,
        IMAGEX_NO_ACLS_OPTION,
        IMAGEX_NORPFIX_OPTION,
        IMAGEX_NOCHECK_OPTION,
        IMAGEX_NO_ACLS_OPTION,

+       IMAGEX_NO_PACK_STREAMS_OPTION,

        IMAGEX_NOT_PIPABLE_OPTION,
        IMAGEX_NOT_PIPABLE_OPTION,

+       IMAGEX_PACK_STREAMS_OPTION,

        IMAGEX_PATH_OPTION,
        IMAGEX_PIPABLE_OPTION,
        IMAGEX_REBUILD_OPTION,
        IMAGEX_PATH_OPTION,
        IMAGEX_PIPABLE_OPTION,
        IMAGEX_REBUILD_OPTION,


@@ -192,6 +194,7 @@ static const struct option capture_or_append_options[] = {
        {T("compress"),    required_argument, NULL, IMAGEX_COMPRESS_OPTION},
        {T("compress-slow"), no_argument,     NULL, IMAGEX_COMPRESS_SLOW_OPTION},
        {T("chunk-size"),  required_argument, NULL, IMAGEX_CHUNK_SIZE_OPTION},
        {T("compress"),    required_argument, NULL, IMAGEX_COMPRESS_OPTION},
        {T("compress-slow"), no_argument,     NULL, IMAGEX_COMPRESS_SLOW_OPTION},
        {T("chunk-size"),  required_argument, NULL, IMAGEX_CHUNK_SIZE_OPTION},

+       {T("pack-streams"), no_argument,      NULL, IMAGEX_PACK_STREAMS_OPTION},

        {T("config"),      required_argument, NULL, IMAGEX_CONFIG_OPTION},
        {T("dereference"), no_argument,       NULL, IMAGEX_DEREFERENCE_OPTION},
        {T("flags"),       required_argument, NULL, IMAGEX_FLAGS_OPTION},
        {T("config"),      required_argument, NULL, IMAGEX_CONFIG_OPTION},
        {T("dereference"), no_argument,       NULL, IMAGEX_DEREFERENCE_OPTION},
        {T("flags"),       required_argument, NULL, IMAGEX_FLAGS_OPTION},


@@ -1024,22 +1027,6 @@ stdin_get_text_contents(size_t *num_tchars_ret)
 #define TO_PERCENT(numerator, denominator) \
        (((denominator) == 0) ? 0 : ((numerator) * 100 / (denominator)))
 
 #define TO_PERCENT(numerator, denominator) \
        (((denominator) == 0) ? 0 : ((numerator) * 100 / (denominator)))
 

-/* Given an enumerated value for WIM compression type, return a descriptive
- * string. */
-static const tchar *
-get_data_type(int ctype)
-{
-       switch (ctype) {
-       case WIMLIB_COMPRESSION_TYPE_NONE:
-               return T("uncompressed");
-       case WIMLIB_COMPRESSION_TYPE_LZX:
-               return T("LZX-compressed");
-       case WIMLIB_COMPRESSION_TYPE_XPRESS:
-               return T("XPRESS-compressed");
-       }
-       return NULL;
-}
-

 #define GIBIBYTE_MIN_NBYTES 10000000000ULL
 #define MEBIBYTE_MIN_NBYTES 10000000ULL
 #define KIBIBYTE_MIN_NBYTES 10000ULL
 #define GIBIBYTE_MIN_NBYTES 10000000000ULL
 #define MEBIBYTE_MIN_NBYTES 10000000ULL
 #define KIBIBYTE_MIN_NBYTES 10000ULL


@@ -1079,11 +1066,9 @@ imagex_progress_func(enum wimlib_progress_msg msg,
                                          info->write_streams.total_bytes);
 
                if (info->write_streams.completed_streams == 0) {
                                          info->write_streams.total_bytes);
 
                if (info->write_streams.completed_streams == 0) {

-                       const tchar *data_type;
-
-                       data_type = get_data_type(info->write_streams.compression_type);
-                       imagex_printf(T("Writing %"TS" data using %u thread%"TS"\n"),
-                               data_type, info->write_streams.num_threads,
+                       imagex_printf(T("Writing %"TS"-compressed data using %u thread%"TS"\n"),
+                               wimlib_get_compression_type_string(info->write_streams.compression_type),
+                               info->write_streams.num_threads,

                                (info->write_streams.num_threads == 1) ? T("") : T("s"));
                }
                if (info->write_streams.total_parts <= 1) {
                                (info->write_streams.num_threads == 1) ? T("") : T("s"));
                }
                if (info->write_streams.total_parts <= 1) {


@@ -1753,6 +1738,9 @@ imagex_capture_or_append(int argc, tchar **argv, int cmd)
                        if (chunk_size == UINT32_MAX)
                                goto out_err;
                        break;
                        if (chunk_size == UINT32_MAX)
                                goto out_err;
                        break;

+               case IMAGEX_PACK_STREAMS_OPTION:
+                       write_flags |= WIMLIB_WRITE_FLAG_PACK_STREAMS;
+                       break;

                case IMAGEX_FLAGS_OPTION:
                        flags_element = optarg;
                        break;
                case IMAGEX_FLAGS_OPTION:
                        flags_element = optarg;
                        break;


@@ -2773,6 +2761,7 @@ print_wim_information(const tchar *wimfile, const struct wimlib_wim_info *info)
        tprintf(T("GUID:           0x"));
        print_byte_field(info->guid, sizeof(info->guid));
        tputchar(T('\n'));
        tprintf(T("GUID:           0x"));
        print_byte_field(info->guid, sizeof(info->guid));
        tputchar(T('\n'));

+       tprintf(T("Version:        %u\n"), info->wim_version);

        tprintf(T("Image Count:    %d\n"), info->image_count);
        tprintf(T("Compression:    %"TS"\n"),
                wimlib_get_compression_type_string(info->compression_type));
        tprintf(T("Image Count:    %d\n"), info->image_count);
        tprintf(T("Compression:    %"TS"\n"),
                wimlib_get_compression_type_string(info->compression_type));


@@ -2796,7 +2785,7 @@ print_resource(const struct wimlib_resource_entry *resource,
 {
        tprintf(T("Uncompressed size     = %"PRIu64" bytes\n"),
                resource->uncompressed_size);
 {
        tprintf(T("Uncompressed size     = %"PRIu64" bytes\n"),
                resource->uncompressed_size);

-       if (resource->is_partial) {
+       if (resource->is_packed_streams) {

                tprintf(T("Raw uncompressed size = %"PRIu64" bytes\n"),
                        resource->raw_resource_uncompressed_size);
 
                tprintf(T("Raw uncompressed size = %"PRIu64" bytes\n"),
                        resource->raw_resource_uncompressed_size);
 


@@ -2833,6 +2822,8 @@ print_resource(const struct wimlib_resource_entry *resource,
                tprintf(T("WIM_RESHDR_FLAG_FREE  "));
        if (resource->is_spanned)
                tprintf(T("WIM_RESHDR_FLAG_SPANNED  "));
                tprintf(T("WIM_RESHDR_FLAG_FREE  "));
        if (resource->is_spanned)
                tprintf(T("WIM_RESHDR_FLAG_SPANNED  "));

+       if (resource->is_packed_streams)
+               tprintf(T("WIM_RESHDR_FLAG_PACKED_STREAMS  "));

        tputchar(T('\n'));
        tputchar(T('\n'));
        return 0;
        tputchar(T('\n'));
        tputchar(T('\n'));
        return 0;

diff --git a/src/extract.c b/src/extract.c
index 3438ca73327ba3f29684967a2ba91536027810d8..a7e62697a696617fed01c4bdb3e2f242049346df 100644 (file)
--- a/src/extract.c
+++ b/src/extract.c
@@ -1308,7 +1308,8 @@ need_tmpfile_to_extract(struct wim_lookup_table_entry *lte,
 
 static int
 begin_extract_stream_to_tmpfile(struct wim_lookup_table_entry *lte,
 
 static int
 begin_extract_stream_to_tmpfile(struct wim_lookup_table_entry *lte,

-                               bool is_partial_res, void *_ctx)
+                               bool is_partial_res,
+                               void *_ctx)

 {
        struct apply_ctx *ctx = _ctx;
        int ret;
 {
        struct apply_ctx *ctx = _ctx;
        int ret;


@@ -1320,10 +1321,7 @@ begin_extract_stream_to_tmpfile(struct wim_lookup_table_entry *lte,
                if (ret)
                        return ret;
 
                if (ret)
                        return ret;
 

-               /* Negative return value here means the function was successful,
-                * but the consume_chunk and end_chunk callbacks need not be
-                * called.  */
-               return -1;
+               return BEGIN_STREAM_STATUS_SKIP_STREAM;

        }
 
        DEBUG("Temporary file needed for stream (size=%"PRIu64")", lte->size);
        }
 
        DEBUG("Temporary file needed for stream (size=%"PRIu64")", lte->size);


@@ -1390,7 +1388,7 @@ extract_stream_list(struct apply_ctx *ctx)
                };
                return read_stream_list(&ctx->stream_list,
                                        offsetof(struct wim_lookup_table_entry, extraction_list),
                };
                return read_stream_list(&ctx->stream_list,
                                        offsetof(struct wim_lookup_table_entry, extraction_list),

-                                       0, &cbs);
+                                       &cbs, VERIFY_STREAM_HASHES);

        } else {
                /* Extract the streams in unsorted order.  */
                struct wim_lookup_table_entry *lte;
        } else {
                /* Extract the streams in unsorted order.  */
                struct wim_lookup_table_entry *lte;

diff --git a/src/lookup_table.c b/src/lookup_table.c
index 44ed7966971f51980a7c0977b7629cfc0e07e470..a899bf9581abed3b4badb1c60a55c3a229222311 100644 (file)
--- a/src/lookup_table.c
+++ b/src/lookup_table.c
@@ -91,13 +91,13 @@ clone_lookup_table_entry(const struct wim_lookup_table_entry *old)
        struct wim_lookup_table_entry *new;
 
        new = memdup(old, sizeof(struct wim_lookup_table_entry));
        struct wim_lookup_table_entry *new;
 
        new = memdup(old, sizeof(struct wim_lookup_table_entry));

-       if (!new)
+       if (new == NULL)

                return NULL;
 
        new->extracted_file = NULL;
        switch (new->resource_location) {
        case RESOURCE_IN_WIM:
                return NULL;
 
        new->extracted_file = NULL;
        switch (new->resource_location) {
        case RESOURCE_IN_WIM:

-               list_add(&new->wim_resource_list, &new->rspec->stream_list);
+               list_add(&new->rspec_node, &new->rspec->stream_list);

                break;
 
        case RESOURCE_IN_FILE_ON_DISK:
                break;
 
        case RESOURCE_IN_FILE_ON_DISK:


@@ -110,12 +110,12 @@ clone_lookup_table_entry(const struct wim_lookup_table_entry *old)
                             (void*)&old->staging_file_name);
 #endif
                new->file_on_disk = TSTRDUP(old->file_on_disk);
                             (void*)&old->staging_file_name);
 #endif
                new->file_on_disk = TSTRDUP(old->file_on_disk);

-               if (!new->file_on_disk)
+               if (new->file_on_disk == NULL)

                        goto out_free;
                break;
        case RESOURCE_IN_ATTACHED_BUFFER:
                new->attached_buffer = memdup(old->attached_buffer, old->size);
                        goto out_free;
                break;
        case RESOURCE_IN_ATTACHED_BUFFER:
                new->attached_buffer = memdup(old->attached_buffer, old->size);

-               if (!new->attached_buffer)
+               if (new->attached_buffer == NULL)

                        goto out_free;
                break;
 #ifdef WITH_NTFS_3G
                        goto out_free;
                break;
 #ifdef WITH_NTFS_3G


@@ -123,18 +123,18 @@ clone_lookup_table_entry(const struct wim_lookup_table_entry *old)
                if (old->ntfs_loc) {
                        struct ntfs_location *loc;
                        loc = memdup(old->ntfs_loc, sizeof(struct ntfs_location));
                if (old->ntfs_loc) {
                        struct ntfs_location *loc;
                        loc = memdup(old->ntfs_loc, sizeof(struct ntfs_location));

-                       if (!loc)
+                       if (loc == NULL)

                                goto out_free;
                        loc->path = NULL;
                        loc->stream_name = NULL;
                        new->ntfs_loc = loc;
                        loc->path = STRDUP(old->ntfs_loc->path);
                                goto out_free;
                        loc->path = NULL;
                        loc->stream_name = NULL;
                        new->ntfs_loc = loc;
                        loc->path = STRDUP(old->ntfs_loc->path);

-                       if (!loc->path)
+                       if (loc->path == NULL)

                                goto out_free;
                                goto out_free;

-                       if (loc->stream_name_nchars) {
+                       if (loc->stream_name_nchars != 0) {

                                loc->stream_name = memdup(old->ntfs_loc->stream_name,
                                                          loc->stream_name_nchars * 2);
                                loc->stream_name = memdup(old->ntfs_loc->stream_name,
                                                          loc->stream_name_nchars * 2);

-                               if (!loc->stream_name)
+                               if (loc->stream_name == NULL)

                                        goto out_free;
                        }
                }
                                        goto out_free;
                        }
                }


@@ -155,7 +155,7 @@ free_lookup_table_entry(struct wim_lookup_table_entry *lte)
        if (lte) {
                switch (lte->resource_location) {
                case RESOURCE_IN_WIM:
        if (lte) {
                switch (lte->resource_location) {
                case RESOURCE_IN_WIM:

-                       list_del(&lte->wim_resource_list);
+                       list_del(&lte->rspec_node);

                        if (list_empty(&lte->rspec->stream_list))
                                FREE(lte->rspec);
                        break;
                        if (list_empty(&lte->rspec->stream_list))
                                FREE(lte->rspec);
                        break;


@@ -298,7 +298,7 @@ for_lookup_table_entry(struct wim_lookup_table *table,
 }
 
 /* qsort() callback that sorts streams (represented by `struct
 }
 
 /* qsort() callback that sorts streams (represented by `struct

- * wim_lookup_table_entry's) into an order optimized for reading and writing.
+ * wim_lookup_table_entry's) into an order optimized for reading.

  *
  * Sorting is done primarily by resource location, then secondarily by a
  * per-resource location order.  For example, resources in WIM files are sorted
  *
  * Sorting is done primarily by resource location, then secondarily by a
  * per-resource location order.  For example, resources in WIM files are sorted


@@ -310,7 +310,6 @@ cmp_streams_by_sequential_order(const void *p1, const void *p2)
        const struct wim_lookup_table_entry *lte1, *lte2;
        int v;
        WIMStruct *wim1, *wim2;
        const struct wim_lookup_table_entry *lte1, *lte2;
        int v;
        WIMStruct *wim1, *wim2;

-       u64 offset1, offset2;

 
        lte1 = *(const struct wim_lookup_table_entry**)p1;
        lte2 = *(const struct wim_lookup_table_entry**)p2;
 
        lte1 = *(const struct wim_lookup_table_entry**)p1;
        lte2 = *(const struct wim_lookup_table_entry**)p2;


@@ -338,15 +337,9 @@ cmp_streams_by_sequential_order(const void *p1, const void *p2)
                if (v)
                        return v;
 
                if (v)
                        return v;
 

-               /* Compare by offset.  */
-               offset1 = lte1->rspec->offset_in_wim + lte1->offset_in_res;
-               offset2 = lte2->rspec->offset_in_wim + lte2->offset_in_res;
+               return cmp_u64(lte1->rspec->offset_in_wim + lte1->offset_in_res,
+                              lte2->rspec->offset_in_wim + lte2->offset_in_res);

 
 

-               if (offset1 < offset2)
-                       return -1;
-               if (offset1 > offset2)
-                       return 1;
-               return 0;

        case RESOURCE_IN_FILE_ON_DISK:
 #ifdef WITH_FUSE
        case RESOURCE_IN_STAGING_FILE:
        case RESOURCE_IN_FILE_ON_DISK:
 #ifdef WITH_FUSE
        case RESOURCE_IN_STAGING_FILE:


@@ -370,8 +363,9 @@ cmp_streams_by_sequential_order(const void *p1, const void *p2)
 }
 
 int
 }
 
 int

-sort_stream_list_by_sequential_order(struct list_head *stream_list,
-                                    size_t list_head_offset)
+sort_stream_list(struct list_head *stream_list,
+                size_t list_head_offset,
+                int (*compar)(const void *, const void*))

 {
        struct list_head *cur;
        struct wim_lookup_table_entry **array;
 {
        struct list_head *cur;
        struct wim_lookup_table_entry **array;


@@ -382,10 +376,14 @@ sort_stream_list_by_sequential_order(struct list_head *stream_list,
        list_for_each(cur, stream_list)
                num_streams++;
 
        list_for_each(cur, stream_list)
                num_streams++;
 

+       if (num_streams <= 1)
+               return 0;
+

        array_size = num_streams * sizeof(array[0]);
        array = MALLOC(array_size);
        if (array == NULL)
                return WIMLIB_ERR_NOMEM;
        array_size = num_streams * sizeof(array[0]);
        array = MALLOC(array_size);
        if (array == NULL)
                return WIMLIB_ERR_NOMEM;

+

        cur = stream_list->next;
        for (i = 0; i < num_streams; i++) {
                array[i] = (struct wim_lookup_table_entry*)((u8*)cur -
        cur = stream_list->next;
        for (i = 0; i < num_streams; i++) {
                array[i] = (struct wim_lookup_table_entry*)((u8*)cur -


@@ -393,8 +391,7 @@ sort_stream_list_by_sequential_order(struct list_head *stream_list,
                cur = cur->next;
        }
 
                cur = cur->next;
        }
 

-       qsort(array, num_streams, sizeof(array[0]),
-             cmp_streams_by_sequential_order);
+       qsort(array, num_streams, sizeof(array[0]), compar);

 
        INIT_LIST_HEAD(stream_list);
        for (i = 0; i < num_streams; i++) {
 
        INIT_LIST_HEAD(stream_list);
        for (i = 0; i < num_streams; i++) {


@@ -406,6 +403,15 @@ sort_stream_list_by_sequential_order(struct list_head *stream_list,
        return 0;
 }
 
        return 0;
 }
 

+/* Sort the specified list of streams in an order optimized for reading.  */
+int
+sort_stream_list_by_sequential_order(struct list_head *stream_list,
+                                    size_t list_head_offset)
+{
+       return sort_stream_list(stream_list, list_head_offset,
+                               cmp_streams_by_sequential_order);
+}
+

 
 static int
 add_lte_to_array(struct wim_lookup_table_entry *lte,
 
 static int
 add_lte_to_array(struct wim_lookup_table_entry *lte,


@@ -482,13 +488,13 @@ validate_resource(const struct wim_resource_spec *rspec)
        /* Verify that each stream in the resource has a valid offset and size,
         * and that no streams overlap.  */
        cur_offset = 0;
        /* Verify that each stream in the resource has a valid offset and size,
         * and that no streams overlap.  */
        cur_offset = 0;

-       list_for_each_entry(lte, &rspec->stream_list, wim_resource_list) {
+       list_for_each_entry(lte, &rspec->stream_list, rspec_node) {

                if (lte->offset_in_res + lte->size < lte->size ||
                    lte->offset_in_res + lte->size > rspec->uncompressed_size ||
                    lte->offset_in_res < cur_offset)
                        goto invalid;
 
                if (lte->offset_in_res + lte->size < lte->size ||
                    lte->offset_in_res + lte->size > rspec->uncompressed_size ||
                    lte->offset_in_res < cur_offset)
                        goto invalid;
 

-               cur_offset = lte->offset_in_res;
+               cur_offset = lte->offset_in_res + lte->size;

        }
        return 0;
 
        }
        return 0;
 


@@ -523,6 +529,7 @@ read_wim_lookup_table(WIMStruct *wim)
        struct wim_lookup_table_entry *cur_entry, *duplicate_entry;
        struct wim_resource_spec *cur_rspec;
        void *buf;
        struct wim_lookup_table_entry *cur_entry, *duplicate_entry;
        struct wim_resource_spec *cur_rspec;
        void *buf;

+       bool back_to_back_pack;

 
        DEBUG("Reading lookup table.");
 
 
        DEBUG("Reading lookup table.");
 


@@ -586,11 +593,35 @@ read_wim_lookup_table(WIMStruct *wim)
                        continue;
                }
 
                        continue;
                }
 

+               if (!(reshdr.flags & (WIM_RESHDR_FLAG_PACKED_STREAMS |
+                                     WIM_RESHDR_FLAG_COMPRESSED))) {
+                       if (reshdr.uncompressed_size != reshdr.size_in_wim) {
+                               ERROR("Invalid resource entry!");
+                               ret = WIMLIB_ERR_INVALID_LOOKUP_TABLE_ENTRY;
+                               goto out_free_cur_entry;
+                       }
+               }
+
+               back_to_back_pack = false;

                if (!(reshdr.flags & WIM_RESHDR_FLAG_PACKED_STREAMS) ||
                if (!(reshdr.flags & WIM_RESHDR_FLAG_PACKED_STREAMS) ||

-                   cur_rspec == NULL)
+                   cur_rspec == NULL ||
+                   (back_to_back_pack =
+                    ((reshdr.flags & WIM_RESHDR_FLAG_PACKED_STREAMS) &&
+                     reshdr.uncompressed_size == WIM_PACK_MAGIC_NUMBER &&
+                     cur_rspec != NULL &&
+                     cur_rspec->size_in_wim != 0)))

                {
                        /* Starting new run of streams that share the same WIM
                         * resource.  */
                {
                        /* Starting new run of streams that share the same WIM
                         * resource.  */

+                       struct wim_lookup_table_entry *prev_entry = NULL;
+
+                       if (back_to_back_pack) {
+                               prev_entry = list_entry(cur_rspec->stream_list.prev,
+                                                       struct wim_lookup_table_entry,
+                                                       rspec_node);
+                               lte_unbind_wim_resource_spec(prev_entry);
+                               cur_rspec->uncompressed_size -= prev_entry->size;
+                       }

                        if (cur_rspec != NULL) {
                                ret = validate_resource(cur_rspec);
                                if (ret)
                        if (cur_rspec != NULL) {
                                ret = validate_resource(cur_rspec);
                                if (ret)


@@ -614,32 +645,43 @@ read_wim_lookup_table(WIMStruct *wim)
                        if (reshdr.flags & WIM_RESHDR_FLAG_PACKED_STREAMS) {
                                cur_rspec->size_in_wim = 0;
                                cur_rspec->uncompressed_size = 0;
                        if (reshdr.flags & WIM_RESHDR_FLAG_PACKED_STREAMS) {
                                cur_rspec->size_in_wim = 0;
                                cur_rspec->uncompressed_size = 0;

-                               cur_rspec->flags = 0;
+                               cur_rspec->flags = WIM_RESHDR_FLAG_PACKED_STREAMS;

                        }
                        }

+
+                       if (prev_entry) {
+                               lte_bind_wim_resource_spec(prev_entry, cur_rspec);
+                               cur_rspec->uncompressed_size = prev_entry->size;
+                       }
+               }
+
+               if ((reshdr.flags & WIM_RESHDR_FLAG_PACKED_STREAMS) &&
+                   reshdr.uncompressed_size == WIM_PACK_MAGIC_NUMBER)
+               {
+                       /* Found the specification for the packed resource.
+                        * Transfer the values to the `struct
+                        * wim_resource_spec', and discard the current stream
+                        * since this lookup table entry did not, in fact,
+                        * correspond to a "stream".  */
+
+                       cur_rspec->offset_in_wim = reshdr.offset_in_wim;
+                       cur_rspec->size_in_wim = reshdr.size_in_wim;
+                       cur_rspec->flags = reshdr.flags;
+                       DEBUG("Full pack is %"PRIu64" compressed bytes "
+                             "at file offset %"PRIu64" (flags 0x%02x)",
+                             cur_rspec->size_in_wim,
+                             cur_rspec->offset_in_wim,
+                             cur_rspec->flags);
+                       free_lookup_table_entry(cur_entry);
+                       continue;

                }
 
                if (is_zero_hash(cur_entry->hash)) {
                }
 
                if (is_zero_hash(cur_entry->hash)) {

-                       if (reshdr.flags & WIM_RESHDR_FLAG_PACKED_STREAMS) {
-                               /* Found the specification for the packed resource.
-                                * Transfer the values to the `struct
-                                * wim_resource_spec', and discard the current stream
-                                * since this lookup table entry did not, in fact,
-                                * correspond to a "stream".  */
-                               cur_rspec->offset_in_wim = reshdr.offset_in_wim;
-                               cur_rspec->size_in_wim = reshdr.size_in_wim;
-                               cur_rspec->flags = reshdr.flags;
-                               DEBUG("Full run is %"PRIu64" compressed bytes "
-                                     "at file offset %"PRIu64" (flags 0x%02x)",
-                                     cur_rspec->size_in_wim,
-                                     cur_rspec->offset_in_wim,
-                                     cur_rspec->flags);
-                       }

                        free_lookup_table_entry(cur_entry);
                        continue;
                }
 
                if (reshdr.flags & WIM_RESHDR_FLAG_PACKED_STREAMS) {
                        free_lookup_table_entry(cur_entry);
                        continue;
                }
 
                if (reshdr.flags & WIM_RESHDR_FLAG_PACKED_STREAMS) {

-                       /* Continuing the run with another stream.  */
+                       /* Continuing the pack with another stream.  */

                        DEBUG("Continuing packed run with stream: "
                              "%"PRIu64" uncompressed bytes @ resource offset %"PRIu64")",
                              reshdr.size_in_wim, reshdr.offset_in_wim);
                        DEBUG("Continuing packed run with stream: "
                              "%"PRIu64" uncompressed bytes @ resource offset %"PRIu64")",
                              reshdr.size_in_wim, reshdr.offset_in_wim);


@@ -666,14 +708,16 @@ read_wim_lookup_table(WIMStruct *wim)
 
                if (cur_entry->flags & WIM_RESHDR_FLAG_METADATA) {
                        /* Lookup table entry for a metadata resource */
 
                if (cur_entry->flags & WIM_RESHDR_FLAG_METADATA) {
                        /* Lookup table entry for a metadata resource */

+
+                       /* Metadata entries with no references must be ignored;
+                        * see for example the WinPE WIMs from the WAIK v2.1.
+                        * */
+                       if (cur_entry->refcnt == 0) {
+                               free_lookup_table_entry(cur_entry);
+                               continue;
+                       }
+

                        if (cur_entry->refcnt != 1) {
                        if (cur_entry->refcnt != 1) {

-                               /* Metadata entries with no references must be
-                                * ignored.  See for example the WinPE WIMs from
-                                * WAIK v2.1.  */
-                               if (cur_entry->refcnt == 0) {
-                                       free_lookup_table_entry(cur_entry);
-                                       continue;
-                               }

                                if (wimlib_print_errors) {
                                        ERROR("Found metadata resource with refcnt != 1:");
                                        print_lookup_table_entry(cur_entry, stderr);
                                if (wimlib_print_errors) {
                                        ERROR("Found metadata resource with refcnt != 1:");
                                        print_lookup_table_entry(cur_entry, stderr);


@@ -738,7 +782,7 @@ read_wim_lookup_table(WIMStruct *wim)
        if (cur_rspec != NULL) {
                ret = validate_resource(cur_rspec);
                if (ret)
        if (cur_rspec != NULL) {
                ret = validate_resource(cur_rspec);
                if (ret)

-                       goto out_free_cur_entry;
+                       goto out_free_lookup_table;

        }
 
        if (wim->hdr.part_number == 1 && wim->current_image != wim->hdr.image_count) {
        }
 
        if (wim->hdr.part_number == 1 && wim->current_image != wim->hdr.image_count) {


@@ -767,19 +811,18 @@ out:
        return ret;
 }
 
        return ret;
 }
 

-

 static void
 static void

-write_wim_lookup_table_entry(const struct wim_lookup_table_entry *lte,
-                            struct wim_lookup_table_entry_disk *disk_entry,
-                            u16 part_number)
+put_wim_lookup_table_entry(struct wim_lookup_table_entry_disk *disk_entry,
+                          const struct wim_reshdr *out_reshdr,
+                          u16 part_number, u32 refcnt, const u8 *hash)

 {
 {

-       put_wim_reshdr(&lte->out_reshdr, &disk_entry->reshdr);
+       put_wim_reshdr(out_reshdr, &disk_entry->reshdr);

        disk_entry->part_number = cpu_to_le16(part_number);
        disk_entry->part_number = cpu_to_le16(part_number);

-       disk_entry->refcnt = cpu_to_le32(lte->out_refcnt);
-       copy_hash(disk_entry->hash, lte->hash);
+       disk_entry->refcnt = cpu_to_le32(refcnt);
+       copy_hash(disk_entry->hash, hash);

 }
 
 }
 

-static int
+int

 write_wim_lookup_table_from_stream_list(struct list_head *stream_list,
                                        struct filedes *out_fd,
                                        u16 part_number,
 write_wim_lookup_table_from_stream_list(struct list_head *stream_list,
                                        struct filedes *out_fd,
                                        u16 part_number,


@@ -792,23 +835,68 @@ write_wim_lookup_table_from_stream_list(struct list_head *stream_list,
        struct wim_lookup_table_entry_disk *table_buf;
        struct wim_lookup_table_entry_disk *table_buf_ptr;
        int ret;
        struct wim_lookup_table_entry_disk *table_buf;
        struct wim_lookup_table_entry_disk *table_buf_ptr;
        int ret;

+       u64 prev_res_offset_in_wim = ~0ULL;

 
        table_size = 0;
 
        table_size = 0;

-       list_for_each_entry(lte, stream_list, lookup_table_list)
+       list_for_each_entry(lte, stream_list, lookup_table_list) {

                table_size += sizeof(struct wim_lookup_table_entry_disk);
 
                table_size += sizeof(struct wim_lookup_table_entry_disk);
 

+               if (lte->out_reshdr.flags & WIM_RESHDR_FLAG_PACKED_STREAMS &&
+                   lte->out_res_offset_in_wim != prev_res_offset_in_wim)
+               {
+                       table_size += sizeof(struct wim_lookup_table_entry_disk);
+                       prev_res_offset_in_wim = lte->out_res_offset_in_wim;
+               }
+       }
+

        DEBUG("Writing WIM lookup table (size=%zu, offset=%"PRIu64")",
              table_size, out_fd->offset);
 
        table_buf = MALLOC(table_size);
        DEBUG("Writing WIM lookup table (size=%zu, offset=%"PRIu64")",
              table_size, out_fd->offset);
 
        table_buf = MALLOC(table_size);

-       if (!table_buf) {
+       if (table_buf == NULL) {

                ERROR("Failed to allocate %zu bytes for temporary lookup table",
                      table_size);
                return WIMLIB_ERR_NOMEM;
        }
        table_buf_ptr = table_buf;
                ERROR("Failed to allocate %zu bytes for temporary lookup table",
                      table_size);
                return WIMLIB_ERR_NOMEM;
        }
        table_buf_ptr = table_buf;

-       list_for_each_entry(lte, stream_list, lookup_table_list)
-               write_wim_lookup_table_entry(lte, table_buf_ptr++, part_number);
+
+       prev_res_offset_in_wim = ~0ULL;
+       list_for_each_entry(lte, stream_list, lookup_table_list) {
+
+               put_wim_lookup_table_entry(table_buf_ptr++,
+                                          &lte->out_reshdr,
+                                          part_number,
+                                          lte->out_refcnt,
+                                          lte->hash);
+               if (lte->out_reshdr.flags & WIM_RESHDR_FLAG_PACKED_STREAMS &&
+                   lte->out_res_offset_in_wim != prev_res_offset_in_wim)
+               {
+                       /* Put the main resource entry for the pack.  */
+
+                       struct wim_reshdr reshdr;
+
+                       reshdr.offset_in_wim = lte->out_res_offset_in_wim;
+                       reshdr.size_in_wim = lte->out_res_size_in_wim;
+                       reshdr.uncompressed_size = WIM_PACK_MAGIC_NUMBER;
+                       reshdr.flags = WIM_RESHDR_FLAG_PACKED_STREAMS;
+
+                       DEBUG("Putting main entry for pack: "
+                             "size_in_wim=%"PRIu64", "
+                             "offset_in_wim=%"PRIu64", "
+                             "uncompressed_size=%"PRIu64,
+                             reshdr.size_in_wim,
+                             reshdr.offset_in_wim,
+                             reshdr.uncompressed_size);
+
+                       put_wim_lookup_table_entry(table_buf_ptr++,
+                                                  &reshdr,
+                                                  part_number,
+                                                  1, zero_hash);
+                       prev_res_offset_in_wim = lte->out_res_offset_in_wim;
+               }
+
+       }
+       wimlib_assert((u8*)table_buf_ptr - (u8*)table_buf == table_size);

 
        /* Write the lookup table uncompressed.  Although wimlib can handle a
         * compressed lookup table, MS software cannot.  */
 
        /* Write the lookup table uncompressed.  Although wimlib can handle a
         * compressed lookup table, MS software cannot.  */


@@ -827,94 +915,6 @@ write_wim_lookup_table_from_stream_list(struct list_head *stream_list,
        return ret;
 }
 
        return ret;
 }
 

-static int
-append_lookup_table_entry(struct wim_lookup_table_entry *lte, void *_list)
-{
-       /* Lookup table entries with 'out_refcnt' == 0 correspond to streams not
-        * written and not present in the resulting WIM file, and should not be
-        * included in the lookup table.
-        *
-        * Lookup table entries marked as filtered (EXTERNAL_WIM) with
-        * 'out_refcnt != 0' were referenced as part of the logical write but
-        * correspond to streams that were not in fact written, and should not
-        * be included in the lookup table.
-        *
-        * Lookup table entries marked as filtered (SAME_WIM) with 'out_refcnt
-        * != 0' were referenced as part of the logical write but correspond to
-        * streams that were not in fact written, but nevertheless were already
-        * present in the WIM being overwritten in-place.  These entries must be
-        * included in the lookup table, and the resource information to write
-        * needs to be copied from the resource information read originally.
-        */
-       if (lte->out_refcnt != 0 && !(lte->filtered & FILTERED_EXTERNAL_WIM)) {
-               if (lte->filtered & FILTERED_SAME_WIM)
-                       wim_res_spec_to_hdr(lte->rspec, &lte->out_reshdr);
-               list_add_tail(&lte->lookup_table_list, (struct list_head*)_list);
-       }
-       return 0;
-}
-
-int
-write_wim_lookup_table(WIMStruct *wim, int image, int write_flags,
-                      struct wim_reshdr *out_reshdr,
-                      struct list_head *stream_list_override)
-{
-       int write_resource_flags;
-       struct list_head _stream_list;
-       struct list_head *stream_list;
-
-       if (stream_list_override) {
-               stream_list = stream_list_override;
-       } else {
-               stream_list = &_stream_list;
-               INIT_LIST_HEAD(stream_list);
-       }
-
-       if (!(write_flags & WIMLIB_WRITE_FLAG_NO_METADATA)) {
-               int start_image;
-               int end_image;
-
-               if (image == WIMLIB_ALL_IMAGES) {
-                       start_image = 1;
-                       end_image = wim->hdr.image_count;
-               } else {
-                       start_image = image;
-                       end_image = image;
-               }
-
-               /* Push metadata resource lookup table entries onto the front of
-                * the list in reverse order, so that they're written in order.
-                */
-               for (int i = end_image; i >= start_image; i--) {
-                       struct wim_lookup_table_entry *metadata_lte;
-
-                       metadata_lte = wim->image_metadata[i - 1]->metadata_lte;
-                       metadata_lte->out_refcnt = 1;
-                       metadata_lte->out_reshdr.flags |= WIM_RESHDR_FLAG_METADATA;
-                       list_add(&metadata_lte->lookup_table_list, stream_list);
-               }
-       }
-
-       /* Append additional lookup table entries that need to be written, with
-        * some special handling for streams that have been marked as filtered.
-        */
-       if (!stream_list_override) {
-               for_lookup_table_entry(wim->lookup_table,
-                                      append_lookup_table_entry, stream_list);
-       }
-
-       write_resource_flags = 0;
-       if (write_flags & WIMLIB_WRITE_FLAG_PIPABLE)
-               write_resource_flags |= WIMLIB_WRITE_RESOURCE_FLAG_PIPABLE;
-       return write_wim_lookup_table_from_stream_list(stream_list,
-                                                      &wim->out_fd,
-                                                      wim->hdr.part_number,
-                                                      out_reshdr,
-                                                      write_resource_flags,
-                                                      &wim->lzx_context);
-}
-
-

 int
 lte_zero_real_refcnt(struct wim_lookup_table_entry *lte, void *_ignore)
 {
 int
 lte_zero_real_refcnt(struct wim_lookup_table_entry *lte, void *_ignore)
 {


@@ -950,7 +950,7 @@ print_lookup_table_entry(const struct wim_lookup_table_entry *lte, FILE *out)
 
        tprintf(T("Uncompressed size     = %"PRIu64" bytes\n"),
                lte->size);
 
        tprintf(T("Uncompressed size     = %"PRIu64" bytes\n"),
                lte->size);

-       if (lte_is_partial(lte)) {
+       if (lte->flags & WIM_RESHDR_FLAG_PACKED_STREAMS) {

                tprintf(T("Offset                = %"PRIu64" bytes\n"),
                        lte->offset_in_res);
 
                tprintf(T("Offset                = %"PRIu64" bytes\n"),
                        lte->offset_in_res);
 


@@ -1029,7 +1029,7 @@ lte_to_wimlib_resource_entry(const struct wim_lookup_table_entry *lte,
        wentry->uncompressed_size = lte->size;
        if (lte->resource_location == RESOURCE_IN_WIM) {
                wentry->part_number = lte->rspec->wim->hdr.part_number;
        wentry->uncompressed_size = lte->size;
        if (lte->resource_location == RESOURCE_IN_WIM) {
                wentry->part_number = lte->rspec->wim->hdr.part_number;

-               if (lte_is_partial(lte)) {
+               if (lte->flags & WIM_RESHDR_FLAG_PACKED_STREAMS) {

                        wentry->compressed_size = 0;
                        wentry->offset = lte->offset_in_res;
                } else {
                        wentry->compressed_size = 0;
                        wentry->offset = lte->offset_in_res;
                } else {


@@ -1046,7 +1046,7 @@ lte_to_wimlib_resource_entry(const struct wim_lookup_table_entry *lte,
        wentry->is_metadata = (lte->flags & WIM_RESHDR_FLAG_METADATA) != 0;
        wentry->is_free = (lte->flags & WIM_RESHDR_FLAG_FREE) != 0;
        wentry->is_spanned = (lte->flags & WIM_RESHDR_FLAG_SPANNED) != 0;
        wentry->is_metadata = (lte->flags & WIM_RESHDR_FLAG_METADATA) != 0;
        wentry->is_free = (lte->flags & WIM_RESHDR_FLAG_FREE) != 0;
        wentry->is_spanned = (lte->flags & WIM_RESHDR_FLAG_SPANNED) != 0;

-       wentry->is_partial = lte_is_partial(lte);
+       wentry->is_packed_streams = (lte->flags & WIM_RESHDR_FLAG_PACKED_STREAMS) != 0;

 }
 
 struct iterate_lte_context {
 }
 
 struct iterate_lte_context {


@@ -1438,24 +1438,23 @@ hash_unhashed_stream(struct wim_lookup_table_entry *lte,
        list_del(&lte->unhashed_list);
        if (duplicate_lte) {
                /* We have a duplicate stream.  Transfer the reference counts
        list_del(&lte->unhashed_list);
        if (duplicate_lte) {
                /* We have a duplicate stream.  Transfer the reference counts

-                * from this stream to the duplicate, update the reference to
+                * from this stream to the duplicate and update the reference to

                 * this stream (in an inode or ads_entry) to point to the
                 * this stream (in an inode or ads_entry) to point to the

-                * duplicate, then free this stream. */
+                * duplicate.  The caller is responsible for freeing @lte if
+                * needed.  */

                wimlib_assert(!(duplicate_lte->unhashed));
                wimlib_assert(!(duplicate_lte->unhashed));

+               wimlib_assert(duplicate_lte->size == lte->size);

                duplicate_lte->refcnt += lte->refcnt;
                duplicate_lte->refcnt += lte->refcnt;

-               duplicate_lte->out_refcnt += lte->out_refcnt;
+               lte->refcnt = 0;

                *back_ptr = duplicate_lte;
                *back_ptr = duplicate_lte;

-               free_lookup_table_entry(lte);

                lte = duplicate_lte;
        } else {
                lte = duplicate_lte;
        } else {

-               /* No duplicate stream, so we need to insert
-                * this stream into the lookup table and treat
-                * it as a hashed stream. */
+               /* No duplicate stream, so we need to insert this stream into
+                * the lookup table and treat it as a hashed stream. */

                lookup_table_insert(lookup_table, lte);
                lte->unhashed = 0;
        }
                lookup_table_insert(lookup_table, lte);
                lte->unhashed = 0;
        }

-       if (lte_ret)
-               *lte_ret = lte;
+       *lte_ret = lte;

        return 0;
 }
 
        return 0;
 }
 


@@ -1468,7 +1467,7 @@ lte_clone_if_new(struct wim_lookup_table_entry *lte, void *_lookup_table)
                return 0;  /*  Resource already present.  */
 
        lte = clone_lookup_table_entry(lte);
                return 0;  /*  Resource already present.  */
 
        lte = clone_lookup_table_entry(lte);

-       if (!lte)
+       if (lte == NULL)

                return WIMLIB_ERR_NOMEM;
        lte->out_refcnt = 1;
        lookup_table_insert(lookup_table, lte);
                return WIMLIB_ERR_NOMEM;
        lte->out_refcnt = 1;
        lookup_table_insert(lookup_table, lte);

diff --git a/src/ntfs-3g_capture.c b/src/ntfs-3g_capture.c
index 7a07cd3efe24a9a853d3dfc44c44e3fc3ca077d3..17fa9f06902339cd36d39c640d5368c9542a7b8f 100644 (file)
--- a/src/ntfs-3g_capture.c
+++ b/src/ntfs-3g_capture.c
@@ -75,11 +75,8 @@ open_ntfs_attr(ntfs_inode *ni, struct ntfs_location *loc)
 }
 
 int
 }
 
 int

-read_ntfs_file_prefix(const struct wim_lookup_table_entry *lte,
-                     u64 size,
-                     consume_data_callback_t cb,
-                     void *cb_ctx,
-                     int _ignored_flags)
+read_ntfs_file_prefix(const struct wim_lookup_table_entry *lte, u64 size,
+                     consume_data_callback_t cb, void *cb_ctx)

 {
        struct ntfs_location *loc = lte->ntfs_loc;
        ntfs_volume *vol = loc->ntfs_vol;
 {
        struct ntfs_location *loc = lte->ntfs_loc;
        ntfs_volume *vol = loc->ntfs_vol;

diff --git a/src/resource.c b/src/resource.c
index b8b8b92185c9e82a65146857226655b178b94f15..373279c19b9caa560cb1b5458b821f2b4ea460b5 100644 (file)
--- a/src/resource.c
+++ b/src/resource.c
@@ -118,27 +118,6 @@ struct data_range {
        u64 size;
 };
 
        u64 size;
 };
 

-/* Alternate chunk table format for resources with
- * WIM_RESHDR_FLAG_PACKED_STREAMS set.  */
-struct alt_chunk_table_header_disk {
-       /* Uncompressed size of the resource in bytes.  */
-       le64 res_usize;
-
-       /* Number of bytes each compressed chunk decompresses into, except
-        * possibly the last which decompresses into the remainder.  */
-       le32 chunk_size;
-
-       /* Compression format used for compressed chunks:
-        * 0 = None
-        * 1 = LZX
-        * 2 = XPRESS
-        * 3 = LZMS  */
-       le32 compression_format;
-
-       /* This header is directly followed by a table of compressed sizes of
-        * the chunks.  */
-} _packed_attribute;
-

 /*
  * read_compressed_wim_resource() -
  *
 /*
  * read_compressed_wim_resource() -
  *


@@ -153,16 +132,11 @@ struct alt_chunk_table_header_disk {
  *     Number of ranges in @ranges; must be at least 1.
  * @cb
  *     Callback function to feed the data being read.  Each call provides the
  *     Number of ranges in @ranges; must be at least 1.
  * @cb
  *     Callback function to feed the data being read.  Each call provides the

- *     next chunk of the requested data.  Each chunk will be of nonzero size
- *     and will not cross range boundaries, but otherwise is of unspecified
- *     size.
+ *     next chunk of the requested data, uncompressed.  Each chunk will be of
+ *     nonzero size and will not cross range boundaries, but otherwise will be
+ *     of unspecified size.

  * @cb_ctx
  *     Parameter to pass to @cb_ctx.
  * @cb_ctx
  *     Parameter to pass to @cb_ctx.

- * @raw_chunks_mode
- *     If %true, this function will provide the raw compressed chunks of the
- *     resource rather than the uncompressed data.  In this mode, only a single
- *     data range can be requested, and it must cover the entire uncompressed
- *     resource.

  *
  * Possible return values:
  *
  *
  * Possible return values:
  *


@@ -179,8 +153,7 @@ read_compressed_wim_resource(const struct wim_resource_spec * const rspec,
                             const struct data_range * const ranges,
                             const size_t num_ranges,
                             const consume_data_callback_t cb,
                             const struct data_range * const ranges,
                             const size_t num_ranges,
                             const consume_data_callback_t cb,

-                            void * const cb_ctx,
-                            const bool raw_chunks_mode)
+                            void * const cb_ctx)

 {
        int ret;
        int errno_save;
 {
        int ret;
        int errno_save;


@@ -208,12 +181,6 @@ read_compressed_wim_resource(const struct wim_resource_spec * const rspec,
        for (size_t i = 0; i < num_ranges - 1; i++)
                wimlib_assert(ranges[i].offset + ranges[i].size <= ranges[i + 1].offset);
 
        for (size_t i = 0; i < num_ranges - 1; i++)
                wimlib_assert(ranges[i].offset + ranges[i].size <= ranges[i + 1].offset);
 

-       if (raw_chunks_mode) {
-               wimlib_assert(num_ranges == 1);
-               wimlib_assert(ranges[0].offset == 0);
-               wimlib_assert(ranges[0].size == rspec->uncompressed_size);
-       }
-

        /* Get the offsets of the first and last bytes of the read.  */
        const u64 first_offset = ranges[0].offset;
        const u64 last_offset = ranges[num_ranges - 1].offset + ranges[num_ranges - 1].size - 1;
        /* Get the offsets of the first and last bytes of the read.  */
        const u64 first_offset = ranges[0].offset;
        const u64 last_offset = ranges[num_ranges - 1].offset + ranges[num_ranges - 1].size - 1;


@@ -428,19 +395,17 @@ read_compressed_wim_resource(const struct wim_resource_spec * const rspec,
                ubuf_malloced = true;
        }
 
                ubuf_malloced = true;
        }
 

-       /* Unless the raw compressed data was requested, allocate a temporary
-        * buffer for reading compressed chunks, each of which can be at most
-        * @chunk_size - 1 bytes.  This excludes compressed chunks that are a
-        * full @chunk_size bytes, which are actually stored uncompressed.  */
-       if (!raw_chunks_mode) {
-               if (chunk_size - 1 <= STACK_MAX) {
-                       cbuf = alloca(chunk_size - 1);
-               } else {
-                       cbuf = MALLOC(chunk_size - 1);
-                       if (cbuf == NULL)
-                               goto oom;
-                       cbuf_malloced = true;
-               }
+       /* Allocate a temporary buffer for reading compressed chunks, each of
+        * which can be at most @chunk_size - 1 bytes.  This excludes compressed
+        * chunks that are a full @chunk_size bytes, which are actually stored
+        * uncompressed.  */
+       if (chunk_size - 1 <= STACK_MAX) {
+               cbuf = alloca(chunk_size - 1);
+       } else {
+               cbuf = MALLOC(chunk_size - 1);
+               if (cbuf == NULL)
+                       goto oom;
+               cbuf_malloced = true;

        }
 
        /* Set current data range.  */
        }
 
        /* Set current data range.  */


@@ -513,7 +478,7 @@ read_compressed_wim_resource(const struct wim_resource_spec * const rspec,
                         * function.  */
                        u8 *read_buf;
 
                         * function.  */
                        u8 *read_buf;
 

-                       if (chunk_csize == chunk_usize || raw_chunks_mode)
+                       if (chunk_csize == chunk_usize)

                                read_buf = ubuf;
                        else
                                read_buf = cbuf;
                                read_buf = ubuf;
                        else
                                read_buf = cbuf;


@@ -527,7 +492,7 @@ read_compressed_wim_resource(const struct wim_resource_spec * const rspec,
 
                        if (read_buf == cbuf) {
                                DEBUG("Decompressing chunk %"PRIu64" "
 
                        if (read_buf == cbuf) {
                                DEBUG("Decompressing chunk %"PRIu64" "

-                                     "(csize=%"PRIu64" usize=%"PRIu64"",
+                                     "(csize=%"PRIu32" usize=%"PRIu32")",

                                      i, chunk_csize, chunk_usize);
                                ret = decompress(cbuf,
                                                 chunk_csize,
                                      i, chunk_csize, chunk_usize);
                                ret = decompress(cbuf,
                                                 chunk_csize,


@@ -557,10 +522,7 @@ read_compressed_wim_resource(const struct wim_resource_spec * const rspec,
                                end = min(cur_range_end, chunk_end_offset) - chunk_start_offset;
                                size = end - start;
 
                                end = min(cur_range_end, chunk_end_offset) - chunk_start_offset;
                                size = end - start;
 

-                               if (raw_chunks_mode)
-                                       ret = (*cb)(&ubuf[0], chunk_csize, cb_ctx);
-                               else
-                                       ret = (*cb)(&ubuf[start], size, cb_ctx);
+                               ret = (*cb)(&ubuf[start], size, cb_ctx);

 
                                if (ret)
                                        goto out_free_memory;
 
                                if (ret)
                                        goto out_free_memory;


@@ -658,24 +620,21 @@ bufferer_cb(const void *chunk, size_t size, void *_ctx)
  * read_partial_wim_resource()-
  *
  * Read a range of data from an uncompressed or compressed resource in a WIM
  * read_partial_wim_resource()-
  *
  * Read a range of data from an uncompressed or compressed resource in a WIM

- * file.  Data is fed chunk-by-chunk into the callback function @cb, passing it
- * the argument @cb_ctx.  The chunks are of unspecified size unless the
- * RAW_CHUNKS mode is requested.
- *
- * By default, this function provides the uncompressed data of the resource, and
- * @offset and @size and interpreted relative to the uncompressed contents of
- * the resource.  This behavior can be modified by either of the following
- * flags:
- *
- * WIMLIB_READ_RESOURCE_FLAG_RAW_FULL:
- *     Read @size bytes at @offset of the raw contents of the compressed
- *     resource.  In the case of pipable resources, this excludes the stream
- *     header.  Exclusive with WIMLIB_READ_RESOURCE_FLAG_RAW_CHUNKS.
+ * file.

  *
  *

- * WIMLIB_READ_RESOURCE_FLAG_RAW_CHUNKS:
- *     Read the raw compressed chunks of the compressed resource.  @size must
- *     be the full uncompressed size, @offset must be 0, and @cb_chunk_size
- *     must be the resource chunk size.
+ * @rspec
+ *     Specification of the WIM resource to read from.
+ * @offset
+ *     Offset within the uncompressed resource at which to start reading.
+ * @size
+ *     Number of bytes to read.
+ * @cb
+ *     Callback function to feed the data being read.  Each call provides the
+ *     next chunk of the requested data, uncompressed.  Each chunk will be of
+ *     nonzero size and will not cross range boundaries, but otherwise will be
+ *     of unspecified size.
+ * @cb_ctx
+ *     Parameter to pass to @cb_ctx.

  *
  * Return values:
  *     WIMLIB_ERR_SUCCESS (0)
  *
  * Return values:
  *     WIMLIB_ERR_SUCCESS (0)


@@ -688,47 +647,35 @@ bufferer_cb(const void *chunk, size_t size, void *_ctx)
  */
 static int
 read_partial_wim_resource(const struct wim_resource_spec *rspec,
  */
 static int
 read_partial_wim_resource(const struct wim_resource_spec *rspec,

-                         u64 offset, u64 size, consume_data_callback_t cb,
-                         void *cb_ctx, int flags)
+                         u64 offset, u64 size,
+                         consume_data_callback_t cb, void *cb_ctx)

 {
        /* Sanity checks.  */
 {
        /* Sanity checks.  */

-       if (flags & WIMLIB_READ_RESOURCE_FLAG_RAW_FULL) {
-               wimlib_assert(!(flags & WIMLIB_READ_RESOURCE_FLAG_RAW_CHUNKS));
-               wimlib_assert(offset + size >= offset);
-               wimlib_assert(offset + size <= rspec->size_in_wim);
-       } else if (flags & WIMLIB_READ_RESOURCE_FLAG_RAW_CHUNKS) {
-               wimlib_assert(offset == 0);
-               wimlib_assert(size == rspec->uncompressed_size);
-       } else {
-               wimlib_assert(offset + size >= offset);
-               wimlib_assert(offset + size <= rspec->uncompressed_size);
-       }
+       wimlib_assert(offset + size >= offset);
+       wimlib_assert(offset + size <= rspec->uncompressed_size);

 
        DEBUG("Reading %"PRIu64" @ %"PRIu64" from WIM resource  "
 
        DEBUG("Reading %"PRIu64" @ %"PRIu64" from WIM resource  "

-             "%"PRIu64" => %"PRIu64" @ %"PRIu64" (flags 0x%08x)",
+             "%"PRIu64" => %"PRIu64" @ %"PRIu64,

              size, offset, rspec->uncompressed_size,
              size, offset, rspec->uncompressed_size,

-             rspec->size_in_wim, rspec->offset_in_wim, flags);
+             rspec->size_in_wim, rspec->offset_in_wim);

 
        /* Trivial case.  */
        if (size == 0)
                return 0;
 
 
        /* Trivial case.  */
        if (size == 0)
                return 0;
 

-       if ((flags & WIMLIB_READ_RESOURCE_FLAG_RAW_FULL) ||
-           !resource_is_compressed(rspec))
-       {
-               return read_raw_file_data(&rspec->wim->in_fd,
-                                         size,
-                                         cb,
-                                         cb_ctx,
-                                         rspec->offset_in_wim + offset);
-       } else {
-               bool raw_chunks = (flags & WIMLIB_READ_RESOURCE_FLAG_RAW_CHUNKS);
+       if (resource_is_compressed(rspec)) {

                struct data_range range = {
                        .offset = offset,
                        .size = size,
                };
                return read_compressed_wim_resource(rspec, &range, 1,
                struct data_range range = {
                        .offset = offset,
                        .size = size,
                };
                return read_compressed_wim_resource(rspec, &range, 1,

-                                                   cb, cb_ctx, raw_chunks);
+                                                   cb, cb_ctx);
+       } else {
+               return read_raw_file_data(&rspec->wim->in_fd,
+                                         size,
+                                         cb,
+                                         cb_ctx,
+                                         rspec->offset_in_wim + offset);

        }
 }
 
        }
 }
 


@@ -746,8 +693,7 @@ read_partial_wim_stream_into_buf(const struct wim_lookup_table_entry *lte,
                                         lte->offset_in_res + offset,
                                         size,
                                         bufferer_cb,
                                         lte->offset_in_res + offset,
                                         size,
                                         bufferer_cb,

-                                        &buf,
-                                        0);
+                                        &buf);

 }
 
 /* A consume_data_callback_t implementation that simply ignores the data
 }
 
 /* A consume_data_callback_t implementation that simply ignores the data


@@ -764,25 +710,21 @@ int
 skip_wim_stream(struct wim_lookup_table_entry *lte)
 {
        wimlib_assert(lte->resource_location == RESOURCE_IN_WIM);
 skip_wim_stream(struct wim_lookup_table_entry *lte)
 {
        wimlib_assert(lte->resource_location == RESOURCE_IN_WIM);

-       wimlib_assert(!lte_is_partial(lte));
+       wimlib_assert(!(lte->flags & WIM_RESHDR_FLAG_PACKED_STREAMS));
+       DEBUG("Skipping stream (size=%"PRIu64")", lte->size);

        return read_partial_wim_resource(lte->rspec,
                                         0,
                                         lte->rspec->uncompressed_size,
                                         skip_chunk_cb,
        return read_partial_wim_resource(lte->rspec,
                                         0,
                                         lte->rspec->uncompressed_size,
                                         skip_chunk_cb,

-                                        NULL,
-                                        WIMLIB_READ_RESOURCE_FLAG_RAW_CHUNKS);
+                                        NULL);

 }
 
 static int
 read_wim_stream_prefix(const struct wim_lookup_table_entry *lte, u64 size,
 }
 
 static int
 read_wim_stream_prefix(const struct wim_lookup_table_entry *lte, u64 size,

-                      consume_data_callback_t cb, void *cb_ctx, int flags)
+                      consume_data_callback_t cb, void *cb_ctx)

 {
 {

-       return read_partial_wim_resource(lte->rspec,
-                                        lte->offset_in_res,
-                                        size,
-                                        cb,
-                                        cb_ctx,
-                                        flags);
+       return read_partial_wim_resource(lte->rspec, lte->offset_in_res, size,
+                                        cb, cb_ctx);

 }
 
 #ifndef __WIN32__
 }
 
 #ifndef __WIN32__


@@ -796,8 +738,7 @@ read_wim_stream_prefix(const struct wim_lookup_table_entry *lte, u64 size,
  * encrypted), so Windows uses its own code for its equivalent case.  */
 static int
 read_file_on_disk_prefix(const struct wim_lookup_table_entry *lte, u64 size,
  * encrypted), so Windows uses its own code for its equivalent case.  */
 static int
 read_file_on_disk_prefix(const struct wim_lookup_table_entry *lte, u64 size,

-                        consume_data_callback_t cb, void *cb_ctx,
-                        int _ignored_flags)
+                        consume_data_callback_t cb, void *cb_ctx)

 {
        int ret;
        int raw_fd;
 {
        int ret;
        int raw_fd;


@@ -823,8 +764,7 @@ read_file_on_disk_prefix(const struct wim_lookup_table_entry *lte, u64 size,
  * fact, already located in an in-memory buffer.  */
 static int
 read_buffer_prefix(const struct wim_lookup_table_entry *lte,
  * fact, already located in an in-memory buffer.  */
 static int
 read_buffer_prefix(const struct wim_lookup_table_entry *lte,

-                  u64 size, consume_data_callback_t cb,
-                  void *cb_ctx, int _ignored_flags)
+                  u64 size, consume_data_callback_t cb, void *cb_ctx)

 {
        wimlib_assert(size <= lte->size);
        return (*cb)(lte->attached_buffer, size, cb_ctx);
 {
        wimlib_assert(size <= lte->size);
        return (*cb)(lte->attached_buffer, size, cb_ctx);


@@ -833,7 +773,7 @@ read_buffer_prefix(const struct wim_lookup_table_entry *lte,
 typedef int (*read_stream_prefix_handler_t)(const struct wim_lookup_table_entry *lte,
                                            u64 size,
                                            consume_data_callback_t cb,
 typedef int (*read_stream_prefix_handler_t)(const struct wim_lookup_table_entry *lte,
                                            u64 size,
                                            consume_data_callback_t cb,

-                                           void *cb_ctx, int flags);
+                                           void *cb_ctx);

 
 /*
  * read_stream_prefix()-
 
 /*
  * read_stream_prefix()-


@@ -845,9 +785,6 @@ typedef int (*read_stream_prefix_handler_t)(const struct wim_lookup_table_entry
  * This function feeds the data to a callback function @cb in chunks of
  * unspecified size.
  *
  * This function feeds the data to a callback function @cb in chunks of
  * unspecified size.
  *

- * If the stream is located in a WIM file, @flags can be set as documented in
- * read_partial_wim_resource().  Otherwise @flags are ignored.
- *

  * Returns 0 on success; nonzero on error.  A nonzero value will be returned if
  * the stream data cannot be successfully read (for a number of different
  * reasons, depending on the stream location), or if @cb returned nonzero in
  * Returns 0 on success; nonzero on error.  A nonzero value will be returned if
  * the stream data cannot be successfully read (for a number of different
  * reasons, depending on the stream location), or if @cb returned nonzero in


@@ -855,7 +792,7 @@ typedef int (*read_stream_prefix_handler_t)(const struct wim_lookup_table_entry
  */
 int
 read_stream_prefix(const struct wim_lookup_table_entry *lte, u64 size,
  */
 int
 read_stream_prefix(const struct wim_lookup_table_entry *lte, u64 size,

-                  consume_data_callback_t cb, void *cb_ctx, int flags)
+                  consume_data_callback_t cb, void *cb_ctx)

 {
        static const read_stream_prefix_handler_t handlers[] = {
                [RESOURCE_IN_WIM]             = read_wim_stream_prefix,
 {
        static const read_stream_prefix_handler_t handlers[] = {
                [RESOURCE_IN_WIM]             = read_wim_stream_prefix,


@@ -877,7 +814,7 @@ read_stream_prefix(const struct wim_lookup_table_entry *lte, u64 size,
        };
        wimlib_assert(lte->resource_location < ARRAY_LEN(handlers)
                      && handlers[lte->resource_location] != NULL);
        };
        wimlib_assert(lte->resource_location < ARRAY_LEN(handlers)
                      && handlers[lte->resource_location] != NULL);

-       return handlers[lte->resource_location](lte, size, cb, cb_ctx, flags);
+       return handlers[lte->resource_location](lte, size, cb, cb_ctx);

 }
 
 /* Read the full uncompressed data of the specified stream into the specified
 }
 
 /* Read the full uncompressed data of the specified stream into the specified


@@ -886,7 +823,7 @@ int
 read_full_stream_into_buf(const struct wim_lookup_table_entry *lte, void *_buf)
 {
        u8 *buf = _buf;
 read_full_stream_into_buf(const struct wim_lookup_table_entry *lte, void *_buf)
 {
        u8 *buf = _buf;

-       return read_stream_prefix(lte, lte->size, bufferer_cb, &buf, 0);
+       return read_stream_prefix(lte, lte->size, bufferer_cb, &buf);

 }
 
 /* Retrieve the full uncompressed data of the specified stream.  A buffer large
 }
 
 /* Retrieve the full uncompressed data of the specified stream.  A buffer large


@@ -930,7 +867,6 @@ wim_resource_spec_to_data(struct wim_resource_spec *rspec, void **buf_ret)
        if (lte == NULL)
                return WIMLIB_ERR_NOMEM;
 
        if (lte == NULL)
                return WIMLIB_ERR_NOMEM;
 

-       lte->unhashed = 1;

        lte_bind_wim_resource_spec(lte, rspec);
        lte->flags = rspec->flags;
        lte->size = rspec->uncompressed_size;
        lte_bind_wim_resource_spec(lte, rspec);
        lte->flags = rspec->flags;
        lte->size = rspec->uncompressed_size;


@@ -1000,6 +936,13 @@ streamifier_cb(const void *chunk, size_t size, void *_ctx)
 
        if (ctx->cur_stream_offset == ctx->cur_stream->size) {
                /* Finished reading all the data for a stream.  */
 
        if (ctx->cur_stream_offset == ctx->cur_stream->size) {
                /* Finished reading all the data for a stream.  */

+
+               struct list_head *cur, *next;
+
+               cur = (struct list_head *)
+                       ((u8*)ctx->cur_stream + ctx->list_head_offset);
+               next = cur->next;
+

                DEBUG("End stream (size=%"PRIu64").", ctx->cur_stream->size);
                ret = (*ctx->cbs.end_stream)(ctx->cur_stream, 0,
                                             ctx->cbs.end_stream_ctx);
                DEBUG("End stream (size=%"PRIu64").", ctx->cur_stream->size);
                ret = (*ctx->cbs.end_stream)(ctx->cur_stream, 0,
                                             ctx->cbs.end_stream_ctx);


@@ -1008,12 +951,6 @@ streamifier_cb(const void *chunk, size_t size, void *_ctx)
 
                if (ctx->cur_stream != ctx->final_stream) {
                        /* Advance to next stream.  */
 
                if (ctx->cur_stream != ctx->final_stream) {
                        /* Advance to next stream.  */

-                       struct list_head *cur, *next;
-
-                       cur = (struct list_head *)
-                               ((u8*)ctx->cur_stream + ctx->list_head_offset);
-                       next = cur->next;
-

                        ctx->cur_stream = (struct wim_lookup_table_entry *)
                                        ((u8*)next - ctx->list_head_offset);
 
                        ctx->cur_stream = (struct wim_lookup_table_entry *)
                                        ((u8*)next - ctx->list_head_offset);
 


@@ -1028,6 +965,7 @@ streamifier_cb(const void *chunk, size_t size, void *_ctx)
 
 struct hasher_context {
        SHA_CTX sha_ctx;
 
 struct hasher_context {
        SHA_CTX sha_ctx;

+       int flags;

        struct read_stream_list_callbacks cbs;
 };
 
        struct read_stream_list_callbacks cbs;
 };
 


@@ -1083,26 +1021,30 @@ hasher_end_stream(struct wim_lookup_table_entry *lte, int status, void *_ctx)
        sha1_final(hash, &ctx->sha_ctx);
 
        if (lte->unhashed) {
        sha1_final(hash, &ctx->sha_ctx);
 
        if (lte->unhashed) {

-               /* No SHA1 message digest was previously present for the stream.
-                * Set it to the one just calculated.  */
-               DEBUG("Set SHA1 message digest for stream (size=%"PRIu64").", lte->size);
-               copy_hash(lte->hash, hash);
+               if (ctx->flags & COMPUTE_MISSING_STREAM_HASHES) {
+                       /* No SHA1 message digest was previously present for the
+                        * stream.  Set it to the one just calculated.  */
+                       DEBUG("Set SHA1 message digest for stream "
+                             "(size=%"PRIu64").", lte->size);
+                       copy_hash(lte->hash, hash);
+               }

        } else {
        } else {

-               /* The stream already had a SHA1 message digest present.  Verify
-                * that it is the same as the calculated value.  */
-               if (!hashes_equal(hash, lte->hash)) {
-                       if (wimlib_print_errors) {
-                               ERROR("Invalid SHA1 message digest "
-                                     "on the following WIM stream:");
-                               print_lookup_table_entry(lte, stderr);
-                               if (lte->resource_location == RESOURCE_IN_WIM)
-                                       ERROR("The WIM file appears to be corrupt!");
+               if (ctx->flags & VERIFY_STREAM_HASHES) {
+                       /* The stream already had a SHA1 message digest present.  Verify
+                        * that it is the same as the calculated value.  */
+                       if (!hashes_equal(hash, lte->hash)) {
+                               if (wimlib_print_errors) {
+                                       ERROR("Invalid SHA1 message digest "
+                                             "on the following WIM stream:");
+                                       print_lookup_table_entry(lte, stderr);
+                               }
+                               ret = WIMLIB_ERR_INVALID_RESOURCE_HASH;
+                               errno = EINVAL;
+                               goto out_next_cb;

                        }
                        }

-                       ret = WIMLIB_ERR_INVALID_RESOURCE_HASH;
-                       errno = EINVAL;
-                       goto out_next_cb;
+                       DEBUG("SHA1 message digest okay for "
+                             "stream (size=%"PRIu64").", lte->size);

                }
                }

-               DEBUG("SHA1 message digest okay for stream (size=%"PRIu64").", lte->size);

        }
        ret = 0;
 out_next_cb:
        }
        ret = 0;
 out_next_cb:


@@ -1112,89 +1054,43 @@ out_next_cb:
                return (*ctx->cbs.end_stream)(lte, ret, ctx->cbs.end_stream_ctx);
 }
 
                return (*ctx->cbs.end_stream)(lte, ret, ctx->cbs.end_stream_ctx);
 }
 

-/* Read the full data of the specified stream, passing the data into the
- * specified callbacks (all of which are optional) and either checking or
- * computing the SHA1 message digest of the stream.  */

 static int
 static int

-read_full_stream_with_sha1(struct wim_lookup_table_entry *lte,
-                          const struct read_stream_list_callbacks *cbs)
+read_full_stream_with_cbs(struct wim_lookup_table_entry *lte,
+                         const struct read_stream_list_callbacks *cbs)

 {
        int ret;
 
 {
        int ret;
 

-       struct hasher_context hasher_ctx = {
-               .cbs = *cbs,
-       };
-
-       ret = hasher_begin_stream(lte, false, &hasher_ctx);
+       ret = (*cbs->begin_stream)(lte, false, cbs->begin_stream_ctx);

        if (ret)
                return ret;
 
        if (ret)
                return ret;
 

-       ret = read_stream_prefix(lte, lte->size, hasher_consume_chunk,
-                                &hasher_ctx, 0);
+       ret = read_stream_prefix(lte, lte->size, cbs->consume_chunk,
+                                cbs->consume_chunk_ctx);

 
 

-       return hasher_end_stream(lte, ret, &hasher_ctx);
+       return (*cbs->end_stream)(lte, ret, cbs->end_stream_ctx);

 }
 
 }
 

-struct rechunkifier_context {
-       u8 *buffer;
-       u32 buffer_filled;
-       u32 cb_chunk_size;
-
-       const struct data_range *ranges;
-       size_t num_ranges;
-       size_t cur_range;
-       u64 range_bytes_remaining;
-
-       consume_data_callback_t cb;
-       void *cb_ctx;
-};
-
-/* Wrapper callback for adjusting the data chunk size.  */
+/* Read the full data of the specified stream, passing the data into the
+ * specified callbacks (all of which are optional) and either checking or
+ * computing the SHA1 message digest of the stream.  */

 static int
 static int

-rechunkifier_cb(const void *chunk, size_t size, void *_ctx)
+read_full_stream_with_sha1(struct wim_lookup_table_entry *lte,
+                          const struct read_stream_list_callbacks *cbs)

 {
 {

-       struct rechunkifier_context *ctx = _ctx;
-       const u8 *chunkptr = chunk;
-       size_t bytes_to_copy;
-       int ret;
-
-       wimlib_assert(ctx->cur_range != ctx->num_ranges);
-
-       while (size) {
-
-               /* Append more data to the buffer.  */
-               bytes_to_copy = size;
-
-               if (bytes_to_copy > ctx->cb_chunk_size - ctx->buffer_filled)
-                       bytes_to_copy = ctx->cb_chunk_size - ctx->buffer_filled;
-
-               if (bytes_to_copy > ctx->range_bytes_remaining - ctx->buffer_filled)
-                       bytes_to_copy = ctx->range_bytes_remaining - ctx->buffer_filled;
-
-               memcpy(&ctx->buffer[ctx->buffer_filled], chunkptr, bytes_to_copy);
-
-               ctx->buffer_filled += bytes_to_copy;
-               chunkptr += bytes_to_copy;
-               size -= bytes_to_copy;
-               ctx->range_bytes_remaining -= bytes_to_copy;
-
-               if (ctx->buffer_filled == ctx->cb_chunk_size ||
-                   ctx->range_bytes_remaining == 0)
-               {
-                       /* Maximum chunk size reached, or current range ended.
-                        * Call the next consume_data_callback_t and empty the
-                        * buffer  */
-                       ret = (*ctx->cb)(ctx->buffer, ctx->buffer_filled, ctx->cb_ctx);
-                       if (ret)
-                               return ret;
-                       ctx->buffer_filled = 0;
+       struct hasher_context hasher_ctx = {
+               .flags = VERIFY_STREAM_HASHES | COMPUTE_MISSING_STREAM_HASHES,
+               .cbs = *cbs,
+       };
+       struct read_stream_list_callbacks hasher_cbs = {
+               .begin_stream           = hasher_begin_stream,
+               .begin_stream_ctx       = &hasher_ctx,
+               .consume_chunk          = hasher_consume_chunk,
+               .consume_chunk_ctx      = &hasher_ctx,
+               .end_stream             = hasher_end_stream,
+               .end_stream_ctx         = &hasher_ctx,

 
 

-                       if (ctx->range_bytes_remaining == 0 &&
-                           ++ctx->cur_range != ctx->num_ranges)
-                               ctx->range_bytes_remaining = ctx->ranges[ctx->cur_range].size;
-               }
-       }
-       return 0;
+       };
+       return read_full_stream_with_cbs(lte, &hasher_cbs);

 }
 
 /*
 }
 
 /*


@@ -1210,11 +1106,10 @@ rechunkifier_cb(const void *chunk, size_t size, void *_ctx)
  * @list_head_offset
  *     Offset of the `struct list_head' within each `struct
  *     wim_lookup_table_entry' that makes up the @stream_list.
  * @list_head_offset
  *     Offset of the `struct list_head' within each `struct
  *     wim_lookup_table_entry' that makes up the @stream_list.

- * @cb_chunk_size
- *     Size of chunks to provide to @consume_chunk.  For a given stream, all
- *     the chunks will be this size, except possibly the last which will be the
- *     remainder.  If @
- * @cbs TODO
+ * @cbs
+ *     Callback functions to accept the stream data.
+ * @flags
+ *

  *
  * Returns 0 on success; a nonzero error code on failure.  Failure can occur due
  * to an error reading the data or due to an error status being returned by any
  *
  * Returns 0 on success; a nonzero error code on failure.  Failure can occur due
  * to an error reading the data or due to an error status being returned by any


@@ -1223,16 +1118,39 @@ rechunkifier_cb(const void *chunk, size_t size, void *_ctx)
 int
 read_stream_list(struct list_head *stream_list,
                 size_t list_head_offset,
 int
 read_stream_list(struct list_head *stream_list,
                 size_t list_head_offset,

-                u32 cb_chunk_size,
-                const struct read_stream_list_callbacks *cbs)
+                const struct read_stream_list_callbacks *cbs,
+                int flags)

 {
        int ret;
        struct list_head *cur, *next;
        struct wim_lookup_table_entry *lte;
 {
        int ret;
        struct list_head *cur, *next;
        struct wim_lookup_table_entry *lte;

+       struct hasher_context *hasher_ctx;
+       struct read_stream_list_callbacks *sink_cbs;

 
 

-       ret = sort_stream_list_by_sequential_order(stream_list, list_head_offset);
-       if (ret)
-               return ret;
+       if (!(flags & STREAM_LIST_ALREADY_SORTED)) {
+               ret = sort_stream_list_by_sequential_order(stream_list, list_head_offset);
+               if (ret)
+                       return ret;
+       }
+
+       if (flags & (VERIFY_STREAM_HASHES | COMPUTE_MISSING_STREAM_HASHES)) {
+               hasher_ctx = alloca(sizeof(*hasher_ctx));
+               *hasher_ctx = (struct hasher_context) {
+                       .flags  = flags,
+                       .cbs    = *cbs,
+               };
+               sink_cbs = alloca(sizeof(*sink_cbs));
+               *sink_cbs = (struct read_stream_list_callbacks) {
+                       .begin_stream           = hasher_begin_stream,
+                       .begin_stream_ctx       = hasher_ctx,
+                       .consume_chunk          = hasher_consume_chunk,
+                       .consume_chunk_ctx      = hasher_ctx,
+                       .end_stream             = hasher_end_stream,
+                       .end_stream_ctx         = hasher_ctx,
+               };
+       } else {
+               sink_cbs = (struct read_stream_list_callbacks*)cbs;
+       }

 
        for (cur = stream_list->next, next = cur->next;
             cur != stream_list;
 
        for (cur = stream_list->next, next = cur->next;
             cur != stream_list;


@@ -1240,7 +1158,9 @@ read_stream_list(struct list_head *stream_list,
        {
                lte = (struct wim_lookup_table_entry*)((u8*)cur - list_head_offset);
 
        {
                lte = (struct wim_lookup_table_entry*)((u8*)cur - list_head_offset);
 

-               if (lte_is_partial(lte)) {
+               if (lte->flags & WIM_RESHDR_FLAG_PACKED_STREAMS &&
+                   lte->size != lte->rspec->uncompressed_size)
+               {

 
                        struct wim_lookup_table_entry *lte_next, *lte_last;
                        struct list_head *next2;
 
                        struct wim_lookup_table_entry *lte_next, *lte_last;
                        struct list_head *next2;


@@ -1296,73 +1216,8 @@ read_stream_list(struct list_head *stream_list,
                                        }
                                }
 
                                        }
                                }
 

-                               /* Set up a chain of callbacks.
-                                *
-                                * The first level is the
-                                * streamifier_cb,
-                                * which takes in chunks of data and divides
-                                * them into the constituent streams.
-                                *
-                                * The second level are the SHA1 message digest
-                                * callbacks, which checksum each stream.
-                                *
-                                * rechunkifier_cb handles dividing the read
-                                * data into chunks of maximum size
-                                * @cb_chunk_size.  If @cb_chunk_size is 0, then
-                                * this callback is not needed.
-                                *
-                                * Finally, the last level of callbacks are
-                                * @cbs, passed as arguments to this function.
-                                */
-
-                               struct rechunkifier_context *rechunkifier_ctx = NULL;
-                               consume_data_callback_t last_cb;
-                               void *last_cb_ctx;
-
-                               if (cb_chunk_size != 0) {
-                                       rechunkifier_ctx = alloca(sizeof(*rechunkifier_ctx));
-                                       *rechunkifier_ctx = (struct rechunkifier_context) {
-                                               .buffer                 = MALLOC(cb_chunk_size),
-                                               .buffer_filled          = 0,
-                                               .cb_chunk_size          = cb_chunk_size,
-                                               .ranges                 = ranges,
-                                               .num_ranges             = stream_count,
-                                               .cur_range              = 0,
-                                               .range_bytes_remaining  = ranges[0].size,
-                                               .cb                     = cbs->consume_chunk,
-                                               .cb_ctx                 = cbs->consume_chunk_ctx,
-                                       };
-
-                                       if (rechunkifier_ctx->buffer == NULL)
-                                               return WIMLIB_ERR_NOMEM;
-                                       last_cb = rechunkifier_cb;
-                                       last_cb_ctx = rechunkifier_ctx;
-                               } else {
-                                       rechunkifier_ctx = NULL;
-                                       last_cb = cbs->consume_chunk;
-                                       last_cb_ctx = cbs->consume_chunk_ctx;
-                               }
-
-                               struct hasher_context hasher_ctx = {
-                                       .cbs = {
-                                               .begin_stream           = cbs->begin_stream,
-                                               .begin_stream_ctx       = cbs->begin_stream_ctx,
-                                               .consume_chunk          = last_cb,
-                                               .consume_chunk_ctx      = last_cb_ctx,
-                                               .end_stream             = cbs->end_stream,
-                                               .end_stream_ctx         = cbs->end_stream_ctx,
-                                       },
-                               };
-

                                struct streamifier_context streamifier_ctx = {
                                struct streamifier_context streamifier_ctx = {

-                                       .cbs = {
-                                               .begin_stream           = hasher_begin_stream,
-                                               .begin_stream_ctx       = &hasher_ctx,
-                                               .consume_chunk          = hasher_consume_chunk,
-                                               .consume_chunk_ctx      = &hasher_ctx,
-                                               .end_stream             = hasher_end_stream,
-                                               .end_stream_ctx         = &hasher_ctx,
-                                       },
+                                       .cbs                    = *sink_cbs,

                                        .cur_stream             = lte,
                                        .cur_stream_offset      = 0,
                                        .final_stream           = lte_last,
                                        .cur_stream             = lte,
                                        .cur_stream_offset      = 0,
                                        .final_stream           = lte_last,


@@ -1373,10 +1228,7 @@ read_stream_list(struct list_head *stream_list,
                                                                   ranges,
                                                                   stream_count,
                                                                   streamifier_cb,
                                                                   ranges,
                                                                   stream_count,
                                                                   streamifier_cb,

-                                                                  &streamifier_ctx,
-                                                                  false);
-                               if (rechunkifier_ctx != NULL)
-                                       FREE(rechunkifier_ctx->buffer);
+                                                                  &streamifier_ctx);

 
                                if (ret) {
                                        if (streamifier_ctx.cur_stream_offset != 0) {
 
                                if (ret) {
                                        if (streamifier_ctx.cur_stream_offset != 0) {


@@ -1391,8 +1243,8 @@ read_stream_list(struct list_head *stream_list,
                        }
                }
 
                        }
                }
 

-               ret = read_full_stream_with_sha1(lte, cbs);
-               if (ret > 0)
+               ret = read_full_stream_with_cbs(lte, sink_cbs);
+               if (ret && ret != BEGIN_STREAM_STATUS_SKIP_STREAM)

                        return ret;
        }
        return 0;
                        return ret;
        }
        return 0;


@@ -1419,7 +1271,7 @@ extract_stream(struct wim_lookup_table_entry *lte, u64 size,
        } else {
                /* Don't do SHA1.  */
                return read_stream_prefix(lte, size, extract_chunk,
        } else {
                /* Don't do SHA1.  */
                return read_stream_prefix(lte, size, extract_chunk,

-                                         extract_chunk_arg, 0);
+                                         extract_chunk_arg);

        }
 }
 
        }
 }
 


@@ -1499,7 +1351,6 @@ get_wim_reshdr(const struct wim_reshdr_disk *disk_reshdr,
                               ((u64)disk_reshdr->size_in_wim[6] << 48));
        reshdr->uncompressed_size = le64_to_cpu(disk_reshdr->uncompressed_size);
        reshdr->flags = disk_reshdr->flags;
                               ((u64)disk_reshdr->size_in_wim[6] << 48));
        reshdr->uncompressed_size = le64_to_cpu(disk_reshdr->uncompressed_size);
        reshdr->flags = disk_reshdr->flags;

-       return 0;

 }
 
 /* Translates a WIM resource header from an in-memory format into the on-disk
 }
 
 /* Translates a WIM resource header from an in-memory format into the on-disk

diff --git a/src/split.c b/src/split.c
index 167105eb47a68ec982741158624deb5956c92480..01bb75fc817f5ba3f211b659252e61c7c5750bf5 100644 (file)
--- a/src/split.c
+++ b/src/split.c
@@ -148,8 +148,13 @@ add_stream_to_swm(struct wim_lookup_table_entry *lte, void *_swm_info)
        struct swm_info *swm_info = _swm_info;
        u64 stream_size;
 
        struct swm_info *swm_info = _swm_info;
        u64 stream_size;
 

-       /* TODO */
-       stream_size = lte->rspec->size_in_wim;
+       /* We want the compressed size of the stream, but use the uncompressed
+        * size if the compressed size is not available or not relevant.  */
+       if (lte->resource_location == RESOURCE_IN_WIM &&
+           (!lte_is_partial(lte) || lte->rspec->size_in_wim < lte->size))
+               stream_size = lte->rspec->size_in_wim;
+       else
+               stream_size = lte->size;

 
        /* - Start first part if no parts have been started so far;
         * - Start next part if adding this stream exceeds maximum part size,
 
        /* - Start first part if no parts have been started so far;
         * - Start next part if adding this stream exceeds maximum part size,

diff --git a/src/wim.c b/src/wim.c
index 11bb2e50a616b79c397d9c30b0b5c5d3a8d84c98..3d173f9aefa0b38ac721fa8ac8f99aff8cc9b131 100644 (file)
--- a/src/wim.c
+++ b/src/wim.c
@@ -140,8 +140,7 @@ wim_chunk_size_valid(u32 chunk_size, int ctype)
                return order >= 15 && order <= 26;
 
        case WIMLIB_COMPRESSION_TYPE_LZMS:
                return order >= 15 && order <= 26;
 
        case WIMLIB_COMPRESSION_TYPE_LZMS:

-               /* TODO */
-               return 131072;
+               return order >= 15 && order <= 26;

        }
        return false;
 }
        }
        return false;
 }


@@ -836,9 +835,12 @@ wim_checksum_unhashed_streams(WIMStruct *wim)
                struct wim_lookup_table_entry *lte, *tmp;
                struct wim_image_metadata *imd = wim->image_metadata[i];
                image_for_each_unhashed_stream_safe(lte, tmp, imd) {
                struct wim_lookup_table_entry *lte, *tmp;
                struct wim_image_metadata *imd = wim->image_metadata[i];
                image_for_each_unhashed_stream_safe(lte, tmp, imd) {

-                       ret = hash_unhashed_stream(lte, wim->lookup_table, NULL);
+                       struct wim_lookup_table_entry *new_lte;
+                       ret = hash_unhashed_stream(lte, wim->lookup_table, &new_lte);

                        if (ret)
                                return ret;
                        if (ret)
                                return ret;

+                       if (new_lte != lte)
+                               free_lookup_table_entry(lte);

                }
        }
        return 0;
                }
        }
        return 0;

diff --git a/src/write.c b/src/write.c
index e7107ab1d392bb0a740d304737677117e7f6a8a2..95365744d9b435bf754fb08c3c528fabea77697e 100644 (file)
--- a/src/write.c
+++ b/src/write.c
@@ -34,6 +34,7 @@
 #  include <sys/file.h>
 #endif
 
 #  include <sys/file.h>
 #endif
 

+#include "wimlib/compress_chunks.h"

 #include "wimlib/endianness.h"
 #include "wimlib/error.h"
 #include "wimlib/file_io.h"
 #include "wimlib/endianness.h"
 #include "wimlib/error.h"
 #include "wimlib/file_io.h"


@@ -45,17 +46,8 @@
 #include "wimlib/write.h"
 #include "wimlib/xml.h"
 
 #include "wimlib/write.h"
 #include "wimlib/xml.h"
 

-#ifdef __WIN32__
-#  include "wimlib/win32.h" /* win32_get_number_of_processors() */
-#endif
-
-#ifdef ENABLE_MULTITHREADED_COMPRESSION
-#  include <pthread.h>
-#endif
-

 #include <errno.h>
 #include <fcntl.h>
 #include <errno.h>
 #include <fcntl.h>

-#include <limits.h>

 #include <stdlib.h>
 #include <unistd.h>
 
 #include <stdlib.h>
 #include <unistd.h>
 


@@ -63,10 +55,88 @@
 #  include <alloca.h>
 #endif
 
 #  include <alloca.h>
 #endif
 

+static inline int
+write_flags_to_resource_flags(int write_flags)
+{
+       int write_resource_flags = 0;
+
+       if (write_flags & WIMLIB_WRITE_FLAG_RECOMPRESS)
+               write_resource_flags |= WIMLIB_WRITE_RESOURCE_FLAG_RECOMPRESS;
+       if (write_flags & WIMLIB_WRITE_FLAG_PIPABLE)
+               write_resource_flags |= WIMLIB_WRITE_RESOURCE_FLAG_PIPABLE;
+       if (write_flags & WIMLIB_WRITE_FLAG_PACK_STREAMS)
+               write_resource_flags |= WIMLIB_WRITE_RESOURCE_FLAG_PACK_STREAMS;
+       return write_resource_flags;
+}
+
+struct filter_context {
+       int write_flags;
+       WIMStruct *wim;
+};
+
+/* Determine specified stream should be filtered out from the write.
+ *
+ * Return values:
+ *
+ *  < 0 : The stream should be hard-filtered; that is, not included in the
+ *        output WIM at all.
+ *    0 : The stream should not be filtered out.
+ *  > 0 : The stream should be soft-filtered; that is, it already exists in the
+ *       WIM file and may not need to be written again.
+ */
+static int
+stream_filtered(const struct wim_lookup_table_entry *lte,
+               const struct filter_context *ctx)
+{
+       int write_flags = ctx->write_flags;
+       WIMStruct *wim = ctx->wim;
+
+       if (ctx == NULL)
+               return 0;
+
+       if (write_flags & WIMLIB_WRITE_FLAG_OVERWRITE &&
+           lte->resource_location == RESOURCE_IN_WIM &&
+           lte->rspec->wim == wim)
+               return 1;
+
+       if (write_flags & WIMLIB_WRITE_FLAG_SKIP_EXTERNAL_WIMS &&
+           lte->resource_location == RESOURCE_IN_WIM &&
+           lte->rspec->wim != wim)
+               return -1;
+
+       return 0;
+}
+
+static bool
+stream_hard_filtered(const struct wim_lookup_table_entry *lte,
+                    struct filter_context *ctx)
+{
+       return stream_filtered(lte, ctx) < 0;
+}
+
+static inline int
+may_soft_filter_streams(const struct filter_context *ctx)
+{
+       if (ctx == NULL)
+               return 0;
+       return ctx->write_flags & WIMLIB_WRITE_FLAG_OVERWRITE;
+}
+
+static inline int
+may_hard_filter_streams(const struct filter_context *ctx)
+{
+       if (ctx == NULL)
+               return 0;
+       return ctx->write_flags & WIMLIB_WRITE_FLAG_SKIP_EXTERNAL_WIMS;
+}
+
+static inline int
+may_filter_streams(const struct filter_context *ctx)
+{
+       return (may_soft_filter_streams(ctx) ||
+               may_hard_filter_streams(ctx));
+}

 
 

-#ifndef __WIN32__
-#  include <sys/uio.h> /* for `struct iovec' */
-#endif

 
 /* Return true if the specified resource is compressed and the compressed data
  * can be reused with the specified output parameters.  */
 
 /* Return true if the specified resource is compressed and the compressed data
  * can be reused with the specified output parameters.  */


@@ -74,184 +144,95 @@ static bool
 can_raw_copy(const struct wim_lookup_table_entry *lte,
             int write_resource_flags, int out_ctype, u32 out_chunk_size)
 {
 can_raw_copy(const struct wim_lookup_table_entry *lte,
             int write_resource_flags, int out_ctype, u32 out_chunk_size)
 {

+       const struct wim_resource_spec *rspec;
+

        if (write_resource_flags & WIMLIB_WRITE_RESOURCE_FLAG_RECOMPRESS)
                return false;
        if (write_resource_flags & WIMLIB_WRITE_RESOURCE_FLAG_RECOMPRESS)
                return false;

-       if (lte->resource_location != RESOURCE_IN_WIM)
-               return false;
+

        if (out_ctype == WIMLIB_COMPRESSION_TYPE_NONE)
                return false;
        if (out_ctype == WIMLIB_COMPRESSION_TYPE_NONE)
                return false;

-       if (lte->rspec->flags & WIM_RESHDR_FLAG_PACKED_STREAMS)
-               return false;
-       if (!(lte->rspec->flags & WIM_RESHDR_FLAG_COMPRESSED))
-               return false;
-       if (lte->rspec->wim->compression_type != out_ctype)
-               return false;
-       if (lte->rspec->wim->chunk_size != out_chunk_size)
+
+       if (lte->resource_location != RESOURCE_IN_WIM)

                return false;
                return false;

-       return true;
-}

 
 

+       rspec = lte->rspec;

 
 

-/* Return true if the specified resource must be recompressed when the specified
- * output parameters are used.  */
-static bool
-must_compress_stream(const struct wim_lookup_table_entry *lte,
-                    int write_resource_flags, int out_ctype, u32 out_chunk_size)
-{
-       return (out_ctype != WIMLIB_COMPRESSION_TYPE_NONE
-               && ((write_resource_flags & WIMLIB_WRITE_RESOURCE_FLAG_RECOMPRESS)
-                   || !can_raw_copy(lte, write_resource_flags,
-                                    out_ctype, out_chunk_size)));
-}
-
-static unsigned
-compress_chunk(const void * uncompressed_data,
-              unsigned uncompressed_len,
-              void *compressed_data,
-              int out_ctype,
-              struct wimlib_lzx_context *comp_ctx)
-{
-       switch (out_ctype) {
-       case WIMLIB_COMPRESSION_TYPE_XPRESS:
-               return wimlib_xpress_compress(uncompressed_data,
-                                             uncompressed_len,
-                                             compressed_data);
-       case WIMLIB_COMPRESSION_TYPE_LZX:
-               return wimlib_lzx_compress2(uncompressed_data,
-                                           uncompressed_len,
-                                           compressed_data,
-                                           comp_ctx);
-       case WIMLIB_COMPRESSION_TYPE_LZMS:
-               /* TODO */
-               WARNING("LZMS compression not yet implemented!");
-               return 0;
+       if (rspec->is_pipable != !!(write_resource_flags & WIMLIB_WRITE_RESOURCE_FLAG_PIPABLE))
+               return false;

 
 

-       default:
-               wimlib_assert(0);
-               return 0;
+
+       if (rspec->flags & WIM_RESHDR_FLAG_COMPRESSED) {
+               /* Normal compressed resource: Must use same compression type
+                * and chunk size.  */
+               return (rspec->wim->compression_type == out_ctype &&
+                       rspec->wim->chunk_size == out_chunk_size);

        }
        }

-}

 
 

-/* Chunk table that's located at the beginning of each compressed resource in
- * the WIM.  (This is not the on-disk format; the on-disk format just has an
- * array of offsets.) */
-struct chunk_table {
-       u64 original_resource_size;
-       u64 num_chunks;
-       u64 table_disk_size;
-       unsigned bytes_per_chunk_entry;
-       void *cur_offset_p;
-       union {
-               u32 cur_offset_u32;
-               u64 cur_offset_u64;
-       };
-       /* Beginning of chunk offsets, in either 32-bit or 64-bit little endian
-        * integers, including the first offset of 0, which will not be written.
-        * */
-       u8 offsets[] _aligned_attribute(8);
-};
+       if ((rspec->flags & WIM_RESHDR_FLAG_PACKED_STREAMS) &&
+           (write_resource_flags & WIMLIB_WRITE_RESOURCE_FLAG_PACK_STREAMS))
+       {
+               /* Packed resource: Such resources may contain multiple streams,
+                * and in general only a subset of them need to be written.  As
+                * a heuristic, re-use the raw data if at least half the
+                * uncompressed size is being written.  */

 
 

-/* Allocate and initializes a chunk table, then reserve space for it in the
- * output file unless writing a pipable resource.  */
-static int
-begin_wim_resource_chunk_tab(const struct wim_lookup_table_entry *lte,
-                            struct filedes *out_fd,
-                            u32 out_chunk_size,
-                            struct chunk_table **chunk_tab_ret,
-                            int resource_flags)
-{
-       u64 size;
-       u64 num_chunks;
-       unsigned bytes_per_chunk_entry;
-       size_t alloc_size;
-       struct chunk_table *chunk_tab;
-       int ret;
+               /* Note: packed resources contain a header that specifies the
+                * compression type and chunk size; therefore we don't need to
+                * check if they are compatible with @out_ctype and
+                * @out_chunk_size.  */

 
 

-       size = lte->size;
-       num_chunks = DIV_ROUND_UP(size, out_chunk_size);
-       bytes_per_chunk_entry = (size > (1ULL << 32)) ? 8 : 4;
-       alloc_size = sizeof(struct chunk_table) + num_chunks * sizeof(u64);
-       chunk_tab = CALLOC(1, alloc_size);
+               struct wim_lookup_table_entry *res_stream;
+               u64 write_size = 0;

 
 

-       if (!chunk_tab) {
-               ERROR("Failed to allocate chunk table for %"PRIu64" byte "
-                     "resource", size);
-               return WIMLIB_ERR_NOMEM;
-       }
-       chunk_tab->num_chunks = num_chunks;
-       chunk_tab->original_resource_size = size;
-       chunk_tab->bytes_per_chunk_entry = bytes_per_chunk_entry;
-       chunk_tab->table_disk_size = chunk_tab->bytes_per_chunk_entry *
-                                    (num_chunks - 1);
-       chunk_tab->cur_offset_p = chunk_tab->offsets;
-
-       /* We don't know the correct offsets yet; so just write zeroes to
-        * reserve space for the table, so we can go back to it later after
-        * we've written the compressed chunks following it.
-        *
-        * Special case: if writing a pipable WIM, compressed resources are in a
-        * modified format (see comment above write_pipable_wim()) and do not
-        * have a chunk table at the beginning, so don't reserve any space for
-        * one.  */
-       if (!(resource_flags & WIMLIB_WRITE_RESOURCE_FLAG_PIPABLE)) {
-               ret = full_write(out_fd, chunk_tab->offsets,
-                                chunk_tab->table_disk_size);
-               if (ret) {
-                       ERROR_WITH_ERRNO("Failed to write chunk table in compressed "
-                                        "file resource");
-                       FREE(chunk_tab);
-                       return ret;
-               }
+               list_for_each_entry(res_stream, &rspec->stream_list, rspec_node)
+                       if (res_stream->will_be_in_output_wim)
+                               write_size += res_stream->size;
+
+               return (write_size > rspec->uncompressed_size / 2);

        }
        }

-       *chunk_tab_ret = chunk_tab;
-       return 0;
+       return false;

 }
 
 }
 

-/* Add the offset for the next chunk to the chunk table being constructed for a
- * compressed stream. */
-static void
-chunk_tab_record_chunk(struct chunk_table *chunk_tab, unsigned out_chunk_size)
+static u8
+filter_resource_flags(u8 flags)

 {
 {

-       if (chunk_tab->bytes_per_chunk_entry == 4) {
-               *(le32*)chunk_tab->cur_offset_p = cpu_to_le32(chunk_tab->cur_offset_u32);
-               chunk_tab->cur_offset_p = (le32*)chunk_tab->cur_offset_p + 1;
-               chunk_tab->cur_offset_u32 += out_chunk_size;
-       } else {
-               *(le64*)chunk_tab->cur_offset_p = cpu_to_le64(chunk_tab->cur_offset_u64);
-               chunk_tab->cur_offset_p = (le64*)chunk_tab->cur_offset_p + 1;
-               chunk_tab->cur_offset_u64 += out_chunk_size;
-       }
+       return (flags & ~(WIM_RESHDR_FLAG_PACKED_STREAMS |
+                         WIM_RESHDR_FLAG_COMPRESSED |
+                         WIM_RESHDR_FLAG_SPANNED |
+                         WIM_RESHDR_FLAG_FREE));

 }
 
 }
 

-/* Finishes a WIM chunk table and writes it to the output file at the correct
- * offset.  */
-static int
-finish_wim_resource_chunk_tab(struct chunk_table *chunk_tab,
-                             struct filedes *out_fd,
-                             off_t res_start_offset,
-                             int write_resource_flags)
+static void
+stream_set_out_reshdr_for_reuse(struct wim_lookup_table_entry *lte)

 {
 {

-       int ret;
+       const struct wim_resource_spec *rspec;

 
 

-       if (write_resource_flags & WIMLIB_WRITE_RESOURCE_FLAG_PIPABLE) {
-               ret = full_write(out_fd,
-                                chunk_tab->offsets +
-                                        chunk_tab->bytes_per_chunk_entry,
-                                chunk_tab->table_disk_size);
+       wimlib_assert(lte->resource_location == RESOURCE_IN_WIM);
+       rspec = lte->rspec;
+
+       if (rspec->flags & WIM_RESHDR_FLAG_PACKED_STREAMS) {
+
+               wimlib_assert(lte->flags & WIM_RESHDR_FLAG_PACKED_STREAMS);
+
+               lte->out_reshdr.offset_in_wim = lte->offset_in_res;
+               lte->out_reshdr.uncompressed_size = 0;
+               lte->out_reshdr.size_in_wim = lte->size;
+
+               lte->out_res_offset_in_wim = rspec->offset_in_wim;
+               lte->out_res_size_in_wim = rspec->size_in_wim;
+               lte->out_res_uncompressed_size = rspec->uncompressed_size;

        } else {
        } else {

-               ret = full_pwrite(out_fd,
-                                 chunk_tab->offsets +
-                                         chunk_tab->bytes_per_chunk_entry,
-                                 chunk_tab->table_disk_size,
-                                 res_start_offset);
+               wimlib_assert(!(lte->flags & WIM_RESHDR_FLAG_PACKED_STREAMS));
+
+               lte->out_reshdr.offset_in_wim = rspec->offset_in_wim;
+               lte->out_reshdr.uncompressed_size = rspec->uncompressed_size;
+               lte->out_reshdr.size_in_wim = rspec->size_in_wim;

        }
        }

-       if (ret)
-               ERROR_WITH_ERRNO("Write error");
-       return ret;
+       lte->out_reshdr.flags = lte->flags;

 }
 
 }
 

-/* Write the header for a stream in a pipable WIM.
- */
+
+/* Write the header for a stream in a pipable WIM.  */

 static int
 write_pwm_stream_header(const struct wim_lookup_table_entry *lte,
                        struct filedes *out_fd,
 static int
 write_pwm_stream_header(const struct wim_lookup_table_entry *lte,
                        struct filedes *out_fd,


@@ -270,15 +251,16 @@ write_pwm_stream_header(const struct wim_lookup_table_entry *lte,
                copy_hash(stream_hdr.hash, lte->hash);
        }
 
                copy_hash(stream_hdr.hash, lte->hash);
        }
 

-       reshdr_flags = lte->flags & ~(WIM_RESHDR_FLAG_COMPRESSED | WIM_RESHDR_FLAG_PACKED_STREAMS);
+       reshdr_flags = filter_resource_flags(lte->flags);

        reshdr_flags |= additional_reshdr_flags;
        stream_hdr.flags = cpu_to_le32(reshdr_flags);
        ret = full_write(out_fd, &stream_hdr, sizeof(stream_hdr));
        if (ret)
        reshdr_flags |= additional_reshdr_flags;
        stream_hdr.flags = cpu_to_le32(reshdr_flags);
        ret = full_write(out_fd, &stream_hdr, sizeof(stream_hdr));
        if (ret)

-               ERROR_WITH_ERRNO("Error writing stream header");
+               ERROR_WITH_ERRNO("Write error");

        return ret;
 }
 
        return ret;
 }
 

+#if 0

 static int
 seek_and_truncate(struct filedes *out_fd, off_t offset)
 {
 static int
 seek_and_truncate(struct filedes *out_fd, off_t offset)
 {


@@ -290,1530 +272,1119 @@ seek_and_truncate(struct filedes *out_fd, off_t offset)
        }
        return 0;
 }
        }
        return 0;
 }

+#endif

 
 

-static int
-finalize_and_check_sha1(SHA_CTX *sha_ctx, struct wim_lookup_table_entry *lte)
-{
-       u8 md[SHA1_HASH_SIZE];
-
-       sha1_final(md, sha_ctx);
-       if (lte->unhashed) {
-               copy_hash(lte->hash, md);
-       } else if (!hashes_equal(md, lte->hash)) {
-               ERROR("WIM resource has incorrect hash!");
-               if (lte_filename_valid(lte)) {
-                       ERROR("We were reading it from \"%"TS"\"; maybe "
-                             "it changed while we were reading it.",
-                             lte->file_on_disk);
+struct write_streams_progress_data {
+       wimlib_progress_func_t progress_func;
+       union wimlib_progress_info progress;
+       uint64_t next_progress;
+       WIMStruct *prev_wim_part;
+};
+
+static void
+do_write_streams_progress(struct write_streams_progress_data *progress_data,
+                         struct wim_lookup_table_entry *lte,
+                         bool stream_discarded)
+{
+       union wimlib_progress_info *progress = &progress_data->progress;
+       bool new_wim_part;
+
+       if (stream_discarded) {
+               progress->write_streams.total_bytes -= lte->size;
+               if (progress_data->next_progress != ~(uint64_t)0 &&
+                   progress_data->next_progress > progress->write_streams.total_bytes)
+               {
+                       progress_data->next_progress = progress->write_streams.total_bytes;
+               }
+       } else {
+               progress->write_streams.completed_bytes += lte->size;
+       }
+       new_wim_part = false;
+       if (lte->resource_location == RESOURCE_IN_WIM &&
+           lte->rspec->wim != progress_data->prev_wim_part)
+       {
+               if (progress_data->prev_wim_part) {
+                       new_wim_part = true;
+                       progress->write_streams.completed_parts++;
+               }
+               progress_data->prev_wim_part = lte->rspec->wim;
+       }
+       progress->write_streams.completed_streams++;
+       if (progress_data->progress_func
+           && (progress->write_streams.completed_bytes >= progress_data->next_progress
+               || new_wim_part))
+       {
+               progress_data->progress_func(WIMLIB_PROGRESS_MSG_WRITE_STREAMS,
+                                            progress);
+               if (progress_data->next_progress == progress->write_streams.total_bytes) {
+                       progress_data->next_progress = ~(uint64_t)0;
+               } else {
+                       progress_data->next_progress =
+                               min(progress->write_streams.total_bytes,
+                                   progress->write_streams.completed_bytes +
+                                       progress->write_streams.total_bytes / 100);

                }
                }

-               return WIMLIB_ERR_INVALID_RESOURCE_HASH;

        }
        }

-       return 0;

 }
 
 }
 

-struct write_resource_ctx {
+struct write_streams_ctx {
+       /* File descriptor the streams are being written to.  */
+       struct filedes *out_fd;
+
+       /* Lookup table for the WIMStruct on whose behalf the streams are being
+        * written.  */
+       struct wim_lookup_table *lookup_table;
+
+       /* Compression format to use.  */

        int out_ctype;
        int out_ctype;

+
+       /* Maximum uncompressed chunk size in compressed resources to use.  */

        u32 out_chunk_size;
        u32 out_chunk_size;

-       struct wimlib_lzx_context *comp_ctx;
-       struct chunk_table *chunk_tab;
-       struct filedes *out_fd;
-       SHA_CTX sha_ctx;
-       bool doing_sha;
-       int resource_flags;
-};

 
 

-static int
-write_resource_cb(const void *chunk, size_t chunk_size, void *_ctx)
-{
-       struct write_resource_ctx *ctx = _ctx;
-       const void *out_chunk;
-       unsigned out_chunk_size;
-       int ret;
-       void *compressed_chunk = NULL;
-       unsigned compressed_size;
-       bool compressed_chunk_malloced = false;
+       /* Flags that affect how the streams will be written.  */
+       int write_resource_flags;

 
 

-       if (ctx->doing_sha)
-               sha1_update(&ctx->sha_ctx, chunk, chunk_size);
+       /* Data used for issuing WRITE_STREAMS progress.  */
+       struct write_streams_progress_data progress_data;

 
 

-       out_chunk = chunk;
-       out_chunk_size = chunk_size;
-       if (ctx->out_ctype != WIMLIB_COMPRESSION_TYPE_NONE) {
+       struct filter_context *filter_ctx;

 
 

-               /* Compress the chunk.  */
-               if (chunk_size <= STACK_MAX) {
-                       compressed_chunk = alloca(chunk_size);
-               } else {
-                       compressed_chunk = MALLOC(chunk_size);
-                       if (compressed_chunk == NULL)
-                               return WIMLIB_ERR_NOMEM;
-                       compressed_chunk_malloced = true;
-               }
+       /* Upper bound on the total number of bytes that need to be compressed.
+        * */
+       u64 num_bytes_to_compress;

 
 

-               compressed_size = compress_chunk(chunk, chunk_size,
-                                                compressed_chunk,
-                                                ctx->out_ctype,
-                                                ctx->comp_ctx);
-               /* Use compressed data if compression to less than input size
-                * was successful.  */
-               if (compressed_size) {
-                       out_chunk = compressed_chunk;
-                       out_chunk_size = compressed_size;
-               }
-       }
+       /* Pointer to the chunk_compressor implementation being used for
+        * compressing chunks of data, or NULL if chunks are being written
+        * uncompressed.  */
+       struct chunk_compressor *compressor;

 
 

-       if (ctx->chunk_tab) {
-               /* Update chunk table accounting.  */
-               chunk_tab_record_chunk(ctx->chunk_tab, out_chunk_size);
+       /* Buffer for dividing the read data into chunks of size
+        * @out_chunk_size.  */
+       u8 *chunk_buf;

 
 

-               /* If writing compressed chunks to a pipable WIM, before the
-                * chunk data write a chunk header that provides the compressed
-                * chunk size.  */
-               if (ctx->resource_flags & WIMLIB_WRITE_RESOURCE_FLAG_PIPABLE) {
-                       struct pwm_chunk_hdr chunk_hdr = {
-                               .compressed_size = cpu_to_le32(out_chunk_size),
-                       };
-                       ret = full_write(ctx->out_fd, &chunk_hdr,
-                                        sizeof(chunk_hdr));
-                       if (ret)
-                               goto error;
-               }
-       }
+       /* Number of bytes in @chunk_buf that are currently filled.  */
+       size_t chunk_buf_filled;

 
 

-       /* Write the chunk data.  */
-       ret = full_write(ctx->out_fd, out_chunk, out_chunk_size);
-       if (ret)
-               goto error;
+       /* List of streams that currently have chunks being compressed.  */
+       struct list_head pending_streams;

 
 

-out_free_memory:
-       if (compressed_chunk_malloced)
-               FREE(compressed_chunk);
-       return ret;
+       /* Set to true if the stream currently being read was a duplicate, and
+        * therefore the corresponding stream entry needs to be freed once the
+        * read finishes.  (In this case we add the duplicate entry to
+        * pending_streams rather than the entry being read.)  */
+       bool stream_was_duplicate;

 
 

-error:
-       ERROR_WITH_ERRNO("Failed to write WIM resource chunk");
-       goto out_free_memory;
+       /* Current uncompressed offset in the resource being read.  */
+       u64 cur_read_res_offset;
+
+       /* Uncompressed size of the resource currently being read.  */
+       u64 cur_read_res_size;
+
+       /* Current uncompressed offset in the resource being written.  */
+       u64 cur_write_res_offset;
+
+       /* Uncompressed size of resource currently being written.  */
+       u64 cur_write_res_size;
+
+       /* Array that is filled in with compressed chunk sizes as a resource is
+        * being written.  */
+       u64 *chunk_csizes;
+
+       /* Index of next entry in @chunk_csizes to fill in.  */
+       size_t chunk_index;
+
+       /* Number of entries in @chunk_csizes currently allocated.  */
+       size_t num_alloc_chunks;
+
+       /* Offset in the output file of the start of the chunks of the resource
+        * currently being written.  */
+       u64 chunks_start_offset;
+};
+
+static u64
+get_chunk_entry_size(u64 res_size)
+{
+       if (res_size <= UINT32_MAX)
+               return 4;
+       else
+               return 8;

 }
 
 }
 

-/*
- * write_wim_resource()-
- *
- * Write a resource to an output WIM.
- *
- * @lte:
- *     Lookup table entry for the resource, which could be in another WIM, in
- *     an external file, or in another location.
- *
- * @out_fd:
- *     File descriptor opened to the output WIM.
- *
- * @out_ctype:
- *     One of the WIMLIB_COMPRESSION_TYPE_* constants to indicate which
- *     compression algorithm to use.
- *
- * @out_chunk_size:
- *     Compressed chunk size to use.
- *
- * @out_reshdr:
- *     On success, this is filled in with the offset, flags, compressed size,
- *     and uncompressed size of the resource in the output WIM.
- *
- * @resource_flags:
- *     * WIMLIB_WRITE_RESOURCE_FLAG_RECOMPRESS to force data to be recompressed even
- *       if it could otherwise be copied directly from the input;
- *     * WIMLIB_WRITE_RESOURCE_FLAG_PIPABLE if writing a resource for a pipable WIM
- *       (and the output file descriptor may be a pipe).
- *
- * @comp_ctx:
- *     Location of LZX compression context pointer, which will be allocated or
- *     updated if needed.  (Initialize to NULL.)
- *
- * Additional notes:  The SHA1 message digest of the uncompressed data is
- * calculated (except when doing a raw copy --- see below).  If the @unhashed
- * flag is set on the lookup table entry, this message digest is simply copied
- * to it; otherwise, the message digest is compared with the existing one, and
- * this function will fail if they do not match.
- */
+/* Reserve space for the chunk table and prepare to accumulate the chunk table
+ * in memory.  */

 static int
 static int

-write_wim_resource(struct wim_lookup_table_entry *lte,
-                  struct filedes *out_fd, int out_ctype,
-                  u32 out_chunk_size,
-                  struct wim_reshdr *out_reshdr,
-                  int resource_flags,
-                  struct wimlib_lzx_context **comp_ctx)
+begin_chunk_table(struct write_streams_ctx *ctx, u64 res_expected_size)

 {
 {

-       struct write_resource_ctx write_ctx;
-       off_t res_start_offset;
-       u64 read_size;
+       u64 expected_num_chunks;
+       u64 expected_num_chunk_entries;
+       size_t reserve_size;

        int ret;
 
        int ret;
 

-       /* Mask out any irrelevant flags, since this function also uses this
-        * variable to store WIMLIB_READ_RESOURCE flags.  */
-       resource_flags &= WIMLIB_WRITE_RESOURCE_MASK;
-
-       /* Get current position in output WIM.  */
-       res_start_offset = out_fd->offset;
-
-       /* If we are not forcing the data to be recompressed, and the input
-        * resource is located in a WIM with a compression mode compatible with
-        * the output, we can simply copy the compressed data without
-        * recompressing it.  This also means we must skip calculating the SHA1,
-        * as we never will see the uncompressed data.  */
-       if (can_raw_copy(lte, resource_flags, out_ctype, out_chunk_size)) {
-               /* Normally, for raw copies we can request a RAW_FULL read, but
-                * if we're reading from a pipable resource and writing a
-                * non-pipable resource or vice versa, then a RAW_CHUNKS read
-                * needs to be requested so that the written resource can be
-                * appropriately formatted.  However, in neither case is any
-                * actual decompression needed.  */
-               if (lte->rspec->is_pipable == !!(resource_flags &
-                                                WIMLIB_WRITE_RESOURCE_FLAG_PIPABLE))
-               {
-                       resource_flags |= WIMLIB_READ_RESOURCE_FLAG_RAW_FULL;
-                       read_size = lte->rspec->size_in_wim;
-               } else {
-                       resource_flags |= WIMLIB_READ_RESOURCE_FLAG_RAW_CHUNKS;
-                       read_size = lte->size;
+       /* Calculate the number of chunks and chunk entries that should be
+        * needed for the resource.  These normally will be the final values,
+        * but in PACKED_STREAMS mode some of the streams we're planning to
+        * write into the resource may be duplicates, and therefore discarded,
+        * potentially decreasing the number of chunk entries needed.  */
+       expected_num_chunks = DIV_ROUND_UP(res_expected_size, ctx->out_chunk_size);
+       expected_num_chunk_entries = expected_num_chunks;
+       if (!(ctx->write_resource_flags & WIMLIB_WRITE_RESOURCE_FLAG_PACK_STREAMS))
+               expected_num_chunk_entries--;
+
+       /* Make sure the chunk_csizes array is long enough to store the
+        * compressed size of each chunk.  */
+       if (expected_num_chunks > ctx->num_alloc_chunks) {
+               u64 new_length = expected_num_chunks + 50;
+
+               if ((size_t)new_length != new_length) {
+                       ERROR("Resource size too large (%"PRIu64" bytes!",
+                             res_expected_size);
+                       return WIMLIB_ERR_NOMEM;

                }
                }

-               write_ctx.doing_sha = false;
-       } else {
-               write_ctx.doing_sha = true;
-               sha1_init(&write_ctx.sha_ctx);
-               read_size = lte->size;
-       }

 
 

-       /* Set the output compression mode and initialize chunk table if needed.
-        */
-       write_ctx.out_ctype = WIMLIB_COMPRESSION_TYPE_NONE;
-       write_ctx.out_chunk_size = out_chunk_size;
-       write_ctx.chunk_tab = NULL;
-       if (out_ctype != WIMLIB_COMPRESSION_TYPE_NONE) {
-               wimlib_assert(out_chunk_size > 0);
-               if (!(resource_flags & WIMLIB_READ_RESOURCE_FLAG_RAW)) {
-                       /* Compression needed.  */
-                       write_ctx.out_ctype = out_ctype;
-                       if (out_ctype == WIMLIB_COMPRESSION_TYPE_LZX) {
-                               ret = wimlib_lzx_alloc_context(out_chunk_size,
-                                                              NULL, comp_ctx);
-                               if (ret)
-                                       goto out;
-                       }
-                       write_ctx.comp_ctx = *comp_ctx;
-               }
-               if (!(resource_flags & WIMLIB_READ_RESOURCE_FLAG_RAW_FULL)) {
-                       /* Chunk table needed.  */
-                       ret = begin_wim_resource_chunk_tab(lte, out_fd,
-                                                          out_chunk_size,
-                                                          &write_ctx.chunk_tab,
-                                                          resource_flags);
-                       if (ret)
-                               goto out;
+               FREE(ctx->chunk_csizes);
+               ctx->chunk_csizes = MALLOC(new_length * sizeof(ctx->chunk_csizes[0]));
+               if (ctx->chunk_csizes == NULL) {
+                       ctx->num_alloc_chunks = 0;
+                       return WIMLIB_ERR_NOMEM;

                }
                }

-       }
-
-       /* If writing a pipable resource, write the stream header and update
-        * @res_start_offset to be the end of the stream header.  */
-       if (resource_flags & WIMLIB_WRITE_RESOURCE_FLAG_PIPABLE) {
-               int reshdr_flags = 0;
-               if (out_ctype != WIMLIB_COMPRESSION_TYPE_NONE)
-                       reshdr_flags |= WIM_RESHDR_FLAG_COMPRESSED;
-               ret = write_pwm_stream_header(lte, out_fd, reshdr_flags);
+               ctx->num_alloc_chunks = new_length;
+       }
+
+       ctx->chunk_index = 0;
+
+       if (!(ctx->write_resource_flags & WIMLIB_WRITE_RESOURCE_FLAG_PIPABLE)) {
+               /* Reserve space for the chunk table in the output file.  In the
+                * case of packed resources this reserves the upper bound for
+                * the needed space, not necessarily the exact space which will
+                * prove to be needed.  At this point, we just use @chunk_csizes
+                * for a buffer of 0's because the actual compressed chunk sizes
+                * are unknown.  */
+               reserve_size = expected_num_chunk_entries * get_chunk_entry_size(res_expected_size);
+               if (ctx->write_resource_flags & WIMLIB_WRITE_RESOURCE_FLAG_PACK_STREAMS)
+                       reserve_size += sizeof(struct alt_chunk_table_header_disk);
+               memset(ctx->chunk_csizes, 0, reserve_size);
+               ret = full_write(ctx->out_fd, ctx->chunk_csizes, reserve_size);

                if (ret)
                if (ret)

-                       goto out_free_chunk_tab;
-               res_start_offset = out_fd->offset;
+                       return ret;

        }
        }

+       return 0;
+}

 
 

-       /* Write the entire resource by reading the entire resource and feeding
-        * the data through write_resource_cb().  */
-       write_ctx.out_fd = out_fd;
-       write_ctx.resource_flags = resource_flags;
-try_write_again:
-       ret = read_stream_prefix(lte, read_size, write_resource_cb,
-                                &write_ctx, resource_flags);
-       if (ret)
-               goto out_free_chunk_tab;
+static int
+begin_write_resource(struct write_streams_ctx *ctx, u64 res_expected_size)
+{
+       int ret;

 
 

-       /* Verify SHA1 message digest of the resource, or set the hash for the
-        * first time. */
-       if (write_ctx.doing_sha) {
-               ret = finalize_and_check_sha1(&write_ctx.sha_ctx, lte);
-               if (ret)
-                       goto out_free_chunk_tab;
-       }
+       wimlib_assert(res_expected_size != 0);

 
 

-       /* Write chunk table if needed.  */
-       if (write_ctx.chunk_tab) {
-               ret = finish_wim_resource_chunk_tab(write_ctx.chunk_tab,
-                                                   out_fd,
-                                                   res_start_offset,
-                                                   resource_flags);
+       if (ctx->compressor != NULL) {
+               ret = begin_chunk_table(ctx, res_expected_size);

                if (ret)
                if (ret)

-                       goto out_free_chunk_tab;
+                       return ret;

        }
 
        }
 

-       /* Fill in out_reshdr with information about the newly written
-        * resource.  */
-       out_reshdr->size_in_wim   = out_fd->offset - res_start_offset;
-       out_reshdr->flags         = lte->flags & ~WIM_RESHDR_FLAG_PACKED_STREAMS;
-       if (out_ctype == WIMLIB_COMPRESSION_TYPE_NONE)
-               out_reshdr->flags &= ~WIM_RESHDR_FLAG_COMPRESSED;
-       else
-               out_reshdr->flags |= WIM_RESHDR_FLAG_COMPRESSED;
-       out_reshdr->offset_in_wim  = res_start_offset;
-       out_reshdr->uncompressed_size = lte->size;
-
-       /* Check for resources compressed to greater than their original size
-        * and write them uncompressed instead.  (But never do this if writing
-        * to a pipe, and don't bother if we did a raw copy.)  */
-       if (out_reshdr->size_in_wim > out_reshdr->uncompressed_size &&
-           !(resource_flags & (WIMLIB_WRITE_RESOURCE_FLAG_PIPABLE |
-                               WIMLIB_READ_RESOURCE_FLAG_RAW)))
-       {
-               DEBUG("Compressed %"PRIu64" => %"PRIu64" bytes; "
-                     "writing uncompressed instead",
-                     out_reshdr->uncompressed_size, out_reshdr->size_in_wim);
-               ret = seek_and_truncate(out_fd, res_start_offset);
-               if (ret)
-                       goto out_free_chunk_tab;
-               out_ctype = WIMLIB_COMPRESSION_TYPE_NONE;
-               FREE(write_ctx.chunk_tab);
-               write_ctx.out_ctype = WIMLIB_COMPRESSION_TYPE_NONE;
-               write_ctx.chunk_tab = NULL;
-               write_ctx.doing_sha = false;
-               goto try_write_again;
-       }
-       if (resource_flags & WIMLIB_READ_RESOURCE_FLAG_RAW) {
-               DEBUG("Copied raw compressed data "
-                     "(%"PRIu64" => %"PRIu64" bytes @ +%"PRIu64", flags=0x%02x)",
-                     out_reshdr->uncompressed_size, out_reshdr->size_in_wim,
-                     out_reshdr->offset_in_wim, out_reshdr->flags);
-       } else if (out_ctype != WIMLIB_COMPRESSION_TYPE_NONE) {
-               DEBUG("Wrote compressed resource "
-                     "(%"PRIu64" => %"PRIu64" bytes @ +%"PRIu64", flags=0x%02x)",
-                     out_reshdr->uncompressed_size, out_reshdr->size_in_wim,
-                     out_reshdr->offset_in_wim, out_reshdr->flags);
-       } else {
-               DEBUG("Wrote uncompressed resource "
-                     "(%"PRIu64" bytes @ +%"PRIu64", flags=0x%02x)",
-                     out_reshdr->uncompressed_size,
-                     out_reshdr->offset_in_wim, out_reshdr->flags);
-       }
-       ret = 0;
-out_free_chunk_tab:
-       FREE(write_ctx.chunk_tab);
-out:
-       return ret;
+       /* Output file descriptor is now positioned at the offset at which to
+        * write the first chunk of the resource.  */
+       ctx->chunks_start_offset = ctx->out_fd->offset;
+       ctx->cur_write_res_offset = 0;
+       ctx->cur_write_res_size = res_expected_size;
+       return 0;

 }
 
 }
 

-/* Like write_wim_resource(), but the resource is specified by a buffer of
- * uncompressed data rather a lookup table entry.  Also writes the SHA1 message
- * digest of the buffer to @hash_ret if it is non-NULL.  */
-int
-write_wim_resource_from_buffer(const void *buf, size_t buf_size,
-                              int reshdr_flags, struct filedes *out_fd,
-                              int out_ctype,
-                              u32 out_chunk_size,
-                              struct wim_reshdr *out_reshdr,
-                              u8 *hash_ret, int write_resource_flags,
-                              struct wimlib_lzx_context **comp_ctx)
+static int
+end_chunk_table(struct write_streams_ctx *ctx, u64 res_actual_size,
+               u64 *res_start_offset_ret, u64 *res_store_size_ret)

 {
 {

+       size_t actual_num_chunks;
+       size_t actual_num_chunk_entries;
+       size_t chunk_entry_size;

        int ret;
        int ret;

-       struct wim_lookup_table_entry *lte;

 
 

-       /* Set up a temporary lookup table entry to provide to
-        * write_wim_resource().  */
+       actual_num_chunks = ctx->chunk_index;
+       actual_num_chunk_entries = actual_num_chunks;
+       if (!(ctx->write_resource_flags & WIMLIB_WRITE_RESOURCE_FLAG_PACK_STREAMS))
+               actual_num_chunk_entries--;

 
 

-       lte = new_lookup_table_entry();
-       if (lte == NULL)
-               return WIMLIB_ERR_NOMEM;
+       chunk_entry_size = get_chunk_entry_size(res_actual_size);

 
 

-       lte->resource_location  = RESOURCE_IN_ATTACHED_BUFFER;
-       lte->attached_buffer    = (void*)buf;
-       lte->size               = buf_size;
-       lte->flags              = reshdr_flags;
+       typedef le64 __attribute__((may_alias)) aliased_le64_t;
+       typedef le32 __attribute__((may_alias)) aliased_le32_t;

 
 

-       if (write_resource_flags & WIMLIB_WRITE_RESOURCE_FLAG_PIPABLE) {
-               sha1_buffer(buf, buf_size, lte->hash);
-               lte->unhashed = 0;
+       if (chunk_entry_size == 4) {
+               aliased_le32_t *entries = (aliased_le32_t*)ctx->chunk_csizes;
+
+               if (ctx->write_resource_flags & WIMLIB_WRITE_RESOURCE_FLAG_PACK_STREAMS) {
+                       for (size_t i = 0; i < actual_num_chunk_entries; i++)
+                               entries[i] = cpu_to_le32(ctx->chunk_csizes[i]);
+               } else {
+                       u32 offset = ctx->chunk_csizes[0];
+                       for (size_t i = 0; i < actual_num_chunk_entries; i++) {
+                               u32 next_size = ctx->chunk_csizes[i + 1];
+                               entries[i] = cpu_to_le32(offset);
+                               offset += next_size;
+                       }
+               }

        } else {
        } else {

-               lte->unhashed = 1;
+               aliased_le64_t *entries = (aliased_le64_t*)ctx->chunk_csizes;
+
+               if (ctx->write_resource_flags & WIMLIB_WRITE_RESOURCE_FLAG_PACK_STREAMS) {
+                       for (size_t i = 0; i < actual_num_chunk_entries; i++)
+                               entries[i] = cpu_to_le64(ctx->chunk_csizes[i]);
+               } else {
+                       u64 offset = ctx->chunk_csizes[0];
+                       for (size_t i = 0; i < actual_num_chunk_entries; i++) {
+                               u64 next_size = ctx->chunk_csizes[i + 1];
+                               entries[i] = cpu_to_le64(offset);
+                               offset += next_size;
+                       }
+               }

        }
 
        }
 

-       ret = write_wim_resource(lte, out_fd, out_ctype, out_chunk_size,
-                                out_reshdr, write_resource_flags, comp_ctx);
-       if (ret)
-               goto out_free_lte;
-       if (hash_ret)
-               copy_hash(hash_ret, lte->hash);
-       ret = 0;
-out_free_lte:
-       lte->resource_location = RESOURCE_NONEXISTENT;
-       free_lookup_table_entry(lte);
-       return ret;
-}
+       size_t chunk_table_size = actual_num_chunk_entries * chunk_entry_size;
+       u64 res_start_offset;
+       u64 res_end_offset = ctx->out_fd->offset;

 
 

+       if (ctx->write_resource_flags & WIMLIB_WRITE_RESOURCE_FLAG_PIPABLE) {
+               ret = full_write(ctx->out_fd, ctx->chunk_csizes, chunk_table_size);
+               if (ret)
+                       goto error;
+               res_end_offset = ctx->out_fd->offset;
+               res_start_offset = ctx->chunks_start_offset;
+       } else {
+               res_end_offset = ctx->out_fd->offset;

 
 

-#ifdef ENABLE_MULTITHREADED_COMPRESSION
+               u64 chunk_table_offset;

 
 

-/* Blocking shared queue (solves the producer-consumer problem) */
-struct shared_queue {
-       unsigned size;
-       unsigned front;
-       unsigned back;
-       unsigned filled_slots;
-       void **array;
-       pthread_mutex_t lock;
-       pthread_cond_t msg_avail_cond;
-       pthread_cond_t space_avail_cond;
-};
+               chunk_table_offset = ctx->chunks_start_offset - chunk_table_size;

 
 

-static int
-shared_queue_init(struct shared_queue *q, unsigned size)
-{
-       wimlib_assert(size != 0);
-       q->array = CALLOC(sizeof(q->array[0]), size);
-       if (!q->array)
-               goto err;
-       q->filled_slots = 0;
-       q->front = 0;
-       q->back = size - 1;
-       q->size = size;
-       if (pthread_mutex_init(&q->lock, NULL)) {
-               ERROR_WITH_ERRNO("Failed to initialize mutex");
-               goto err;
-       }
-       if (pthread_cond_init(&q->msg_avail_cond, NULL)) {
-               ERROR_WITH_ERRNO("Failed to initialize condition variable");
-               goto err_destroy_lock;
-       }
-       if (pthread_cond_init(&q->space_avail_cond, NULL)) {
-               ERROR_WITH_ERRNO("Failed to initialize condition variable");
-               goto err_destroy_msg_avail_cond;
+               if (ctx->write_resource_flags & WIMLIB_WRITE_RESOURCE_FLAG_PACK_STREAMS) {
+                       struct alt_chunk_table_header_disk hdr;
+
+                       hdr.res_usize = cpu_to_le64(res_actual_size);
+                       hdr.chunk_size = cpu_to_le32(ctx->out_chunk_size);
+                       hdr.compression_format = cpu_to_le32(ctx->out_ctype);
+
+                       BUILD_BUG_ON(WIMLIB_COMPRESSION_TYPE_LZX != 1);
+                       BUILD_BUG_ON(WIMLIB_COMPRESSION_TYPE_XPRESS != 2);
+                       BUILD_BUG_ON(WIMLIB_COMPRESSION_TYPE_LZMS != 3);
+
+                       ret = full_pwrite(ctx->out_fd, &hdr, sizeof(hdr),
+                                         chunk_table_offset - sizeof(hdr));
+                       if (ret)
+                               goto error;
+                       res_start_offset = chunk_table_offset - sizeof(hdr);
+               } else {
+                       res_start_offset = chunk_table_offset;
+               }
+
+               ret = full_pwrite(ctx->out_fd, ctx->chunk_csizes,
+                                 chunk_table_size, chunk_table_offset);
+               if (ret)
+                       goto error;

        }
        }

+
+       *res_start_offset_ret = res_start_offset;
+       *res_store_size_ret = res_end_offset - res_start_offset;
+

        return 0;
        return 0;

-err_destroy_msg_avail_cond:
-       pthread_cond_destroy(&q->msg_avail_cond);
-err_destroy_lock:
-       pthread_mutex_destroy(&q->lock);
-err:
-       return WIMLIB_ERR_NOMEM;
-}

 
 

-static void
-shared_queue_destroy(struct shared_queue *q)
-{
-       FREE(q->array);
-       pthread_mutex_destroy(&q->lock);
-       pthread_cond_destroy(&q->msg_avail_cond);
-       pthread_cond_destroy(&q->space_avail_cond);
+error:
+       ERROR_WITH_ERRNO("Write error");
+       return ret;

 }
 
 }
 

-static void
-shared_queue_put(struct shared_queue *q, void *obj)
+/* Finish writing a WIM resource by writing or updating the chunk table (if not
+ * writing the data uncompressed) and loading its metadata into @out_reshdr.  */
+static int
+end_write_resource(struct write_streams_ctx *ctx, struct wim_reshdr *out_reshdr)

 {
 {

-       pthread_mutex_lock(&q->lock);
-       while (q->filled_slots == q->size)
-               pthread_cond_wait(&q->space_avail_cond, &q->lock);
+       int ret;
+       u64 res_size_in_wim;
+       u64 res_uncompressed_size;
+       u64 res_offset_in_wim;

 
 

-       q->back = (q->back + 1) % q->size;
-       q->array[q->back] = obj;
-       q->filled_slots++;
+       wimlib_assert(ctx->cur_write_res_size == ctx->cur_write_res_offset);
+       res_uncompressed_size = ctx->cur_write_res_size;

 
 

-       pthread_cond_broadcast(&q->msg_avail_cond);
-       pthread_mutex_unlock(&q->lock);
+       if (ctx->compressor) {
+               ret = end_chunk_table(ctx, res_uncompressed_size,
+                                     &res_offset_in_wim, &res_size_in_wim);
+               if (ret)
+                       return ret;
+       } else {
+               res_offset_in_wim = ctx->chunks_start_offset;
+               res_size_in_wim = ctx->out_fd->offset - res_offset_in_wim;
+       }
+       out_reshdr->uncompressed_size = res_uncompressed_size;
+       out_reshdr->size_in_wim = res_size_in_wim;
+       out_reshdr->offset_in_wim = res_offset_in_wim;
+       DEBUG("Finished writing resource: %"PRIu64" => %"PRIu64" @ %"PRIu64"",
+             res_uncompressed_size, res_size_in_wim, res_offset_in_wim);
+       return 0;

 }
 
 }
 

-static void *
-shared_queue_get(struct shared_queue *q)
+/* Begin processing a stream for writing.  */
+static int
+write_stream_begin_read(struct wim_lookup_table_entry *lte,
+                       bool is_partial_res, void *_ctx)

 {
 {

-       void *obj;
+       struct write_streams_ctx *ctx = _ctx;
+       int ret;

 
 

-       pthread_mutex_lock(&q->lock);
-       while (q->filled_slots == 0)
-               pthread_cond_wait(&q->msg_avail_cond, &q->lock);
+       wimlib_assert(lte->size > 0);

 
 

-       obj = q->array[q->front];
-       q->array[q->front] = NULL;
-       q->front = (q->front + 1) % q->size;
-       q->filled_slots--;
+       ctx->cur_read_res_offset = 0;
+       ctx->cur_read_res_size = lte->size;

 
 

-       pthread_cond_broadcast(&q->space_avail_cond);
-       pthread_mutex_unlock(&q->lock);
-       return obj;
-}
+       /* As an optimization, we allow some streams to be "unhashed", meaning
+        * their SHA1 message digests are unknown.  This is the case with
+        * streams that are added by scanning a directry tree with
+        * wimlib_add_image(), for example.  Since WIM uses single-instance
+        * streams, we don't know whether such each such stream really need to
+        * written until it is actually checksummed, unless it has a unique
+        * size.  In such cases we read and checksum the stream in this
+        * function, thereby advancing ahead of read_stream_list(), which will
+        * still provide the data again to write_stream_process_chunk().  This
+        * is okay because an unhashed stream cannot be in a WIM resource, which
+        * might be costly to decompress.  */
+       ctx->stream_was_duplicate = false;
+       if (ctx->lookup_table != NULL && lte->unhashed && !lte->unique_size) {

 
 

-struct compressor_thread_params {
-       struct shared_queue *res_to_compress_queue;
-       struct shared_queue *compressed_res_queue;
-       int out_ctype;
-       struct wimlib_lzx_context *comp_ctx;
-};
+               wimlib_assert(!is_partial_res);
+               wimlib_assert(ctx->lookup_table != NULL);

 
 

-#define MAX_CHUNKS_PER_MSG 2
+               struct wim_lookup_table_entry *lte_new;

 
 

-struct message {
-       struct wim_lookup_table_entry *lte;
-       u32 out_chunk_size;
-       u8 *uncompressed_chunks[MAX_CHUNKS_PER_MSG];
-       u8 *compressed_chunks[MAX_CHUNKS_PER_MSG];
-       unsigned uncompressed_chunk_sizes[MAX_CHUNKS_PER_MSG];
-       struct iovec out_chunks[MAX_CHUNKS_PER_MSG];
-       unsigned num_chunks;
-       struct list_head list;
-       bool complete;
-       u64 begin_chunk;
-};
+               ret = hash_unhashed_stream(lte, ctx->lookup_table, &lte_new);
+               if (ret)
+                       return ret;
+               if (lte_new != lte) {
+                       /* Duplicate stream detected.  */

 
 

-static void
-compress_chunks(struct message *msg, int out_ctype,
-               struct wimlib_lzx_context *comp_ctx)
-{
-       for (unsigned i = 0; i < msg->num_chunks; i++) {
-               unsigned len;
-
-               len = compress_chunk(msg->uncompressed_chunks[i],
-                                    msg->uncompressed_chunk_sizes[i],
-                                    msg->compressed_chunks[i],
-                                    out_ctype,
-                                    comp_ctx);
-
-               void *out_chunk;
-               unsigned out_len;
-               if (len) {
-                       /* To be written compressed */
-                       out_chunk = msg->compressed_chunks[i];
-                       out_len = len;
-               } else {
-                       /* To be written uncompressed */
-                       out_chunk = msg->uncompressed_chunks[i];
-                       out_len = msg->uncompressed_chunk_sizes[i];
+                       if (lte_new->will_be_in_output_wim ||
+                           stream_filtered(lte_new, ctx->filter_ctx))
+                       {
+                               /* The duplicate stream is already being
+                                * included in the output WIM, or it would be
+                                * filtered out if it had been.  Skip writing
+                                * this stream (and reading it again) entirely,
+                                * passing its output reference count to the
+                                * duplicate stream in the former case.  */
+                               DEBUG("Discarding duplicate stream of "
+                                     "length %"PRIu64, lte->size);
+                               do_write_streams_progress(&ctx->progress_data,
+                                                         lte, true);
+                               list_del(&lte->write_streams_list);
+                               list_del(&lte->lookup_table_list);
+                               if (lte_new->will_be_in_output_wim)
+                                       lte_new->out_refcnt += lte->out_refcnt;
+                               if (ctx->write_resource_flags & WIMLIB_WRITE_RESOURCE_FLAG_PACK_STREAMS)
+                                       ctx->cur_write_res_size -= lte->size;
+                               free_lookup_table_entry(lte);
+                               return BEGIN_STREAM_STATUS_SKIP_STREAM;
+                       } else {
+                               /* The duplicate stream can validly be written,
+                                * but was not marked as such.  Discard the
+                                * current stream entry and use the duplicate,
+                                * but actually freeing the current entry must
+                                * wait until read_stream_list() has finished
+                                * reading its data.  */
+                               DEBUG("Stream duplicate, but not already "
+                                     "selected for writing.");
+                               list_replace(&lte->write_streams_list,
+                                            &lte_new->write_streams_list);
+                               list_replace(&lte->lookup_table_list,
+                                            &lte_new->lookup_table_list);
+                               lte_new->out_refcnt = lte->out_refcnt;
+                               lte_new->will_be_in_output_wim = 1;
+                               ctx->stream_was_duplicate = true;
+                               lte = lte_new;
+                       }

                }
                }

-               msg->out_chunks[i].iov_base = out_chunk;
-               msg->out_chunks[i].iov_len = out_len;

        }
        }

+       list_move_tail(&lte->write_streams_list, &ctx->pending_streams);
+       return 0;

 }
 
 }
 

-/* Compressor thread routine.  This is a lot simpler than the main thread
- * routine: just repeatedly get a group of chunks from the
- * res_to_compress_queue, compress them, and put them in the
- * compressed_res_queue.  A NULL pointer indicates that the thread should stop.
- * */
-static void *
-compressor_thread_proc(void *arg)
+/* Write the next chunk of (typically compressed) data to the output WIM,
+ * handling the writing of the chunk table.  */
+static int
+write_chunk(struct write_streams_ctx *ctx, const void *cchunk,
+           size_t csize, size_t usize)

 {
 {

-       struct compressor_thread_params *params = arg;
-       struct shared_queue *res_to_compress_queue = params->res_to_compress_queue;
-       struct shared_queue *compressed_res_queue = params->compressed_res_queue;
-       struct message *msg;
+       int ret;

 
 

-       DEBUG("Compressor thread ready");
-       while ((msg = shared_queue_get(res_to_compress_queue)) != NULL) {
-               compress_chunks(msg, params->out_ctype, params->comp_ctx);
-               shared_queue_put(compressed_res_queue, msg);
-       }
-       DEBUG("Compressor thread terminating");
-       return NULL;
-}
-#endif /* ENABLE_MULTITHREADED_COMPRESSION */
+       if (ctx->cur_write_res_offset == 0 &&
+           !(ctx->write_resource_flags & WIMLIB_WRITE_RESOURCE_FLAG_PACK_STREAMS))
+       {
+               /* Starting to write a new stream in non-packed mode.  */

 
 

-struct write_streams_progress_data {
-       wimlib_progress_func_t progress_func;
-       union wimlib_progress_info progress;
-       uint64_t next_progress;
-       WIMStruct *prev_wim_part;
-};
+               struct wim_lookup_table_entry *lte;

 
 

-static void
-do_write_streams_progress(struct write_streams_progress_data *progress_data,
-                         struct wim_lookup_table_entry *lte,
-                         bool stream_discarded)
-{
-       union wimlib_progress_info *progress = &progress_data->progress;
-       bool new_wim_part;
+               lte = list_entry(ctx->pending_streams.next,
+                                struct wim_lookup_table_entry, write_streams_list);

 
 

-       if (stream_discarded) {
-               progress->write_streams.total_bytes -= lte->size;
-               if (progress_data->next_progress != ~(uint64_t)0 &&
-                   progress_data->next_progress > progress->write_streams.total_bytes)
-               {
-                       progress_data->next_progress = progress->write_streams.total_bytes;
-               }
-       } else {
-               progress->write_streams.completed_bytes += lte->size;
-       }
-       new_wim_part = false;
-       if (lte->resource_location == RESOURCE_IN_WIM &&
-           lte->rspec->wim != progress_data->prev_wim_part)
-       {
-               if (progress_data->prev_wim_part) {
-                       new_wim_part = true;
-                       progress->write_streams.completed_parts++;
+               if (ctx->write_resource_flags & WIMLIB_WRITE_RESOURCE_FLAG_PIPABLE) {
+                       int additional_reshdr_flags = 0;
+                       if (ctx->compressor != NULL)
+                               additional_reshdr_flags |= WIM_RESHDR_FLAG_COMPRESSED;
+
+                       DEBUG("Writing pipable WIM stream header "
+                             "(offset=%"PRIu64")", ctx->out_fd->offset);
+
+                       ret = write_pwm_stream_header(lte, ctx->out_fd,
+                                                     additional_reshdr_flags);
+                       if (ret)
+                               return ret;

                }
                }

-               progress_data->prev_wim_part = lte->rspec->wim;
+
+               ret = begin_write_resource(ctx, lte->size);
+               if (ret)
+                       return ret;

        }
        }

-       progress->write_streams.completed_streams++;
-       if (progress_data->progress_func
-           && (progress->write_streams.completed_bytes >= progress_data->next_progress
-               || new_wim_part))
-       {
-               progress_data->progress_func(WIMLIB_PROGRESS_MSG_WRITE_STREAMS,
-                                            progress);
-               if (progress_data->next_progress == progress->write_streams.total_bytes) {
-                       progress_data->next_progress = ~(uint64_t)0;
-               } else {
-                       progress_data->next_progress =
-                               min(progress->write_streams.total_bytes,
-                                   progress->write_streams.completed_bytes +
-                                       progress->write_streams.total_bytes / 100);
+
+       if (ctx->compressor != NULL) {
+               /* Record the compresed chunk size.  */
+               wimlib_assert(ctx->chunk_index < ctx->num_alloc_chunks);
+               ctx->chunk_csizes[ctx->chunk_index++] = csize;
+
+              /* If writing a pipable WIM, before the chunk data write a chunk
+               * header that provides the compressed chunk size.  */
+               if (ctx->write_resource_flags & WIMLIB_WRITE_RESOURCE_FLAG_PIPABLE) {
+                       struct pwm_chunk_hdr chunk_hdr = {
+                               .compressed_size = cpu_to_le32(csize),
+                       };
+                       ret = full_write(ctx->out_fd, &chunk_hdr,
+                                        sizeof(chunk_hdr));
+                       if (ret)
+                               goto error;

                }
        }
                }
        }

-}

 
 

-struct serial_write_stream_ctx {
-       struct filedes *out_fd;
-       int out_ctype;
-       u32 out_chunk_size;
-       struct wimlib_lzx_context **comp_ctx;
-       int write_resource_flags;
-};
+       /* Write the chunk data.  */
+       ret = full_write(ctx->out_fd, cchunk, csize);
+       if (ret)
+               goto error;

 
 

-static int
-serial_write_stream(struct wim_lookup_table_entry *lte, void *_ctx)
-{
-       struct serial_write_stream_ctx *ctx = _ctx;
-       return write_wim_resource(lte, ctx->out_fd,
-                                 ctx->out_ctype,
-                                 ctx->out_chunk_size,
-                                 &lte->out_reshdr,
-                                 ctx->write_resource_flags,
-                                 ctx->comp_ctx);
-}
+       ctx->cur_write_res_offset += usize;

 
 

+       if (ctx->cur_write_res_offset == ctx->cur_write_res_size &&
+           !(ctx->write_resource_flags & WIMLIB_WRITE_RESOURCE_FLAG_PACK_STREAMS))
+       {
+               struct wim_lookup_table_entry *lte;

 
 

-/* Write a list of streams, taking into account that some streams may be
- * duplicates that are checksummed and discarded on the fly, and also delegating
- * the actual writing of a stream to a function @write_stream_cb, which is
- * passed the context @write_stream_ctx. */
-static int
-do_write_stream_list(struct list_head *stream_list,
-                    struct wim_lookup_table *lookup_table,
-                    int (*write_stream_cb)(struct wim_lookup_table_entry *, void *),
-                    void *write_stream_ctx,
-                    struct write_streams_progress_data *progress_data)
-{
-       int ret = 0;
-       struct wim_lookup_table_entry *lte;
-       bool stream_discarded;
-
-       /* For each stream in @stream_list ... */
-       while (!list_empty(stream_list)) {
-               stream_discarded = false;
-               lte = container_of(stream_list->next,
-                                  struct wim_lookup_table_entry,
-                                  write_streams_list);
-               list_del(&lte->write_streams_list);
-               if (lte->unhashed && !lte->unique_size) {
-                       /* Unhashed stream that shares a size with some other
-                        * stream in the WIM we are writing.  The stream must be
-                        * checksummed to know if we need to write it or not. */
-                       struct wim_lookup_table_entry *tmp;
-                       u32 orig_out_refcnt = lte->out_refcnt;
-
-                       ret = hash_unhashed_stream(lte, lookup_table, &tmp);
-                       if (ret)
-                               break;
-                       if (tmp != lte) {
-                               /* We found a duplicate stream.  'lte' was
-                                * freed, so replace it with the duplicate.  */
-                               lte = tmp;
-
-                               /* 'out_refcnt' was transferred to the
-                                * duplicate, and we can detect if the duplicate
-                                * stream was already referenced for writing by
-                                * checking if its 'out_refcnt' is higher than
-                                * that of the original stream.  In such cases,
-                                * the current stream can be discarded.  We can
-                                * also discard the current stream if it was
-                                * previously marked as filtered (e.g. already
-                                * present in the WIM being written).  */
-                               if (lte->out_refcnt > orig_out_refcnt ||
-                                   lte->filtered) {
-                                       DEBUG("Discarding duplicate stream of "
-                                             "length %"PRIu64,
-                                             lte->size);
-                                       lte->no_progress = 0;
-                                       stream_discarded = true;
-                                       goto skip_to_progress;
-                               }
-                       }
-               }
+               lte = list_entry(ctx->pending_streams.next,
+                                struct wim_lookup_table_entry, write_streams_list);
+               wimlib_assert(ctx->cur_write_res_offset == lte->size);
+
+               /* Finished writing a stream in non-packed mode.  */

 
 

-               /* Here, @lte is either a hashed stream or an unhashed stream
-                * with a unique size.  In either case we know that the stream
-                * has to be written.  In either case the SHA1 message digest
-                * will be calculated over the stream while writing it; however,
-                * in the former case this is done merely to check the data,
-                * while in the latter case this is done because we do not have
-                * the SHA1 message digest yet.  */
-               wimlib_assert(lte->out_refcnt != 0);
-               lte->deferred = 0;
-               lte->no_progress = 0;
-               ret = (*write_stream_cb)(lte, write_stream_ctx);
+               do_write_streams_progress(&ctx->progress_data, lte, false);
+
+               ret = end_write_resource(ctx, &lte->out_reshdr);

                if (ret)
                if (ret)

-                       break;
-               /* In parallel mode, some streams are deferred for later,
-                * serialized processing; ignore them here.  */
-               if (lte->deferred)
-                       continue;
-               if (lte->unhashed) {
-                       list_del(&lte->unhashed_list);
-                       lookup_table_insert(lookup_table, lte);
-                       lte->unhashed = 0;
-               }
-       skip_to_progress:
-               if (!lte->no_progress) {
-                       do_write_streams_progress(progress_data,
-                                                 lte, stream_discarded);
-               }
+                       return ret;
+
+               wimlib_assert(lte->out_reshdr.uncompressed_size == lte->size);
+
+               lte->out_reshdr.flags = filter_resource_flags(lte->flags);
+               if (ctx->compressor != NULL)
+                       lte->out_reshdr.flags |= WIM_RESHDR_FLAG_COMPRESSED;
+
+               list_del(&lte->write_streams_list);
+               ctx->cur_write_res_offset = 0;

        }
        }

+
+       return 0;
+
+error:
+       ERROR_WITH_ERRNO("Write error");

        return ret;
 }
 
 static int
        return ret;
 }
 
 static int

-do_write_stream_list_serial(struct list_head *stream_list,
-                           struct wim_lookup_table *lookup_table,
-                           struct filedes *out_fd,
-                           int out_ctype,
-                           u32 out_chunk_size,
-                           struct wimlib_lzx_context **comp_ctx,
-                           int write_resource_flags,
-                           struct write_streams_progress_data *progress_data)
-{
-       struct serial_write_stream_ctx ctx = {
-               .out_fd = out_fd,
-               .out_ctype = out_ctype,
-               .out_chunk_size = out_chunk_size,
-               .write_resource_flags = write_resource_flags,
-               .comp_ctx = comp_ctx,
-       };
-       return do_write_stream_list(stream_list,
-                                   lookup_table,
-                                   serial_write_stream,
-                                   &ctx,
-                                   progress_data);
-}
-
-static inline int
-write_flags_to_resource_flags(int write_flags)
+submit_chunk_for_compression(struct write_streams_ctx *ctx,
+                            const void *chunk, size_t size)

 {
 {

-       int resource_flags = 0;
+       /* While we are unable to submit the chunk for compression (due to too
+        * many chunks already outstanding), retrieve and write the next
+        * compressed chunk.  */
+       while (!ctx->compressor->submit_chunk(ctx->compressor, chunk, size)) {
+               const void *cchunk;
+               unsigned csize;
+               unsigned usize;
+               bool bret;
+               int ret;

 
 

-       if (write_flags & WIMLIB_WRITE_FLAG_RECOMPRESS)
-               resource_flags |= WIMLIB_WRITE_RESOURCE_FLAG_RECOMPRESS;
-       if (write_flags & WIMLIB_WRITE_FLAG_PIPABLE)
-               resource_flags |= WIMLIB_WRITE_RESOURCE_FLAG_PIPABLE;
-       return resource_flags;
-}
+               bret = ctx->compressor->get_chunk(ctx->compressor,
+                                                 &cchunk, &csize, &usize);

 
 

-static int
-write_stream_list_serial(struct list_head *stream_list,
-                        struct wim_lookup_table *lookup_table,
-                        struct filedes *out_fd,
-                        int out_ctype,
-                        u32 out_chunk_size,
-                        struct wimlib_lzx_context **comp_ctx,
-                        int write_resource_flags,
-                        struct write_streams_progress_data *progress_data)
-{
-       union wimlib_progress_info *progress = &progress_data->progress;
-       DEBUG("Writing stream list of size %"PRIu64" (serial version)",
-             progress->write_streams.total_streams);
-       progress->write_streams.num_threads = 1;
-       if (progress_data->progress_func) {
-               progress_data->progress_func(WIMLIB_PROGRESS_MSG_WRITE_STREAMS,
-                                            progress);
+               wimlib_assert(bret);
+
+               ret = write_chunk(ctx, cchunk, csize, usize);
+               if (ret)
+                       return ret;

        }
        }

-       return do_write_stream_list_serial(stream_list,
-                                          lookup_table,
-                                          out_fd,
-                                          out_ctype,
-                                          out_chunk_size,
-                                          comp_ctx,
-                                          write_resource_flags,
-                                          progress_data);
+       return 0;

 }
 
 }
 

-#ifdef ENABLE_MULTITHREADED_COMPRESSION
+/* Process the next chunk of data to be written to a WIM resource.  */

 static int
 static int

-write_wim_chunks(struct message *msg, struct filedes *out_fd,
-                struct chunk_table *chunk_tab,
-                int write_resource_flags)
+write_stream_process_chunk(const void *chunk, size_t size, void *_ctx)

 {
 {

-       struct iovec *vecs;
-       struct pwm_chunk_hdr *chunk_hdrs;
-       unsigned nvecs;
+       struct write_streams_ctx *ctx = _ctx;

        int ret;
        int ret;

+       const u8 *chunkptr, *chunkend;

 
 

-       for (unsigned i = 0; i < msg->num_chunks; i++)
-               chunk_tab_record_chunk(chunk_tab, msg->out_chunks[i].iov_len);
+       wimlib_assert(size != 0);

 
 

-       if (!(write_resource_flags & WIMLIB_WRITE_RESOURCE_FLAG_PIPABLE)) {
-               nvecs = msg->num_chunks;
-               vecs = msg->out_chunks;
-       } else {
-               /* Special case:  If writing a compressed resource to a pipable
-                * WIM, prefix each compressed chunk with a header that gives
-                * its compressed size.  */
-               nvecs = msg->num_chunks * 2;
-               vecs = alloca(nvecs * sizeof(vecs[0]));
-               chunk_hdrs = alloca(msg->num_chunks * sizeof(chunk_hdrs[0]));
-
-               for (unsigned i = 0; i < msg->num_chunks; i++) {
-                       chunk_hdrs[i].compressed_size = cpu_to_le32(msg->out_chunks[i].iov_len);
-                       vecs[i * 2].iov_base = &chunk_hdrs[i];
-                       vecs[i * 2].iov_len = sizeof(chunk_hdrs[i]);
-                       vecs[i * 2 + 1].iov_base = msg->out_chunks[i].iov_base;
-                       vecs[i * 2 + 1].iov_len = msg->out_chunks[i].iov_len;
+       if (ctx->compressor == NULL) {
+               /* Write chunk uncompressed.  */
+                ret = write_chunk(ctx, chunk, size, size);
+                if (ret)
+                        return ret;
+                ctx->cur_read_res_offset += size;
+                return 0;
+       }
+
+       /* Submit the chunk for compression, but take into account that the
+        * @size the chunk was provided in may not correspond to the
+        * @out_chunk_size being used for compression.  */
+       chunkptr = chunk;
+       chunkend = chunkptr + size;
+       do {
+               const u8 *resized_chunk;
+               size_t needed_chunk_size;
+
+               if (ctx->write_resource_flags & WIMLIB_WRITE_RESOURCE_FLAG_PACK_STREAMS) {
+                       needed_chunk_size = ctx->out_chunk_size;
+               } else {
+                       u64 res_bytes_remaining;
+
+                       res_bytes_remaining = ctx->cur_read_res_size -
+                                             ctx->cur_read_res_offset;
+                       needed_chunk_size = min(ctx->out_chunk_size,
+                                               ctx->chunk_buf_filled +
+                                                       res_bytes_remaining);

                }
                }

-       }
-       ret = full_writev(out_fd, vecs, nvecs);
-       if (ret)
-               ERROR_WITH_ERRNO("Write error");
-       return ret;
-}

 
 

-struct main_writer_thread_ctx {
-       struct list_head *stream_list;
-       struct wim_lookup_table *lookup_table;
-       struct filedes *out_fd;
-       off_t res_start_offset;
-       int out_ctype;
-       u32 out_chunk_size;
-       struct wimlib_lzx_context **comp_ctx;
-       int write_resource_flags;
-       struct shared_queue *res_to_compress_queue;
-       struct shared_queue *compressed_res_queue;
-       size_t num_messages;
-       struct write_streams_progress_data *progress_data;
-
-       struct list_head available_msgs;
-       struct list_head outstanding_streams;
-       struct list_head serial_streams;
-       size_t num_outstanding_messages;
-
-       SHA_CTX next_sha_ctx;
-       u64 next_chunk;
-       u64 next_num_chunks;
-       struct wim_lookup_table_entry *next_lte;
-
-       struct message *msgs;
-       struct message *next_msg;
-       struct chunk_table *cur_chunk_tab;
-};
+               if (ctx->chunk_buf_filled == 0 &&
+                   chunkend - chunkptr >= needed_chunk_size)
+               {
+                       /* No intermediate buffering needed.  */
+                       resized_chunk = chunkptr;
+                       chunkptr += needed_chunk_size;
+                       ctx->cur_read_res_offset += needed_chunk_size;
+               } else {
+                       /* Intermediate buffering needed.  */
+                       size_t bytes_consumed;

 
 

-static int
-init_message(struct message *msg, u32 out_chunk_size)
-{
-       msg->out_chunk_size = out_chunk_size;
-       for (size_t i = 0; i < MAX_CHUNKS_PER_MSG; i++) {
-               msg->compressed_chunks[i] = MALLOC(out_chunk_size);
-               msg->uncompressed_chunks[i] = MALLOC(out_chunk_size);
-               if (msg->compressed_chunks[i] == NULL ||
-                   msg->uncompressed_chunks[i] == NULL)
-                       return WIMLIB_ERR_NOMEM;
-       }
-       return 0;
-}
+                       bytes_consumed = min(chunkend - chunkptr,
+                                            needed_chunk_size - ctx->chunk_buf_filled);

 
 

-static void
-destroy_message(struct message *msg)
-{
-       for (size_t i = 0; i < MAX_CHUNKS_PER_MSG; i++) {
-               FREE(msg->compressed_chunks[i]);
-               FREE(msg->uncompressed_chunks[i]);
-       }
-}
+                       memcpy(&ctx->chunk_buf[ctx->chunk_buf_filled],
+                              chunkptr, bytes_consumed);

 
 

-static void
-free_messages(struct message *msgs, size_t num_messages)
-{
-       if (msgs) {
-               for (size_t i = 0; i < num_messages; i++)
-                       destroy_message(&msgs[i]);
-               FREE(msgs);
-       }
-}
+                       resized_chunk = ctx->chunk_buf;

 
 

-static struct message *
-allocate_messages(size_t num_messages, u32 out_chunk_size)
-{
-       struct message *msgs;
+                       chunkptr += bytes_consumed;
+                       ctx->cur_read_res_offset += bytes_consumed;
+                       ctx->chunk_buf_filled += bytes_consumed;
+                       if (ctx->chunk_buf_filled == needed_chunk_size) {
+                               resized_chunk = ctx->chunk_buf;
+                               ctx->chunk_buf_filled = 0;
+                       } else {
+                               break;
+                       }

 
 

-       msgs = CALLOC(num_messages, sizeof(struct message));
-       if (msgs == NULL)
-               return NULL;
-       for (size_t i = 0; i < num_messages; i++) {
-               if (init_message(&msgs[i], out_chunk_size)) {
-                       free_messages(msgs, num_messages);
-                       return NULL;

                }
                }

+
+               ret = submit_chunk_for_compression(ctx, resized_chunk,
+                                                  needed_chunk_size);
+               if (ret)
+                       return ret;
+
+       } while (chunkptr != chunkend);
+       return 0;
+}
+
+/* Finish processing a stream for writing.  It may not have been completely
+ * written yet, as the chunk_compressor implementation may still have chunks
+ * buffered or being compressed.  */
+static int
+write_stream_end_read(struct wim_lookup_table_entry *lte, int status, void *_ctx)
+{
+       struct write_streams_ctx *ctx = _ctx;
+       if (status == 0)
+               wimlib_assert(ctx->cur_read_res_offset == ctx->cur_read_res_size);
+       if (ctx->stream_was_duplicate) {
+               free_lookup_table_entry(lte);
+       } else if (lte->unhashed && ctx->lookup_table != NULL) {
+               list_del(&lte->unhashed_list);
+               lookup_table_insert(ctx->lookup_table, lte);
+               lte->unhashed = 0;

        }
        }

-       return msgs;
+       return status;

 }
 
 }
 

+/* Compute statistics about a list of streams that will be written.
+ *
+ * Assumes the streams are sorted such that all streams located in each distinct
+ * WIM (specified by WIMStruct) are together.  */

 static void
 static void

-main_writer_thread_destroy_ctx(struct main_writer_thread_ctx *ctx)
+compute_stream_list_stats(struct list_head *stream_list,
+                         struct write_streams_ctx *ctx)

 {
 {

-       while (ctx->num_outstanding_messages--)
-               shared_queue_get(ctx->compressed_res_queue);
-       free_messages(ctx->msgs, ctx->num_messages);
-       FREE(ctx->cur_chunk_tab);
-}
+       struct wim_lookup_table_entry *lte;
+       u64 total_bytes = 0;
+       u64 num_streams = 0;
+       u64 total_parts = 0;
+       WIMStruct *prev_wim_part = NULL;

 
 

-static int
-main_writer_thread_init_ctx(struct main_writer_thread_ctx *ctx)
+       list_for_each_entry(lte, stream_list, write_streams_list) {
+               num_streams++;
+               total_bytes += lte->size;
+               if (lte->resource_location == RESOURCE_IN_WIM) {
+                       if (prev_wim_part != lte->rspec->wim) {
+                               prev_wim_part = lte->rspec->wim;
+                               total_parts++;
+                       }
+               }
+       }
+       ctx->progress_data.progress.write_streams.total_bytes       = total_bytes;
+       ctx->progress_data.progress.write_streams.total_streams     = num_streams;
+       ctx->progress_data.progress.write_streams.completed_bytes   = 0;
+       ctx->progress_data.progress.write_streams.completed_streams = 0;
+       ctx->progress_data.progress.write_streams.compression_type  = ctx->out_ctype;
+       ctx->progress_data.progress.write_streams.total_parts       = total_parts;
+       ctx->progress_data.progress.write_streams.completed_parts   = 0;
+       ctx->progress_data.next_progress = 0;
+       ctx->progress_data.prev_wim_part = NULL;
+}
+
+/* Find streams in @stream_list that can be copied to the output WIM in raw form
+ * rather than compressed.  Delete these streams from @stream_list, and move one
+ * per resource to @raw_copy_resources.  Return the total uncompressed size of
+ * the streams that need to be compressed.  */
+static u64
+find_raw_copy_resources(struct list_head *stream_list,
+                       int write_resource_flags,
+                       int out_ctype,
+                       u32 out_chunk_size,
+                       struct list_head *raw_copy_resources)

 {
 {

-       /* Pre-allocate all the buffers that will be needed to do the chunk
-        * compression. */
-       ctx->msgs = allocate_messages(ctx->num_messages, ctx->out_chunk_size);
-       if (ctx->msgs == NULL)
-               return WIMLIB_ERR_NOMEM;
-
-       /* Initially, all the messages are available to use. */
-       INIT_LIST_HEAD(&ctx->available_msgs);
-       for (size_t i = 0; i < ctx->num_messages; i++)
-               list_add_tail(&ctx->msgs[i].list, &ctx->available_msgs);
-
-       /* outstanding_streams is the list of streams that currently have had
-        * chunks sent off for compression.
-        *
-        * The first stream in outstanding_streams is the stream that is
-        * currently being written.
-        *
-        * The last stream in outstanding_streams is the stream that is
-        * currently being read and having chunks fed to the compressor threads.
-        * */
-       INIT_LIST_HEAD(&ctx->outstanding_streams);
-       ctx->num_outstanding_messages = 0;
+       struct wim_lookup_table_entry *lte, *tmp;
+       u64 num_bytes_to_compress = 0;

 
 

-       /* Message currently being prepared.  */
-       ctx->next_msg = NULL;
+       INIT_LIST_HEAD(raw_copy_resources);

 
 

-       /* Resources that don't need any chunks compressed are added to this
-        * list and written directly by the main thread.  */
-       INIT_LIST_HEAD(&ctx->serial_streams);
+       /* Initialize temporary raw_copy_ok flag.  */
+       list_for_each_entry(lte, stream_list, write_streams_list)
+               if (lte->resource_location == RESOURCE_IN_WIM)
+                       lte->rspec->raw_copy_ok = 0;

 
 

-       /* Pointer to chunk table for stream currently being written.  */
-       ctx->cur_chunk_tab = NULL;
+       list_for_each_entry_safe(lte, tmp, stream_list, write_streams_list) {
+               if (lte->resource_location == RESOURCE_IN_WIM &&
+                   lte->rspec->raw_copy_ok)
+               {
+                       list_del(&lte->write_streams_list);
+               } else if (can_raw_copy(lte, write_resource_flags,
+                                out_ctype, out_chunk_size))
+               {
+                       lte->rspec->raw_copy_ok = 1;
+                       list_move_tail(&lte->write_streams_list,
+                                      raw_copy_resources);
+               } else {
+                       num_bytes_to_compress += lte->size;
+               }
+       }

 
 

-       return 0;
+       return num_bytes_to_compress;

 }
 
 }
 

+/* Copy a raw compressed resource located in another WIM file to the WIM file
+ * being written.  */

 static int
 static int

-receive_compressed_chunks(struct main_writer_thread_ctx *ctx)
+write_raw_copy_resource(struct wim_resource_spec *in_rspec,
+                       struct filedes *out_fd)

 {
 {

-       struct message *msg;
-       struct wim_lookup_table_entry *cur_lte;
+       u64 cur_read_offset;
+       u64 end_read_offset;
+       u8 buf[BUFFER_SIZE];
+       size_t bytes_to_read;

        int ret;
        int ret;

+       struct filedes *in_fd;
+       struct wim_lookup_table_entry *lte;
+       u64 out_offset_in_wim;

 
 

-       wimlib_assert(!list_empty(&ctx->outstanding_streams));
-       wimlib_assert(ctx->num_outstanding_messages != 0);
-
-       cur_lte = container_of(ctx->outstanding_streams.next,
-                              struct wim_lookup_table_entry,
-                              being_compressed_list);
-
-       /* Get the next message from the queue and process it.
-        * The message will contain 1 or more data chunks that have been
-        * compressed.  */
-       msg = shared_queue_get(ctx->compressed_res_queue);
-       msg->complete = true;
-       --ctx->num_outstanding_messages;
-
-       /* Is this the next chunk in the current resource?  If it's not (i.e.,
-        * an earlier chunk in a same or different resource hasn't been
-        * compressed yet), do nothing, and keep this message around until all
-        * earlier chunks are received.
-        *
-        * Otherwise, write all the chunks we can.  */
-       while (cur_lte != NULL &&
-              !list_empty(&cur_lte->msg_list)
-              && (msg = container_of(cur_lte->msg_list.next,
-                                     struct message,
-                                     list))->complete)
-       {
-               list_move(&msg->list, &ctx->available_msgs);
-               if (msg->begin_chunk == 0) {
-                       /* First set of chunks.  */
+       DEBUG("Copying raw compressed data (size_in_wim=%"PRIu64", "
+             "uncompressed_size=%"PRIu64")",
+             in_rspec->size_in_wim, in_rspec->uncompressed_size);

 
 

-                       /* Write pipable WIM stream header if needed.  */
-                       if (ctx->write_resource_flags &
-                           WIMLIB_WRITE_RESOURCE_FLAG_PIPABLE)
-                       {
-                               ret = write_pwm_stream_header(cur_lte, ctx->out_fd,
-                                                             WIM_RESHDR_FLAG_COMPRESSED);
-                               if (ret)
-                                       return ret;
-                       }
+       /* Copy the raw data.  */
+       cur_read_offset = in_rspec->offset_in_wim;
+       end_read_offset = cur_read_offset + in_rspec->size_in_wim;

 
 

-                       /* Save current offset.  */
-                       ctx->res_start_offset = ctx->out_fd->offset;
+       out_offset_in_wim = out_fd->offset;

 
 

-                       /* Begin building the chunk table, and leave space for
-                        * it if needed.  */
-                       ret = begin_wim_resource_chunk_tab(cur_lte,
-                                                          ctx->out_fd,
-                                                          ctx->out_chunk_size,
-                                                          &ctx->cur_chunk_tab,
-                                                          ctx->write_resource_flags);
-                       if (ret)
-                               return ret;
-               }
+       if (in_rspec->is_pipable) {
+               if (cur_read_offset < sizeof(struct pwm_stream_hdr))
+                       return WIMLIB_ERR_INVALID_PIPABLE_WIM;
+               cur_read_offset -= sizeof(struct pwm_stream_hdr);
+               out_offset_in_wim += sizeof(struct pwm_stream_hdr);
+       }
+       in_fd = &in_rspec->wim->in_fd;
+       while (cur_read_offset != end_read_offset) {

 
 

-               /* Write the compressed chunks from the message.  */
-               ret = write_wim_chunks(msg, ctx->out_fd, ctx->cur_chunk_tab,
-                                      ctx->write_resource_flags);
+               bytes_to_read = min(sizeof(buf), end_read_offset - cur_read_offset);
+
+               ret = full_pread(in_fd, buf, bytes_to_read, cur_read_offset);

                if (ret)
                        return ret;
 
                if (ret)
                        return ret;
 

-               /* Was this the last chunk of the stream?  If so, finish the
-                * stream by writing the chunk table.  */
-               if (list_empty(&cur_lte->msg_list) &&
-                   msg->begin_chunk + msg->num_chunks == ctx->cur_chunk_tab->num_chunks)
-               {
-                       u64 res_csize;
-
-                       ret = finish_wim_resource_chunk_tab(ctx->cur_chunk_tab,
-                                                           ctx->out_fd,
-                                                           ctx->res_start_offset,
-                                                           ctx->write_resource_flags);
-                       if (ret)
-                               return ret;
-
-                       list_del(&cur_lte->being_compressed_list);
-
-                       res_csize = ctx->out_fd->offset - ctx->res_start_offset;
-
-                       FREE(ctx->cur_chunk_tab);
-                       ctx->cur_chunk_tab = NULL;
-
-                       /* Check for resources compressed to greater than or
-                        * equal to their original size and write them
-                        * uncompressed instead.  (But never do this if writing
-                        * to a pipe.)  */
-                       if (res_csize >= cur_lte->size &&
-                           !(ctx->write_resource_flags & WIMLIB_WRITE_RESOURCE_FLAG_PIPABLE))
-                       {
-                               DEBUG("Compressed %"PRIu64" => %"PRIu64" bytes; "
-                                     "writing uncompressed instead",
-                                     cur_lte->size, res_csize);
-                               ret = seek_and_truncate(ctx->out_fd, ctx->res_start_offset);
-                               if (ret)
-                                       return ret;
-                               ret = write_wim_resource(cur_lte,
-                                                        ctx->out_fd,
-                                                        WIMLIB_COMPRESSION_TYPE_NONE,
-                                                        0,
-                                                        &cur_lte->out_reshdr,
-                                                        ctx->write_resource_flags,
-                                                        ctx->comp_ctx);
-                               if (ret)
-                                       return ret;
-                       } else {
-                               cur_lte->out_reshdr.size_in_wim =
-                                       res_csize;
-
-                               cur_lte->out_reshdr.uncompressed_size =
-                                       cur_lte->size;
-
-                               cur_lte->out_reshdr.offset_in_wim =
-                                       ctx->res_start_offset;
-
-                               cur_lte->out_reshdr.flags =
-                                       cur_lte->flags |
-                                               WIM_RESHDR_FLAG_COMPRESSED;
+               ret = full_write(out_fd, buf, bytes_to_read);
+               if (ret)
+                       return ret;

 
 

-                               DEBUG("Wrote compressed resource "
-                                     "(%"PRIu64" => %"PRIu64" bytes @ +%"PRIu64", flags=0x%02x)",
-                                     cur_lte->out_reshdr.uncompressed_size,
-                                     cur_lte->out_reshdr.size_in_wim,
-                                     cur_lte->out_reshdr.offset_in_wim,
-                                     cur_lte->out_reshdr.flags);
-                       }
+               cur_read_offset += bytes_to_read;
+       }

 
 

-                       do_write_streams_progress(ctx->progress_data,
-                                                 cur_lte, false);
-
-                       /* Since we just finished writing a stream, write any
-                        * streams that have been added to the serial_streams
-                        * list for direct writing by the main thread (e.g.
-                        * resources that don't need to be compressed because
-                        * the desired compression type is the same as the
-                        * previous compression type).  */
-                       if (!list_empty(&ctx->serial_streams)) {
-                               ret = do_write_stream_list_serial(&ctx->serial_streams,
-                                                                 ctx->lookup_table,
-                                                                 ctx->out_fd,
-                                                                 ctx->out_ctype,
-                                                                 ctx->out_chunk_size,
-                                                                 ctx->comp_ctx,
-                                                                 ctx->write_resource_flags,
-                                                                 ctx->progress_data);
-                               if (ret)
-                                       return ret;
-                       }
+       list_for_each_entry(lte, &in_rspec->stream_list, rspec_node) {
+               if (lte->will_be_in_output_wim) {
+                       stream_set_out_reshdr_for_reuse(lte);
+                       if (in_rspec->flags & WIM_RESHDR_FLAG_PACKED_STREAMS)
+                               lte->out_res_offset_in_wim = out_offset_in_wim;
+                       else
+                               lte->out_reshdr.offset_in_wim = out_offset_in_wim;

 
 

-                       /* Advance to the next stream to write.  */
-                       if (list_empty(&ctx->outstanding_streams)) {
-                               cur_lte = NULL;
-                       } else {
-                               cur_lte = container_of(ctx->outstanding_streams.next,
-                                                      struct wim_lookup_table_entry,
-                                                      being_compressed_list);
-                       }

                }
        }
        return 0;
 }
 
                }
        }
        return 0;
 }
 

-/* Called when the main thread has read a new chunk of data.  */
+/* Copy a list of raw compressed resources located other WIM file(s) to the WIM
+ * file being written.  */

 static int
 static int

-main_writer_thread_cb(const void *chunk, size_t chunk_size, void *_ctx)
+write_raw_copy_resources(struct list_head *raw_copy_resources,
+                        struct filedes *out_fd)

 {
 {

-       struct main_writer_thread_ctx *ctx = _ctx;
+       struct wim_lookup_table_entry *lte;

        int ret;
        int ret;

-       struct message *next_msg;
-       u64 next_chunk_in_msg;
-
-       /* Update SHA1 message digest for the stream currently being read by the
-        * main thread. */
-       sha1_update(&ctx->next_sha_ctx, chunk, chunk_size);
-
-       /* We send chunks of data to the compressor chunks in batches which we
-        * refer to as "messages".  @next_msg is the message that is currently
-        * being prepared to send off.  If it is NULL, that indicates that we
-        * need to start a new message. */
-       next_msg = ctx->next_msg;
-       if (!next_msg) {
-               /* We need to start a new message.  First check to see if there
-                * is a message available in the list of available messages.  If
-                * so, we can just take one.  If not, all the messages (there is
-                * a fixed number of them, proportional to the number of
-                * threads) have been sent off to the compressor threads, so we
-                * receive messages from the compressor threads containing
-                * compressed chunks of data.
-                *
-                * We may need to receive multiple messages before one is
-                * actually available to use because messages received that are
-                * *not* for the very next set of chunks to compress must be
-                * buffered until it's time to write those chunks. */
-               while (list_empty(&ctx->available_msgs)) {
-                       ret = receive_compressed_chunks(ctx);
-                       if (ret)
-                               return ret;
-               }

 
 

-               next_msg = container_of(ctx->available_msgs.next,
-                                       struct message, list);
-               list_del(&next_msg->list);
-               next_msg->complete = false;
-               next_msg->begin_chunk = ctx->next_chunk;
-               next_msg->num_chunks = min(MAX_CHUNKS_PER_MSG,
-                                          ctx->next_num_chunks - ctx->next_chunk);
-               ctx->next_msg = next_msg;
-       }
-
-       /* Fill in the next chunk to compress */
-       next_chunk_in_msg = ctx->next_chunk - next_msg->begin_chunk;
-
-       next_msg->uncompressed_chunk_sizes[next_chunk_in_msg] = chunk_size;
-       memcpy(next_msg->uncompressed_chunks[next_chunk_in_msg],
-              chunk, chunk_size);
-       ctx->next_chunk++;
-       if (++next_chunk_in_msg == next_msg->num_chunks) {
-               /* Send off an array of chunks to compress */
-               list_add_tail(&next_msg->list, &ctx->next_lte->msg_list);
-               shared_queue_put(ctx->res_to_compress_queue, next_msg);
-               ++ctx->num_outstanding_messages;
-               ctx->next_msg = NULL;
+       list_for_each_entry(lte, raw_copy_resources, write_streams_list) {
+               ret = write_raw_copy_resource(lte->rspec, out_fd);
+               if (ret)
+                       return ret;

        }
        return 0;
 }
 
        }
        return 0;
 }
 

+/* Wait for and write all chunks pending in the compressor.  */

 static int
 static int

-main_writer_thread_finish(void *_ctx)
+finish_remaining_chunks(struct write_streams_ctx *ctx)

 {
 {

-       struct main_writer_thread_ctx *ctx = _ctx;
+       const void *cdata;
+       unsigned csize;
+       unsigned usize;

        int ret;
        int ret;

-       while (ctx->num_outstanding_messages != 0) {
-               ret = receive_compressed_chunks(ctx);
+
+       if (ctx->compressor == NULL)
+               return 0;
+
+       if (ctx->chunk_buf_filled != 0) {
+               ret = submit_chunk_for_compression(ctx, ctx->chunk_buf,
+                                                  ctx->chunk_buf_filled);

                if (ret)
                        return ret;
        }
                if (ret)
                        return ret;
        }

-       wimlib_assert(list_empty(&ctx->outstanding_streams));
-       return do_write_stream_list_serial(&ctx->serial_streams,
-                                          ctx->lookup_table,
-                                          ctx->out_fd,
-                                          ctx->out_ctype,
-                                          ctx->out_chunk_size,
-                                          ctx->comp_ctx,
-                                          ctx->write_resource_flags,
-                                          ctx->progress_data);
-}
-
-static int
-submit_stream_for_compression(struct wim_lookup_table_entry *lte,
-                             struct main_writer_thread_ctx *ctx)
-{
-       int ret;

 
 

-       /* Read the entire stream @lte, feeding its data chunks to the
-        * compressor threads.  Also SHA1-sum the stream; this is required in
-        * the case that @lte is unhashed, and a nice additional verification
-        * when @lte is already hashed.  */
-       sha1_init(&ctx->next_sha_ctx);
-       ctx->next_chunk = 0;
-       ctx->next_num_chunks = DIV_ROUND_UP(lte->size, ctx->out_chunk_size);
-       ctx->next_lte = lte;
-       INIT_LIST_HEAD(&lte->msg_list);
-       list_add_tail(&lte->being_compressed_list, &ctx->outstanding_streams);
-       ret = read_stream_prefix(lte, lte->size, main_writer_thread_cb, ctx, 0);
-       if (ret)
-               return ret;
-       wimlib_assert(ctx->next_chunk == ctx->next_num_chunks);
-       return finalize_and_check_sha1(&ctx->next_sha_ctx, lte);
+       while (ctx->compressor->get_chunk(ctx->compressor, &cdata, &csize, &usize)) {
+               ret = write_chunk(ctx, cdata, csize, usize);
+               if (ret)
+                       return ret;
+       }
+       return 0;

 }
 
 }
 

-static int
-main_thread_process_next_stream(struct wim_lookup_table_entry *lte, void *_ctx)
+static void
+remove_zero_length_streams(struct list_head *stream_list)

 {
 {

-       struct main_writer_thread_ctx *ctx = _ctx;
-       int ret;
+       struct wim_lookup_table_entry *lte, *tmp;

 
 

-       if (lte->size < 1000 ||
-           !must_compress_stream(lte, ctx->write_resource_flags,
-                                 ctx->out_ctype, ctx->out_chunk_size))
-       {
-               /* Stream is too small or isn't being compressed.  Process it by
-                * the main thread when we have a chance.  We can't necessarily
-                * process it right here, as the main thread could be in the
-                * middle of writing a different stream.  */
-               list_add_tail(&lte->write_streams_list, &ctx->serial_streams);
-               lte->deferred = 1;
-               ret = 0;
-       } else {
-               ret = submit_stream_for_compression(lte, ctx);
+       list_for_each_entry_safe(lte, tmp, stream_list, write_streams_list) {
+               wimlib_assert(lte->will_be_in_output_wim);
+               if (lte->size == 0) {
+                       list_del(&lte->write_streams_list);
+                       lte->out_reshdr.offset_in_wim = 0;
+                       lte->out_reshdr.size_in_wim = 0;
+                       lte->out_reshdr.uncompressed_size = 0;
+                       lte->out_reshdr.flags = filter_resource_flags(lte->flags);
+               }

        }
        }

-       lte->no_progress = 1;
-       return ret;
-}
-
-static long
-get_default_num_threads(void)
-{
-#ifdef __WIN32__
-       return win32_get_number_of_processors();
-#else
-       return sysconf(_SC_NPROCESSORS_ONLN);
-#endif

 }
 
 }
 

-/* Equivalent to write_stream_list_serial(), except this takes a @num_threads
- * parameter and will perform compression using that many threads.  Falls
- * back to write_stream_list_serial() on certain errors, such as a failure to
- * create the number of threads requested.
- *
- * High level description of the algorithm for writing compressed streams in
- * parallel:  We perform compression on chunks rather than on full files.  The
- * currently executing thread becomes the main thread and is entirely in charge
- * of reading the data to compress (which may be in any location understood by
- * the resource code--- such as in an external file being captured, or in
- * another WIM file from which an image is being exported) and actually writing
- * the compressed data to the output file.  Additional threads are "compressor
- * threads" and all execute the compressor_thread_proc, where they repeatedly
- * retrieve buffers of data from the main thread, compress them, and hand them
- * back to the main thread.
- *
- * Certain streams, such as streams that do not need to be compressed (e.g.
- * input compression type same as output compression type) or streams of very
- * small size are placed in a list (main_writer_thread_ctx.serial_list) and
- * handled entirely by the main thread at an appropriate time.
- *
- * At any given point in time, multiple streams may be having chunks compressed
- * concurrently.  The stream that the main thread is currently *reading* may be
- * later in the list that the stream that the main thread is currently
- * *writing*.  */

 static int
 static int

-write_stream_list_parallel(struct list_head *stream_list,
-                          struct wim_lookup_table *lookup_table,
-                          struct filedes *out_fd,
-                          int out_ctype,
-                          u32 out_chunk_size,
-                          struct wimlib_lzx_context **comp_ctx,
-                          int write_resource_flags,
-                          struct write_streams_progress_data *progress_data,
-                          unsigned num_threads)
+write_stream_list(struct list_head *stream_list,
+                 struct filedes *out_fd,
+                 int write_resource_flags,
+                 int out_ctype,
+                 u32 out_chunk_size,
+                 unsigned num_threads,
+                 struct wim_lookup_table *lookup_table,
+                 struct filter_context *filter_ctx,
+                 struct wimlib_lzx_context **comp_ctx,
+                 wimlib_progress_func_t progress_func)

 {
        int ret;
 {
        int ret;

-       struct shared_queue res_to_compress_queue;
-       struct shared_queue compressed_res_queue;
-       pthread_t *compressor_threads = NULL;
-       union wimlib_progress_info *progress = &progress_data->progress;
-       unsigned num_started_threads;
-       bool can_retry = true;
-
-       if (num_threads == 0) {
-               long nthreads = get_default_num_threads();
-               if (nthreads < 1 || nthreads > UINT_MAX) {
-                       WARNING("Could not determine number of processors! Assuming 1");
-                       goto out_serial_quiet;
-               } else if (nthreads == 1) {
-                       goto out_serial_quiet;
-               } else {
-                       num_threads = nthreads;
-               }
+       struct write_streams_ctx ctx;
+       struct list_head raw_copy_resources;
+
+       remove_zero_length_streams(stream_list);
+
+       if (list_empty(stream_list)) {
+               DEBUG("No streams to write.");
+               return 0;

        }
 
        }
 

-       DEBUG("Writing stream list of size %"PRIu64" "
-             "(parallel version, num_threads=%u)",
-             progress->write_streams.total_streams, num_threads);
+       memset(&ctx, 0, sizeof(ctx));
+
+       /* Pre-sorting the streams is required for compute_stream_list_stats().
+        * Afterwards, read_stream_list() need not sort them again.  */
+       ret = sort_stream_list_by_sequential_order(stream_list,
+                                                  offsetof(struct wim_lookup_table_entry,
+                                                           write_streams_list));
+       if (ret)
+               return ret;

 
 

-       progress->write_streams.num_threads = num_threads;
+       ctx.out_fd = out_fd;
+       ctx.lookup_table = lookup_table;
+       ctx.out_ctype = out_ctype;
+       ctx.out_chunk_size = out_chunk_size;
+       ctx.write_resource_flags = write_resource_flags;
+       ctx.filter_ctx = filter_ctx;

 
 

-       static const size_t MESSAGES_PER_THREAD = 2;
-       size_t queue_size = (size_t)(num_threads * MESSAGES_PER_THREAD);
+       if (out_chunk_size <= STACK_MAX) {
+               ctx.chunk_buf = alloca(out_chunk_size);
+       } else {
+               ctx.chunk_buf = MALLOC(out_chunk_size);
+               if (ctx.chunk_buf == NULL) {
+                       ret = WIMLIB_ERR_NOMEM;
+                       goto out_destroy_context;
+               }
+       }
+       ctx.chunk_buf_filled = 0;

 
 

-       DEBUG("Initializing shared queues (queue_size=%zu)", queue_size);
+       compute_stream_list_stats(stream_list, &ctx);

 
 

-       ret = shared_queue_init(&res_to_compress_queue, queue_size);
-       if (ret)
-               goto out_serial;
+       ctx.progress_data.progress_func = progress_func;

 
 

-       ret = shared_queue_init(&compressed_res_queue, queue_size);
-       if (ret)
-               goto out_destroy_res_to_compress_queue;
+       ctx.num_bytes_to_compress = find_raw_copy_resources(stream_list,
+                                                           write_resource_flags,
+                                                           out_ctype,
+                                                           out_chunk_size,
+                                                           &raw_copy_resources);

 
 

-       struct compressor_thread_params *params;
+       DEBUG("Writing stream list "
+             "(offset = %"PRIu64", write_resource_flags=0x%08x, "
+             "out_ctype=%d, out_chunk_size=%u, num_threads=%u, "
+             "total_bytes=%"PRIu64", num_bytes_to_compress=%"PRIu64")",
+             out_fd->offset, write_resource_flags,
+             out_ctype, out_chunk_size, num_threads,
+             ctx.progress_data.progress.write_streams.total_bytes,
+             ctx.num_bytes_to_compress);

 
 

-       params = CALLOC(num_threads, sizeof(params[0]));
-       if (params == NULL) {
-               ret = WIMLIB_ERR_NOMEM;
-               goto out_destroy_compressed_res_queue;
+       if (ctx.num_bytes_to_compress == 0) {
+               DEBUG("No compression needed; skipping to raw copy!");
+               goto out_write_raw_copy_resources;

        }
 
        }
 

-       for (unsigned i = 0; i < num_threads; i++) {
-               params[i].res_to_compress_queue = &res_to_compress_queue;
-               params[i].compressed_res_queue = &compressed_res_queue;
-               params[i].out_ctype = out_ctype;
-               if (out_ctype == WIMLIB_COMPRESSION_TYPE_LZX) {
-                       ret = wimlib_lzx_alloc_context(out_chunk_size,
-                                                      NULL, &params[i].comp_ctx);
-                       if (ret)
-                               goto out_free_params;
+       /* Unless uncompressed output was required, allocate a chunk_compressor
+        * to do compression.  There are serial and parallel implementations of
+        * the chunk_compressor interface.  We default to parallel using the
+        * specified number of threads, unless the upper bound on the number
+        * bytes needing to be compressed is less 2000000 (heuristic value).  */
+       if (out_ctype != WIMLIB_COMPRESSION_TYPE_NONE) {
+
+               if (ctx.num_bytes_to_compress >= 2000000) {
+                       ret = new_parallel_chunk_compressor(out_ctype,
+                                                           out_chunk_size,
+                                                           num_threads, 0,
+                                                           &ctx.compressor);
+                       if (ret) {
+                               DEBUG("Couldn't create parallel chunk compressor "
+                                     "(status %d)", ret);
+                       }

                }
                }

-       }

 
 

-       compressor_threads = MALLOC(num_threads * sizeof(pthread_t));
-       if (compressor_threads == NULL) {
-               ret = WIMLIB_ERR_NOMEM;
-               goto out_free_params;
-       }
-
-       for (unsigned i = 0; i < num_threads; i++) {
-               DEBUG("pthread_create thread %u of %u", i + 1, num_threads);
-               ret = pthread_create(&compressor_threads[i], NULL,
-                                    compressor_thread_proc, &params[i]);
-               if (ret) {
-                       errno = ret;
-                       ret = -1;
-                       ERROR_WITH_ERRNO("Failed to create compressor "
-                                        "thread %u of %u",
-                                        i + 1, num_threads);
-                       num_started_threads = i;
-                       goto out_join;
+               if (ctx.compressor == NULL) {
+                       if (out_ctype == WIMLIB_COMPRESSION_TYPE_LZX) {
+                               ret = wimlib_lzx_alloc_context(out_chunk_size,
+                                                              NULL,
+                                                              comp_ctx);
+                               if (ret)
+                                       goto out_destroy_context;
+                       }
+                       ret = new_serial_chunk_compressor(out_ctype, out_chunk_size,
+                                                         *comp_ctx, &ctx.compressor);
+                       if (ret)
+                               goto out_destroy_context;

                }
        }
                }
        }

-       num_started_threads = num_threads;

 
 

-       if (progress_data->progress_func) {
-               progress_data->progress_func(WIMLIB_PROGRESS_MSG_WRITE_STREAMS,
-                                            progress);
+       if (ctx.compressor)
+               ctx.progress_data.progress.write_streams.num_threads = ctx.compressor->num_threads;
+       else
+               ctx.progress_data.progress.write_streams.num_threads = 1;
+
+       DEBUG("Actually using %u threads",
+             ctx.progress_data.progress.write_streams.num_threads);
+
+       INIT_LIST_HEAD(&ctx.pending_streams);
+
+       if (write_resource_flags & WIMLIB_WRITE_RESOURCE_FLAG_PACK_STREAMS) {
+               ret = begin_write_resource(&ctx, ctx.num_bytes_to_compress);
+               if (ret)
+                       goto out_destroy_context;

        }
 
        }
 

-       struct main_writer_thread_ctx ctx;
+       /* Read the list of streams needing to be compressed, using the
+        * specified callbacks to execute processing of the data.  */

 
 

-       memset(&ctx, 0, sizeof(ctx));
+       struct read_stream_list_callbacks cbs = {
+               .begin_stream           = write_stream_begin_read,
+               .begin_stream_ctx       = &ctx,
+               .consume_chunk          = write_stream_process_chunk,
+               .consume_chunk_ctx      = &ctx,
+               .end_stream             = write_stream_end_read,
+               .end_stream_ctx         = &ctx,
+       };
+
+       ret = read_stream_list(stream_list,
+                              offsetof(struct wim_lookup_table_entry, write_streams_list),
+                              &cbs,
+                              STREAM_LIST_ALREADY_SORTED |
+                                       VERIFY_STREAM_HASHES |
+                                       COMPUTE_MISSING_STREAM_HASHES);

 
 

-       ctx.stream_list           = stream_list;
-       ctx.lookup_table          = lookup_table;
-       ctx.out_fd                = out_fd;
-       ctx.out_ctype             = out_ctype;
-       ctx.out_chunk_size        = out_chunk_size;
-       ctx.comp_ctx              = comp_ctx;
-       ctx.res_to_compress_queue = &res_to_compress_queue;
-       ctx.compressed_res_queue  = &compressed_res_queue;
-       ctx.num_messages          = queue_size;
-       ctx.write_resource_flags  = write_resource_flags;
-       ctx.progress_data         = progress_data;
-       ret = main_writer_thread_init_ctx(&ctx);

        if (ret)
        if (ret)

-               goto out_join;
+               goto out_destroy_context;

 
 

-       can_retry = false;
-       ret = do_write_stream_list(stream_list, lookup_table,
-                                  main_thread_process_next_stream,
-                                  &ctx, progress_data);
+       ret = finish_remaining_chunks(&ctx);

        if (ret)
        if (ret)

-               goto out_destroy_ctx;
-
-       /* The main thread has finished reading all streams that are going to be
-        * compressed in parallel, and it now needs to wait for all remaining
-        * chunks to be compressed so that the remaining streams can actually be
-        * written to the output file.  Furthermore, any remaining streams that
-        * had processing deferred to the main thread need to be handled.  These
-        * tasks are done by the main_writer_thread_finish() function.  */
-       ret = main_writer_thread_finish(&ctx);
-out_destroy_ctx:
-       main_writer_thread_destroy_ctx(&ctx);
-out_join:
-       for (unsigned i = 0; i < num_started_threads; i++)
-               shared_queue_put(&res_to_compress_queue, NULL);
-
-       for (unsigned i = 0; i < num_started_threads; i++) {
-               if (pthread_join(compressor_threads[i], NULL)) {
-                       WARNING_WITH_ERRNO("Failed to join compressor "
-                                          "thread %u of %u",
-                                          i + 1, num_threads);
+               goto out_destroy_context;
+
+       if (write_resource_flags & WIMLIB_WRITE_RESOURCE_FLAG_PACK_STREAMS) {
+               struct wim_reshdr reshdr;
+               struct wim_lookup_table_entry *lte;
+               u64 offset_in_res;
+
+               ret = end_write_resource(&ctx, &reshdr);
+               if (ret)
+                       goto out_destroy_context;
+
+               DEBUG("Ending packed resource: %lu %lu %lu.",
+                     reshdr.offset_in_wim,
+                     reshdr.size_in_wim,
+                     reshdr.uncompressed_size);
+
+               offset_in_res = 0;
+               list_for_each_entry(lte, &ctx.pending_streams, write_streams_list) {
+                       lte->out_reshdr.size_in_wim = lte->size;
+                       lte->out_reshdr.flags = filter_resource_flags(lte->flags);
+                       lte->out_reshdr.flags |= WIM_RESHDR_FLAG_PACKED_STREAMS;
+                       lte->out_reshdr.uncompressed_size = 0;
+                       lte->out_reshdr.offset_in_wim = offset_in_res;
+                       lte->out_res_offset_in_wim = reshdr.offset_in_wim;
+                       lte->out_res_size_in_wim = reshdr.size_in_wim;
+                       lte->out_res_uncompressed_size = reshdr.uncompressed_size;
+                       offset_in_res += lte->size;

                }
                }

+               wimlib_assert(offset_in_res == reshdr.uncompressed_size);

        }
        }

-       FREE(compressor_threads);
-out_free_params:
-       for (unsigned i = 0; i < num_threads; i++)
-               wimlib_lzx_free_context(params[i].comp_ctx);
-       FREE(params);
-out_destroy_compressed_res_queue:
-       shared_queue_destroy(&compressed_res_queue);
-out_destroy_res_to_compress_queue:
-       shared_queue_destroy(&res_to_compress_queue);
-       if (!can_retry || (ret >= 0 && ret != WIMLIB_ERR_NOMEM))
-               return ret;
-out_serial:
-       WARNING("Falling back to single-threaded compression");
-out_serial_quiet:
-       return write_stream_list_serial(stream_list,
-                                       lookup_table,
-                                       out_fd,
-                                       out_ctype,
-                                       out_chunk_size,
-                                       comp_ctx,
-                                       write_resource_flags,
-                                       progress_data);

 
 

+out_write_raw_copy_resources:
+       /* Copy any compressed resources for which the raw data can be reused
+        * without decompression.  */
+       ret = write_raw_copy_resources(&raw_copy_resources, ctx.out_fd);
+
+out_destroy_context:
+       if (out_chunk_size > STACK_MAX)
+               FREE(ctx.chunk_buf);
+       FREE(ctx.chunk_csizes);
+       if (ctx.compressor)
+               ctx.compressor->destroy(ctx.compressor);
+       DEBUG("Done (ret=%d)", ret);
+       return ret;

 }
 }

-#endif

 
 

-/* Write a list of streams to a WIM (@out_fd) using the compression type
- * @out_ctype, chunk size @out_chunk_size, and up to @num_threads compressor
- * threads.  */

 static int
 static int

-write_stream_list(struct list_head *stream_list,
-                 struct wim_lookup_table *lookup_table,
-                 struct filedes *out_fd, int out_ctype,
-                 u32 out_chunk_size,
-                 struct wimlib_lzx_context **comp_ctx,
-                 int write_flags,
-                 unsigned num_threads, wimlib_progress_func_t progress_func)
+write_wim_resource(struct wim_lookup_table_entry *lte,
+                  struct filedes *out_fd,
+                  int out_ctype,
+                  u32 out_chunk_size,
+                  int write_resource_flags,
+                  struct wimlib_lzx_context **comp_ctx)
+{
+       LIST_HEAD(stream_list);
+       list_add(&lte->write_streams_list, &stream_list);
+       lte->will_be_in_output_wim = 1;
+       return write_stream_list(&stream_list,
+                                out_fd,
+                                write_resource_flags & ~WIMLIB_WRITE_RESOURCE_FLAG_PACK_STREAMS,
+                                out_ctype,
+                                out_chunk_size,
+                                1,
+                                NULL,
+                                NULL,
+                                comp_ctx,
+                                NULL);
+}
+
+int
+write_wim_resource_from_buffer(const void *buf, size_t buf_size,
+                              int reshdr_flags, struct filedes *out_fd,
+                              int out_ctype,
+                              u32 out_chunk_size,
+                              struct wim_reshdr *out_reshdr,
+                              u8 *hash,
+                              int write_resource_flags,
+                              struct wimlib_lzx_context **comp_ctx)

 {
        int ret;
 {
        int ret;

-       int write_resource_flags;
-       u64 total_bytes;
-       u64 total_compression_bytes;
-       unsigned total_parts;
-       WIMStruct *prev_wim_part;
-       size_t num_streams;

        struct wim_lookup_table_entry *lte;
        struct wim_lookup_table_entry *lte;

-       struct write_streams_progress_data progress_data;

 
 

-       if (list_empty(stream_list)) {
-               DEBUG("No streams to write.");
-               return 0;
-       }
+       /* Set up a temporary lookup table entry to provide to
+        * write_wim_resource().  */

 
 

-       write_resource_flags = write_flags_to_resource_flags(write_flags);
+       lte = new_lookup_table_entry();
+       if (lte == NULL)
+               return WIMLIB_ERR_NOMEM;

 
 

-       DEBUG("Writing stream list (offset = %"PRIu64", write_resource_flags=0x%08x)",
-             out_fd->offset, write_resource_flags);
+       lte->resource_location  = RESOURCE_IN_ATTACHED_BUFFER;
+       lte->attached_buffer    = (void*)buf;
+       lte->size               = buf_size;
+       lte->flags              = reshdr_flags;

 
 

-       /* Sort the stream list into a good order for reading.  */
-       ret = sort_stream_list_by_sequential_order(stream_list,
-                                                  offsetof(struct wim_lookup_table_entry,
-                                                           write_streams_list));
+       if (write_resource_flags & WIMLIB_WRITE_RESOURCE_FLAG_PIPABLE) {
+               sha1_buffer(buf, buf_size, lte->hash);
+               lte->unhashed = 0;
+       } else {
+               lte->unhashed = 1;
+       }
+
+       ret = write_wim_resource(lte, out_fd, out_ctype, out_chunk_size,
+                                write_resource_flags, comp_ctx);

        if (ret)
        if (ret)

-               return ret;
+               goto out_free_lte;

 
 

-       /* Calculate the total size of the streams to be written.  Note: this
-        * will be the uncompressed size, as we may not know the compressed size
-        * yet, and also this will assume that every unhashed stream will be
-        * written (which will not necessarily be the case).  */
-       total_bytes = 0;
-       total_compression_bytes = 0;
-       num_streams = 0;
-       total_parts = 0;
-       prev_wim_part = NULL;
-       list_for_each_entry(lte, stream_list, write_streams_list) {
-               num_streams++;
-               total_bytes += lte->size;
-               if (must_compress_stream(lte, write_resource_flags,
-                                        out_ctype, out_chunk_size))
-                       total_compression_bytes += lte->size;
-               if (lte->resource_location == RESOURCE_IN_WIM) {
-                       if (prev_wim_part != lte->rspec->wim) {
-                               prev_wim_part = lte->rspec->wim;
-                               total_parts++;
-                       }
-               }
-       }
+       copy_reshdr(out_reshdr, &lte->out_reshdr);

 
 

-       memset(&progress_data, 0, sizeof(progress_data));
-       progress_data.progress_func = progress_func;
-
-       progress_data.progress.write_streams.total_bytes       = total_bytes;
-       progress_data.progress.write_streams.total_streams     = num_streams;
-       progress_data.progress.write_streams.completed_bytes   = 0;
-       progress_data.progress.write_streams.completed_streams = 0;
-       progress_data.progress.write_streams.num_threads       = num_threads;
-       progress_data.progress.write_streams.compression_type  = out_ctype;
-       progress_data.progress.write_streams.total_parts       = total_parts;
-       progress_data.progress.write_streams.completed_parts   = 0;
-
-       progress_data.next_progress = 0;
-       progress_data.prev_wim_part = NULL;
-
-#ifdef ENABLE_MULTITHREADED_COMPRESSION
-       if (total_compression_bytes >= 2000000 && num_threads != 1)
-               ret = write_stream_list_parallel(stream_list,
-                                                lookup_table,
-                                                out_fd,
-                                                out_ctype,
-                                                out_chunk_size,
-                                                comp_ctx,
-                                                write_resource_flags,
-                                                &progress_data,
-                                                num_threads);
-       else
-#endif
-               ret = write_stream_list_serial(stream_list,
-                                              lookup_table,
-                                              out_fd,
-                                              out_ctype,
-                                              out_chunk_size,
-                                              comp_ctx,
-                                              write_resource_flags,
-                                              &progress_data);
-       if (ret == 0)
-               DEBUG("Successfully wrote stream list.");
-       else
-               DEBUG("Failed to write stream list (ret=%d).", ret);
+       if (hash)
+               copy_hash(hash, lte->hash);
+       ret = 0;
+out_free_lte:
+       lte->resource_location = RESOURCE_NONEXISTENT;
+       free_lookup_table_entry(lte);

        return ret;
 }
 
        return ret;
 }
 


@@ -1827,7 +1398,7 @@ static int
 init_stream_size_table(struct stream_size_table *tab, size_t capacity)
 {
        tab->array = CALLOC(capacity, sizeof(tab->array[0]));
 init_stream_size_table(struct stream_size_table *tab, size_t capacity)
 {
        tab->array = CALLOC(capacity, sizeof(tab->array[0]));

-       if (!tab->array)
+       if (tab->array == NULL)

                return WIMLIB_ERR_NOMEM;
        tab->num_entries = 0;
        tab->capacity = capacity;
                return WIMLIB_ERR_NOMEM;
        tab->num_entries = 0;
        tab->capacity = capacity;


@@ -1871,40 +1442,42 @@ struct find_streams_ctx {
 };
 
 static void
 };
 
 static void

-lte_reference_for_logical_write(struct wim_lookup_table_entry *lte,
-                               struct find_streams_ctx *ctx,
-                               unsigned nref)
+reference_stream_for_write(struct wim_lookup_table_entry *lte,
+                          struct list_head *stream_list, u32 nref)

 {
 {

-       if (lte->out_refcnt == 0) {
-               stream_size_table_insert(lte, &ctx->stream_size_tab);
-               list_add_tail(&lte->write_streams_list, &ctx->stream_list);
+       if (!lte->will_be_in_output_wim) {
+               lte->out_refcnt = 0;
+               list_add_tail(&lte->write_streams_list, stream_list);
+               lte->will_be_in_output_wim = 1;

        }
        lte->out_refcnt += nref;
 }
 
 static int
        }
        lte->out_refcnt += nref;
 }
 
 static int

-do_lte_full_reference_for_logical_write(struct wim_lookup_table_entry *lte,
-                                       void *_ctx)
+fully_reference_stream_for_write(struct wim_lookup_table_entry *lte,
+                                void *_stream_list)

 {
 {

-       struct find_streams_ctx *ctx = _ctx;
-       lte->out_refcnt = 0;
-       lte_reference_for_logical_write(lte, ctx,
-                                       (lte->refcnt ? lte->refcnt : 1));
+       struct list_head *stream_list = _stream_list;
+       lte->will_be_in_output_wim = 0;
+       reference_stream_for_write(lte, stream_list, lte->refcnt);

        return 0;
 }
 
 static int
        return 0;
 }
 
 static int

-inode_find_streams_to_write(struct wim_inode *inode,
-                           struct wim_lookup_table *table,
-                           struct find_streams_ctx *ctx)
+inode_find_streams_to_reference(const struct wim_inode *inode,
+                               const struct wim_lookup_table *table,
+                               struct list_head *stream_list)

 {
        struct wim_lookup_table_entry *lte;
        unsigned i;
 
 {
        struct wim_lookup_table_entry *lte;
        unsigned i;
 

+       wimlib_assert(inode->i_nlink > 0);
+

        for (i = 0; i <= inode->i_num_ads; i++) {
                lte = inode_stream_lte(inode, i, table);
                if (lte)
        for (i = 0; i <= inode->i_num_ads; i++) {
                lte = inode_stream_lte(inode, i, table);
                if (lte)

-                       lte_reference_for_logical_write(lte, ctx, inode->i_nlink);
+                       reference_stream_for_write(lte, stream_list,
+                                                  inode->i_nlink);

                else if (!is_zero_hash(inode_stream_hash(inode, i)))
                        return WIMLIB_ERR_RESOURCE_NOT_FOUND;
        }
                else if (!is_zero_hash(inode_stream_hash(inode, i)))
                        return WIMLIB_ERR_RESOURCE_NOT_FOUND;
        }


@@ -1912,52 +1485,47 @@ inode_find_streams_to_write(struct wim_inode *inode,
 }
 
 static int
 }
 
 static int

-image_find_streams_to_write(WIMStruct *wim)
+do_stream_set_not_in_output_wim(struct wim_lookup_table_entry *lte, void *_ignore)
+{
+       lte->will_be_in_output_wim = 0;
+       return 0;
+}
+
+static int
+image_find_streams_to_reference(WIMStruct *wim)

 {
 {

-       struct find_streams_ctx *ctx;

        struct wim_image_metadata *imd;
        struct wim_inode *inode;
        struct wim_lookup_table_entry *lte;
        struct wim_image_metadata *imd;
        struct wim_inode *inode;
        struct wim_lookup_table_entry *lte;

+       struct list_head *stream_list;

        int ret;
 
        int ret;
 

-       ctx = wim->private;

        imd = wim_get_current_image_metadata(wim);
 
        image_for_each_unhashed_stream(lte, imd)
        imd = wim_get_current_image_metadata(wim);
 
        image_for_each_unhashed_stream(lte, imd)

-               lte->out_refcnt = 0;
+               lte->will_be_in_output_wim = 0;

 
 

-       /* Go through this image's inodes to find any streams that have not been
-        * found yet. */
+       stream_list = wim->private;

        image_for_each_inode(inode, imd) {
        image_for_each_inode(inode, imd) {

-               ret = inode_find_streams_to_write(inode, wim->lookup_table, ctx);
+               ret = inode_find_streams_to_reference(inode,
+                                                     wim->lookup_table,
+                                                     stream_list);

                if (ret)
                        return ret;
        }
        return 0;
 }
 
                if (ret)
                        return ret;
        }
        return 0;
 }
 

-/*
- * Build a list of streams (via `struct wim_lookup_table_entry's) included in
- * the "logical write" of the WIM, meaning all streams that are referenced at
- * least once by dentries in the the image(s) being written.  'out_refcnt' on
- * each stream being included in the logical write is set to the number of
- * references from dentries in the image(s).  Furthermore, 'unique_size' on each
- * stream being included in the logical write is set to indicate whether that
- * stream has a unique size relative to the streams being included in the
- * logical write.  Still furthermore, 'part_number' on each stream being
- * included in the logical write is set to the part number given in the
- * in-memory header of @p wim.
- *
- * This is considered a "logical write" because it does not take into account
- * filtering out streams already present in the WIM (in the case of an in place
- * overwrite) or present in other WIMs (in case of creating delta WIM).
- */

 static int
 static int

-prepare_logical_stream_list(WIMStruct *wim, int image, bool streams_ok,
-                           struct find_streams_ctx *ctx)
+prepare_unfiltered_list_of_streams_in_output_wim(WIMStruct *wim,
+                                                int image,
+                                                int streams_ok,
+                                                struct list_head *stream_list_ret)

 {
        int ret;
 
 {
        int ret;
 

+       INIT_LIST_HEAD(stream_list_ret);
+

        if (streams_ok && (image == WIMLIB_ALL_IMAGES ||
                           (image == 1 && wim->hdr.image_count == 1)))
        {
        if (streams_ok && (image == WIMLIB_ALL_IMAGES ||
                           (image == 1 && wim->hdr.image_count == 1)))
        {


@@ -1968,18 +1536,21 @@ prepare_logical_stream_list(WIMStruct *wim, int image, bool streams_ok,
                unsigned i;
 
                for_lookup_table_entry(wim->lookup_table,
                unsigned i;
 
                for_lookup_table_entry(wim->lookup_table,

-                                      do_lte_full_reference_for_logical_write, ctx);
+                                      fully_reference_stream_for_write,
+                                      stream_list_ret);
+

                for (i = 0; i < wim->hdr.image_count; i++) {
                        imd = wim->image_metadata[i];
                        image_for_each_unhashed_stream(lte, imd)
                for (i = 0; i < wim->hdr.image_count; i++) {
                        imd = wim->image_metadata[i];
                        image_for_each_unhashed_stream(lte, imd)

-                               do_lte_full_reference_for_logical_write(lte, ctx);
+                               fully_reference_stream_for_write(lte, stream_list_ret);

                }
        } else {
                /* Slow case:  Walk through the images being written and
                 * determine the streams referenced.  */
                }
        } else {
                /* Slow case:  Walk through the images being written and
                 * determine the streams referenced.  */

-               for_lookup_table_entry(wim->lookup_table, lte_zero_out_refcnt, NULL);
-               wim->private = ctx;
-               ret = for_image(wim, image, image_find_streams_to_write);
+               for_lookup_table_entry(wim->lookup_table,
+                                      do_stream_set_not_in_output_wim, NULL);
+               wim->private = stream_list_ret;
+               ret = for_image(wim, image, image_find_streams_to_reference);

                if (ret)
                        return ret;
        }
                if (ret)
                        return ret;
        }


@@ -1987,129 +1558,203 @@ prepare_logical_stream_list(WIMStruct *wim, int image, bool streams_ok,
        return 0;
 }
 
        return 0;
 }
 

+struct insert_other_if_hard_filtered_ctx {
+       struct stream_size_table *tab;
+       struct filter_context *filter_ctx;
+};
+

 static int
 static int

-process_filtered_stream(struct wim_lookup_table_entry *lte, void *_ctx)
-{
-       struct find_streams_ctx *ctx = _ctx;
-       u16 filtered = 0;
-
-       /* Calculate and set lte->filtered.  */
-       if (lte->resource_location == RESOURCE_IN_WIM) {
-               if (lte->rspec->wim == ctx->wim &&
-                   (ctx->write_flags & WIMLIB_WRITE_FLAG_OVERWRITE))
-                       filtered |= FILTERED_SAME_WIM;
-               if (lte->rspec->wim != ctx->wim &&
-                   (ctx->write_flags & WIMLIB_WRITE_FLAG_SKIP_EXTERNAL_WIMS))
-                       filtered |= FILTERED_EXTERNAL_WIM;
-       }
-       lte->filtered = filtered;
-
-       /* Filtered streams get inserted into the stream size table too, unless
-        * they already were.  This is because streams that are checksummed
-        * on-the-fly during the write should not be written if they are
-        * duplicates of filtered stream.  */
-       if (lte->filtered && lte->out_refcnt == 0)
-               stream_size_table_insert(lte, &ctx->stream_size_tab);
+insert_other_if_hard_filtered(struct wim_lookup_table_entry *lte, void *_ctx)
+{
+       struct insert_other_if_hard_filtered_ctx *ctx = _ctx;
+
+       if (!lte->will_be_in_output_wim &&
+           stream_hard_filtered(lte, ctx->filter_ctx))
+               stream_size_table_insert(lte, ctx->tab);

        return 0;
 }
 
 static int
        return 0;
 }
 
 static int

-mark_stream_not_filtered(struct wim_lookup_table_entry *lte, void *_ignore)
+determine_stream_size_uniquity(struct list_head *stream_list,
+                              struct wim_lookup_table *lt,
+                              struct filter_context *filter_ctx)

 {
 {

-       lte->filtered = 0;
+       int ret;
+       struct stream_size_table tab;
+       struct wim_lookup_table_entry *lte;
+
+       ret = init_stream_size_table(&tab, lt->capacity);
+       if (ret)
+               return ret;
+
+       if (may_hard_filter_streams(filter_ctx)) {
+               struct insert_other_if_hard_filtered_ctx ctx = {
+                       .tab = &tab,
+                       .filter_ctx = filter_ctx,
+               };
+               for_lookup_table_entry(lt, insert_other_if_hard_filtered, &ctx);
+       }
+
+       list_for_each_entry(lte, stream_list, write_streams_list)
+               stream_size_table_insert(lte, &tab);
+
+       destroy_stream_size_table(&tab);

        return 0;
 }
 
        return 0;
 }
 

-/* Given the list of streams to include in a logical write of a WIM, handle
- * filtering out streams already present in the WIM or already present in
- * external WIMs, depending on the write flags provided.  */

 static void
 static void

-handle_stream_filtering(struct find_streams_ctx *ctx)
+filter_stream_list_for_write(struct list_head *stream_list,
+                            struct filter_context *filter_ctx)

 {
        struct wim_lookup_table_entry *lte, *tmp;
 
 {
        struct wim_lookup_table_entry *lte, *tmp;
 

-       if (!(ctx->write_flags & (WIMLIB_WRITE_FLAG_OVERWRITE |
-                                 WIMLIB_WRITE_FLAG_SKIP_EXTERNAL_WIMS)))
+       list_for_each_entry_safe(lte, tmp,
+                                stream_list, write_streams_list)

        {
        {

-               for_lookup_table_entry(ctx->wim->lookup_table,
-                                      mark_stream_not_filtered, ctx);
-               return;
-       }
+               int status = stream_filtered(lte, filter_ctx);

 
 

-       for_lookup_table_entry(ctx->wim->lookup_table,
-                              process_filtered_stream, ctx);
-
-       /* Streams in logical write list that were filtered can be removed.  */
-       list_for_each_entry_safe(lte, tmp, &ctx->stream_list,
-                                write_streams_list)
-               if (lte->filtered)
+               if (status == 0) {
+                       /* Not filtered.  */
+                       continue;
+               } else {
+                       if (status > 0) {
+                               /* Soft filtered.  */
+                       } else {
+                               /* Hard filtered.  */
+                               lte->will_be_in_output_wim = 0;
+                               list_del(&lte->lookup_table_list);
+                       }

                        list_del(&lte->write_streams_list);
                        list_del(&lte->write_streams_list);

+               }
+       }

 }
 
 }
 

-/* Prepares list of streams to write for the specified WIM image(s).  This wraps
- * around prepare_logical_stream_list() to handle filtering out streams already
- * present in the WIM or already present in external WIMs, depending on the
- * write flags provided.
+/*
+ * prepare_stream_list_for_write() -
+ *
+ * Prepare the list of streams to write for writing a WIM containing the
+ * specified image(s) with the specified write flags.
+ *
+ * @wim
+ *     The WIMStruct on whose behalf the write is occurring.
+ *
+ * @image
+ *     Image(s) from the WIM to write; may be WIMLIB_ALL_IMAGES.
+ *
+ * @write_flags
+ *     WIMLIB_WRITE_FLAG_* flags for the write operation:

  *
  *

- * Note: some additional data is stored in each `struct wim_lookup_table_entry':
+ *     STREAMS_OK:  For writes of all images, assume that all streams in the
+ *     lookup table of @wim and the per-image lists of unhashed streams should
+ *     be taken as-is, and image metadata should not be searched for
+ *     references.  This does not exclude filtering with OVERWRITE and
+ *     SKIP_EXTERNAL_WIMS, below.

  *
  *

- * - 'out_refcnt' is set to the number of references found for the logical write.
- *    This will be nonzero on all streams in the list returned by this function,
- *    but will also be nonzero on streams not in the list that were included in
- *    the logical write list, but filtered out from the returned list.
- * - 'filtered' is set to nonzero if the stream was filtered.  Filtered streams
- *   are not included in the list of streams returned by this function.
- * - 'unique_size' is set if the stream has a unique size among all streams in
- *   the logical write plus any filtered streams in the entire WIM that could
- *   potentially turn out to have the same checksum as a yet-to-be-checksummed
- *   stream being written.
+ *     OVERWRITE:  Streams already present in @wim shall not be returned in
+ *     @stream_list_ret.
+ *
+ *     SKIP_EXTERNAL_WIMS:  Streams already present in a WIM file, but not
+ *     @wim, shall be be returned in neither @stream_list_ret nor
+ *     @lookup_table_list_ret.
+ *
+ * @stream_list_ret
+ *     List of streams, linked by write_streams_list, that need to be written
+ *     will be returned here.
+ *
+ *     Note that this function assumes that unhashed streams will be written;
+ *     it does not take into account that they may become duplicates when
+ *     actually hashed.
+ *
+ * @lookup_table_list_ret
+ *     List of streams, linked by lookup_table_list, that need to be included
+ *     in the WIM's lookup table will be returned here.  This will be a
+ *     superset of the streams in @stream_list_ret.
+ *
+ *     This list will be a proper superset of @stream_list_ret if and only if
+ *     WIMLIB_WRITE_FLAG_OVERWRITE was specified in @write_flags and some of
+ *     the streams that would otherwise need to be written were already located
+ *     in the WIM file.
+ *
+ *     All streams in this list will have @out_refcnt set to the number of
+ *     references to the stream in the output WIM.  If
+ *     WIMLIB_WRITE_FLAG_STREAMS_OK was specified in @write_flags, @out_refcnt
+ *     may be as low as 0.
+ *
+ * @filter_ctx_ret
+ *     A context for queries of stream filter status with stream_filtered() is
+ *     returned in this location.
+ *
+ * In addition, @will_be_in_output_wim will be set to 1 in all stream entries
+ * inserted into @lookup_table_list_ret and to 0 in all stream entries in the
+ * lookup table of @wim not inserted into @lookup_table_list_ret.
+ *
+ * Still furthermore, @unique_size will be set to 1 on all stream entries in
+ * @stream_list_ret that have unique size among all stream entries in
+ * @stream_list_ret and among all stream entries in the lookup table of @wim
+ * that are ineligible for being written due to filtering.
+ *
+ * Returns 0 on success; nonzero on read error, memory allocation error, or
+ * otherwise.

  */
 static int
  */
 static int

-prepare_stream_list(WIMStruct *wim, int image, int write_flags,
-                   struct list_head *stream_list)
+prepare_stream_list_for_write(WIMStruct *wim, int image,
+                             int write_flags,
+                             struct list_head *stream_list_ret,
+                             struct list_head *lookup_table_list_ret,
+                             struct filter_context *filter_ctx_ret)

 {
        int ret;
 {
        int ret;

-       bool streams_ok;
-       struct find_streams_ctx ctx;
+       struct wim_lookup_table_entry *lte;
+
+       filter_ctx_ret->write_flags = write_flags;
+       filter_ctx_ret->wim = wim;

 
 

-       INIT_LIST_HEAD(&ctx.stream_list);
-       ret = init_stream_size_table(&ctx.stream_size_tab,
-                                    wim->lookup_table->capacity);
+       ret = prepare_unfiltered_list_of_streams_in_output_wim(
+                               wim,
+                               image,
+                               write_flags & WIMLIB_WRITE_FLAG_STREAMS_OK,
+                               stream_list_ret);

        if (ret)
                return ret;
        if (ret)
                return ret;

-       ctx.write_flags = write_flags;
-       ctx.wim = wim;

 
 

-       streams_ok = ((write_flags & WIMLIB_WRITE_FLAG_STREAMS_OK) != 0);
+       INIT_LIST_HEAD(lookup_table_list_ret);
+       list_for_each_entry(lte, stream_list_ret, write_streams_list)
+               list_add_tail(&lte->lookup_table_list, lookup_table_list_ret);

 
 

-       ret = prepare_logical_stream_list(wim, image, streams_ok, &ctx);
+       ret = determine_stream_size_uniquity(stream_list_ret, wim->lookup_table,
+                                            filter_ctx_ret);

        if (ret)
        if (ret)

-               goto out_destroy_table;
+               return ret;

 
 

-       handle_stream_filtering(&ctx);
-       list_transfer(&ctx.stream_list, stream_list);
-       ret = 0;
-out_destroy_table:
-       destroy_stream_size_table(&ctx.stream_size_tab);
-       return ret;
+       if (may_filter_streams(filter_ctx_ret))
+               filter_stream_list_for_write(stream_list_ret, filter_ctx_ret);
+
+       return 0;

 }
 
 static int
 write_wim_streams(WIMStruct *wim, int image, int write_flags,
                  unsigned num_threads,
                  wimlib_progress_func_t progress_func,
 }
 
 static int
 write_wim_streams(WIMStruct *wim, int image, int write_flags,
                  unsigned num_threads,
                  wimlib_progress_func_t progress_func,

-                 struct list_head *stream_list_override)
+                 struct list_head *stream_list_override,
+                 struct list_head *lookup_table_list_ret)

 {
        int ret;
        struct list_head _stream_list;
        struct list_head *stream_list;
        struct wim_lookup_table_entry *lte;
 {
        int ret;
        struct list_head _stream_list;
        struct list_head *stream_list;
        struct wim_lookup_table_entry *lte;

+       struct filter_context _filter_ctx;
+       struct filter_context *filter_ctx;

 
        if (stream_list_override == NULL) {
                /* Normal case: prepare stream list from image(s) being written.
                 */
                stream_list = &_stream_list;
 
        if (stream_list_override == NULL) {
                /* Normal case: prepare stream list from image(s) being written.
                 */
                stream_list = &_stream_list;

-               ret = prepare_stream_list(wim, image, write_flags, stream_list);
+               filter_ctx = &_filter_ctx;
+               ret = prepare_stream_list_for_write(wim, image, write_flags,
+                                                   stream_list,
+                                                   lookup_table_list_ret,
+                                                   filter_ctx);

                if (ret)
                        return ret;
        } else {
                if (ret)
                        return ret;
        } else {


@@ -2117,18 +1762,25 @@ write_wim_streams(WIMStruct *wim, int image, int write_flags,
                 * use stream list already explicitly provided.  Use existing
                 * reference counts.  */
                stream_list = stream_list_override;
                 * use stream list already explicitly provided.  Use existing
                 * reference counts.  */
                stream_list = stream_list_override;

-               list_for_each_entry(lte, stream_list, write_streams_list)
-                       lte->out_refcnt = (lte->refcnt ? lte->refcnt : 1);
+               filter_ctx = NULL;
+               INIT_LIST_HEAD(lookup_table_list_ret);
+               list_for_each_entry(lte, stream_list, write_streams_list) {
+                       lte->out_refcnt = lte->refcnt;
+                       lte->will_be_in_output_wim = 1;
+                       lte->unique_size = 0;
+                       list_add_tail(&lte->lookup_table_list, lookup_table_list_ret);
+               }

        }
 
        return write_stream_list(stream_list,
        }
 
        return write_stream_list(stream_list,

-                                wim->lookup_table,

                                 &wim->out_fd,
                                 &wim->out_fd,

+                                write_flags_to_resource_flags(write_flags),

                                 wim->out_compression_type,
                                 wim->out_chunk_size,
                                 wim->out_compression_type,
                                 wim->out_chunk_size,

-                                &wim->lzx_context,
-                                write_flags,

                                 num_threads,
                                 num_threads,

+                                wim->lookup_table,
+                                filter_ctx,
+                                &wim->lzx_context,

                                 progress_func);
 }
 
                                 progress_func);
 }
 


@@ -2148,6 +1800,8 @@ write_wim_metadata_resources(WIMStruct *wim, int image, int write_flags,
 
        write_resource_flags = write_flags_to_resource_flags(write_flags);
 
 
        write_resource_flags = write_flags_to_resource_flags(write_flags);
 

+       write_resource_flags &= ~WIMLIB_WRITE_RESOURCE_FLAG_PACK_STREAMS;
+

        DEBUG("Writing metadata resources (offset=%"PRIu64")",
              wim->out_fd.offset);
 
        DEBUG("Writing metadata resources (offset=%"PRIu64")",
              wim->out_fd.offset);
 


@@ -2177,8 +1831,7 @@ write_wim_metadata_resources(WIMStruct *wim, int image, int write_flags,
                } else if (write_flags & WIMLIB_WRITE_FLAG_OVERWRITE) {
                        DEBUG("Image %u was not modified; re-using existing "
                              "metadata resource.", i);
                } else if (write_flags & WIMLIB_WRITE_FLAG_OVERWRITE) {
                        DEBUG("Image %u was not modified; re-using existing "
                              "metadata resource.", i);

-                       wim_res_spec_to_hdr(imd->metadata_lte->rspec,
-                                           &imd->metadata_lte->out_reshdr);
+                       stream_set_out_reshdr_for_reuse(imd->metadata_lte);

                        ret = 0;
                } else {
                        DEBUG("Image %u was not modified; copying existing "
                        ret = 0;
                } else {
                        DEBUG("Image %u was not modified; copying existing "


@@ -2187,7 +1840,6 @@ write_wim_metadata_resources(WIMStruct *wim, int image, int write_flags,
                                                 &wim->out_fd,
                                                 wim->out_compression_type,
                                                 wim->out_chunk_size,
                                                 &wim->out_fd,
                                                 wim->out_compression_type,
                                                 wim->out_chunk_size,

-                                                &imd->metadata_lte->out_reshdr,

                                                 write_resource_flags,
                                                 &wim->lzx_context);
                }
                                                 write_resource_flags,
                                                 &wim->lzx_context);
                }


@@ -2229,6 +1881,90 @@ close_wim_writable(WIMStruct *wim, int write_flags)
        return ret;
 }
 
        return ret;
 }
 

+static int
+cmp_streams_by_out_rspec(const void *p1, const void *p2)
+{
+       const struct wim_lookup_table_entry *lte1, *lte2;
+
+       lte1 = *(const struct wim_lookup_table_entry**)p1;
+       lte2 = *(const struct wim_lookup_table_entry**)p2;
+
+       if (lte1->out_reshdr.flags & WIM_RESHDR_FLAG_PACKED_STREAMS) {
+               if (lte2->out_reshdr.flags & WIM_RESHDR_FLAG_PACKED_STREAMS) {
+                       if (lte1->out_res_offset_in_wim != lte2->out_res_offset_in_wim)
+                               return cmp_u64(lte1->out_res_offset_in_wim,
+                                              lte2->out_res_offset_in_wim);
+               } else {
+                       return 1;
+               }
+       } else {
+               if (lte2->out_reshdr.flags & WIM_RESHDR_FLAG_PACKED_STREAMS)
+                       return -1;
+       }
+       return cmp_u64(lte1->out_reshdr.offset_in_wim, 
+                      lte2->out_reshdr.offset_in_wim);
+}
+
+static int
+write_wim_lookup_table(WIMStruct *wim, int image, int write_flags,
+                      struct wim_reshdr *out_reshdr,
+                      struct list_head *lookup_table_list)
+{
+       int ret;
+
+       /* Set output resource metadata for streams already present in WIM.  */
+       if (write_flags & WIMLIB_WRITE_FLAG_OVERWRITE) {
+               struct wim_lookup_table_entry *lte;
+               list_for_each_entry(lte, lookup_table_list, lookup_table_list)
+               {
+                       if (lte->resource_location == RESOURCE_IN_WIM &&
+                           lte->rspec->wim == wim)
+                       {
+                               stream_set_out_reshdr_for_reuse(lte);
+                       }
+               }
+       }
+
+       ret = sort_stream_list(lookup_table_list,
+                              offsetof(struct wim_lookup_table_entry, lookup_table_list),
+                              cmp_streams_by_out_rspec);
+       if (ret)
+               return ret;
+
+       /* Add entries for metadata resources.  */
+       if (!(write_flags & WIMLIB_WRITE_FLAG_NO_METADATA)) {
+               int start_image;
+               int end_image;
+
+               if (image == WIMLIB_ALL_IMAGES) {
+                       start_image = 1;
+                       end_image = wim->hdr.image_count;
+               } else {
+                       start_image = image;
+                       end_image = image;
+               }
+
+               /* Push metadata resource lookup table entries onto the front of
+                * the list in reverse order, so that they're written in order.
+                */
+               for (int i = end_image; i >= start_image; i--) {
+                       struct wim_lookup_table_entry *metadata_lte;
+
+                       metadata_lte = wim->image_metadata[i - 1]->metadata_lte;
+                       wimlib_assert(metadata_lte->out_reshdr.flags & WIM_RESHDR_FLAG_METADATA);
+                       metadata_lte->out_refcnt = 1;
+                       list_add(&metadata_lte->lookup_table_list, lookup_table_list);
+               }
+       }
+
+       return write_wim_lookup_table_from_stream_list(lookup_table_list,
+                                                      &wim->out_fd,
+                                                      wim->hdr.part_number,
+                                                      out_reshdr,
+                                                      write_flags_to_resource_flags(write_flags),
+                                                      &wim->lzx_context);
+}
+

 /*
  * finish_write():
  *
 /*
  * finish_write():
  *


@@ -2278,7 +2014,7 @@ close_wim_writable(WIMStruct *wim, int write_flags)
 static int
 finish_write(WIMStruct *wim, int image, int write_flags,
             wimlib_progress_func_t progress_func,
 static int
 finish_write(WIMStruct *wim, int image, int write_flags,
             wimlib_progress_func_t progress_func,

-            struct list_head *stream_list_override)
+            struct list_head *lookup_table_list)

 {
        int ret;
        off_t hdr_offset;
 {
        int ret;
        off_t hdr_offset;


@@ -2300,8 +2036,8 @@ finish_write(WIMStruct *wim, int image, int write_flags,
                zero_reshdr(&wim->hdr.boot_metadata_reshdr);
        } else {
                copy_reshdr(&wim->hdr.boot_metadata_reshdr,
                zero_reshdr(&wim->hdr.boot_metadata_reshdr);
        } else {
                copy_reshdr(&wim->hdr.boot_metadata_reshdr,

-                           &wim->image_metadata[wim->hdr.boot_idx- 1
-                                       ]->metadata_lte->out_reshdr);
+                           &wim->image_metadata[
+                               wim->hdr.boot_idx - 1]->metadata_lte->out_reshdr);

        }
 
        /* Write lookup table.  (Save old position first.)  */
        }
 
        /* Write lookup table.  (Save old position first.)  */


@@ -2310,7 +2046,7 @@ finish_write(WIMStruct *wim, int image, int write_flags,
        if (!(write_flags & WIMLIB_WRITE_FLAG_NO_LOOKUP_TABLE)) {
                ret = write_wim_lookup_table(wim, image, write_flags,
                                             &wim->hdr.lookup_table_reshdr,
        if (!(write_flags & WIMLIB_WRITE_FLAG_NO_LOOKUP_TABLE)) {
                ret = write_wim_lookup_table(wim, image, write_flags,
                                             &wim->hdr.lookup_table_reshdr,

-                                            stream_list_override);
+                                            lookup_table_list);

                if (ret)
                        return ret;
        }
                if (ret)
                        return ret;
        }


@@ -2497,7 +2233,8 @@ lock_wim(WIMStruct *wim, int fd)
 static int
 write_pipable_wim(WIMStruct *wim, int image, int write_flags,
                  unsigned num_threads, wimlib_progress_func_t progress_func,
 static int
 write_pipable_wim(WIMStruct *wim, int image, int write_flags,
                  unsigned num_threads, wimlib_progress_func_t progress_func,

-                 struct list_head *stream_list_override)
+                 struct list_head *stream_list_override,
+                 struct list_head *lookup_table_list_ret)

 {
        int ret;
        struct wim_reshdr xml_reshdr;
 {
        int ret;
        struct wim_reshdr xml_reshdr;


@@ -2538,7 +2275,8 @@ write_pipable_wim(WIMStruct *wim, int image, int write_flags,
        /* Write streams needed for the image(s) being included in the output
         * WIM, or streams needed for the split WIM part.  */
        return write_wim_streams(wim, image, write_flags, num_threads,
        /* Write streams needed for the image(s) being included in the output
         * WIM, or streams needed for the split WIM part.  */
        return write_wim_streams(wim, image, write_flags, num_threads,

-                                progress_func, stream_list_override);
+                                progress_func, stream_list_override,
+                                lookup_table_list_ret);

 
        /* The lookup table, XML data, and header at end are handled by
         * finish_write().  */
 
        /* The lookup table, XML data, and header at end are handled by
         * finish_write().  */


@@ -2560,7 +2298,7 @@ write_wim_part(WIMStruct *wim,
 {
        int ret;
        struct wim_header hdr_save;
 {
        int ret;
        struct wim_header hdr_save;

-       struct list_head lt_stream_list_override;
+       struct list_head lookup_table_list;

 
        if (total_parts == 1)
                DEBUG("Writing standalone WIM.");
 
        if (total_parts == 1)
                DEBUG("Writing standalone WIM.");


@@ -2575,26 +2313,46 @@ write_wim_part(WIMStruct *wim,
        else
                DEBUG("Path: \"%"TS"\"", (const tchar*)path_or_fd);
        DEBUG("Write flags: 0x%08x", write_flags);
        else
                DEBUG("Path: \"%"TS"\"", (const tchar*)path_or_fd);
        DEBUG("Write flags: 0x%08x", write_flags);

+

        if (write_flags & WIMLIB_WRITE_FLAG_CHECK_INTEGRITY)
                DEBUG("\tCHECK_INTEGRITY");
        if (write_flags & WIMLIB_WRITE_FLAG_CHECK_INTEGRITY)
                DEBUG("\tCHECK_INTEGRITY");

+

        if (write_flags & WIMLIB_WRITE_FLAG_REBUILD)
                DEBUG("\tREBUILD");
        if (write_flags & WIMLIB_WRITE_FLAG_REBUILD)
                DEBUG("\tREBUILD");

+

        if (write_flags & WIMLIB_WRITE_FLAG_RECOMPRESS)
                DEBUG("\tRECOMPRESS");
        if (write_flags & WIMLIB_WRITE_FLAG_RECOMPRESS)
                DEBUG("\tRECOMPRESS");

+

        if (write_flags & WIMLIB_WRITE_FLAG_FSYNC)
                DEBUG("\tFSYNC");
        if (write_flags & WIMLIB_WRITE_FLAG_FSYNC)
                DEBUG("\tFSYNC");

+

        if (write_flags & WIMLIB_WRITE_FLAG_SOFT_DELETE)
                DEBUG("\tFSYNC");
        if (write_flags & WIMLIB_WRITE_FLAG_SOFT_DELETE)
                DEBUG("\tFSYNC");

+

        if (write_flags & WIMLIB_WRITE_FLAG_IGNORE_READONLY_FLAG)
                DEBUG("\tIGNORE_READONLY_FLAG");
        if (write_flags & WIMLIB_WRITE_FLAG_IGNORE_READONLY_FLAG)
                DEBUG("\tIGNORE_READONLY_FLAG");

+

        if (write_flags & WIMLIB_WRITE_FLAG_PIPABLE)
                DEBUG("\tPIPABLE");
        if (write_flags & WIMLIB_WRITE_FLAG_PIPABLE)
                DEBUG("\tPIPABLE");

+
+       if (write_flags & WIMLIB_WRITE_FLAG_NOT_PIPABLE)
+               DEBUG("\tNOT_PIPABLE");
+
+       if (write_flags & WIMLIB_WRITE_FLAG_PACK_STREAMS)
+               DEBUG("\tPACK_STREAMS");
+
+       if (write_flags & WIMLIB_WRITE_FLAG_NO_PACK_STREAMS)
+               DEBUG("\tNO_PACK_STREAMS");
+

        if (write_flags & WIMLIB_WRITE_FLAG_FILE_DESCRIPTOR)
                DEBUG("\tFILE_DESCRIPTOR");
        if (write_flags & WIMLIB_WRITE_FLAG_FILE_DESCRIPTOR)
                DEBUG("\tFILE_DESCRIPTOR");

+

        if (write_flags & WIMLIB_WRITE_FLAG_NO_METADATA)
                DEBUG("\tNO_METADATA");
        if (write_flags & WIMLIB_WRITE_FLAG_NO_METADATA)
                DEBUG("\tNO_METADATA");

+

        if (write_flags & WIMLIB_WRITE_FLAG_USE_EXISTING_TOTALBYTES)
                DEBUG("\tUSE_EXISTING_TOTALBYTES");
        if (write_flags & WIMLIB_WRITE_FLAG_USE_EXISTING_TOTALBYTES)
                DEBUG("\tUSE_EXISTING_TOTALBYTES");

+

        if (num_threads == 0)
                DEBUG("Number of threads: autodetect");
        else
        if (num_threads == 0)
                DEBUG("Number of threads: autodetect");
        else


@@ -2634,19 +2392,42 @@ write_wim_part(WIMStruct *wim,
                                    WIMLIB_WRITE_FLAG_NOT_PIPABLE))
                return WIMLIB_ERR_INVALID_PARAM;
 
                                    WIMLIB_WRITE_FLAG_NOT_PIPABLE))
                return WIMLIB_ERR_INVALID_PARAM;
 

+       if ((write_flags & (WIMLIB_WRITE_FLAG_PACK_STREAMS |
+                           WIMLIB_WRITE_FLAG_NO_PACK_STREAMS))
+                               == (WIMLIB_WRITE_FLAG_PACK_STREAMS |
+                                   WIMLIB_WRITE_FLAG_NO_PACK_STREAMS))
+               return WIMLIB_ERR_INVALID_PARAM;
+

        /* Save previous header, then start initializing the new one.  */
        memcpy(&hdr_save, &wim->hdr, sizeof(struct wim_header));
 
        /* Save previous header, then start initializing the new one.  */
        memcpy(&hdr_save, &wim->hdr, sizeof(struct wim_header));
 

-       /* Set default integrity and pipable flags.  */
+       /* Set default integrity, pipable, and packed stream flags.  */

        if (!(write_flags & (WIMLIB_WRITE_FLAG_PIPABLE |
                             WIMLIB_WRITE_FLAG_NOT_PIPABLE)))
        if (!(write_flags & (WIMLIB_WRITE_FLAG_PIPABLE |
                             WIMLIB_WRITE_FLAG_NOT_PIPABLE)))

-               if (wim_is_pipable(wim))
+               if (wim_is_pipable(wim)) {
+                       DEBUG("WIM is pipable; default to PIPABLE.");

                        write_flags |= WIMLIB_WRITE_FLAG_PIPABLE;
                        write_flags |= WIMLIB_WRITE_FLAG_PIPABLE;

+               }

 
        if (!(write_flags & (WIMLIB_WRITE_FLAG_CHECK_INTEGRITY |
                             WIMLIB_WRITE_FLAG_NO_CHECK_INTEGRITY)))
 
        if (!(write_flags & (WIMLIB_WRITE_FLAG_CHECK_INTEGRITY |
                             WIMLIB_WRITE_FLAG_NO_CHECK_INTEGRITY)))

-               if (wim_has_integrity_table(wim))
+               if (wim_has_integrity_table(wim)) {
+                       DEBUG("Integrity table present; default to CHECK_INTEGRITY.");

                        write_flags |= WIMLIB_WRITE_FLAG_CHECK_INTEGRITY;
                        write_flags |= WIMLIB_WRITE_FLAG_CHECK_INTEGRITY;

+               }
+
+       if (!(write_flags & (WIMLIB_WRITE_FLAG_PACK_STREAMS |
+                            WIMLIB_WRITE_FLAG_NO_PACK_STREAMS)))
+               if (wim->hdr.wim_version == WIM_VERSION_PACKED_STREAMS) {
+                       DEBUG("WIM version 3584; default to PACK_STREAMS.");
+                       write_flags |= WIMLIB_WRITE_FLAG_PACK_STREAMS;
+               }
+
+       if ((write_flags & (WIMLIB_WRITE_FLAG_PIPABLE |
+                           WIMLIB_WRITE_FLAG_PACK_STREAMS))
+                                   == (WIMLIB_WRITE_FLAG_PIPABLE |
+                                       WIMLIB_WRITE_FLAG_PACK_STREAMS))
+               return WIMLIB_ERR_INVALID_PARAM;

 
        /* Set appropriate magic number.  */
        if (write_flags & WIMLIB_WRITE_FLAG_PIPABLE)
 
        /* Set appropriate magic number.  */
        if (write_flags & WIMLIB_WRITE_FLAG_PIPABLE)


@@ -2654,6 +2435,12 @@ write_wim_part(WIMStruct *wim,
        else
                wim->hdr.magic = WIM_MAGIC;
 
        else
                wim->hdr.magic = WIM_MAGIC;
 

+       /* Set appropriate version number.  */
+       if (write_flags & WIMLIB_WRITE_FLAG_PACK_STREAMS)
+               wim->hdr.wim_version = WIM_VERSION_PACKED_STREAMS;
+       else
+               wim->hdr.wim_version = WIM_VERSION_DEFAULT;
+

        /* Clear header flags that will be set automatically.  */
        wim->hdr.flags &= ~(WIM_HDR_FLAG_METADATA_ONLY          |
                            WIM_HDR_FLAG_RESOURCE_ONLY          |
        /* Clear header flags that will be set automatically.  */
        wim->hdr.flags &= ~(WIM_HDR_FLAG_METADATA_ONLY          |
                            WIM_HDR_FLAG_RESOURCE_ONLY          |


@@ -2740,22 +2527,12 @@ write_wim_part(WIMStruct *wim,
        if (ret)
                goto out_restore_hdr;
 
        if (ret)
                goto out_restore_hdr;
 

-       if (stream_list_override) {
-               struct wim_lookup_table_entry *lte;
-               INIT_LIST_HEAD(&lt_stream_list_override);
-               list_for_each_entry(lte, stream_list_override,
-                                   write_streams_list)
-               {
-                       list_add_tail(&lte->lookup_table_list,
-                                     &lt_stream_list_override);
-               }
-       }
-

        /* Write metadata resources and streams.  */
        if (!(write_flags & WIMLIB_WRITE_FLAG_PIPABLE)) {
                /* Default case: create a normal (non-pipable) WIM.  */
                ret = write_wim_streams(wim, image, write_flags, num_threads,
        /* Write metadata resources and streams.  */
        if (!(write_flags & WIMLIB_WRITE_FLAG_PIPABLE)) {
                /* Default case: create a normal (non-pipable) WIM.  */
                ret = write_wim_streams(wim, image, write_flags, num_threads,

-                                       progress_func, stream_list_override);
+                                       progress_func, stream_list_override,
+                                       &lookup_table_list);

                if (ret)
                        goto out_restore_hdr;
 
                if (ret)
                        goto out_restore_hdr;
 


@@ -2766,18 +2543,17 @@ write_wim_part(WIMStruct *wim,
        } else {
                /* Non-default case: create pipable WIM.  */
                ret = write_pipable_wim(wim, image, write_flags, num_threads,
        } else {
                /* Non-default case: create pipable WIM.  */
                ret = write_pipable_wim(wim, image, write_flags, num_threads,

-                                       progress_func, stream_list_override);
+                                       progress_func, stream_list_override,
+                                       &lookup_table_list);

                if (ret)
                        goto out_restore_hdr;
                write_flags |= WIMLIB_WRITE_FLAG_HEADER_AT_END;
        }
 
                if (ret)
                        goto out_restore_hdr;
                write_flags |= WIMLIB_WRITE_FLAG_HEADER_AT_END;
        }
 

-       if (stream_list_override)
-               stream_list_override = &lt_stream_list_override;

 
        /* Write lookup table, XML data, and (optional) integrity table.  */
        ret = finish_write(wim, image, write_flags, progress_func,
 
        /* Write lookup table, XML data, and (optional) integrity table.  */
        ret = finish_write(wim, image, write_flags, progress_func,

-                          stream_list_override);
+                          &lookup_table_list);

 out_restore_hdr:
        memcpy(&wim->hdr, &hdr_save, sizeof(struct wim_header));
        (void)close_wim_writable(wim, write_flags);
 out_restore_hdr:
        memcpy(&wim->hdr, &hdr_save, sizeof(struct wim_header));
        (void)close_wim_writable(wim, write_flags);


@@ -2932,10 +2708,12 @@ overwrite_wim_inplace(WIMStruct *wim, int write_flags,
                      wimlib_progress_func_t progress_func)
 {
        int ret;
                      wimlib_progress_func_t progress_func)
 {
        int ret;

-       struct list_head stream_list;

        off_t old_wim_end;
        u64 old_lookup_table_end, old_xml_begin, old_xml_end;
        struct wim_header hdr_save;
        off_t old_wim_end;
        u64 old_lookup_table_end, old_xml_begin, old_xml_end;
        struct wim_header hdr_save;

+       struct list_head stream_list;
+       struct list_head lookup_table_list;
+       struct filter_context filter_ctx;

 
        DEBUG("Overwriting `%"TS"' in-place", wim->filename);
 
 
        DEBUG("Overwriting `%"TS"' in-place", wim->filename);
 


@@ -2945,6 +2723,12 @@ overwrite_wim_inplace(WIMStruct *wim, int write_flags,
                if (wim_has_integrity_table(wim))
                        write_flags |= WIMLIB_WRITE_FLAG_CHECK_INTEGRITY;
 
                if (wim_has_integrity_table(wim))
                        write_flags |= WIMLIB_WRITE_FLAG_CHECK_INTEGRITY;
 

+       /* Set default packed flag.  */
+       if (!(write_flags & (WIMLIB_WRITE_FLAG_PACK_STREAMS |
+                            WIMLIB_WRITE_FLAG_NO_PACK_STREAMS)))
+               if (wim->hdr.wim_version == WIM_VERSION_PACKED_STREAMS)
+                       write_flags |= WIMLIB_WRITE_FLAG_PACK_STREAMS;
+

        /* Set additional flags for overwrite.  */
        write_flags |= WIMLIB_WRITE_FLAG_OVERWRITE |
                       WIMLIB_WRITE_FLAG_STREAMS_OK;
        /* Set additional flags for overwrite.  */
        write_flags |= WIMLIB_WRITE_FLAG_OVERWRITE |
                       WIMLIB_WRITE_FLAG_STREAMS_OK;


@@ -2998,8 +2782,9 @@ overwrite_wim_inplace(WIMStruct *wim, int write_flags,
        if (ret)
                return ret;
 
        if (ret)
                return ret;
 

-       ret = prepare_stream_list(wim, WIMLIB_ALL_IMAGES, write_flags,
-                                 &stream_list);
+       ret = prepare_stream_list_for_write(wim, WIMLIB_ALL_IMAGES, write_flags,
+                                           &stream_list, &lookup_table_list,
+                                           &filter_ctx);

        if (ret)
                return ret;
 
        if (ret)
                return ret;
 


@@ -3029,13 +2814,14 @@ overwrite_wim_inplace(WIMStruct *wim, int write_flags,
        }
 
        ret = write_stream_list(&stream_list,
        }
 
        ret = write_stream_list(&stream_list,

-                               wim->lookup_table,

                                &wim->out_fd,
                                &wim->out_fd,

+                               write_flags_to_resource_flags(write_flags),

                                wim->compression_type,
                                wim->chunk_size,
                                wim->compression_type,
                                wim->chunk_size,

-                               &wim->lzx_context,
-                               write_flags,

                                num_threads,
                                num_threads,

+                               wim->lookup_table,
+                               &filter_ctx,
+                               &wim->lzx_context,

                                progress_func);
        if (ret)
                goto out_truncate;
                                progress_func);
        if (ret)
                goto out_truncate;


@@ -3047,7 +2833,7 @@ overwrite_wim_inplace(WIMStruct *wim, int write_flags,
 
        write_flags |= WIMLIB_WRITE_FLAG_REUSE_INTEGRITY_TABLE;
        ret = finish_write(wim, WIMLIB_ALL_IMAGES, write_flags,
 
        write_flags |= WIMLIB_WRITE_FLAG_REUSE_INTEGRITY_TABLE;
        ret = finish_write(wim, WIMLIB_ALL_IMAGES, write_flags,

-                          progress_func, NULL);
+                          progress_func, &lookup_table_list);

        if (ret)
                goto out_truncate;
 
        if (ret)
                goto out_truncate;
 


@@ -3126,6 +2912,28 @@ overwrite_wim_via_tmpfile(WIMStruct *wim, int write_flags,
        return 0;
 }
 
        return 0;
 }
 

+static bool
+can_overwrite_wim_inplace(const WIMStruct *wim, int write_flags)
+{
+       if (write_flags & WIMLIB_WRITE_FLAG_REBUILD)
+               return false;
+
+       if (wim->deletion_occurred && !(write_flags & WIMLIB_WRITE_FLAG_SOFT_DELETE))
+               return false;
+
+       if (wim_is_pipable(wim) || (write_flags & WIMLIB_WRITE_FLAG_PIPABLE))
+               return false;
+
+       if (wim->hdr.wim_version != WIM_VERSION_PACKED_STREAMS) {
+               if (wim->compression_type != wim->out_compression_type)
+                       return false;
+               if (wim->chunk_size != wim->out_chunk_size)
+                       return false;
+       }
+
+       return true;
+}
+

 /* API function documented in wimlib.h  */
 WIMLIBAPI int
 wimlib_overwrite(WIMStruct *wim, int write_flags,
 /* API function documented in wimlib.h  */
 WIMLIBAPI int
 wimlib_overwrite(WIMStruct *wim, int write_flags,


@@ -3151,13 +2959,7 @@ wimlib_overwrite(WIMStruct *wim, int write_flags,
        if (ret)
                return ret;
 
        if (ret)
                return ret;
 

-       if ((!wim->deletion_occurred || (write_flags & WIMLIB_WRITE_FLAG_SOFT_DELETE))
-           && !(write_flags & (WIMLIB_WRITE_FLAG_REBUILD |
-                               WIMLIB_WRITE_FLAG_PIPABLE))
-           && !(wim_is_pipable(wim))
-           && wim->compression_type == wim->out_compression_type
-           && wim->chunk_size == wim->out_chunk_size)
-       {
+       if (can_overwrite_wim_inplace(wim, write_flags)) {

                ret = overwrite_wim_inplace(wim, write_flags, num_threads,
                                            progress_func);
                if (ret != WIMLIB_ERR_RESOURCE_ORDER)
                ret = overwrite_wim_inplace(wim, write_flags, num_threads,
                                            progress_func);
                if (ret != WIMLIB_ERR_RESOURCE_ORDER)

diff --git a/src/xml.c b/src/xml.c
index 4a8bda9a56d0c5a99ce76f7cb81541bab1beea94..920eeeb5e1194db05f90a1c88a0d61ed607fbd4e 100644 (file)
--- a/src/xml.c
+++ b/src/xml.c
@@ -1524,7 +1524,7 @@ write_wim_xml_data(WIMStruct *wim, int image, u64 total_bytes,
                                             write_resource_flags,
                                             &wim->lzx_context);
        FREE(xml_data);
                                             write_resource_flags,
                                             &wim->lzx_context);
        FREE(xml_data);

-       DEBUG("ret=%d");
+       DEBUG("ret=%d", ret);

        return ret;
 }
 
        return ret;
 }