Improve write streams performance and handling of joins

Provide the number of completed WIM parts and total WIM parts in
WIMLIB_PROGRESS_MSG_WRITE_STREAMS, to replace WIMLIB_PROGRESS_MSG_JOIN.
Furthermore, write streams in a more intelligent order; e.g. sorting by WIM part
number and file offset when relevant.

diff --git a/NEWS b/NEWS
index 23b79dbca29614d7a0ff50e5d4d558e3aa3d4833..e8281cc2e2766b4b25b61a7fb55124869583b8f4 100644 (file)
--- a/NEWS
+++ b/NEWS
@@ -37,8 +37,11 @@ Version 1.5.0:
        The test suite no longer fails when run in a locale where the decimal
        separator is not a period.
 
        The test suite no longer fails when run in a locale where the decimal
        separator is not a period.
 

-       WIMLIB_PROGRESS_MSG_EXTRACT_DENTRY has been removed and
-       WIMLIB_EXTRACT_FLAG_VERBOSE re-reserved for future use.
+       From the library, WIMLIB_PROGRESS_MSG_EXTRACT_DENTRY has been removed
+       and WIMLIB_EXTRACT_FLAG_VERBOSE re-reserved for future use.  Also,
+       WIMLIB_PROGRESS_MSG_JOIN_STREAMS has been removed, but
+       WIMLIB_PROGRESS_MSG_WRITE_STREAMS will be received instead and now
+       provides WIM part numbers.

 
        The extraction code has been rewritten and it will now be easier to
        support new features on all supported backends (currently Win32, UNIX,
 
        The extraction code has been rewritten and it will now be easier to
        support new features on all supported backends (currently Win32, UNIX,

diff --git a/include/wimlib.h b/include/wimlib.h
index a7e068aee74f09149bbf3d58d15d062ec42ea260..6ab50ce414454e42a3a7b50d7ea6fad810c6f330 100644 (file)
--- a/include/wimlib.h
+++ b/include/wimlib.h
@@ -327,9 +327,10 @@ enum wimlib_progress_msg {
         * ::wimlib_progress_info.integrity. */
        WIMLIB_PROGRESS_MSG_CALC_INTEGRITY,
 
         * ::wimlib_progress_info.integrity. */
        WIMLIB_PROGRESS_MSG_CALC_INTEGRITY,
 

-       /** A wimlib_join() operation is in progress.  @a info will point to
-        * ::wimlib_progress_info.join. */
-       WIMLIB_PROGRESS_MSG_JOIN_STREAMS,
+       /** Reserved.  (Previously used for WIMLIB_PROGRESS_MSG_JOIN_STREAMS,
+        * but in wimlib v1.5.0 this was removed to simplify the code and now
+        * you'll get ::WIMLIB_PROGRESS_MSG_WRITE_STREAMS messages instead.)  */
+       WIMLIB_PROGRESS_MSG_RESERVED,

 
        /** A wimlib_split() operation is in progress, and a new split part is
         * about to be started.  @a info will point to
 
        /** A wimlib_split() operation is in progress, and a new split part is
         * about to be started.  @a info will point to


@@ -371,6 +372,7 @@ union wimlib_progress_info {
                 * (The actual number of bytes will be less if the data is being
                 * written compressed.) */
                uint64_t total_bytes;
                 * (The actual number of bytes will be less if the data is being
                 * written compressed.) */
                uint64_t total_bytes;

+

                /** Number of streams that are going to be written. */
                uint64_t total_streams;
 
                /** Number of streams that are going to be written. */
                uint64_t total_streams;
 


@@ -384,7 +386,7 @@ union wimlib_progress_info {
                uint64_t completed_streams;
 
                /** Number of threads that are being used to compress resources
                uint64_t completed_streams;
 
                /** Number of threads that are being used to compress resources

-                * (if applicable). */
+                * (if applicable).  */

                unsigned num_threads;
 
                /** The compression type being used to write the streams; either
                unsigned num_threads;
 
                /** The compression type being used to write the streams; either


@@ -393,8 +395,13 @@ union wimlib_progress_info {
                 * ::WIMLIB_COMPRESSION_TYPE_LZX. */
                int      compression_type;
 
                 * ::WIMLIB_COMPRESSION_TYPE_LZX. */
                int      compression_type;
 

-               /** Library internal use only. */
-               uint64_t _private;
+               /** Number of split WIM parts from which streams are being
+                * written (may be 0 if irrelevant).  */
+               unsigned total_parts;
+
+               /** Number of split WIM parts from which streams have been
+                * written (may be 0 if irrelevant).  */
+               unsigned completed_parts;

        } write_streams;
 
        /** Valid on messages ::WIMLIB_PROGRESS_MSG_SCAN_BEGIN and
        } write_streams;
 
        /** Valid on messages ::WIMLIB_PROGRESS_MSG_SCAN_BEGIN and


@@ -518,25 +525,6 @@ union wimlib_progress_info {
                const wimlib_tchar *filename;
        } integrity;
 
                const wimlib_tchar *filename;
        } integrity;
 

-       /** Valid on messages ::WIMLIB_PROGRESS_MSG_JOIN_STREAMS. */
-       struct wimlib_progress_info_join {
-               /** Total number of bytes of compressed data contained in all
-                * the split WIM part's file and metadata resources. */
-               uint64_t total_bytes;
-
-               /** Number of bytes that have been copied to the joined WIM so
-                * far.  Will be 0 initially, and equal to @a total_bytes at the
-                * end. */
-               uint64_t completed_bytes;
-
-               /** Number of split WIM parts that have had all their file and
-                * metadata resources copied over to the joined WIM so far. */
-               unsigned completed_parts;
-
-               /** Number of split WIM parts. */
-               unsigned total_parts;
-       } join;
-

        /** Valid on messages ::WIMLIB_PROGRESS_MSG_SPLIT_BEGIN_PART and
         * ::WIMLIB_PROGRESS_MSG_SPLIT_END_PART. */
        struct wimlib_progress_info_split {
        /** Valid on messages ::WIMLIB_PROGRESS_MSG_SPLIT_BEGIN_PART and
         * ::WIMLIB_PROGRESS_MSG_SPLIT_END_PART. */
        struct wimlib_progress_info_split {

diff --git a/include/wimlib/lookup_table.h b/include/wimlib/lookup_table.h
index 28862d5a3b9b753ea53e88c87881f923bfe38782..481794daf74d20849612eb1c23132b9f27da6223 100644 (file)
--- a/include/wimlib/lookup_table.h
+++ b/include/wimlib/lookup_table.h
@@ -375,7 +375,8 @@ for_lookup_table_entry(struct wim_lookup_table *table,
                       void *arg);
 
 extern int
                       void *arg);
 
 extern int

-cmp_streams_by_wim_position(const void *p1, const void *p2);
+sort_stream_list_by_sequential_order(struct list_head *stream_list,
+                                    size_t list_head_offset);

 
 extern int
 for_lookup_table_entry_pos_sorted(struct wim_lookup_table *table,
 
 extern int
 for_lookup_table_entry_pos_sorted(struct wim_lookup_table *table,

diff --git a/programs/imagex.c b/programs/imagex.c
index e99202b1effdbf659a43b4b38096844bfeca8570..767b67240c6fc448a008454a2d4b7df7b598a9d8 100644 (file)
--- a/programs/imagex.c
+++ b/programs/imagex.c
@@ -985,6 +985,7 @@ imagex_progress_func(enum wimlib_progress_msg msg,
                unit_shift = get_unit(info->write_streams.total_bytes, &unit_name);
                percent_done = TO_PERCENT(info->write_streams.completed_bytes,
                                          info->write_streams.total_bytes);
                unit_shift = get_unit(info->write_streams.total_bytes, &unit_name);
                percent_done = TO_PERCENT(info->write_streams.completed_bytes,
                                          info->write_streams.total_bytes);

+

                if (info->write_streams.completed_streams == 0) {
                        const tchar *data_type;
 
                if (info->write_streams.completed_streams == 0) {
                        const tchar *data_type;
 


@@ -993,13 +994,28 @@ imagex_progress_func(enum wimlib_progress_msg msg,
                                data_type, info->write_streams.num_threads,
                                (info->write_streams.num_threads == 1) ? T("") : T("s"));
                }
                                data_type, info->write_streams.num_threads,
                                (info->write_streams.num_threads == 1) ? T("") : T("s"));
                }

-               imagex_printf(T("\r%"PRIu64" %"TS" of %"PRIu64" %"TS" (uncompressed) "
-                       "written (%u%% done)"),
-                       info->write_streams.completed_bytes >> unit_shift,
-                       unit_name,
-                       info->write_streams.total_bytes >> unit_shift,
-                       unit_name,
-                       percent_done);
+               if (info->write_streams.total_parts <= 1) {
+                       imagex_printf(T("\r%"PRIu64" %"TS" of %"PRIu64" %"TS" (uncompressed) "
+                               "written (%u%% done)"),
+                               info->write_streams.completed_bytes >> unit_shift,
+                               unit_name,
+                               info->write_streams.total_bytes >> unit_shift,
+                               unit_name,
+                               percent_done);
+               } else {
+                       imagex_printf(T("\rWriting resources from part %u of %u: "
+                                 "%"PRIu64 " %"TS" of %"PRIu64" %"TS" (%u%%) written"),
+                               (info->write_streams.completed_parts ==
+                                       info->write_streams.total_parts) ?
+                                               info->write_streams.completed_parts :
+                                               info->write_streams.completed_parts + 1,
+                               info->write_streams.total_parts,
+                               info->write_streams.completed_bytes >> unit_shift,
+                               unit_name,
+                               info->write_streams.total_bytes >> unit_shift,
+                               unit_name,
+                               percent_done);
+               }

                if (info->write_streams.completed_bytes >= info->write_streams.total_bytes)
                        imagex_printf(T("\n"));
                break;
                if (info->write_streams.completed_bytes >= info->write_streams.total_bytes)
                        imagex_printf(T("\n"));
                break;


@@ -1019,8 +1035,6 @@ imagex_progress_func(enum wimlib_progress_msg msg,
                else
                        imagex_printf(T("Scanning \"%"TS"\"\n"), info->scan.cur_path);
                break;
                else
                        imagex_printf(T("Scanning \"%"TS"\"\n"), info->scan.cur_path);
                break;

-       /*case WIMLIB_PROGRESS_MSG_SCAN_END:*/
-               /*break;*/

        case WIMLIB_PROGRESS_MSG_VERIFY_INTEGRITY:
                unit_shift = get_unit(info->integrity.total_bytes, &unit_name);
                percent_done = TO_PERCENT(info->integrity.completed_bytes,
        case WIMLIB_PROGRESS_MSG_VERIFY_INTEGRITY:
                unit_shift = get_unit(info->integrity.total_bytes, &unit_name);
                percent_done = TO_PERCENT(info->integrity.completed_bytes,


@@ -1070,10 +1084,6 @@ imagex_progress_func(enum wimlib_progress_msg msg,
                        info->extract.wimfile_name,
                        info->extract.target);
                break;
                        info->extract.wimfile_name,
                        info->extract.target);
                break;

-       /*case WIMLIB_PROGRESS_MSG_EXTRACT_DIR_STRUCTURE_BEGIN:*/
-               /*imagex_printf(T("Applying directory structure to %"TS"\n"),*/
-                       /*info->extract.target);*/
-               /*break;*/

        case WIMLIB_PROGRESS_MSG_EXTRACT_STREAMS:
                percent_done = TO_PERCENT(info->extract.completed_bytes,
                                          info->extract.total_bytes);
        case WIMLIB_PROGRESS_MSG_EXTRACT_STREAMS:
                percent_done = TO_PERCENT(info->extract.completed_bytes,
                                          info->extract.total_bytes);


@@ -1098,21 +1108,6 @@ imagex_progress_func(enum wimlib_progress_msg msg,
                                info->extract.target);
                }
                break;
                                info->extract.target);
                }
                break;

-       case WIMLIB_PROGRESS_MSG_JOIN_STREAMS:
-               percent_done = TO_PERCENT(info->join.completed_bytes,
-                                         info->join.total_bytes);
-               unit_shift = get_unit(info->join.total_bytes, &unit_name);
-               imagex_printf(T("Writing resources from part %u of %u: "
-                         "%"PRIu64 " %"TS" of %"PRIu64" %"TS" (%u%%) written\n"),
-                       (info->join.completed_parts == info->join.total_parts) ?
-                       info->join.completed_parts : info->join.completed_parts + 1,
-                       info->join.total_parts,
-                       info->join.completed_bytes >> unit_shift,
-                       unit_name,
-                       info->join.total_bytes >> unit_shift,
-                       unit_name,
-                       percent_done);
-               break;

        case WIMLIB_PROGRESS_MSG_SPLIT_BEGIN_PART:
                percent_done = TO_PERCENT(info->split.completed_bytes,
                                          info->split.total_bytes);
        case WIMLIB_PROGRESS_MSG_SPLIT_BEGIN_PART:
                percent_done = TO_PERCENT(info->split.completed_bytes,
                                          info->split.total_bytes);

diff --git a/src/extract.c b/src/extract.c
index b96367b6a1d934ef879584ee9e53878f46ce2d81..d9e03333e43403f701f337535253d0d4e403aafc 100644 (file)
--- a/src/extract.c
+++ b/src/extract.c
@@ -1337,42 +1337,6 @@ dentry_extract_final(struct wim_dentry *dentry, void *_ctx)
        return extract_timestamps(path, ctx, dentry);
 }
 
        return extract_timestamps(path, ctx, dentry);
 }
 

-/* Sorts a list of streams in ascending order of their offset in the WIM file in
- * order to prepare for sequential extraction.  */
-static int
-sort_stream_list_by_wim_position(struct list_head *stream_list)
-{
-       struct list_head *cur;
-       size_t num_streams;
-       struct wim_lookup_table_entry **array;
-       size_t i;
-       size_t array_size;
-
-       num_streams = 0;
-       list_for_each(cur, stream_list)
-               num_streams++;
-       array_size = num_streams * sizeof(array[0]);
-       array = MALLOC(array_size);
-       if (!array) {
-               ERROR("Failed to allocate %zu bytes to sort stream entries",
-                     array_size);
-               return WIMLIB_ERR_NOMEM;
-       }
-       cur = stream_list->next;
-       for (i = 0; i < num_streams; i++) {
-               array[i] = container_of(cur, struct wim_lookup_table_entry, extraction_list);
-               cur = cur->next;
-       }
-
-       qsort(array, num_streams, sizeof(array[0]), cmp_streams_by_wim_position);
-
-       INIT_LIST_HEAD(stream_list);
-       for (i = 0; i < num_streams; i++)
-               list_add_tail(&array[i]->extraction_list, stream_list);
-       FREE(array);
-       return 0;
-}
-

 /*
  * Extract a WIM dentry to standard output.
  *
 /*
  * Extract a WIM dentry to standard output.
  *


@@ -2033,7 +1997,10 @@ extract_tree(WIMStruct *wim, const tchar *wim_source_path, const tchar *target,
                              WIMLIB_EXTRACT_FLAG_FROM_PIPE))
                                        == WIMLIB_EXTRACT_FLAG_SEQUENTIAL)
        {
                              WIMLIB_EXTRACT_FLAG_FROM_PIPE))
                                        == WIMLIB_EXTRACT_FLAG_SEQUENTIAL)
        {

-               ret = sort_stream_list_by_wim_position(&ctx.stream_list);
+               ret = sort_stream_list_by_sequential_order(
+                               &ctx.stream_list,
+                               offsetof(struct wim_lookup_table_entry,
+                                        extraction_list));

                if (ret)
                        goto out_teardown_stream_list;
        }
                if (ret)
                        goto out_teardown_stream_list;
        }

diff --git a/src/join.c b/src/join.c
index 5cda7739a1f57d2f5a0820b837b01c65a3c5f439..d606b7be6392a71cd20241f11845b91d75f9909b 100644 (file)
--- a/src/join.c
+++ b/src/join.c
@@ -82,10 +82,8 @@ wimlib_join(const tchar * const *swm_names,
        if (ret)
                goto out_free_swms;
 
        if (ret)
                goto out_free_swms;
 

-       ret = wimlib_export_image(swm0, WIMLIB_ALL_IMAGES,
-                                 wim, NULL, NULL, 0,
-                                 swms, num_additional_swms,
-                                 progress_func);
+       ret = wimlib_export_image(swm0, WIMLIB_ALL_IMAGES, wim, NULL, NULL, 0,
+                                 swms, num_additional_swms, progress_func);

        if (ret)
                goto out_free_wim;
 
        if (ret)
                goto out_free_wim;
 

diff --git a/src/lookup_table.c b/src/lookup_table.c
index 3b98553e9bb586d790df154f8de706351bc2ac0e..15f5c0830c382fb501d9dc13ff5dbcc3513f85a8 100644 (file)
--- a/src/lookup_table.c
+++ b/src/lookup_table.c
@@ -288,20 +288,107 @@ for_lookup_table_entry(struct wim_lookup_table *table,
        return 0;
 }
 
        return 0;
 }
 

-int
-cmp_streams_by_wim_position(const void *p1, const void *p2)
+/* qsort() callback that sorts streams (represented by `struct
+ * wim_lookup_table_entry's) into an order optimized for reading and writing.
+ *
+ * Sorting is done primarily by resource location, then secondarily by a
+ * per-resource location order.  For example, resources in WIM files are sorted
+ * primarily by part number, then secondarily by offset, as to implement optimal
+ * reading of either a standalone or split WIM.  */
+static int
+cmp_streams_by_sequential_order(const void *p1, const void *p2)

 {
        const struct wim_lookup_table_entry *lte1, *lte2;
 {
        const struct wim_lookup_table_entry *lte1, *lte2;

+       int v;
+

        lte1 = *(const struct wim_lookup_table_entry**)p1;
        lte2 = *(const struct wim_lookup_table_entry**)p2;
        lte1 = *(const struct wim_lookup_table_entry**)p1;
        lte2 = *(const struct wim_lookup_table_entry**)p2;

-       if (lte1->resource_entry.offset < lte2->resource_entry.offset)
-               return -1;
-       else if (lte1->resource_entry.offset > lte2->resource_entry.offset)
-               return 1;
-       else
+
+       v = (int)lte1->resource_location - (int)lte2->resource_location;
+
+       /* Different resource locations?  */
+       if (v)
+               return v;
+
+       switch (lte1->resource_location) {
+       case RESOURCE_IN_WIM:
+
+               /* Different (possibly split) WIMs?  */
+               if (lte1->wim != lte2->wim) {
+                       v = memcmp(lte1->wim->hdr.guid, lte2->wim->hdr.guid,
+                                  WIM_GID_LEN);
+                       if (v)
+                               return v;
+               }
+
+               /* Different part numbers in the same WIM?  */
+               v = (int)lte1->wim->hdr.part_number - (int)lte2->wim->hdr.part_number;
+               if (v)
+                       return v;
+
+               /* Compare by offset.  */
+               if (lte1->resource_entry.offset < lte2->resource_entry.offset)
+                       return -1;
+               else if (lte1->resource_entry.offset > lte2->resource_entry.offset)
+                       return 1;

                return 0;
                return 0;

+       case RESOURCE_IN_FILE_ON_DISK:
+#ifdef __WIN32__
+       case RESOURCE_WIN32_ENCRYPTED:
+#endif
+               /* Compare files by path: just a heuristic that will place files
+                * in the same directory next to each other.  */
+               return tstrcmp(lte1->file_on_disk, lte2->file_on_disk);
+#ifdef WITH_NTFS_3G
+       case RESOURCE_IN_NTFS_VOLUME:
+               return tstrcmp(lte1->ntfs_loc->path, lte2->ntfs_loc->path);
+#endif
+       default:
+               /* No additional sorting order defined for this resource
+                * location (e.g. RESOURCE_IN_ATTACHED_BUFFER); simply compare
+                * everything equal to each other.  */
+               return 0;
+       }
+}
+
+int
+sort_stream_list_by_sequential_order(struct list_head *stream_list,
+                                    size_t list_head_offset)
+{
+       struct list_head *cur;
+       struct wim_lookup_table_entry **array;
+       size_t i;
+       size_t array_size;
+       size_t num_streams = 0;
+
+       list_for_each(cur, stream_list)
+               num_streams++;
+
+       array_size = num_streams * sizeof(array[0]);
+       array = MALLOC(array_size);
+       if (!array)
+               return WIMLIB_ERR_NOMEM;
+       cur = stream_list->next;
+       for (i = 0; i < num_streams; i++) {
+               array[i] = (struct wim_lookup_table_entry*)((u8*)cur -
+                                                           list_head_offset);
+               cur = cur->next;
+       }
+
+       qsort(array, num_streams, sizeof(array[0]),
+             cmp_streams_by_sequential_order);
+
+       INIT_LIST_HEAD(stream_list);
+       for (i = 0; i < num_streams; i++) {
+               list_add_tail((struct list_head*)
+                              ((u8*)array[i] + list_head_offset),
+                             stream_list);
+       }
+       FREE(array);
+       return 0;

 }
 
 }
 

+

 static int
 add_lte_to_array(struct wim_lookup_table_entry *lte,
                 void *_pp)
 static int
 add_lte_to_array(struct wim_lookup_table_entry *lte,
                 void *_pp)


@@ -333,7 +420,7 @@ for_lookup_table_entry_pos_sorted(struct wim_lookup_table *table,
        wimlib_assert(p == lte_array + num_streams);
 
        qsort(lte_array, num_streams, sizeof(lte_array[0]),
        wimlib_assert(p == lte_array + num_streams);
 
        qsort(lte_array, num_streams, sizeof(lte_array[0]),

-             cmp_streams_by_wim_position);
+             cmp_streams_by_sequential_order);

        ret = 0;
        for (size_t i = 0; i < num_streams; i++) {
                ret = visitor(lte_array[i], arg);
        ret = 0;
        for (size_t i = 0; i < num_streams; i++) {
                ret = visitor(lte_array[i], arg);

diff --git a/src/write.c b/src/write.c
index 3cac5a2868a7d8ef7c9f3ea90e853e3de824b9fe..e4a0f3ad45712c6ce8ab5a5da345b9aeed0c0bf8 100644 (file)
--- a/src/write.c
+++ b/src/write.c
@@ -53,17 +53,16 @@
 #  include <pthread.h>
 #endif
 
 #  include <pthread.h>
 #endif
 

-#include <unistd.h>
-#include <fcntl.h>

 #include <errno.h>
 #include <errno.h>

+#include <fcntl.h>
+#include <limits.h>
+#include <stdlib.h>
+#include <unistd.h>

 
 #ifdef HAVE_ALLOCA_H
 #  include <alloca.h>
 
 #ifdef HAVE_ALLOCA_H
 #  include <alloca.h>

-#else
-#  include <stdlib.h>

 #endif
 
 #endif
 

-#include <limits.h>

 
 #ifndef __WIN32__
 #  include <sys/uio.h> /* for `struct iovec' */
 
 #ifndef __WIN32__
 #  include <sys/uio.h> /* for `struct iovec' */


@@ -683,7 +682,6 @@ struct message {
        u8 *compressed_chunks[MAX_CHUNKS_PER_MSG];
        unsigned uncompressed_chunk_sizes[MAX_CHUNKS_PER_MSG];
        struct iovec out_chunks[MAX_CHUNKS_PER_MSG];
        u8 *compressed_chunks[MAX_CHUNKS_PER_MSG];
        unsigned uncompressed_chunk_sizes[MAX_CHUNKS_PER_MSG];
        struct iovec out_chunks[MAX_CHUNKS_PER_MSG];

-       size_t total_out_bytes;

        unsigned num_chunks;
        struct list_head list;
        bool complete;
        unsigned num_chunks;
        struct list_head list;
        bool complete;


@@ -693,7 +691,6 @@ struct message {
 static void
 compress_chunks(struct message *msg, compress_func_t compress)
 {
 static void
 compress_chunks(struct message *msg, compress_func_t compress)
 {

-       msg->total_out_bytes = 0;

        for (unsigned i = 0; i < msg->num_chunks; i++) {
                unsigned len = compress(msg->uncompressed_chunks[i],
                                        msg->uncompressed_chunk_sizes[i],
        for (unsigned i = 0; i < msg->num_chunks; i++) {
                unsigned len = compress(msg->uncompressed_chunks[i],
                                        msg->uncompressed_chunk_sizes[i],


@@ -711,7 +708,6 @@ compress_chunks(struct message *msg, compress_func_t compress)
                }
                msg->out_chunks[i].iov_base = out_chunk;
                msg->out_chunks[i].iov_len = out_len;
                }
                msg->out_chunks[i].iov_base = out_chunk;
                msg->out_chunks[i].iov_len = out_len;

-               msg->total_out_bytes += out_len;

        }
 }
 
        }
 }
 


@@ -739,32 +735,52 @@ compressor_thread_proc(void *arg)
 }
 #endif /* ENABLE_MULTITHREADED_COMPRESSION */
 
 }
 #endif /* ENABLE_MULTITHREADED_COMPRESSION */
 

+struct write_streams_progress_data {
+       wimlib_progress_func_t progress_func;
+       union wimlib_progress_info progress;
+       uint64_t next_progress;
+       WIMStruct *prev_wim_part;
+};
+

 static void
 static void

-do_write_streams_progress(union wimlib_progress_info *progress,
-                         wimlib_progress_func_t progress_func,
-                         uint64_t size_added,
+do_write_streams_progress(struct write_streams_progress_data *progress_data,
+                         struct wim_lookup_table_entry *lte,

                          bool stream_discarded)
 {
                          bool stream_discarded)
 {

+       union wimlib_progress_info *progress = &progress_data->progress;
+       bool new_wim_part;
+

        if (stream_discarded) {
        if (stream_discarded) {

-               progress->write_streams.total_bytes -= size_added;
-               if (progress->write_streams._private != ~(uint64_t)0 &&
-                   progress->write_streams._private > progress->write_streams.total_bytes)
+               progress->write_streams.total_bytes -= wim_resource_size(lte);
+               if (progress_data->next_progress != ~(uint64_t)0 &&
+                   progress_data->next_progress > progress->write_streams.total_bytes)

                {
                {

-                       progress->write_streams._private = progress->write_streams.total_bytes;
+                       progress_data->next_progress = progress->write_streams.total_bytes;

                }
        } else {
                }
        } else {

-               progress->write_streams.completed_bytes += size_added;
+               progress->write_streams.completed_bytes += wim_resource_size(lte);
+       }
+       new_wim_part = false;
+       if (lte->resource_location == RESOURCE_IN_WIM &&
+           lte->wim != progress_data->prev_wim_part)
+       {
+               if (progress_data->prev_wim_part) {
+                       new_wim_part = true;
+                       progress->write_streams.completed_parts++;
+               }
+               progress_data->prev_wim_part = lte->wim;

        }
        progress->write_streams.completed_streams++;
        }
        progress->write_streams.completed_streams++;

-       if (progress_func &&
-           progress->write_streams.completed_bytes >= progress->write_streams._private)
+       if (progress_data->progress_func
+           && (progress->write_streams.completed_bytes >= progress_data->next_progress
+               || new_wim_part))

        {
        {

-               progress_func(WIMLIB_PROGRESS_MSG_WRITE_STREAMS,
-                             progress);
-               if (progress->write_streams._private == progress->write_streams.total_bytes) {
-                       progress->write_streams._private = ~(uint64_t)0;
+               progress_data->progress_func(WIMLIB_PROGRESS_MSG_WRITE_STREAMS,
+                                            progress);
+               if (progress_data->next_progress == progress->write_streams.total_bytes) {
+                       progress_data->next_progress = ~(uint64_t)0;

                } else {
                } else {

-                       progress->write_streams._private =
+                       progress_data->next_progress =

                                min(progress->write_streams.total_bytes,
                                    progress->write_streams.completed_bytes +
                                        progress->write_streams.total_bytes / 100);
                                min(progress->write_streams.total_bytes,
                                    progress->write_streams.completed_bytes +
                                        progress->write_streams.total_bytes / 100);


@@ -787,6 +803,7 @@ serial_write_stream(struct wim_lookup_table_entry *lte, void *_ctx)
                                  ctx->write_resource_flags);
 }
 
                                  ctx->write_resource_flags);
 }
 

+

 /* Write a list of streams, taking into account that some streams may be
  * duplicates that are checksummed and discarded on the fly, and also delegating
  * the actual writing of a stream to a function @write_stream_cb, which is
 /* Write a list of streams, taking into account that some streams may be
  * duplicates that are checksummed and discarded on the fly, and also delegating
  * the actual writing of a stream to a function @write_stream_cb, which is


@@ -796,8 +813,7 @@ do_write_stream_list(struct list_head *stream_list,
                     struct wim_lookup_table *lookup_table,
                     int (*write_stream_cb)(struct wim_lookup_table_entry *, void *),
                     void *write_stream_ctx,
                     struct wim_lookup_table *lookup_table,
                     int (*write_stream_cb)(struct wim_lookup_table_entry *, void *),
                     void *write_stream_ctx,

-                    wimlib_progress_func_t progress_func,
-                    union wimlib_progress_info *progress)
+                    struct write_streams_progress_data *progress_data)

 {
        int ret = 0;
        struct wim_lookup_table_entry *lte;
 {
        int ret = 0;
        struct wim_lookup_table_entry *lte;


@@ -860,10 +876,8 @@ do_write_stream_list(struct list_head *stream_list,
                }
        skip_to_progress:
                if (!lte->no_progress) {
                }
        skip_to_progress:
                if (!lte->no_progress) {

-                       do_write_streams_progress(progress,
-                                                 progress_func,
-                                                 wim_resource_size(lte),
-                                                 stream_discarded);
+                       do_write_streams_progress(progress_data,
+                                                 lte, stream_discarded);

                }
        }
        return ret;
                }
        }
        return ret;


@@ -875,8 +889,7 @@ do_write_stream_list_serial(struct list_head *stream_list,
                            struct filedes *out_fd,
                            int out_ctype,
                            int write_resource_flags,
                            struct filedes *out_fd,
                            int out_ctype,
                            int write_resource_flags,

-                           wimlib_progress_func_t progress_func,
-                           union wimlib_progress_info *progress)
+                           struct write_streams_progress_data *progress_data)

 {
        struct serial_write_stream_ctx ctx = {
                .out_fd = out_fd,
 {
        struct serial_write_stream_ctx ctx = {
                .out_fd = out_fd,


@@ -887,8 +900,7 @@ do_write_stream_list_serial(struct list_head *stream_list,
                                    lookup_table,
                                    serial_write_stream,
                                    &ctx,
                                    lookup_table,
                                    serial_write_stream,
                                    &ctx,

-                                   progress_func,
-                                   progress);
+                                   progress_data);

 }
 
 static inline int
 }
 
 static inline int


@@ -909,21 +921,22 @@ write_stream_list_serial(struct list_head *stream_list,
                         struct filedes *out_fd,
                         int out_ctype,
                         int write_resource_flags,
                         struct filedes *out_fd,
                         int out_ctype,
                         int write_resource_flags,

-                        wimlib_progress_func_t progress_func,
-                        union wimlib_progress_info *progress)
+                        struct write_streams_progress_data *progress_data)

 {
 {

+       union wimlib_progress_info *progress = &progress_data->progress;

        DEBUG("Writing stream list of size %"PRIu64" (serial version)",
              progress->write_streams.total_streams);
        progress->write_streams.num_threads = 1;
        DEBUG("Writing stream list of size %"PRIu64" (serial version)",
              progress->write_streams.total_streams);
        progress->write_streams.num_threads = 1;

-       if (progress_func)
-               progress_func(WIMLIB_PROGRESS_MSG_WRITE_STREAMS, progress);
+       if (progress_data->progress_func) {
+               progress_data->progress_func(WIMLIB_PROGRESS_MSG_WRITE_STREAMS,
+                                            progress);
+       }

        return do_write_stream_list_serial(stream_list,
                                           lookup_table,
                                           out_fd,
                                           out_ctype,
                                           write_resource_flags,
        return do_write_stream_list_serial(stream_list,
                                           lookup_table,
                                           out_fd,
                                           out_ctype,
                                           write_resource_flags,

-                                          progress_func,
-                                          progress);
+                                          progress_data);

 }
 
 #ifdef ENABLE_MULTITHREADED_COMPRESSION
 }
 
 #ifdef ENABLE_MULTITHREADED_COMPRESSION


@@ -935,7 +948,6 @@ write_wim_chunks(struct message *msg, struct filedes *out_fd,
        struct iovec *vecs;
        struct pwm_chunk_hdr *chunk_hdrs;
        unsigned nvecs;
        struct iovec *vecs;
        struct pwm_chunk_hdr *chunk_hdrs;
        unsigned nvecs;

-       size_t nbytes;

        int ret;
 
        for (unsigned i = 0; i < msg->num_chunks; i++)
        int ret;
 
        for (unsigned i = 0; i < msg->num_chunks; i++)


@@ -944,7 +956,6 @@ write_wim_chunks(struct message *msg, struct filedes *out_fd,
        if (!(write_resource_flags & WIMLIB_WRITE_RESOURCE_FLAG_PIPABLE)) {
                nvecs = msg->num_chunks;
                vecs = msg->out_chunks;
        if (!(write_resource_flags & WIMLIB_WRITE_RESOURCE_FLAG_PIPABLE)) {
                nvecs = msg->num_chunks;
                vecs = msg->out_chunks;

-               nbytes = msg->total_out_bytes;

        } else {
                /* Special case:  If writing a compressed resource to a pipable
                 * WIM, prefix each compressed chunk with a header that gives
        } else {
                /* Special case:  If writing a compressed resource to a pipable
                 * WIM, prefix each compressed chunk with a header that gives


@@ -960,7 +971,6 @@ write_wim_chunks(struct message *msg, struct filedes *out_fd,
                        vecs[i * 2 + 1].iov_base = msg->out_chunks[i].iov_base;
                        vecs[i * 2 + 1].iov_len = msg->out_chunks[i].iov_len;
                }
                        vecs[i * 2 + 1].iov_base = msg->out_chunks[i].iov_base;
                        vecs[i * 2 + 1].iov_len = msg->out_chunks[i].iov_len;
                }

-               nbytes = msg->total_out_bytes + msg->num_chunks * sizeof(chunk_hdrs[0]);

        }
        ret = full_writev(out_fd, vecs, nvecs);
        if (ret)
        }
        ret = full_writev(out_fd, vecs, nvecs);
        if (ret)


@@ -978,8 +988,7 @@ struct main_writer_thread_ctx {
        struct shared_queue *res_to_compress_queue;
        struct shared_queue *compressed_res_queue;
        size_t num_messages;
        struct shared_queue *res_to_compress_queue;
        struct shared_queue *compressed_res_queue;
        size_t num_messages;

-       wimlib_progress_func_t progress_func;
-       union wimlib_progress_info *progress;
+       struct write_streams_progress_data *progress_data;

 
        struct list_head available_msgs;
        struct list_head outstanding_streams;
 
        struct list_head available_msgs;
        struct list_head outstanding_streams;


@@ -1221,10 +1230,8 @@ receive_compressed_chunks(struct main_writer_thread_ctx *ctx)
                                      cur_lte->output_resource_entry.flags);
                        }
 
                                      cur_lte->output_resource_entry.flags);
                        }
 

-                       do_write_streams_progress(ctx->progress,
-                                                 ctx->progress_func,
-                                                 wim_resource_size(cur_lte),
-                                                 false);
+                       do_write_streams_progress(ctx->progress_data,
+                                                 cur_lte, false);

 
                        /* Since we just finished writing a stream, write any
                         * streams that have been added to the serial_streams
 
                        /* Since we just finished writing a stream, write any
                         * streams that have been added to the serial_streams


@@ -1238,8 +1245,7 @@ receive_compressed_chunks(struct main_writer_thread_ctx *ctx)
                                                                  ctx->out_fd,
                                                                  ctx->out_ctype,
                                                                  ctx->write_resource_flags,
                                                                  ctx->out_fd,
                                                                  ctx->out_ctype,
                                                                  ctx->write_resource_flags,

-                                                                 ctx->progress_func,
-                                                                 ctx->progress);
+                                                                 ctx->progress_data);

                                if (ret)
                                        return ret;
                        }
                                if (ret)
                                        return ret;
                        }


@@ -1337,8 +1343,7 @@ main_writer_thread_finish(void *_ctx)
                                           ctx->out_fd,
                                           ctx->out_ctype,
                                           ctx->write_resource_flags,
                                           ctx->out_fd,
                                           ctx->out_ctype,
                                           ctx->write_resource_flags,

-                                          ctx->progress_func,
-                                          ctx->progress);
+                                          ctx->progress_data);

 }
 
 static int
 }
 
 static int


@@ -1433,14 +1438,14 @@ write_stream_list_parallel(struct list_head *stream_list,
                           struct filedes *out_fd,
                           int out_ctype,
                           int write_resource_flags,
                           struct filedes *out_fd,
                           int out_ctype,
                           int write_resource_flags,

-                          wimlib_progress_func_t progress_func,
-                          union wimlib_progress_info *progress,
+                          struct write_streams_progress_data *progress_data,

                           unsigned num_threads)
 {
        int ret;
        struct shared_queue res_to_compress_queue;
        struct shared_queue compressed_res_queue;
        pthread_t *compressor_threads = NULL;
                           unsigned num_threads)
 {
        int ret;
        struct shared_queue res_to_compress_queue;
        struct shared_queue compressed_res_queue;
        pthread_t *compressor_threads = NULL;

+       union wimlib_progress_info *progress = &progress_data->progress;

 
        if (num_threads == 0) {
                long nthreads = get_default_num_threads();
 
        if (num_threads == 0) {
                long nthreads = get_default_num_threads();


@@ -1498,8 +1503,10 @@ write_stream_list_parallel(struct list_head *stream_list,
                }
        }
 
                }
        }
 

-       if (progress_func)
-               progress_func(WIMLIB_PROGRESS_MSG_WRITE_STREAMS, progress);
+       if (progress_data->progress_func) {
+               progress_data->progress_func(WIMLIB_PROGRESS_MSG_WRITE_STREAMS,
+                                            progress);
+       }

 
        struct main_writer_thread_ctx ctx;
        ctx.stream_list           = stream_list;
 
        struct main_writer_thread_ctx ctx;
        ctx.stream_list           = stream_list;


@@ -1510,14 +1517,13 @@ write_stream_list_parallel(struct list_head *stream_list,
        ctx.compressed_res_queue  = &compressed_res_queue;
        ctx.num_messages          = queue_size;
        ctx.write_resource_flags  = write_resource_flags;
        ctx.compressed_res_queue  = &compressed_res_queue;
        ctx.num_messages          = queue_size;
        ctx.write_resource_flags  = write_resource_flags;

-       ctx.progress_func         = progress_func;
-       ctx.progress              = progress;
+       ctx.progress_data         = progress_data;

        ret = main_writer_thread_init_ctx(&ctx);
        if (ret)
                goto out_join;
        ret = do_write_stream_list(stream_list, lookup_table,
                                   main_thread_process_next_stream,
        ret = main_writer_thread_init_ctx(&ctx);
        if (ret)
                goto out_join;
        ret = do_write_stream_list(stream_list, lookup_table,
                                   main_thread_process_next_stream,

-                                  &ctx, progress_func, progress);
+                                  &ctx, progress_data);

        if (ret)
                goto out_destroy_ctx;
 
        if (ret)
                goto out_destroy_ctx;
 


@@ -1556,8 +1562,7 @@ out_serial_quiet:
                                        out_fd,
                                        out_ctype,
                                        write_resource_flags,
                                        out_fd,
                                        out_ctype,
                                        write_resource_flags,

-                                       progress_func,
-                                       progress);
+                                       progress_data);

 
 }
 #endif
 
 }
 #endif


@@ -1576,9 +1581,11 @@ write_stream_list(struct list_head *stream_list,
        size_t num_streams = 0;
        u64 total_bytes = 0;
        u64 total_compression_bytes = 0;
        size_t num_streams = 0;
        u64 total_bytes = 0;
        u64 total_compression_bytes = 0;

-       union wimlib_progress_info progress;
+       struct write_streams_progress_data progress_data;

        int ret;
        int write_resource_flags;
        int ret;
        int write_resource_flags;

+       unsigned total_parts = 0;
+       WIMStruct *prev_wim_part = NULL;

 
        if (list_empty(stream_list))
                return 0;
 
        if (list_empty(stream_list))
                return 0;


@@ -1587,6 +1594,10 @@ write_stream_list(struct list_head *stream_list,
 
        DEBUG("write_resource_flags=0x%08x", write_resource_flags);
 
 
        DEBUG("write_resource_flags=0x%08x", write_resource_flags);
 

+       sort_stream_list_by_sequential_order(stream_list,
+                                            offsetof(struct wim_lookup_table_entry,
+                                                     write_streams_list));
+

        /* Calculate the total size of the streams to be written.  Note: this
         * will be the uncompressed size, as we may not know the compressed size
         * yet, and also this will assume that every unhashed stream will be
        /* Calculate the total size of the streams to be written.  Note: this
         * will be the uncompressed size, as we may not know the compressed size
         * yet, and also this will assume that every unhashed stream will be


@@ -1600,14 +1611,29 @@ write_stream_list(struct list_head *stream_list,
                {
                        total_compression_bytes += wim_resource_size(lte);
                }
                {
                        total_compression_bytes += wim_resource_size(lte);
                }

+               if (lte->resource_location == RESOURCE_IN_WIM) {
+                       if (prev_wim_part != lte->wim) {
+                               prev_wim_part = lte->wim;
+                               total_parts++;
+                       }
+               }
+

        }
        }

-       progress.write_streams.total_bytes       = total_bytes;
-       progress.write_streams.total_streams     = num_streams;
-       progress.write_streams.completed_bytes   = 0;
-       progress.write_streams.completed_streams = 0;
-       progress.write_streams.num_threads       = num_threads;
-       progress.write_streams.compression_type  = out_ctype;
-       progress.write_streams._private          = 0;
+
+       memset(&progress_data, 0, sizeof(progress_data));
+       progress_data.progress_func = progress_func;
+
+       progress_data.progress.write_streams.total_bytes       = total_bytes;
+       progress_data.progress.write_streams.total_streams     = num_streams;
+       progress_data.progress.write_streams.completed_bytes   = 0;
+       progress_data.progress.write_streams.completed_streams = 0;
+       progress_data.progress.write_streams.num_threads       = num_threads;
+       progress_data.progress.write_streams.compression_type  = out_ctype;
+       progress_data.progress.write_streams.total_parts       = total_parts;
+       progress_data.progress.write_streams.completed_parts   = 0;
+
+       progress_data.next_progress = 0;
+       progress_data.prev_wim_part = NULL;

 
 #ifdef ENABLE_MULTITHREADED_COMPRESSION
        if (total_compression_bytes >= 2000000 && num_threads != 1)
 
 #ifdef ENABLE_MULTITHREADED_COMPRESSION
        if (total_compression_bytes >= 2000000 && num_threads != 1)


@@ -1616,8 +1642,7 @@ write_stream_list(struct list_head *stream_list,
                                                 out_fd,
                                                 out_ctype,
                                                 write_resource_flags,
                                                 out_fd,
                                                 out_ctype,
                                                 write_resource_flags,

-                                                progress_func,
-                                                &progress,
+                                                &progress_data,

                                                 num_threads);
        else
 #endif
                                                 num_threads);
        else
 #endif


@@ -1626,8 +1651,7 @@ write_stream_list(struct list_head *stream_list,
                                               out_fd,
                                               out_ctype,
                                               write_resource_flags,
                                               out_fd,
                                               out_ctype,
                                               write_resource_flags,

-                                              progress_func,
-                                              &progress);
+                                              &progress_data);

        if (ret == 0)
                DEBUG("Successfully wrote stream list.");
        else
        if (ret == 0)
                DEBUG("Successfully wrote stream list.");
        else