]> wimlib.net Git - wimlib/commitdiff
Improve write streams performance and handling of joins
authorEric Biggers <ebiggers3@gmail.com>
Thu, 15 Aug 2013 19:08:00 +0000 (14:08 -0500)
committerEric Biggers <ebiggers3@gmail.com>
Thu, 15 Aug 2013 19:11:24 +0000 (14:11 -0500)
Provide the number of completed WIM parts and total WIM parts in
WIMLIB_PROGRESS_MSG_WRITE_STREAMS, to replace WIMLIB_PROGRESS_MSG_JOIN.
Furthermore, write streams in a more intelligent order; e.g. sorting by WIM part
number and file offset when relevant.

NEWS
include/wimlib.h
include/wimlib/lookup_table.h
programs/imagex.c
src/extract.c
src/join.c
src/lookup_table.c
src/write.c

diff --git a/NEWS b/NEWS
index 23b79dbca29614d7a0ff50e5d4d558e3aa3d4833..e8281cc2e2766b4b25b61a7fb55124869583b8f4 100644 (file)
--- a/NEWS
+++ b/NEWS
@@ -37,8 +37,11 @@ Version 1.5.0:
        The test suite no longer fails when run in a locale where the decimal
        separator is not a period.
 
-       WIMLIB_PROGRESS_MSG_EXTRACT_DENTRY has been removed and
-       WIMLIB_EXTRACT_FLAG_VERBOSE re-reserved for future use.
+       From the library, WIMLIB_PROGRESS_MSG_EXTRACT_DENTRY has been removed
+       and WIMLIB_EXTRACT_FLAG_VERBOSE re-reserved for future use.  Also,
+       WIMLIB_PROGRESS_MSG_JOIN_STREAMS has been removed, but
+       WIMLIB_PROGRESS_MSG_WRITE_STREAMS will be received instead and now
+       provides WIM part numbers.
 
        The extraction code has been rewritten and it will now be easier to
        support new features on all supported backends (currently Win32, UNIX,
index a7e068aee74f09149bbf3d58d15d062ec42ea260..6ab50ce414454e42a3a7b50d7ea6fad810c6f330 100644 (file)
@@ -327,9 +327,10 @@ enum wimlib_progress_msg {
         * ::wimlib_progress_info.integrity. */
        WIMLIB_PROGRESS_MSG_CALC_INTEGRITY,
 
-       /** A wimlib_join() operation is in progress.  @a info will point to
-        * ::wimlib_progress_info.join. */
-       WIMLIB_PROGRESS_MSG_JOIN_STREAMS,
+       /** Reserved.  (Previously used for WIMLIB_PROGRESS_MSG_JOIN_STREAMS,
+        * but in wimlib v1.5.0 this was removed to simplify the code and now
+        * you'll get ::WIMLIB_PROGRESS_MSG_WRITE_STREAMS messages instead.)  */
+       WIMLIB_PROGRESS_MSG_RESERVED,
 
        /** A wimlib_split() operation is in progress, and a new split part is
         * about to be started.  @a info will point to
@@ -371,6 +372,7 @@ union wimlib_progress_info {
                 * (The actual number of bytes will be less if the data is being
                 * written compressed.) */
                uint64_t total_bytes;
+
                /** Number of streams that are going to be written. */
                uint64_t total_streams;
 
@@ -384,7 +386,7 @@ union wimlib_progress_info {
                uint64_t completed_streams;
 
                /** Number of threads that are being used to compress resources
-                * (if applicable). */
+                * (if applicable).  */
                unsigned num_threads;
 
                /** The compression type being used to write the streams; either
@@ -393,8 +395,13 @@ union wimlib_progress_info {
                 * ::WIMLIB_COMPRESSION_TYPE_LZX. */
                int      compression_type;
 
-               /** Library internal use only. */
-               uint64_t _private;
+               /** Number of split WIM parts from which streams are being
+                * written (may be 0 if irrelevant).  */
+               unsigned total_parts;
+
+               /** Number of split WIM parts from which streams have been
+                * written (may be 0 if irrelevant).  */
+               unsigned completed_parts;
        } write_streams;
 
        /** Valid on messages ::WIMLIB_PROGRESS_MSG_SCAN_BEGIN and
@@ -518,25 +525,6 @@ union wimlib_progress_info {
                const wimlib_tchar *filename;
        } integrity;
 
-       /** Valid on messages ::WIMLIB_PROGRESS_MSG_JOIN_STREAMS. */
-       struct wimlib_progress_info_join {
-               /** Total number of bytes of compressed data contained in all
-                * the split WIM part's file and metadata resources. */
-               uint64_t total_bytes;
-
-               /** Number of bytes that have been copied to the joined WIM so
-                * far.  Will be 0 initially, and equal to @a total_bytes at the
-                * end. */
-               uint64_t completed_bytes;
-
-               /** Number of split WIM parts that have had all their file and
-                * metadata resources copied over to the joined WIM so far. */
-               unsigned completed_parts;
-
-               /** Number of split WIM parts. */
-               unsigned total_parts;
-       } join;
-
        /** Valid on messages ::WIMLIB_PROGRESS_MSG_SPLIT_BEGIN_PART and
         * ::WIMLIB_PROGRESS_MSG_SPLIT_END_PART. */
        struct wimlib_progress_info_split {
index 28862d5a3b9b753ea53e88c87881f923bfe38782..481794daf74d20849612eb1c23132b9f27da6223 100644 (file)
@@ -375,7 +375,8 @@ for_lookup_table_entry(struct wim_lookup_table *table,
                       void *arg);
 
 extern int
-cmp_streams_by_wim_position(const void *p1, const void *p2);
+sort_stream_list_by_sequential_order(struct list_head *stream_list,
+                                    size_t list_head_offset);
 
 extern int
 for_lookup_table_entry_pos_sorted(struct wim_lookup_table *table,
index e99202b1effdbf659a43b4b38096844bfeca8570..767b67240c6fc448a008454a2d4b7df7b598a9d8 100644 (file)
@@ -985,6 +985,7 @@ imagex_progress_func(enum wimlib_progress_msg msg,
                unit_shift = get_unit(info->write_streams.total_bytes, &unit_name);
                percent_done = TO_PERCENT(info->write_streams.completed_bytes,
                                          info->write_streams.total_bytes);
+
                if (info->write_streams.completed_streams == 0) {
                        const tchar *data_type;
 
@@ -993,13 +994,28 @@ imagex_progress_func(enum wimlib_progress_msg msg,
                                data_type, info->write_streams.num_threads,
                                (info->write_streams.num_threads == 1) ? T("") : T("s"));
                }
-               imagex_printf(T("\r%"PRIu64" %"TS" of %"PRIu64" %"TS" (uncompressed) "
-                       "written (%u%% done)"),
-                       info->write_streams.completed_bytes >> unit_shift,
-                       unit_name,
-                       info->write_streams.total_bytes >> unit_shift,
-                       unit_name,
-                       percent_done);
+               if (info->write_streams.total_parts <= 1) {
+                       imagex_printf(T("\r%"PRIu64" %"TS" of %"PRIu64" %"TS" (uncompressed) "
+                               "written (%u%% done)"),
+                               info->write_streams.completed_bytes >> unit_shift,
+                               unit_name,
+                               info->write_streams.total_bytes >> unit_shift,
+                               unit_name,
+                               percent_done);
+               } else {
+                       imagex_printf(T("\rWriting resources from part %u of %u: "
+                                 "%"PRIu64 " %"TS" of %"PRIu64" %"TS" (%u%%) written"),
+                               (info->write_streams.completed_parts ==
+                                       info->write_streams.total_parts) ?
+                                               info->write_streams.completed_parts :
+                                               info->write_streams.completed_parts + 1,
+                               info->write_streams.total_parts,
+                               info->write_streams.completed_bytes >> unit_shift,
+                               unit_name,
+                               info->write_streams.total_bytes >> unit_shift,
+                               unit_name,
+                               percent_done);
+               }
                if (info->write_streams.completed_bytes >= info->write_streams.total_bytes)
                        imagex_printf(T("\n"));
                break;
@@ -1019,8 +1035,6 @@ imagex_progress_func(enum wimlib_progress_msg msg,
                else
                        imagex_printf(T("Scanning \"%"TS"\"\n"), info->scan.cur_path);
                break;
-       /*case WIMLIB_PROGRESS_MSG_SCAN_END:*/
-               /*break;*/
        case WIMLIB_PROGRESS_MSG_VERIFY_INTEGRITY:
                unit_shift = get_unit(info->integrity.total_bytes, &unit_name);
                percent_done = TO_PERCENT(info->integrity.completed_bytes,
@@ -1070,10 +1084,6 @@ imagex_progress_func(enum wimlib_progress_msg msg,
                        info->extract.wimfile_name,
                        info->extract.target);
                break;
-       /*case WIMLIB_PROGRESS_MSG_EXTRACT_DIR_STRUCTURE_BEGIN:*/
-               /*imagex_printf(T("Applying directory structure to %"TS"\n"),*/
-                       /*info->extract.target);*/
-               /*break;*/
        case WIMLIB_PROGRESS_MSG_EXTRACT_STREAMS:
                percent_done = TO_PERCENT(info->extract.completed_bytes,
                                          info->extract.total_bytes);
@@ -1098,21 +1108,6 @@ imagex_progress_func(enum wimlib_progress_msg msg,
                                info->extract.target);
                }
                break;
-       case WIMLIB_PROGRESS_MSG_JOIN_STREAMS:
-               percent_done = TO_PERCENT(info->join.completed_bytes,
-                                         info->join.total_bytes);
-               unit_shift = get_unit(info->join.total_bytes, &unit_name);
-               imagex_printf(T("Writing resources from part %u of %u: "
-                         "%"PRIu64 " %"TS" of %"PRIu64" %"TS" (%u%%) written\n"),
-                       (info->join.completed_parts == info->join.total_parts) ?
-                       info->join.completed_parts : info->join.completed_parts + 1,
-                       info->join.total_parts,
-                       info->join.completed_bytes >> unit_shift,
-                       unit_name,
-                       info->join.total_bytes >> unit_shift,
-                       unit_name,
-                       percent_done);
-               break;
        case WIMLIB_PROGRESS_MSG_SPLIT_BEGIN_PART:
                percent_done = TO_PERCENT(info->split.completed_bytes,
                                          info->split.total_bytes);
index b96367b6a1d934ef879584ee9e53878f46ce2d81..d9e03333e43403f701f337535253d0d4e403aafc 100644 (file)
@@ -1337,42 +1337,6 @@ dentry_extract_final(struct wim_dentry *dentry, void *_ctx)
        return extract_timestamps(path, ctx, dentry);
 }
 
-/* Sorts a list of streams in ascending order of their offset in the WIM file in
- * order to prepare for sequential extraction.  */
-static int
-sort_stream_list_by_wim_position(struct list_head *stream_list)
-{
-       struct list_head *cur;
-       size_t num_streams;
-       struct wim_lookup_table_entry **array;
-       size_t i;
-       size_t array_size;
-
-       num_streams = 0;
-       list_for_each(cur, stream_list)
-               num_streams++;
-       array_size = num_streams * sizeof(array[0]);
-       array = MALLOC(array_size);
-       if (!array) {
-               ERROR("Failed to allocate %zu bytes to sort stream entries",
-                     array_size);
-               return WIMLIB_ERR_NOMEM;
-       }
-       cur = stream_list->next;
-       for (i = 0; i < num_streams; i++) {
-               array[i] = container_of(cur, struct wim_lookup_table_entry, extraction_list);
-               cur = cur->next;
-       }
-
-       qsort(array, num_streams, sizeof(array[0]), cmp_streams_by_wim_position);
-
-       INIT_LIST_HEAD(stream_list);
-       for (i = 0; i < num_streams; i++)
-               list_add_tail(&array[i]->extraction_list, stream_list);
-       FREE(array);
-       return 0;
-}
-
 /*
  * Extract a WIM dentry to standard output.
  *
@@ -2033,7 +1997,10 @@ extract_tree(WIMStruct *wim, const tchar *wim_source_path, const tchar *target,
                              WIMLIB_EXTRACT_FLAG_FROM_PIPE))
                                        == WIMLIB_EXTRACT_FLAG_SEQUENTIAL)
        {
-               ret = sort_stream_list_by_wim_position(&ctx.stream_list);
+               ret = sort_stream_list_by_sequential_order(
+                               &ctx.stream_list,
+                               offsetof(struct wim_lookup_table_entry,
+                                        extraction_list));
                if (ret)
                        goto out_teardown_stream_list;
        }
index 5cda7739a1f57d2f5a0820b837b01c65a3c5f439..d606b7be6392a71cd20241f11845b91d75f9909b 100644 (file)
@@ -82,10 +82,8 @@ wimlib_join(const tchar * const *swm_names,
        if (ret)
                goto out_free_swms;
 
-       ret = wimlib_export_image(swm0, WIMLIB_ALL_IMAGES,
-                                 wim, NULL, NULL, 0,
-                                 swms, num_additional_swms,
-                                 progress_func);
+       ret = wimlib_export_image(swm0, WIMLIB_ALL_IMAGES, wim, NULL, NULL, 0,
+                                 swms, num_additional_swms, progress_func);
        if (ret)
                goto out_free_wim;
 
index 3b98553e9bb586d790df154f8de706351bc2ac0e..15f5c0830c382fb501d9dc13ff5dbcc3513f85a8 100644 (file)
@@ -288,20 +288,107 @@ for_lookup_table_entry(struct wim_lookup_table *table,
        return 0;
 }
 
-int
-cmp_streams_by_wim_position(const void *p1, const void *p2)
+/* qsort() callback that sorts streams (represented by `struct
+ * wim_lookup_table_entry's) into an order optimized for reading and writing.
+ *
+ * Sorting is done primarily by resource location, then secondarily by a
+ * per-resource location order.  For example, resources in WIM files are sorted
+ * primarily by part number, then secondarily by offset, as to implement optimal
+ * reading of either a standalone or split WIM.  */
+static int
+cmp_streams_by_sequential_order(const void *p1, const void *p2)
 {
        const struct wim_lookup_table_entry *lte1, *lte2;
+       int v;
+
        lte1 = *(const struct wim_lookup_table_entry**)p1;
        lte2 = *(const struct wim_lookup_table_entry**)p2;
-       if (lte1->resource_entry.offset < lte2->resource_entry.offset)
-               return -1;
-       else if (lte1->resource_entry.offset > lte2->resource_entry.offset)
-               return 1;
-       else
+
+       v = (int)lte1->resource_location - (int)lte2->resource_location;
+
+       /* Different resource locations?  */
+       if (v)
+               return v;
+
+       switch (lte1->resource_location) {
+       case RESOURCE_IN_WIM:
+
+               /* Different (possibly split) WIMs?  */
+               if (lte1->wim != lte2->wim) {
+                       v = memcmp(lte1->wim->hdr.guid, lte2->wim->hdr.guid,
+                                  WIM_GID_LEN);
+                       if (v)
+                               return v;
+               }
+
+               /* Different part numbers in the same WIM?  */
+               v = (int)lte1->wim->hdr.part_number - (int)lte2->wim->hdr.part_number;
+               if (v)
+                       return v;
+
+               /* Compare by offset.  */
+               if (lte1->resource_entry.offset < lte2->resource_entry.offset)
+                       return -1;
+               else if (lte1->resource_entry.offset > lte2->resource_entry.offset)
+                       return 1;
                return 0;
+       case RESOURCE_IN_FILE_ON_DISK:
+#ifdef __WIN32__
+       case RESOURCE_WIN32_ENCRYPTED:
+#endif
+               /* Compare files by path: just a heuristic that will place files
+                * in the same directory next to each other.  */
+               return tstrcmp(lte1->file_on_disk, lte2->file_on_disk);
+#ifdef WITH_NTFS_3G
+       case RESOURCE_IN_NTFS_VOLUME:
+               return tstrcmp(lte1->ntfs_loc->path, lte2->ntfs_loc->path);
+#endif
+       default:
+               /* No additional sorting order defined for this resource
+                * location (e.g. RESOURCE_IN_ATTACHED_BUFFER); simply compare
+                * everything equal to each other.  */
+               return 0;
+       }
+}
+
+int
+sort_stream_list_by_sequential_order(struct list_head *stream_list,
+                                    size_t list_head_offset)
+{
+       struct list_head *cur;
+       struct wim_lookup_table_entry **array;
+       size_t i;
+       size_t array_size;
+       size_t num_streams = 0;
+
+       list_for_each(cur, stream_list)
+               num_streams++;
+
+       array_size = num_streams * sizeof(array[0]);
+       array = MALLOC(array_size);
+       if (!array)
+               return WIMLIB_ERR_NOMEM;
+       cur = stream_list->next;
+       for (i = 0; i < num_streams; i++) {
+               array[i] = (struct wim_lookup_table_entry*)((u8*)cur -
+                                                           list_head_offset);
+               cur = cur->next;
+       }
+
+       qsort(array, num_streams, sizeof(array[0]),
+             cmp_streams_by_sequential_order);
+
+       INIT_LIST_HEAD(stream_list);
+       for (i = 0; i < num_streams; i++) {
+               list_add_tail((struct list_head*)
+                              ((u8*)array[i] + list_head_offset),
+                             stream_list);
+       }
+       FREE(array);
+       return 0;
 }
 
+
 static int
 add_lte_to_array(struct wim_lookup_table_entry *lte,
                 void *_pp)
@@ -333,7 +420,7 @@ for_lookup_table_entry_pos_sorted(struct wim_lookup_table *table,
        wimlib_assert(p == lte_array + num_streams);
 
        qsort(lte_array, num_streams, sizeof(lte_array[0]),
-             cmp_streams_by_wim_position);
+             cmp_streams_by_sequential_order);
        ret = 0;
        for (size_t i = 0; i < num_streams; i++) {
                ret = visitor(lte_array[i], arg);
index 3cac5a2868a7d8ef7c9f3ea90e853e3de824b9fe..e4a0f3ad45712c6ce8ab5a5da345b9aeed0c0bf8 100644 (file)
 #  include <pthread.h>
 #endif
 
-#include <unistd.h>
-#include <fcntl.h>
 #include <errno.h>
+#include <fcntl.h>
+#include <limits.h>
+#include <stdlib.h>
+#include <unistd.h>
 
 #ifdef HAVE_ALLOCA_H
 #  include <alloca.h>
-#else
-#  include <stdlib.h>
 #endif
 
-#include <limits.h>
 
 #ifndef __WIN32__
 #  include <sys/uio.h> /* for `struct iovec' */
@@ -683,7 +682,6 @@ struct message {
        u8 *compressed_chunks[MAX_CHUNKS_PER_MSG];
        unsigned uncompressed_chunk_sizes[MAX_CHUNKS_PER_MSG];
        struct iovec out_chunks[MAX_CHUNKS_PER_MSG];
-       size_t total_out_bytes;
        unsigned num_chunks;
        struct list_head list;
        bool complete;
@@ -693,7 +691,6 @@ struct message {
 static void
 compress_chunks(struct message *msg, compress_func_t compress)
 {
-       msg->total_out_bytes = 0;
        for (unsigned i = 0; i < msg->num_chunks; i++) {
                unsigned len = compress(msg->uncompressed_chunks[i],
                                        msg->uncompressed_chunk_sizes[i],
@@ -711,7 +708,6 @@ compress_chunks(struct message *msg, compress_func_t compress)
                }
                msg->out_chunks[i].iov_base = out_chunk;
                msg->out_chunks[i].iov_len = out_len;
-               msg->total_out_bytes += out_len;
        }
 }
 
@@ -739,32 +735,52 @@ compressor_thread_proc(void *arg)
 }
 #endif /* ENABLE_MULTITHREADED_COMPRESSION */
 
+struct write_streams_progress_data {
+       wimlib_progress_func_t progress_func;
+       union wimlib_progress_info progress;
+       uint64_t next_progress;
+       WIMStruct *prev_wim_part;
+};
+
 static void
-do_write_streams_progress(union wimlib_progress_info *progress,
-                         wimlib_progress_func_t progress_func,
-                         uint64_t size_added,
+do_write_streams_progress(struct write_streams_progress_data *progress_data,
+                         struct wim_lookup_table_entry *lte,
                          bool stream_discarded)
 {
+       union wimlib_progress_info *progress = &progress_data->progress;
+       bool new_wim_part;
+
        if (stream_discarded) {
-               progress->write_streams.total_bytes -= size_added;
-               if (progress->write_streams._private != ~(uint64_t)0 &&
-                   progress->write_streams._private > progress->write_streams.total_bytes)
+               progress->write_streams.total_bytes -= wim_resource_size(lte);
+               if (progress_data->next_progress != ~(uint64_t)0 &&
+                   progress_data->next_progress > progress->write_streams.total_bytes)
                {
-                       progress->write_streams._private = progress->write_streams.total_bytes;
+                       progress_data->next_progress = progress->write_streams.total_bytes;
                }
        } else {
-               progress->write_streams.completed_bytes += size_added;
+               progress->write_streams.completed_bytes += wim_resource_size(lte);
+       }
+       new_wim_part = false;
+       if (lte->resource_location == RESOURCE_IN_WIM &&
+           lte->wim != progress_data->prev_wim_part)
+       {
+               if (progress_data->prev_wim_part) {
+                       new_wim_part = true;
+                       progress->write_streams.completed_parts++;
+               }
+               progress_data->prev_wim_part = lte->wim;
        }
        progress->write_streams.completed_streams++;
-       if (progress_func &&
-           progress->write_streams.completed_bytes >= progress->write_streams._private)
+       if (progress_data->progress_func
+           && (progress->write_streams.completed_bytes >= progress_data->next_progress
+               || new_wim_part))
        {
-               progress_func(WIMLIB_PROGRESS_MSG_WRITE_STREAMS,
-                             progress);
-               if (progress->write_streams._private == progress->write_streams.total_bytes) {
-                       progress->write_streams._private = ~(uint64_t)0;
+               progress_data->progress_func(WIMLIB_PROGRESS_MSG_WRITE_STREAMS,
+                                            progress);
+               if (progress_data->next_progress == progress->write_streams.total_bytes) {
+                       progress_data->next_progress = ~(uint64_t)0;
                } else {
-                       progress->write_streams._private =
+                       progress_data->next_progress =
                                min(progress->write_streams.total_bytes,
                                    progress->write_streams.completed_bytes +
                                        progress->write_streams.total_bytes / 100);
@@ -787,6 +803,7 @@ serial_write_stream(struct wim_lookup_table_entry *lte, void *_ctx)
                                  ctx->write_resource_flags);
 }
 
+
 /* Write a list of streams, taking into account that some streams may be
  * duplicates that are checksummed and discarded on the fly, and also delegating
  * the actual writing of a stream to a function @write_stream_cb, which is
@@ -796,8 +813,7 @@ do_write_stream_list(struct list_head *stream_list,
                     struct wim_lookup_table *lookup_table,
                     int (*write_stream_cb)(struct wim_lookup_table_entry *, void *),
                     void *write_stream_ctx,
-                    wimlib_progress_func_t progress_func,
-                    union wimlib_progress_info *progress)
+                    struct write_streams_progress_data *progress_data)
 {
        int ret = 0;
        struct wim_lookup_table_entry *lte;
@@ -860,10 +876,8 @@ do_write_stream_list(struct list_head *stream_list,
                }
        skip_to_progress:
                if (!lte->no_progress) {
-                       do_write_streams_progress(progress,
-                                                 progress_func,
-                                                 wim_resource_size(lte),
-                                                 stream_discarded);
+                       do_write_streams_progress(progress_data,
+                                                 lte, stream_discarded);
                }
        }
        return ret;
@@ -875,8 +889,7 @@ do_write_stream_list_serial(struct list_head *stream_list,
                            struct filedes *out_fd,
                            int out_ctype,
                            int write_resource_flags,
-                           wimlib_progress_func_t progress_func,
-                           union wimlib_progress_info *progress)
+                           struct write_streams_progress_data *progress_data)
 {
        struct serial_write_stream_ctx ctx = {
                .out_fd = out_fd,
@@ -887,8 +900,7 @@ do_write_stream_list_serial(struct list_head *stream_list,
                                    lookup_table,
                                    serial_write_stream,
                                    &ctx,
-                                   progress_func,
-                                   progress);
+                                   progress_data);
 }
 
 static inline int
@@ -909,21 +921,22 @@ write_stream_list_serial(struct list_head *stream_list,
                         struct filedes *out_fd,
                         int out_ctype,
                         int write_resource_flags,
-                        wimlib_progress_func_t progress_func,
-                        union wimlib_progress_info *progress)
+                        struct write_streams_progress_data *progress_data)
 {
+       union wimlib_progress_info *progress = &progress_data->progress;
        DEBUG("Writing stream list of size %"PRIu64" (serial version)",
              progress->write_streams.total_streams);
        progress->write_streams.num_threads = 1;
-       if (progress_func)
-               progress_func(WIMLIB_PROGRESS_MSG_WRITE_STREAMS, progress);
+       if (progress_data->progress_func) {
+               progress_data->progress_func(WIMLIB_PROGRESS_MSG_WRITE_STREAMS,
+                                            progress);
+       }
        return do_write_stream_list_serial(stream_list,
                                           lookup_table,
                                           out_fd,
                                           out_ctype,
                                           write_resource_flags,
-                                          progress_func,
-                                          progress);
+                                          progress_data);
 }
 
 #ifdef ENABLE_MULTITHREADED_COMPRESSION
@@ -935,7 +948,6 @@ write_wim_chunks(struct message *msg, struct filedes *out_fd,
        struct iovec *vecs;
        struct pwm_chunk_hdr *chunk_hdrs;
        unsigned nvecs;
-       size_t nbytes;
        int ret;
 
        for (unsigned i = 0; i < msg->num_chunks; i++)
@@ -944,7 +956,6 @@ write_wim_chunks(struct message *msg, struct filedes *out_fd,
        if (!(write_resource_flags & WIMLIB_WRITE_RESOURCE_FLAG_PIPABLE)) {
                nvecs = msg->num_chunks;
                vecs = msg->out_chunks;
-               nbytes = msg->total_out_bytes;
        } else {
                /* Special case:  If writing a compressed resource to a pipable
                 * WIM, prefix each compressed chunk with a header that gives
@@ -960,7 +971,6 @@ write_wim_chunks(struct message *msg, struct filedes *out_fd,
                        vecs[i * 2 + 1].iov_base = msg->out_chunks[i].iov_base;
                        vecs[i * 2 + 1].iov_len = msg->out_chunks[i].iov_len;
                }
-               nbytes = msg->total_out_bytes + msg->num_chunks * sizeof(chunk_hdrs[0]);
        }
        ret = full_writev(out_fd, vecs, nvecs);
        if (ret)
@@ -978,8 +988,7 @@ struct main_writer_thread_ctx {
        struct shared_queue *res_to_compress_queue;
        struct shared_queue *compressed_res_queue;
        size_t num_messages;
-       wimlib_progress_func_t progress_func;
-       union wimlib_progress_info *progress;
+       struct write_streams_progress_data *progress_data;
 
        struct list_head available_msgs;
        struct list_head outstanding_streams;
@@ -1221,10 +1230,8 @@ receive_compressed_chunks(struct main_writer_thread_ctx *ctx)
                                      cur_lte->output_resource_entry.flags);
                        }
 
-                       do_write_streams_progress(ctx->progress,
-                                                 ctx->progress_func,
-                                                 wim_resource_size(cur_lte),
-                                                 false);
+                       do_write_streams_progress(ctx->progress_data,
+                                                 cur_lte, false);
 
                        /* Since we just finished writing a stream, write any
                         * streams that have been added to the serial_streams
@@ -1238,8 +1245,7 @@ receive_compressed_chunks(struct main_writer_thread_ctx *ctx)
                                                                  ctx->out_fd,
                                                                  ctx->out_ctype,
                                                                  ctx->write_resource_flags,
-                                                                 ctx->progress_func,
-                                                                 ctx->progress);
+                                                                 ctx->progress_data);
                                if (ret)
                                        return ret;
                        }
@@ -1337,8 +1343,7 @@ main_writer_thread_finish(void *_ctx)
                                           ctx->out_fd,
                                           ctx->out_ctype,
                                           ctx->write_resource_flags,
-                                          ctx->progress_func,
-                                          ctx->progress);
+                                          ctx->progress_data);
 }
 
 static int
@@ -1433,14 +1438,14 @@ write_stream_list_parallel(struct list_head *stream_list,
                           struct filedes *out_fd,
                           int out_ctype,
                           int write_resource_flags,
-                          wimlib_progress_func_t progress_func,
-                          union wimlib_progress_info *progress,
+                          struct write_streams_progress_data *progress_data,
                           unsigned num_threads)
 {
        int ret;
        struct shared_queue res_to_compress_queue;
        struct shared_queue compressed_res_queue;
        pthread_t *compressor_threads = NULL;
+       union wimlib_progress_info *progress = &progress_data->progress;
 
        if (num_threads == 0) {
                long nthreads = get_default_num_threads();
@@ -1498,8 +1503,10 @@ write_stream_list_parallel(struct list_head *stream_list,
                }
        }
 
-       if (progress_func)
-               progress_func(WIMLIB_PROGRESS_MSG_WRITE_STREAMS, progress);
+       if (progress_data->progress_func) {
+               progress_data->progress_func(WIMLIB_PROGRESS_MSG_WRITE_STREAMS,
+                                            progress);
+       }
 
        struct main_writer_thread_ctx ctx;
        ctx.stream_list           = stream_list;
@@ -1510,14 +1517,13 @@ write_stream_list_parallel(struct list_head *stream_list,
        ctx.compressed_res_queue  = &compressed_res_queue;
        ctx.num_messages          = queue_size;
        ctx.write_resource_flags  = write_resource_flags;
-       ctx.progress_func         = progress_func;
-       ctx.progress              = progress;
+       ctx.progress_data         = progress_data;
        ret = main_writer_thread_init_ctx(&ctx);
        if (ret)
                goto out_join;
        ret = do_write_stream_list(stream_list, lookup_table,
                                   main_thread_process_next_stream,
-                                  &ctx, progress_func, progress);
+                                  &ctx, progress_data);
        if (ret)
                goto out_destroy_ctx;
 
@@ -1556,8 +1562,7 @@ out_serial_quiet:
                                        out_fd,
                                        out_ctype,
                                        write_resource_flags,
-                                       progress_func,
-                                       progress);
+                                       progress_data);
 
 }
 #endif
@@ -1576,9 +1581,11 @@ write_stream_list(struct list_head *stream_list,
        size_t num_streams = 0;
        u64 total_bytes = 0;
        u64 total_compression_bytes = 0;
-       union wimlib_progress_info progress;
+       struct write_streams_progress_data progress_data;
        int ret;
        int write_resource_flags;
+       unsigned total_parts = 0;
+       WIMStruct *prev_wim_part = NULL;
 
        if (list_empty(stream_list))
                return 0;
@@ -1587,6 +1594,10 @@ write_stream_list(struct list_head *stream_list,
 
        DEBUG("write_resource_flags=0x%08x", write_resource_flags);
 
+       sort_stream_list_by_sequential_order(stream_list,
+                                            offsetof(struct wim_lookup_table_entry,
+                                                     write_streams_list));
+
        /* Calculate the total size of the streams to be written.  Note: this
         * will be the uncompressed size, as we may not know the compressed size
         * yet, and also this will assume that every unhashed stream will be
@@ -1600,14 +1611,29 @@ write_stream_list(struct list_head *stream_list,
                {
                        total_compression_bytes += wim_resource_size(lte);
                }
+               if (lte->resource_location == RESOURCE_IN_WIM) {
+                       if (prev_wim_part != lte->wim) {
+                               prev_wim_part = lte->wim;
+                               total_parts++;
+                       }
+               }
+
        }
-       progress.write_streams.total_bytes       = total_bytes;
-       progress.write_streams.total_streams     = num_streams;
-       progress.write_streams.completed_bytes   = 0;
-       progress.write_streams.completed_streams = 0;
-       progress.write_streams.num_threads       = num_threads;
-       progress.write_streams.compression_type  = out_ctype;
-       progress.write_streams._private          = 0;
+
+       memset(&progress_data, 0, sizeof(progress_data));
+       progress_data.progress_func = progress_func;
+
+       progress_data.progress.write_streams.total_bytes       = total_bytes;
+       progress_data.progress.write_streams.total_streams     = num_streams;
+       progress_data.progress.write_streams.completed_bytes   = 0;
+       progress_data.progress.write_streams.completed_streams = 0;
+       progress_data.progress.write_streams.num_threads       = num_threads;
+       progress_data.progress.write_streams.compression_type  = out_ctype;
+       progress_data.progress.write_streams.total_parts       = total_parts;
+       progress_data.progress.write_streams.completed_parts   = 0;
+
+       progress_data.next_progress = 0;
+       progress_data.prev_wim_part = NULL;
 
 #ifdef ENABLE_MULTITHREADED_COMPRESSION
        if (total_compression_bytes >= 2000000 && num_threads != 1)
@@ -1616,8 +1642,7 @@ write_stream_list(struct list_head *stream_list,
                                                 out_fd,
                                                 out_ctype,
                                                 write_resource_flags,
-                                                progress_func,
-                                                &progress,
+                                                &progress_data,
                                                 num_threads);
        else
 #endif
@@ -1626,8 +1651,7 @@ write_stream_list(struct list_head *stream_list,
                                               out_fd,
                                               out_ctype,
                                               write_resource_flags,
-                                              progress_func,
-                                              &progress);
+                                              &progress_data);
        if (ret == 0)
                DEBUG("Successfully wrote stream list.");
        else