Improve write streams performance and handling of joins
authorEric Biggers <ebiggers3@gmail.com>
Thu, 15 Aug 2013 19:08:00 +0000 (14:08 -0500)
committerEric Biggers <ebiggers3@gmail.com>
Thu, 15 Aug 2013 19:11:24 +0000 (14:11 -0500)
Provide the number of completed WIM parts and total WIM parts in
WIMLIB_PROGRESS_MSG_WRITE_STREAMS, to replace WIMLIB_PROGRESS_MSG_JOIN.
Furthermore, write streams in a more intelligent order; e.g. sorting by WIM part
number and file offset when relevant.

NEWS
include/wimlib.h
include/wimlib/lookup_table.h
programs/imagex.c
src/extract.c
src/join.c
src/lookup_table.c
src/write.c

diff --git a/NEWS b/NEWS
index 23b79db..e8281cc 100644 (file)
--- a/NEWS
+++ b/NEWS
@@ -37,8 +37,11 @@ Version 1.5.0:
        The test suite no longer fails when run in a locale where the decimal
        separator is not a period.
 
-       WIMLIB_PROGRESS_MSG_EXTRACT_DENTRY has been removed and
-       WIMLIB_EXTRACT_FLAG_VERBOSE re-reserved for future use.
+       From the library, WIMLIB_PROGRESS_MSG_EXTRACT_DENTRY has been removed
+       and WIMLIB_EXTRACT_FLAG_VERBOSE re-reserved for future use.  Also,
+       WIMLIB_PROGRESS_MSG_JOIN_STREAMS has been removed, but
+       WIMLIB_PROGRESS_MSG_WRITE_STREAMS will be received instead and now
+       provides WIM part numbers.
 
        The extraction code has been rewritten and it will now be easier to
        support new features on all supported backends (currently Win32, UNIX,
index a7e068a..6ab50ce 100644 (file)
@@ -327,9 +327,10 @@ enum wimlib_progress_msg {
         * ::wimlib_progress_info.integrity. */
        WIMLIB_PROGRESS_MSG_CALC_INTEGRITY,
 
-       /** A wimlib_join() operation is in progress.  @a info will point to
-        * ::wimlib_progress_info.join. */
-       WIMLIB_PROGRESS_MSG_JOIN_STREAMS,
+       /** Reserved.  (Previously used for WIMLIB_PROGRESS_MSG_JOIN_STREAMS,
+        * but in wimlib v1.5.0 this was removed to simplify the code and now
+        * you'll get ::WIMLIB_PROGRESS_MSG_WRITE_STREAMS messages instead.)  */
+       WIMLIB_PROGRESS_MSG_RESERVED,
 
        /** A wimlib_split() operation is in progress, and a new split part is
         * about to be started.  @a info will point to
@@ -371,6 +372,7 @@ union wimlib_progress_info {
                 * (The actual number of bytes will be less if the data is being
                 * written compressed.) */
                uint64_t total_bytes;
+
                /** Number of streams that are going to be written. */
                uint64_t total_streams;
 
@@ -384,7 +386,7 @@ union wimlib_progress_info {
                uint64_t completed_streams;
 
                /** Number of threads that are being used to compress resources
-                * (if applicable). */
+                * (if applicable).  */
                unsigned num_threads;
 
                /** The compression type being used to write the streams; either
@@ -393,8 +395,13 @@ union wimlib_progress_info {
                 * ::WIMLIB_COMPRESSION_TYPE_LZX. */
                int      compression_type;
 
-               /** Library internal use only. */
-               uint64_t _private;
+               /** Number of split WIM parts from which streams are being
+                * written (may be 0 if irrelevant).  */
+               unsigned total_parts;
+
+               /** Number of split WIM parts from which streams have been
+                * written (may be 0 if irrelevant).  */
+               unsigned completed_parts;
        } write_streams;
 
        /** Valid on messages ::WIMLIB_PROGRESS_MSG_SCAN_BEGIN and
@@ -518,25 +525,6 @@ union wimlib_progress_info {
                const wimlib_tchar *filename;
        } integrity;
 
-       /** Valid on messages ::WIMLIB_PROGRESS_MSG_JOIN_STREAMS. */
-       struct wimlib_progress_info_join {
-               /** Total number of bytes of compressed data contained in all
-                * the split WIM part's file and metadata resources. */
-               uint64_t total_bytes;
-
-               /** Number of bytes that have been copied to the joined WIM so
-                * far.  Will be 0 initially, and equal to @a total_bytes at the
-                * end. */
-               uint64_t completed_bytes;
-
-               /** Number of split WIM parts that have had all their file and
-                * metadata resources copied over to the joined WIM so far. */
-               unsigned completed_parts;
-
-               /** Number of split WIM parts. */
-               unsigned total_parts;
-       } join;
-
        /** Valid on messages ::WIMLIB_PROGRESS_MSG_SPLIT_BEGIN_PART and
         * ::WIMLIB_PROGRESS_MSG_SPLIT_END_PART. */
        struct wimlib_progress_info_split {
index 28862d5..481794d 100644 (file)
@@ -375,7 +375,8 @@ for_lookup_table_entry(struct wim_lookup_table *table,
                       void *arg);
 
 extern int
-cmp_streams_by_wim_position(const void *p1, const void *p2);
+sort_stream_list_by_sequential_order(struct list_head *stream_list,
+                                    size_t list_head_offset);
 
 extern int
 for_lookup_table_entry_pos_sorted(struct wim_lookup_table *table,
index e99202b..767b672 100644 (file)
@@ -985,6 +985,7 @@ imagex_progress_func(enum wimlib_progress_msg msg,
                unit_shift = get_unit(info->write_streams.total_bytes, &unit_name);
                percent_done = TO_PERCENT(info->write_streams.completed_bytes,
                                          info->write_streams.total_bytes);
+
                if (info->write_streams.completed_streams == 0) {
                        const tchar *data_type;
 
@@ -993,13 +994,28 @@ imagex_progress_func(enum wimlib_progress_msg msg,
                                data_type, info->write_streams.num_threads,
                                (info->write_streams.num_threads == 1) ? T("") : T("s"));
                }
-               imagex_printf(T("\r%"PRIu64" %"TS" of %"PRIu64" %"TS" (uncompressed) "
-                       "written (%u%% done)"),
-                       info->write_streams.completed_bytes >> unit_shift,
-                       unit_name,
-                       info->write_streams.total_bytes >> unit_shift,
-                       unit_name,
-                       percent_done);
+               if (info->write_streams.total_parts <= 1) {
+                       imagex_printf(T("\r%"PRIu64" %"TS" of %"PRIu64" %"TS" (uncompressed) "
+                               "written (%u%% done)"),
+                               info->write_streams.completed_bytes >> unit_shift,
+                               unit_name,
+                               info->write_streams.total_bytes >> unit_shift,
+                               unit_name,
+                               percent_done);
+               } else {
+                       imagex_printf(T("\rWriting resources from part %u of %u: "
+                                 "%"PRIu64 " %"TS" of %"PRIu64" %"TS" (%u%%) written"),
+                               (info->write_streams.completed_parts ==
+                                       info->write_streams.total_parts) ?
+                                               info->write_streams.completed_parts :
+                                               info->write_streams.completed_parts + 1,
+                               info->write_streams.total_parts,
+                               info->write_streams.completed_bytes >> unit_shift,
+                               unit_name,
+                               info->write_streams.total_bytes >> unit_shift,
+                               unit_name,
+                               percent_done);
+               }
                if (info->write_streams.completed_bytes >= info->write_streams.total_bytes)
                        imagex_printf(T("\n"));
                break;
@@ -1019,8 +1035,6 @@ imagex_progress_func(enum wimlib_progress_msg msg,
                else
                        imagex_printf(T("Scanning \"%"TS"\"\n"), info->scan.cur_path);
                break;
-       /*case WIMLIB_PROGRESS_MSG_SCAN_END:*/
-               /*break;*/
        case WIMLIB_PROGRESS_MSG_VERIFY_INTEGRITY:
                unit_shift = get_unit(info->integrity.total_bytes, &unit_name);
                percent_done = TO_PERCENT(info->integrity.completed_bytes,
@@ -1070,10 +1084,6 @@ imagex_progress_func(enum wimlib_progress_msg msg,
                        info->extract.wimfile_name,
                        info->extract.target);
                break;
-       /*case WIMLIB_PROGRESS_MSG_EXTRACT_DIR_STRUCTURE_BEGIN:*/
-               /*imagex_printf(T("Applying directory structure to %"TS"\n"),*/
-                       /*info->extract.target);*/
-               /*break;*/
        case WIMLIB_PROGRESS_MSG_EXTRACT_STREAMS:
                percent_done = TO_PERCENT(info->extract.completed_bytes,
                                          info->extract.total_bytes);
@@ -1098,21 +1108,6 @@ imagex_progress_func(enum wimlib_progress_msg msg,
                                info->extract.target);
                }
                break;
-       case WIMLIB_PROGRESS_MSG_JOIN_STREAMS:
-               percent_done = TO_PERCENT(info->join.completed_bytes,
-                                         info->join.total_bytes);
-               unit_shift = get_unit(info->join.total_bytes, &unit_name);
-               imagex_printf(T("Writing resources from part %u of %u: "
-                         "%"PRIu64 " %"TS" of %"PRIu64" %"TS" (%u%%) written\n"),
-                       (info->join.completed_parts == info->join.total_parts) ?
-                       info->join.completed_parts : info->join.completed_parts + 1,
-                       info->join.total_parts,
-                       info->join.completed_bytes >> unit_shift,
-                       unit_name,
-                       info->join.total_bytes >> unit_shift,
-                       unit_name,
-                       percent_done);
-               break;
        case WIMLIB_PROGRESS_MSG_SPLIT_BEGIN_PART:
                percent_done = TO_PERCENT(info->split.completed_bytes,
                                          info->split.total_bytes);
index b96367b..d9e0333 100644 (file)
@@ -1337,42 +1337,6 @@ dentry_extract_final(struct wim_dentry *dentry, void *_ctx)
        return extract_timestamps(path, ctx, dentry);
 }
 
-/* Sorts a list of streams in ascending order of their offset in the WIM file in
- * order to prepare for sequential extraction.  */
-static int
-sort_stream_list_by_wim_position(struct list_head *stream_list)
-{
-       struct list_head *cur;
-       size_t num_streams;
-       struct wim_lookup_table_entry **array;
-       size_t i;
-       size_t array_size;
-
-       num_streams = 0;
-       list_for_each(cur, stream_list)
-               num_streams++;
-       array_size = num_streams * sizeof(array[0]);
-       array = MALLOC(array_size);
-       if (!array) {
-               ERROR("Failed to allocate %zu bytes to sort stream entries",
-                     array_size);
-               return WIMLIB_ERR_NOMEM;
-       }
-       cur = stream_list->next;
-       for (i = 0; i < num_streams; i++) {
-               array[i] = container_of(cur, struct wim_lookup_table_entry, extraction_list);
-               cur = cur->next;
-       }
-
-       qsort(array, num_streams, sizeof(array[0]), cmp_streams_by_wim_position);
-
-       INIT_LIST_HEAD(stream_list);
-       for (i = 0; i < num_streams; i++)
-               list_add_tail(&array[i]->extraction_list, stream_list);
-       FREE(array);
-       return 0;
-}
-
 /*
  * Extract a WIM dentry to standard output.
  *
@@ -2033,7 +1997,10 @@ extract_tree(WIMStruct *wim, const tchar *wim_source_path, const tchar *target,
                              WIMLIB_EXTRACT_FLAG_FROM_PIPE))
                                        == WIMLIB_EXTRACT_FLAG_SEQUENTIAL)
        {
-               ret = sort_stream_list_by_wim_position(&ctx.stream_list);
+               ret = sort_stream_list_by_sequential_order(
+                               &ctx.stream_list,
+                               offsetof(struct wim_lookup_table_entry,
+                                        extraction_list));
                if (ret)
                        goto out_teardown_stream_list;
        }
index 5cda773..d606b7b 100644 (file)
@@ -82,10 +82,8 @@ wimlib_join(const tchar * const *swm_names,
        if (ret)
                goto out_free_swms;
 
-       ret = wimlib_export_image(swm0, WIMLIB_ALL_IMAGES,
-                                 wim, NULL, NULL, 0,
-                                 swms, num_additional_swms,
-                                 progress_func);
+       ret = wimlib_export_image(swm0, WIMLIB_ALL_IMAGES, wim, NULL, NULL, 0,
+                                 swms, num_additional_swms, progress_func);
        if (ret)
                goto out_free_wim;
 
index 3b98553..15f5c08 100644 (file)
@@ -288,20 +288,107 @@ for_lookup_table_entry(struct wim_lookup_table *table,
        return 0;
 }
 
-int
-cmp_streams_by_wim_position(const void *p1, const void *p2)
+/* qsort() callback that sorts streams (represented by `struct
+ * wim_lookup_table_entry's) into an order optimized for reading and writing.
+ *
+ * Sorting is done primarily by resource location, then secondarily by a
+ * per-resource location order.  For example, resources in WIM files are sorted
+ * primarily by part number, then secondarily by offset, as to implement optimal
+ * reading of either a standalone or split WIM.  */
+static int
+cmp_streams_by_sequential_order(const void *p1, const void *p2)
 {
        const struct wim_lookup_table_entry *lte1, *lte2;
+       int v;
+
        lte1 = *(const struct wim_lookup_table_entry**)p1;
        lte2 = *(const struct wim_lookup_table_entry**)p2;
-       if (lte1->resource_entry.offset < lte2->resource_entry.offset)
-               return -1;
-       else if (lte1->resource_entry.offset > lte2->resource_entry.offset)
-               return 1;
-       else
+
+       v = (int)lte1->resource_location - (int)lte2->resource_location;
+
+       /* Different resource locations?  */
+       if (v)
+               return v;
+
+       switch (lte1->resource_location) {
+       case RESOURCE_IN_WIM:
+
+               /* Different (possibly split) WIMs?  */
+               if (lte1->wim != lte2->wim) {
+                       v = memcmp(lte1->wim->hdr.guid, lte2->wim->hdr.guid,
+                                  WIM_GID_LEN);
+                       if (v)
+                               return v;
+               }
+
+               /* Different part numbers in the same WIM?  */
+               v = (int)lte1->wim->hdr.part_number - (int)lte2->wim->hdr.part_number;
+               if (v)
+                       return v;
+
+               /* Compare by offset.  */
+               if (lte1->resource_entry.offset < lte2->resource_entry.offset)
+                       return -1;
+               else if (lte1->resource_entry.offset > lte2->resource_entry.offset)
+                       return 1;
                return 0;
+       case RESOURCE_IN_FILE_ON_DISK:
+#ifdef __WIN32__
+       case RESOURCE_WIN32_ENCRYPTED:
+#endif
+               /* Compare files by path: just a heuristic that will place files
+                * in the same directory next to each other.  */
+               return tstrcmp(lte1->file_on_disk, lte2->file_on_disk);
+#ifdef WITH_NTFS_3G
+       case RESOURCE_IN_NTFS_VOLUME:
+               return tstrcmp(lte1->ntfs_loc->path, lte2->ntfs_loc->path);
+#endif
+       default:
+               /* No additional sorting order defined for this resource
+                * location (e.g. RESOURCE_IN_ATTACHED_BUFFER); simply compare
+                * everything equal to each other.  */
+               return 0;
+       }
+}
+
+int
+sort_stream_list_by_sequential_order(struct list_head *stream_list,
+                                    size_t list_head_offset)
+{
+       struct list_head *cur;
+       struct wim_lookup_table_entry **array;
+       size_t i;
+       size_t array_size;
+       size_t num_streams = 0;
+
+       list_for_each(cur, stream_list)
+               num_streams++;
+
+       array_size = num_streams * sizeof(array[0]);
+       array = MALLOC(array_size);
+       if (!array)
+               return WIMLIB_ERR_NOMEM;
+       cur = stream_list->next;
+       for (i = 0; i < num_streams; i++) {
+               array[i] = (struct wim_lookup_table_entry*)((u8*)cur -
+                                                           list_head_offset);
+               cur = cur->next;
+       }
+
+       qsort(array, num_streams, sizeof(array[0]),
+             cmp_streams_by_sequential_order);
+
+       INIT_LIST_HEAD(stream_list);
+       for (i = 0; i < num_streams; i++) {
+               list_add_tail((struct list_head*)
+                              ((u8*)array[i] + list_head_offset),
+                             stream_list);
+       }
+       FREE(array);
+       return 0;
 }
 
+
 static int
 add_lte_to_array(struct wim_lookup_table_entry *lte,
                 void *_pp)
@@ -333,7 +420,7 @@ for_lookup_table_entry_pos_sorted(struct wim_lookup_table *table,
        wimlib_assert(p == lte_array + num_streams);
 
        qsort(lte_array, num_streams, sizeof(lte_array[0]),
-             cmp_streams_by_wim_position);
+             cmp_streams_by_sequential_order);
        ret = 0;
        for (size_t i = 0; i < num_streams; i++) {
                ret = visitor(lte_array[i], arg);
index 3cac5a2..e4a0f3a 100644 (file)
 #  include <pthread.h>
 #endif
 
-#include <unistd.h>
-#include <fcntl.h>
 #include <errno.h>
+#include <fcntl.h>
+#include <limits.h>
+#include <stdlib.h>
+#include <unistd.h>
 
 #ifdef HAVE_ALLOCA_H
 #  include <alloca.h>
-#else
-#  include <stdlib.h>
 #endif
 
-#include <limits.h>
 
 #ifndef __WIN32__
 #  include <sys/uio.h> /* for `struct iovec' */
@@ -683,7 +682,6 @@ struct message {
        u8 *compressed_chunks[MAX_CHUNKS_PER_MSG];
        unsigned uncompressed_chunk_sizes[MAX_CHUNKS_PER_MSG];
        struct iovec out_chunks[MAX_CHUNKS_PER_MSG];
-       size_t total_out_bytes;
        unsigned num_chunks;
        struct list_head list;
        bool complete;
@@ -693,7 +691,6 @@ struct message {
 static void
 compress_chunks(struct message *msg, compress_func_t compress)
 {
-       msg->total_out_bytes = 0;
        for (unsigned i = 0; i < msg->num_chunks; i++) {
                unsigned len = compress(msg->uncompressed_chunks[i],
                                        msg->uncompressed_chunk_sizes[i],
@@ -711,7 +708,6 @@ compress_chunks(struct message *msg, compress_func_t compress)
                }
                msg->out_chunks[i].iov_base = out_chunk;
                msg->out_chunks[i].iov_len = out_len;
-               msg->total_out_bytes += out_len;
        }
 }
 
@@ -739,32 +735,52 @@ compressor_thread_proc(void *arg)
 }
 #endif /* ENABLE_MULTITHREADED_COMPRESSION */
 
+struct write_streams_progress_data {
+       wimlib_progress_func_t progress_func;
+       union wimlib_progress_info progress;
+       uint64_t next_progress;
+       WIMStruct *prev_wim_part;
+};
+
 static void
-do_write_streams_progress(union wimlib_progress_info *progress,
-                         wimlib_progress_func_t progress_func,
-                         uint64_t size_added,
+do_write_streams_progress(struct write_streams_progress_data *progress_data,
+                         struct wim_lookup_table_entry *lte,
                          bool stream_discarded)
 {
+       union wimlib_progress_info *progress = &progress_data->progress;
+       bool new_wim_part;
+
        if (stream_discarded) {
-               progress->write_streams.total_bytes -= size_added;
-               if (progress->write_streams._private != ~(uint64_t)0 &&
-                   progress->write_streams._private > progress->write_streams.total_bytes)
+               progress->write_streams.total_bytes -= wim_resource_size(lte);
+               if (progress_data->next_progress != ~(uint64_t)0 &&
+                   progress_data->next_progress > progress->write_streams.total_bytes)
                {
-                       progress->write_streams._private = progress->write_streams.total_bytes;
+                       progress_data->next_progress = progress->write_streams.total_bytes;
                }
        } else {
-               progress->write_streams.completed_bytes += size_added;
+               progress->write_streams.completed_bytes += wim_resource_size(lte);
+       }
+       new_wim_part = false;
+       if (lte->resource_location == RESOURCE_IN_WIM &&
+           lte->wim != progress_data->prev_wim_part)
+       {
+               if (progress_data->prev_wim_part) {
+                       new_wim_part = true;
+                       progress->write_streams.completed_parts++;
+               }
+               progress_data->prev_wim_part = lte->wim;
        }
        progress->write_streams.completed_streams++;
-       if (progress_func &&
-           progress->write_streams.completed_bytes >= progress->write_streams._private)
+       if (progress_data->progress_func
+           && (progress->write_streams.completed_bytes >= progress_data->next_progress
+               || new_wim_part))
        {
-               progress_func(WIMLIB_PROGRESS_MSG_WRITE_STREAMS,
-                             progress);
-               if (progress->write_streams._private == progress->write_streams.total_bytes) {
-                       progress->write_streams._private = ~(uint64_t)0;
+               progress_data->progress_func(WIMLIB_PROGRESS_MSG_WRITE_STREAMS,
+                                            progress);
+               if (progress_data->next_progress == progress->write_streams.total_bytes) {
+                       progress_data->next_progress = ~(uint64_t)0;
                } else {
-                       progress->write_streams._private =
+                       progress_data->next_progress =
                                min(progress->write_streams.total_bytes,
                                    progress->write_streams.completed_bytes +
                                        progress->write_streams.total_bytes / 100);
@@ -787,6 +803,7 @@ serial_write_stream(struct wim_lookup_table_entry *lte, void *_ctx)
                                  ctx->write_resource_flags);
 }
 
+
 /* Write a list of streams, taking into account that some streams may be
  * duplicates that are checksummed and discarded on the fly, and also delegating
  * the actual writing of a stream to a function @write_stream_cb, which is
@@ -796,8 +813,7 @@ do_write_stream_list(struct list_head *stream_list,
                     struct wim_lookup_table *lookup_table,
                     int (*write_stream_cb)(struct wim_lookup_table_entry *, void *),
                     void *write_stream_ctx,
-                    wimlib_progress_func_t progress_func,
-                    union wimlib_progress_info *progress)
+                    struct write_streams_progress_data *progress_data)
 {
        int ret = 0;
        struct wim_lookup_table_entry *lte;
@@ -860,10 +876,8 @@ do_write_stream_list(struct list_head *stream_list,
                }
        skip_to_progress:
                if (!lte->no_progress) {
-                       do_write_streams_progress(progress,
-                                                 progress_func,
-                                                 wim_resource_size(lte),
-                                                 stream_discarded);
+                       do_write_streams_progress(progress_data,
+                                                 lte, stream_discarded);
                }
        }
        return ret;
@@ -875,8 +889,7 @@ do_write_stream_list_serial(struct list_head *stream_list,
                            struct filedes *out_fd,
                            int out_ctype,
                            int write_resource_flags,
-                           wimlib_progress_func_t progress_func,
-                           union wimlib_progress_info *progress)
+                           struct write_streams_progress_data *progress_data)
 {
        struct serial_write_stream_ctx ctx = {
                .out_fd = out_fd,
@@ -887,8 +900,7 @@ do_write_stream_list_serial(struct list_head *stream_list,
                                    lookup_table,
                                    serial_write_stream,
                                    &ctx,
-                                   progress_func,
-                                   progress);
+                                   progress_data);
 }
 
 static inline int
@@ -909,21 +921,22 @@ write_stream_list_serial(struct list_head *stream_list,
                         struct filedes *out_fd,
                         int out_ctype,
                         int write_resource_flags,
-                        wimlib_progress_func_t progress_func,
-                        union wimlib_progress_info *progress)
+                        struct write_streams_progress_data *progress_data)
 {
+       union wimlib_progress_info *progress = &progress_data->progress;
        DEBUG("Writing stream list of size %"PRIu64" (serial version)",
              progress->write_streams.total_streams);
        progress->write_streams.num_threads = 1;
-       if (progress_func)
-               progress_func(WIMLIB_PROGRESS_MSG_WRITE_STREAMS, progress);
+       if (progress_data->progress_func) {
+               progress_data->progress_func(WIMLIB_PROGRESS_MSG_WRITE_STREAMS,
+                                            progress);
+       }
        return do_write_stream_list_serial(stream_list,
                                           lookup_table,
                                           out_fd,
                                           out_ctype,
                                           write_resource_flags,
-                                          progress_func,
-                                          progress);
+                                          progress_data);
 }
 
 #ifdef ENABLE_MULTITHREADED_COMPRESSION
@@ -935,7 +948,6 @@ write_wim_chunks(struct message *msg, struct filedes *out_fd,
        struct iovec *vecs;
        struct pwm_chunk_hdr *chunk_hdrs;
        unsigned nvecs;
-       size_t nbytes;
        int ret;
 
        for (unsigned i = 0; i < msg->num_chunks; i++)
@@ -944,7 +956,6 @@ write_wim_chunks(struct message *msg, struct filedes *out_fd,
        if (!(write_resource_flags & WIMLIB_WRITE_RESOURCE_FLAG_PIPABLE)) {
                nvecs = msg->num_chunks;
                vecs = msg->out_chunks;
-               nbytes = msg->total_out_bytes;
        } else {
                /* Special case:  If writing a compressed resource to a pipable
                 * WIM, prefix each compressed chunk with a header that gives
@@ -960,7 +971,6 @@ write_wim_chunks(struct message *msg, struct filedes *out_fd,
                        vecs[i * 2 + 1].iov_base = msg->out_chunks[i].iov_base;
                        vecs[i * 2 + 1].iov_len = msg->out_chunks[i].iov_len;
                }
-               nbytes = msg->total_out_bytes + msg->num_chunks * sizeof(chunk_hdrs[0]);
        }
        ret = full_writev(out_fd, vecs, nvecs);
        if (ret)
@@ -978,8 +988,7 @@ struct main_writer_thread_ctx {
        struct shared_queue *res_to_compress_queue;
        struct shared_queue *compressed_res_queue;
        size_t num_messages;
-       wimlib_progress_func_t progress_func;
-       union wimlib_progress_info *progress;
+       struct write_streams_progress_data *progress_data;
 
        struct list_head available_msgs;
        struct list_head outstanding_streams;
@@ -1221,10 +1230,8 @@ receive_compressed_chunks(struct main_writer_thread_ctx *ctx)
                                      cur_lte->output_resource_entry.flags);
                        }
 
-                       do_write_streams_progress(ctx->progress,
-                                                 ctx->progress_func,
-                                                 wim_resource_size(cur_lte),
-                                                 false);
+                       do_write_streams_progress(ctx->progress_data,
+                                                 cur_lte, false);
 
                        /* Since we just finished writing a stream, write any
                         * streams that have been added to the serial_streams
@@ -1238,8 +1245,7 @@ receive_compressed_chunks(struct main_writer_thread_ctx *ctx)
                                                                  ctx->out_fd,
                                                                  ctx->out_ctype,
                                                                  ctx->write_resource_flags,
-                                                                 ctx->progress_func,
-                                                                 ctx->progress);
+                                                                 ctx->progress_data);
                                if (ret)
                                        return ret;
                        }
@@ -1337,8 +1343,7 @@ main_writer_thread_finish(void *_ctx)
                                           ctx->out_fd,
                                           ctx->out_ctype,
                                           ctx->write_resource_flags,
-                                          ctx->progress_func,
-                                          ctx->progress);
+                                          ctx->progress_data);
 }
 
 static int
@@ -1433,14 +1438,14 @@ write_stream_list_parallel(struct list_head *stream_list,
                           struct filedes *out_fd,
                           int out_ctype,
                           int write_resource_flags,
-                          wimlib_progress_func_t progress_func,
-                          union wimlib_progress_info *progress,
+                          struct write_streams_progress_data *progress_data,
                           unsigned num_threads)
 {
        int ret;
        struct shared_queue res_to_compress_queue;
        struct shared_queue compressed_res_queue;
        pthread_t *compressor_threads = NULL;
+       union wimlib_progress_info *progress = &progress_data->progress;
 
        if (num_threads == 0) {
                long nthreads = get_default_num_threads();
@@ -1498,8 +1503,10 @@ write_stream_list_parallel(struct list_head *stream_list,
                }
        }
 
-       if (progress_func)
-               progress_func(WIMLIB_PROGRESS_MSG_WRITE_STREAMS, progress);
+       if (progress_data->progress_func) {
+               progress_data->progress_func(WIMLIB_PROGRESS_MSG_WRITE_STREAMS,
+                                            progress);
+       }
 
        struct main_writer_thread_ctx ctx;
        ctx.stream_list           = stream_list;
@@ -1510,14 +1517,13 @@ write_stream_list_parallel(struct list_head *stream_list,
        ctx.compressed_res_queue  = &compressed_res_queue;
        ctx.num_messages          = queue_size;
        ctx.write_resource_flags  = write_resource_flags;
-       ctx.progress_func         = progress_func;
-       ctx.progress              = progress;
+       ctx.progress_data         = progress_data;
        ret = main_writer_thread_init_ctx(&ctx);
        if (ret)
                goto out_join;
        ret = do_write_stream_list(stream_list, lookup_table,
                                   main_thread_process_next_stream,
-                                  &ctx, progress_func, progress);
+                                  &ctx, progress_data);
        if (ret)
                goto out_destroy_ctx;
 
@@ -1556,8 +1562,7 @@ out_serial_quiet:
                                        out_fd,
                                        out_ctype,
                                        write_resource_flags,
-                                       progress_func,
-                                       progress);
+                                       progress_data);
 
 }
 #endif
@@ -1576,9 +1581,11 @@ write_stream_list(struct list_head *stream_list,
        size_t num_streams = 0;
        u64 total_bytes = 0;
        u64 total_compression_bytes = 0;
-       union wimlib_progress_info progress;
+       struct write_streams_progress_data progress_data;
        int ret;
        int write_resource_flags;
+       unsigned total_parts = 0;
+       WIMStruct *prev_wim_part = NULL;
 
        if (list_empty(stream_list))
                return 0;
@@ -1587,6 +1594,10 @@ write_stream_list(struct list_head *stream_list,
 
        DEBUG("write_resource_flags=0x%08x", write_resource_flags);
 
+       sort_stream_list_by_sequential_order(stream_list,
+                                            offsetof(struct wim_lookup_table_entry,
+                                                     write_streams_list));
+
        /* Calculate the total size of the streams to be written.  Note: this
         * will be the uncompressed size, as we may not know the compressed size
         * yet, and also this will assume that every unhashed stream will be
@@ -1600,14 +1611,29 @@ write_stream_list(struct list_head *stream_list,
                {
                        total_compression_bytes += wim_resource_size(lte);
                }
+               if (lte->resource_location == RESOURCE_IN_WIM) {
+                       if (prev_wim_part != lte->wim) {
+                               prev_wim_part = lte->wim;
+                               total_parts++;
+                       }
+               }
+
        }
-       progress.write_streams.total_bytes       = total_bytes;
-       progress.write_streams.total_streams     = num_streams;
-       progress.write_streams.completed_bytes   = 0;
-       progress.write_streams.completed_streams = 0;
-       progress.write_streams.num_threads       = num_threads;
-       progress.write_streams.compression_type  = out_ctype;
-       progress.write_streams._private          = 0;
+
+       memset(&progress_data, 0, sizeof(progress_data));
+       progress_data.progress_func = progress_func;
+
+       progress_data.progress.write_streams.total_bytes       = total_bytes;
+       progress_data.progress.write_streams.total_streams     = num_streams;
+       progress_data.progress.write_streams.completed_bytes   = 0;
+       progress_data.progress.write_streams.completed_streams = 0;
+       progress_data.progress.write_streams.num_threads       = num_threads;
+       progress_data.progress.write_streams.compression_type  = out_ctype;
+       progress_data.progress.write_streams.total_parts       = total_parts;
+       progress_data.progress.write_streams.completed_parts   = 0;
+
+       progress_data.next_progress = 0;
+       progress_data.prev_wim_part = NULL;
 
 #ifdef ENABLE_MULTITHREADED_COMPRESSION
        if (total_compression_bytes >= 2000000 && num_threads != 1)
@@ -1616,8 +1642,7 @@ write_stream_list(struct list_head *stream_list,
                                                 out_fd,
                                                 out_ctype,
                                                 write_resource_flags,
-                                                progress_func,
-                                                &progress,
+                                                &progress_data,
                                                 num_threads);
        else
 #endif
@@ -1626,8 +1651,7 @@ write_stream_list(struct list_head *stream_list,
                                               out_fd,
                                               out_ctype,
                                               write_resource_flags,
-                                              progress_func,
-                                              &progress);
+                                              &progress_data);
        if (ret == 0)
                DEBUG("Successfully wrote stream list.");
        else