Fix slow progress updating for wimsplit

wimsplit only prints a progress message when starting each WIM part.
That could be very infrequently since each part could be gigabytes.

Fix it to update the progress regularly as data is written, like the
other wimlib-imagex commands do.

This required changing the library to report
WIMLIB_PROGRESS_MSG_WRITE_STREAMS messages from wimlib_split() and
include the completed compressed size in them.

Reported at https://www.reddit.com/r/pcmasterrace/comments/hagu4k/wimlibimagex_split_stuck_at_0

diff --git a/include/wimlib.h b/include/wimlib.h
index 71b2c4a65b24e797fff3bb82f1917a021f219f97..47c357d064672834d12f2d2508c69efe628ea4f2 100644 (file)
--- a/include/wimlib.h
+++ b/include/wimlib.h
@@ -649,7 +649,8 @@ enum wimlib_progress_msg {
         * to ::wimlib_progress_info.write_streams.  This message may be
         * received many times while the WIM file is being written or appended
         * to with wimlib_write(), wimlib_overwrite(), or wimlib_write_to_fd().
-        */
+        * Since wimlib v1.13.4 it will also be received when a split WIM part
+        * is being written by wimlib_split().  */
        WIMLIB_PROGRESS_MSG_WRITE_STREAMS = 12,
 
        /** Per-image metadata is about to be written to the WIM file.  @p info
@@ -825,7 +826,8 @@ union wimlib_progress_info {
                /** The number of bytes of file data that have been written so
                 * far.  This starts at 0 and ends at @p total_bytes.  This
                 * number is the uncompressed size; the actual size may be lower
-                * due to compression.  */
+                * due to compression.  See @p completed_compressed_bytes for
+                * the compressed size.  */
                uint64_t completed_bytes;
 
                /** The number of distinct file data "blobs" that have been
@@ -848,6 +850,10 @@ union wimlib_progress_info {
 
                /** This is currently broken and will always be 0.  */
                uint32_t completed_parts;
+
+               /** Since wimlib v1.13.4: Like @p completed_bytes, but counts
+                * the compressed size.  */
+               uint64_t completed_compressed_bytes;
        } write_streams;
 
        /** Valid on messages ::WIMLIB_PROGRESS_MSG_SCAN_BEGIN,
@@ -4305,7 +4311,9 @@ wimlib_set_wim_info(WIMStruct *wim, const struct wimlib_wim_info *info,
  * If a progress function is registered with @p wim, then for each split WIM
  * part that is written it will receive the messages
  * ::WIMLIB_PROGRESS_MSG_SPLIT_BEGIN_PART and
- * ::WIMLIB_PROGRESS_MSG_SPLIT_END_PART.
+ * ::WIMLIB_PROGRESS_MSG_SPLIT_END_PART.  Since wimlib v1.13.4 it will also
+ * receive ::WIMLIB_PROGRESS_MSG_WRITE_STREAMS messages while writing each part;
+ * these messages will report the progress of the current part only.
  */
 extern int
 wimlib_split(WIMStruct *wim,

diff --git a/programs/imagex.c b/programs/imagex.c
index 2d407fac422b885dd6a4882b51abe25f93aec54c..dd43d75c059e494d79f95c5d15371c4f4dd19414 100644 (file)
--- a/programs/imagex.c
+++ b/programs/imagex.c
@@ -1163,6 +1163,31 @@ report_scan_progress(const struct wimlib_progress_info_scan *scan, bool done)
                last_scan_progress = *scan;
        }
 }
+
+static struct wimlib_progress_info_split last_split_progress;
+
+static void
+report_split_progress(uint64_t bytes_completed_in_part)
+{
+       uint64_t completed_bytes = last_split_progress.completed_bytes +
+                                  bytes_completed_in_part;
+       unsigned percent_done = TO_PERCENT(completed_bytes,
+                                          last_split_progress.total_bytes);
+       unsigned unit_shift;
+       const tchar *unit_name;
+
+       unit_shift = get_unit(last_split_progress.total_bytes, &unit_name);
+       imagex_printf(T("\rSplitting WIM: %"PRIu64" %"TS" of "
+                       "%"PRIu64" %"TS" (%u%%) written, part %u of %u"),
+                     completed_bytes >> unit_shift,
+                     unit_name,
+                     last_split_progress.total_bytes >> unit_shift,
+                     unit_name,
+                     percent_done,
+                     last_split_progress.cur_part_number,
+                     last_split_progress.total_parts);
+}
+
 /* Progress callback function passed to various wimlib functions. */
 static enum wimlib_progress_status
 imagex_progress_func(enum wimlib_progress_msg msg,
@@ -1175,6 +1200,12 @@ imagex_progress_func(enum wimlib_progress_msg msg,
 
        switch (msg) {
        case WIMLIB_PROGRESS_MSG_WRITE_STREAMS:
+               if (last_split_progress.total_bytes != 0) {
+                       /* wimlib_split() in progress; use the split-specific
+                        * progress message.  */
+                       report_split_progress(info->write_streams.completed_compressed_bytes);
+                       break;
+               }
                {
                        static bool started;
                        if (!started) {
@@ -1330,26 +1361,9 @@ imagex_progress_func(enum wimlib_progress_msg msg,
                }
                break;
        case WIMLIB_PROGRESS_MSG_SPLIT_BEGIN_PART:
-               percent_done = TO_PERCENT(info->split.completed_bytes,
-                                         info->split.total_bytes);
-               unit_shift = get_unit(info->split.total_bytes, &unit_name);
-               imagex_printf(T("Writing \"%"TS"\" (part %u of %u): %"PRIu64" %"TS" of "
-                         "%"PRIu64" %"TS" (%u%%) written\n"),
-                       info->split.part_name,
-                       info->split.cur_part_number,
-                       info->split.total_parts,
-                       info->split.completed_bytes >> unit_shift,
-                       unit_name,
-                       info->split.total_bytes >> unit_shift,
-                       unit_name,
-                       percent_done);
-               break;
        case WIMLIB_PROGRESS_MSG_SPLIT_END_PART:
-               if (info->split.completed_bytes == info->split.total_bytes) {
-                       imagex_printf(T("Finished writing split WIM part %u of %u\n"),
-                               info->split.cur_part_number,
-                               info->split.total_parts);
-               }
+               last_split_progress = info->split;
+               report_split_progress(0);
                break;
        case WIMLIB_PROGRESS_MSG_UPDATE_END_COMMAND:
                switch (info->update.command->op) {
@@ -4019,6 +4033,7 @@ imagex_split(int argc, tchar **argv, int cmd)
                goto out;
 
        ret = wimlib_split(wim, argv[1], part_size, write_flags);
+       tprintf(T("\nFinished splitting \"%"TS"\"\n"), argv[0]);
        wimlib_free(wim);
 out:
        return ret;

diff --git a/src/split.c b/src/split.c
index 1313a59aa10ef03c36101ed99508db9206fc2034..137585588a98ab5638063233d14eef8cc94c421a 100644 (file)
--- a/src/split.c
+++ b/src/split.c
@@ -96,7 +96,6 @@ write_split_wim(WIMStruct *orig_wim, const tchar *swm_name,
 
        for (part_number = 1; part_number <= swm_info->num_parts; part_number++) {
                int part_write_flags;
-               wimlib_progress_func_t progfunc;
 
                if (part_number != 1) {
                        tsprintf(swm_name_buf + swm_base_name_len,
@@ -118,8 +117,6 @@ write_split_wim(WIMStruct *orig_wim, const tchar *swm_name,
                if (part_number != 1)
                        part_write_flags |= WIMLIB_WRITE_FLAG_NO_METADATA;
 
-               progfunc = orig_wim->progfunc;
-               orig_wim->progfunc = NULL;
                ret = write_wim_part(orig_wim,
                                     progress.split.part_name,
                                     WIMLIB_ALL_IMAGES,
@@ -129,7 +126,6 @@ write_split_wim(WIMStruct *orig_wim, const tchar *swm_name,
                                     swm_info->num_parts,
                                     &swm_info->parts[part_number - 1].blob_list,
                                     guid);
-               orig_wim->progfunc = progfunc;
                if (ret)
                        return ret;
 

diff --git a/src/write.c b/src/write.c
index 4805913de376fab95c1982eae45f0253a6dd13ec..bca5e862a2f1e2549bb471c9a66855b4714f0216 100644 (file)
--- a/src/write.c
+++ b/src/write.c
@@ -301,7 +301,8 @@ struct write_blobs_progress_data {
 
 static int
 do_write_blobs_progress(struct write_blobs_progress_data *progress_data,
-                       u64 complete_size, u32 complete_count, bool discarded)
+                       u64 complete_size, u64 complete_compressed_size,
+                       u32 complete_count, bool discarded)
 {
        union wimlib_progress_info *progress = &progress_data->progress;
        int ret;
@@ -316,6 +317,8 @@ do_write_blobs_progress(struct write_blobs_progress_data *progress_data,
                }
        } else {
                progress->write_streams.completed_bytes += complete_size;
+               progress->write_streams.completed_compressed_bytes +=
+                       complete_compressed_size;
                progress->write_streams.completed_streams += complete_count;
        }
 
@@ -713,7 +716,9 @@ write_blob_begin_read(struct blob_descriptor *blob, void *_ctx)
                                 * output reference count to the duplicate blob
                                 * in the former case.  */
                                ret = do_write_blobs_progress(&ctx->progress_data,
-                                                             blob->size, 1, true);
+                                                             blob->size,
+                                                             blob->size,
+                                                             1, true);
                                list_del(&blob->write_blobs_list);
                                list_del(&blob->blob_table_list);
                                if (new_blob->will_be_in_output_wim)
@@ -867,8 +872,7 @@ write_chunk(struct write_blobs_ctx *ctx, const void *cchunk,
 {
        int ret;
        struct blob_descriptor *blob;
-       u32 completed_blob_count;
-       u32 completed_size;
+       u32 completed_blob_count = 0;
 
        blob = list_entry(ctx->blobs_being_compressed.next,
                          struct blob_descriptor, write_blobs_list);
@@ -915,8 +919,6 @@ write_chunk(struct write_blobs_ctx *ctx, const void *cchunk,
 
        ctx->cur_write_blob_offset += usize;
 
-       completed_size = usize;
-       completed_blob_count = 0;
        if (ctx->write_resource_flags & WRITE_RESOURCE_FLAG_SOLID) {
                /* Wrote chunk in solid mode.  It may have finished multiple
                 * blobs.  */
@@ -973,7 +975,7 @@ write_chunk(struct write_blobs_ctx *ctx, const void *cchunk,
                }
        }
 
-       return do_write_blobs_progress(&ctx->progress_data, completed_size,
+       return do_write_blobs_progress(&ctx->progress_data, usize, csize,
                                       completed_blob_count, false);
 
 write_error:
@@ -1287,15 +1289,18 @@ write_raw_copy_resources(struct list_head *raw_copy_blobs,
                blob->rdesc->raw_copy_ok = 1;
 
        list_for_each_entry(blob, raw_copy_blobs, write_blobs_list) {
+               u64 compressed_size = 0;
+
                if (blob->rdesc->raw_copy_ok) {
                        /* Write each solid resource only one time.  */
                        ret = write_raw_copy_resource(blob->rdesc, out_fd);
                        if (ret)
                                return ret;
                        blob->rdesc->raw_copy_ok = 0;
+                       compressed_size = blob->rdesc->size_in_wim;
                }
                ret = do_write_blobs_progress(progress_data, blob->size,
-                                             1, false);
+                                             compressed_size, 1, false);
                if (ret)
                        return ret;
        }