From 63fb6b74b714d58286823e54a4555b7d85d49163 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Thu, 1 Apr 2021 21:07:53 -0700 Subject: [PATCH] Fix slow progress updating for wimsplit wimsplit only prints a progress message when starting each WIM part. That could be very infrequently since each part could be gigabytes. Fix it to update the progress regularly as data is written, like the other wimlib-imagex commands do. This required changing the library to report WIMLIB_PROGRESS_MSG_WRITE_STREAMS messages from wimlib_split() and include the completed compressed size in them. Reported at https://www.reddit.com/r/pcmasterrace/comments/hagu4k/wimlibimagex_split_stuck_at_0 --- include/wimlib.h | 14 ++++++++++--- programs/imagex.c | 53 ++++++++++++++++++++++++++++++----------------- src/split.c | 4 ---- src/write.c | 21 ++++++++++++------- 4 files changed, 58 insertions(+), 34 deletions(-) diff --git a/include/wimlib.h b/include/wimlib.h index 71b2c4a6..47c357d0 100644 --- a/include/wimlib.h +++ b/include/wimlib.h @@ -649,7 +649,8 @@ enum wimlib_progress_msg { * to ::wimlib_progress_info.write_streams. This message may be * received many times while the WIM file is being written or appended * to with wimlib_write(), wimlib_overwrite(), or wimlib_write_to_fd(). - */ + * Since wimlib v1.13.4 it will also be received when a split WIM part + * is being written by wimlib_split(). */ WIMLIB_PROGRESS_MSG_WRITE_STREAMS = 12, /** Per-image metadata is about to be written to the WIM file. @p info @@ -825,7 +826,8 @@ union wimlib_progress_info { /** The number of bytes of file data that have been written so * far. This starts at 0 and ends at @p total_bytes. This * number is the uncompressed size; the actual size may be lower - * due to compression. */ + * due to compression. See @p completed_compressed_bytes for + * the compressed size. */ uint64_t completed_bytes; /** The number of distinct file data "blobs" that have been @@ -848,6 +850,10 @@ union wimlib_progress_info { /** This is currently broken and will always be 0. */ uint32_t completed_parts; + + /** Since wimlib v1.13.4: Like @p completed_bytes, but counts + * the compressed size. */ + uint64_t completed_compressed_bytes; } write_streams; /** Valid on messages ::WIMLIB_PROGRESS_MSG_SCAN_BEGIN, @@ -4305,7 +4311,9 @@ wimlib_set_wim_info(WIMStruct *wim, const struct wimlib_wim_info *info, * If a progress function is registered with @p wim, then for each split WIM * part that is written it will receive the messages * ::WIMLIB_PROGRESS_MSG_SPLIT_BEGIN_PART and - * ::WIMLIB_PROGRESS_MSG_SPLIT_END_PART. + * ::WIMLIB_PROGRESS_MSG_SPLIT_END_PART. Since wimlib v1.13.4 it will also + * receive ::WIMLIB_PROGRESS_MSG_WRITE_STREAMS messages while writing each part; + * these messages will report the progress of the current part only. */ extern int wimlib_split(WIMStruct *wim, diff --git a/programs/imagex.c b/programs/imagex.c index 2d407fac..dd43d75c 100644 --- a/programs/imagex.c +++ b/programs/imagex.c @@ -1163,6 +1163,31 @@ report_scan_progress(const struct wimlib_progress_info_scan *scan, bool done) last_scan_progress = *scan; } } + +static struct wimlib_progress_info_split last_split_progress; + +static void +report_split_progress(uint64_t bytes_completed_in_part) +{ + uint64_t completed_bytes = last_split_progress.completed_bytes + + bytes_completed_in_part; + unsigned percent_done = TO_PERCENT(completed_bytes, + last_split_progress.total_bytes); + unsigned unit_shift; + const tchar *unit_name; + + unit_shift = get_unit(last_split_progress.total_bytes, &unit_name); + imagex_printf(T("\rSplitting WIM: %"PRIu64" %"TS" of " + "%"PRIu64" %"TS" (%u%%) written, part %u of %u"), + completed_bytes >> unit_shift, + unit_name, + last_split_progress.total_bytes >> unit_shift, + unit_name, + percent_done, + last_split_progress.cur_part_number, + last_split_progress.total_parts); +} + /* Progress callback function passed to various wimlib functions. */ static enum wimlib_progress_status imagex_progress_func(enum wimlib_progress_msg msg, @@ -1175,6 +1200,12 @@ imagex_progress_func(enum wimlib_progress_msg msg, switch (msg) { case WIMLIB_PROGRESS_MSG_WRITE_STREAMS: + if (last_split_progress.total_bytes != 0) { + /* wimlib_split() in progress; use the split-specific + * progress message. */ + report_split_progress(info->write_streams.completed_compressed_bytes); + break; + } { static bool started; if (!started) { @@ -1330,26 +1361,9 @@ imagex_progress_func(enum wimlib_progress_msg msg, } break; case WIMLIB_PROGRESS_MSG_SPLIT_BEGIN_PART: - percent_done = TO_PERCENT(info->split.completed_bytes, - info->split.total_bytes); - unit_shift = get_unit(info->split.total_bytes, &unit_name); - imagex_printf(T("Writing \"%"TS"\" (part %u of %u): %"PRIu64" %"TS" of " - "%"PRIu64" %"TS" (%u%%) written\n"), - info->split.part_name, - info->split.cur_part_number, - info->split.total_parts, - info->split.completed_bytes >> unit_shift, - unit_name, - info->split.total_bytes >> unit_shift, - unit_name, - percent_done); - break; case WIMLIB_PROGRESS_MSG_SPLIT_END_PART: - if (info->split.completed_bytes == info->split.total_bytes) { - imagex_printf(T("Finished writing split WIM part %u of %u\n"), - info->split.cur_part_number, - info->split.total_parts); - } + last_split_progress = info->split; + report_split_progress(0); break; case WIMLIB_PROGRESS_MSG_UPDATE_END_COMMAND: switch (info->update.command->op) { @@ -4019,6 +4033,7 @@ imagex_split(int argc, tchar **argv, int cmd) goto out; ret = wimlib_split(wim, argv[1], part_size, write_flags); + tprintf(T("\nFinished splitting \"%"TS"\"\n"), argv[0]); wimlib_free(wim); out: return ret; diff --git a/src/split.c b/src/split.c index 1313a59a..13758558 100644 --- a/src/split.c +++ b/src/split.c @@ -96,7 +96,6 @@ write_split_wim(WIMStruct *orig_wim, const tchar *swm_name, for (part_number = 1; part_number <= swm_info->num_parts; part_number++) { int part_write_flags; - wimlib_progress_func_t progfunc; if (part_number != 1) { tsprintf(swm_name_buf + swm_base_name_len, @@ -118,8 +117,6 @@ write_split_wim(WIMStruct *orig_wim, const tchar *swm_name, if (part_number != 1) part_write_flags |= WIMLIB_WRITE_FLAG_NO_METADATA; - progfunc = orig_wim->progfunc; - orig_wim->progfunc = NULL; ret = write_wim_part(orig_wim, progress.split.part_name, WIMLIB_ALL_IMAGES, @@ -129,7 +126,6 @@ write_split_wim(WIMStruct *orig_wim, const tchar *swm_name, swm_info->num_parts, &swm_info->parts[part_number - 1].blob_list, guid); - orig_wim->progfunc = progfunc; if (ret) return ret; diff --git a/src/write.c b/src/write.c index 4805913d..bca5e862 100644 --- a/src/write.c +++ b/src/write.c @@ -301,7 +301,8 @@ struct write_blobs_progress_data { static int do_write_blobs_progress(struct write_blobs_progress_data *progress_data, - u64 complete_size, u32 complete_count, bool discarded) + u64 complete_size, u64 complete_compressed_size, + u32 complete_count, bool discarded) { union wimlib_progress_info *progress = &progress_data->progress; int ret; @@ -316,6 +317,8 @@ do_write_blobs_progress(struct write_blobs_progress_data *progress_data, } } else { progress->write_streams.completed_bytes += complete_size; + progress->write_streams.completed_compressed_bytes += + complete_compressed_size; progress->write_streams.completed_streams += complete_count; } @@ -713,7 +716,9 @@ write_blob_begin_read(struct blob_descriptor *blob, void *_ctx) * output reference count to the duplicate blob * in the former case. */ ret = do_write_blobs_progress(&ctx->progress_data, - blob->size, 1, true); + blob->size, + blob->size, + 1, true); list_del(&blob->write_blobs_list); list_del(&blob->blob_table_list); if (new_blob->will_be_in_output_wim) @@ -867,8 +872,7 @@ write_chunk(struct write_blobs_ctx *ctx, const void *cchunk, { int ret; struct blob_descriptor *blob; - u32 completed_blob_count; - u32 completed_size; + u32 completed_blob_count = 0; blob = list_entry(ctx->blobs_being_compressed.next, struct blob_descriptor, write_blobs_list); @@ -915,8 +919,6 @@ write_chunk(struct write_blobs_ctx *ctx, const void *cchunk, ctx->cur_write_blob_offset += usize; - completed_size = usize; - completed_blob_count = 0; if (ctx->write_resource_flags & WRITE_RESOURCE_FLAG_SOLID) { /* Wrote chunk in solid mode. It may have finished multiple * blobs. */ @@ -973,7 +975,7 @@ write_chunk(struct write_blobs_ctx *ctx, const void *cchunk, } } - return do_write_blobs_progress(&ctx->progress_data, completed_size, + return do_write_blobs_progress(&ctx->progress_data, usize, csize, completed_blob_count, false); write_error: @@ -1287,15 +1289,18 @@ write_raw_copy_resources(struct list_head *raw_copy_blobs, blob->rdesc->raw_copy_ok = 1; list_for_each_entry(blob, raw_copy_blobs, write_blobs_list) { + u64 compressed_size = 0; + if (blob->rdesc->raw_copy_ok) { /* Write each solid resource only one time. */ ret = write_raw_copy_resource(blob->rdesc, out_fd); if (ret) return ret; blob->rdesc->raw_copy_ok = 0; + compressed_size = blob->rdesc->size_in_wim; } ret = do_write_blobs_progress(progress_data, blob->size, - 1, false); + compressed_size, 1, false); if (ret) return ret; } -- 2.43.0