Use time-based ratelimiting for data progress messages

Convert the WRITE_STREAMS, EXTRACT_STREAMS, and VERIFY_STREAMS progress
messages over to use time-based ratelimiting.

This avoids cases where progress messages could be printed too quickly
(128 times in a fraction of a second) or too slowly (once per 5 MB
processed which could be arbitrarily slow).

diff --git a/NEWS b/NEWS
index b389840211d26149de254d42697aa1c1d4825abe..21e852a9b31d7cf980d0c2004e71ccc9b4a95cb1 100644 (file)
--- a/NEWS
+++ b/NEWS
@@ -2,6 +2,10 @@ Version 1.13.4-BETA2:
        wimsplit now prints progress messages regularly rather than just once
        per WIM part.
 
+       Progress messages for data processing are now printed at a consistent
+       rate based on time elapsed.  This avoids cases where the messages could
+       be printed too quickly or too slowly.
+
        Added support for a data recovery mode which causes files to be
        extracted even if they are corrupted.  The option is --recover-data for
        wimapply and wimextract, and WIMLIB_EXTRACT_FLAG_RECOVER_DATA for the

diff --git a/include/wimlib/apply.h b/include/wimlib/apply.h
index 2592fae6fd90da5bcd040de7b46313bcf7aa3d39..aaf4b4033a4c0a2f40e013e85d2386ed970d31d2 100644 (file)
--- a/include/wimlib/apply.h
+++ b/include/wimlib/apply.h
@@ -68,7 +68,7 @@ struct apply_ctx {
 
        /* The members below should not be used outside of extract.c  */
        const struct apply_operations *apply_ops;
-       u64 next_progress;
+       u64 last_progress_time;
        unsigned long invalid_sequence;
        unsigned long num_blobs_remaining;
        struct list_head blob_list;

diff --git a/include/wimlib/progress.h b/include/wimlib/progress.h
index 6d39b7e49804fb0089c8234985aa51e978108a00..d2ff37686064740e215213d9c7c72d088dd474fe 100644 (file)
--- a/include/wimlib/progress.h
+++ b/include/wimlib/progress.h
@@ -3,6 +3,7 @@
 
 #include "wimlib.h"
 #include "wimlib/paths.h"
+#include "wimlib/timestamp.h"
 #include "wimlib/types.h"
 
 /* If specified, call the user-provided progress function and check its result.
@@ -34,26 +35,20 @@ extern int
 report_error(wimlib_progress_func_t progfunc,
             void *progctx, int error_code, const tchar *path);
 
-/* Rate-limiting of byte-count based progress messages: update *next_progress_p
- * to the value that completed_bytes needs to reach before the next progress
- * message will be sent.  */
-static inline void
-set_next_progress(u64 completed_bytes, u64 total_bytes, u64 *next_progress_p)
+/* Rate-limiting of byte-count based progress messages.  We update the progress
+ * at most 5 times per second.  */
+static inline bool
+should_update_progress(u64 completed_bytes, u64 total_bytes,
+                      u64 *last_progress_time)
 {
-       if (*next_progress_p < total_bytes) {
-               /*
-                * Send the next message as soon as:
-                *      - another 1/128 of the total has been processed;
-                *      - OR another 5000000 bytes have been processed;
-                *      - OR all bytes have been processed.
-                */
-               *next_progress_p = min(min(completed_bytes + total_bytes / 128,
-                                          completed_bytes + 5000000),
-                                      total_bytes);
-       } else {
-               /* Last message has been sent.  */
-               *next_progress_p = ~0;
-       }
+       u64 now = now_as_wim_timestamp();
+
+       if (completed_bytes < total_bytes &&
+           now - *last_progress_time < 200 * TICKS_PER_MILLISECOND)
+               return false;
+
+       *last_progress_time = now;
+       return true;
 }
 
 /* Windows: temporarily remove the stream name from the path  */

diff --git a/include/wimlib/timestamp.h b/include/wimlib/timestamp.h
index 37666c9aeb3329747aa55ca5348212b7132ee2d7..4ff793e714f909ba0c66baaf690542df0f3df1a8 100644 (file)
--- a/include/wimlib/timestamp.h
+++ b/include/wimlib/timestamp.h
@@ -12,6 +12,28 @@
 
 #include "wimlib/types.h"
 
+/*
+ * Timestamps in WIM files are Windows NT timestamps, or FILETIMEs: 64-bit
+ * values storing the number of 100-nanosecond ticks since January 1, 1601.
+ *
+ * Note: UNIX timestamps are signed; Windows timestamps are not.  Negative UNIX
+ * timestamps represent times before 1970-01-01.  When such a timestamp is
+ * converted to a Windows timestamp, we can preserve the correct date provided
+ * that it is not also before 1601-01-01.
+ */
+
+#define NANOSECONDS_PER_TICK   100
+#define TICKS_PER_SECOND       (1000000000 / NANOSECONDS_PER_TICK)
+#define TICKS_PER_MILLISECOND  (TICKS_PER_SECOND / 1000)
+#define TICKS_PER_MICROSECOND  (TICKS_PER_SECOND / 1000000)
+
+/*
+ * EPOCH_DISTANCE is the number of seconds separating the Windows NT and UNIX
+ * epochs.  This is equal to ((1970-1601)*365+89)*24*60*60.  89 is the number
+ * of leap years between 1970 and 1601.
+ */
+#define EPOCH_DISTANCE         11644473600
+
 struct wimlib_timespec;
 
 extern time_t

diff --git a/src/extract.c b/src/extract.c
index 0e5d4ec4c900c1aab13d23bdd4f905c51afb6b8e..a1a3ea0661b7b2792412a29ec49c8e0a7a808196 100644 (file)
--- a/src/extract.c
+++ b/src/extract.c
@@ -462,15 +462,12 @@ extract_chunk(const struct blob_descriptor *blob, u64 offset,
                        }
                }
        }
-       if (progress->extract.completed_bytes >= ctx->next_progress) {
-
+       if (should_update_progress(progress->extract.completed_bytes,
+                                  progress->extract.total_bytes,
+                                  &ctx->last_progress_time)) {
                ret = extract_progress(ctx, WIMLIB_PROGRESS_MSG_EXTRACT_STREAMS);
                if (ret)
                        return ret;
-
-               set_next_progress(progress->extract.completed_bytes,
-                                 progress->extract.total_bytes,
-                                 &ctx->next_progress);
        }
 
        if (unlikely(filedes_valid(&ctx->tmpfile_fd))) {

diff --git a/src/timestamp.c b/src/timestamp.c
index 51046cdb68084048df29ab185ecd6f03cbeaba33..f79f48670b9bc7fac6604df2f98755f6a437e008 100644 (file)
--- a/src/timestamp.c
+++ b/src/timestamp.c
@@ -28,27 +28,6 @@
 #include "wimlib.h" /* for struct wimlib_timespec */
 #include "wimlib/timestamp.h"
 
-/*
- * Timestamps in WIM files are Windows NT timestamps, or FILETIMEs: 64-bit
- * values storing the number of 100-nanosecond ticks since January 1, 1601.
- *
- * Note: UNIX timestamps are signed; Windows timestamps are not.  Negative UNIX
- * timestamps represent times before 1970-01-01.  When such a timestamp is
- * converted to a Windows timestamp, we can preserve the correct date provided
- * that it is not also before 1601-01-01.
- */
-
-#define NANOSECONDS_PER_TICK   100
-#define TICKS_PER_SECOND       (1000000000 / NANOSECONDS_PER_TICK)
-#define TICKS_PER_MICROSECOND  (TICKS_PER_SECOND / 1000000)
-
-/*
- * EPOCH_DISTANCE is the number of seconds separating the Windows NT and UNIX
- * epochs.  This is equal to ((1970-1601)*365+89)*24*60*60.  89 is the number
- * of leap years between 1970 and 1601.
- */
-#define EPOCH_DISTANCE         11644473600
-
 /* Windows NT timestamps to UNIX timestamps  */
 
 time_t

diff --git a/src/verify.c b/src/verify.c
index 41fcff789919f1b60a0e888ad3598a8a21fb4e37..d50f17b0cfaf5811be67349ab7b7837d638394ee 100644 (file)
--- a/src/verify.c
+++ b/src/verify.c
@@ -43,7 +43,7 @@ struct verify_blob_list_ctx {
        wimlib_progress_func_t progfunc;
        void *progctx;
        union wimlib_progress_info *progress;
-       u64 next_progress;
+       u64 last_progress_time;
 };
 
 static int
@@ -58,17 +58,14 @@ verify_continue_blob(const struct blob_descriptor *blob, u64 offset,
 
        progress->verify_streams.completed_bytes += size;
 
-       if (progress->verify_streams.completed_bytes >= ctx->next_progress) {
-
+       if (should_update_progress(progress->verify_streams.completed_bytes,
+                                  progress->verify_streams.total_bytes,
+                                  &ctx->last_progress_time)) {
                int ret = call_progress(ctx->progfunc,
                                        WIMLIB_PROGRESS_MSG_VERIFY_STREAMS,
                                        progress, ctx->progctx);
                if (ret)
                        return ret;
-
-               set_next_progress(progress->verify_streams.completed_bytes,
-                                 progress->verify_streams.total_bytes,
-                                 &ctx->next_progress);
        }
        return 0;
 }
@@ -161,7 +158,7 @@ wimlib_verify_wim(WIMStruct *wim, int verify_flags)
        ctx.progfunc = wim->progfunc;
        ctx.progctx = wim->progctx;
        ctx.progress = &progress;
-       ctx.next_progress = 0;
+       ctx.last_progress_time = 0;
 
        ret = call_progress(ctx.progfunc, WIMLIB_PROGRESS_MSG_VERIFY_STREAMS,
                            ctx.progress, ctx.progctx);

diff --git a/src/write.c b/src/write.c
index e28069ebba707acd49ad53e5aae3c9ade3a01b8a..86a726d64ec521433595d92308ae97a8370916a8 100644 (file)
--- a/src/write.c
+++ b/src/write.c
@@ -296,7 +296,7 @@ struct write_blobs_progress_data {
        wimlib_progress_func_t progfunc;
        void *progctx;
        union wimlib_progress_info progress;
-       u64 next_progress;
+       u64 last_progress_time;
 };
 
 static int
@@ -310,11 +310,6 @@ do_write_blobs_progress(struct write_blobs_progress_data *progress_data,
        if (discarded) {
                progress->write_streams.total_bytes -= complete_size;
                progress->write_streams.total_streams -= complete_count;
-               if (progress_data->next_progress != ~(u64)0 &&
-                   progress_data->next_progress > progress->write_streams.total_bytes)
-               {
-                       progress_data->next_progress = progress->write_streams.total_bytes;
-               }
        } else {
                progress->write_streams.completed_bytes += complete_size;
                progress->write_streams.completed_compressed_bytes +=
@@ -322,18 +317,15 @@ do_write_blobs_progress(struct write_blobs_progress_data *progress_data,
                progress->write_streams.completed_streams += complete_count;
        }
 
-       if (progress->write_streams.completed_bytes >= progress_data->next_progress) {
-
+       if (should_update_progress(progress->write_streams.completed_bytes,
+                                  progress->write_streams.total_bytes,
+                                  &progress_data->last_progress_time)) {
                ret = call_progress(progress_data->progfunc,
                                    WIMLIB_PROGRESS_MSG_WRITE_STREAMS,
                                    progress,
                                    progress_data->progctx);
                if (ret)
                        return ret;
-
-               set_next_progress(progress->write_streams.completed_bytes,
-                                 progress->write_streams.total_bytes,
-                                 &progress_data->next_progress);
        }
        return 0;
 }
@@ -1158,7 +1150,7 @@ compute_blob_list_stats(struct list_head *blob_list,
        ctx->progress_data.progress.write_streams.compression_type  = ctx->out_ctype;
        ctx->progress_data.progress.write_streams.total_parts       = total_parts;
        ctx->progress_data.progress.write_streams.completed_parts   = 0;
-       ctx->progress_data.next_progress = 0;
+       ctx->progress_data.last_progress_time = 0;
        return 0;
 }