Support file counts for extract file structure and metadata progress
authorEric Biggers <ebiggers3@gmail.com>
Thu, 28 Aug 2014 02:41:39 +0000 (21:41 -0500)
committerEric Biggers <ebiggers3@gmail.com>
Thu, 28 Aug 2014 02:49:51 +0000 (21:49 -0500)
include/wimlib.h
include/wimlib/apply.h
programs/imagex.c
src/extract.c
src/ntfs-3g_apply.c
src/unix_apply.c
src/win32_apply.c

index 6aa36f5..d28fd34 100644 (file)
@@ -552,7 +552,9 @@ enum wimlib_progress_msg {
        /** This message may be sent periodically (not for every file) while
         * files or directories are being created, prior to data stream
         * extraction.  @p info will point to ::wimlib_progress_info.extract.
-        */
+        * In particular, the @p current_file_count and @p end_file_count
+        * members may be used to track the progress of this phase of
+        * extraction.  */
        WIMLIB_PROGRESS_MSG_EXTRACT_FILE_STRUCTURE = 3,
 
        /** File data is currently being extracted.  @p info will point to
@@ -567,7 +569,9 @@ enum wimlib_progress_msg {
        /** This message may be sent periodically (not for every file) while
         * file and directory metadata is being applied, following data stream
         * extraction.  @p info will point to ::wimlib_progress_info.extract.
-        */
+        * In particular, the @p current_file_count and @p end_file_count
+        * members may be used to track the progress of this phase of
+        * extraction.  */
        WIMLIB_PROGRESS_MSG_EXTRACT_METADATA = 6,
 
        /** Confirms that the image has been successfully extracted.  @p info
@@ -984,6 +988,30 @@ union wimlib_progress_info {
                /** Currently only used for
                 * ::WIMLIB_PROGRESS_MSG_EXTRACT_SPWM_PART_BEGIN.  */
                uint8_t guid[WIMLIB_GUID_LEN];
+
+               /** For ::WIMLIB_PROGRESS_MSG_EXTRACT_FILE_STRUCTURE and
+                * ::WIMLIB_PROGRESS_MSG_EXTRACT_METADATA messages, this is the
+                * number of files that have been processed so far.  Once the
+                * corresponding phase of extraction is complete, this value
+                * will be equal to @c end_file_count.  */
+               uint64_t current_file_count;
+
+               /** For ::WIMLIB_PROGRESS_MSG_EXTRACT_FILE_STRUCTURE and
+                * ::WIMLIB_PROGRESS_MSG_EXTRACT_METADATA messages, this is
+                * total number of files that will be processed.
+                *
+                * This number is provided for informational purposes only.
+                * This number will not necessarily be equal to the number of
+                * files actually being extracted.  This is because extraction
+                * backends are free to implement an extraction algorithm that
+                * might be more efficient than processing every file in the
+                * "extract file structure" and "extract metadata" phases.  For
+                * example, the current implementation of the UNIX extraction
+                * backend will create files on-demand during the stream
+                * extraction phase. Therefore, when using that particular
+                * extraction backend, @p end_file_count will only include
+                * directories and empty files.  */
+               uint64_t end_file_count;
        } extract;
 
        /** Valid on messages ::WIMLIB_PROGRESS_MSG_RENAME. */
index 8a1bf87..73ddd87 100644 (file)
@@ -88,21 +88,22 @@ extract_progress(struct apply_ctx *ctx, enum wimlib_progress_msg msg)
 extern int
 do_file_extract_progress(struct apply_ctx *ctx, enum wimlib_progress_msg msg);
 
+#define COUNT_PER_FILE_PROGRESS 256
+
 static inline int
 maybe_do_file_progress(struct apply_ctx *ctx, enum wimlib_progress_msg msg)
 {
+       ctx->progress.extract.current_file_count++;
        if (unlikely(!--ctx->count_until_file_progress))
                return do_file_extract_progress(ctx, msg);
        return 0;
 }
 
-/* Call this to reset the counter for report_file_created() and
- * report_file_metadata_applied().  */
-static inline void
-reset_file_progress(struct apply_ctx *ctx)
-{
-       ctx->count_until_file_progress = 1;
-}
+extern int
+start_file_structure_phase(struct apply_ctx *ctx, uint64_t end_file_count);
+
+extern int
+start_file_metadata_phase(struct apply_ctx *ctx, uint64_t end_file_count);
 
 /* Report that a file was created, prior to stream extraction.  */
 static inline int
@@ -118,6 +119,12 @@ report_file_metadata_applied(struct apply_ctx *ctx)
        return maybe_do_file_progress(ctx, WIMLIB_PROGRESS_MSG_EXTRACT_METADATA);
 }
 
+extern int
+end_file_structure_phase(struct apply_ctx *ctx);
+
+extern int
+end_file_metadata_phase(struct apply_ctx *ctx);
+
 /* Returns any of the aliases of an inode that are being extracted.  */
 #define inode_first_extraction_dentry(inode)           \
        list_first_entry(&(inode)->i_extraction_aliases,        \
index 60b533c..82354fe 100644 (file)
@@ -1160,6 +1160,17 @@ imagex_progress_func(enum wimlib_progress_msg msg,
                         T("NTFS volume") : T("directory")),
                        info->extract.target);
                break;
+       case WIMLIB_PROGRESS_MSG_EXTRACT_FILE_STRUCTURE:
+               if (info->extract.end_file_count >= 2000) {
+                       percent_done = TO_PERCENT(info->extract.current_file_count,
+                                                 info->extract.end_file_count);
+                       imagex_printf(T("\rCreating files: %"PRIu64" of %"PRIu64" (%u%%) done"),
+                                     info->extract.current_file_count,
+                                     info->extract.end_file_count, percent_done);
+                       if (info->extract.current_file_count == info->extract.end_file_count)
+                               imagex_printf(T("\n"));
+               }
+               break;
        case WIMLIB_PROGRESS_MSG_EXTRACT_STREAMS:
                percent_done = TO_PERCENT(info->extract.completed_bytes,
                                          info->extract.total_bytes);
@@ -1174,6 +1185,17 @@ imagex_progress_func(enum wimlib_progress_msg msg,
                if (info->extract.completed_bytes >= info->extract.total_bytes)
                        imagex_printf(T("\n"));
                break;
+       case WIMLIB_PROGRESS_MSG_EXTRACT_METADATA:
+               if (info->extract.end_file_count >= 2000) {
+                       percent_done = TO_PERCENT(info->extract.current_file_count,
+                                                 info->extract.end_file_count);
+                       imagex_printf(T("\rApplying metadata to files: %"PRIu64" of %"PRIu64" (%u%%) done"),
+                                     info->extract.current_file_count,
+                                     info->extract.end_file_count, percent_done);
+                       if (info->extract.current_file_count == info->extract.end_file_count)
+                               imagex_printf(T("\n"));
+               }
+               break;
        case WIMLIB_PROGRESS_MSG_EXTRACT_SPWM_PART_BEGIN:
                if (info->extract.total_parts != 1) {
                        imagex_printf(T("\nReading split pipable WIM part %u of %u\n"),
index 3b04704..8e35994 100644 (file)
 int
 do_file_extract_progress(struct apply_ctx *ctx, enum wimlib_progress_msg msg)
 {
-       ctx->count_until_file_progress = 512;  /* Arbitrary value to limit calls  */
+       ctx->count_until_file_progress = 500;  /* Arbitrary value to limit calls  */
        return extract_progress(ctx, msg);
 }
 
+static int
+start_file_phase(struct apply_ctx *ctx, uint64_t end_file_count, enum wimlib_progress_msg msg)
+{
+       ctx->progress.extract.current_file_count = 0;
+       ctx->progress.extract.end_file_count = end_file_count;
+       return do_file_extract_progress(ctx, msg);
+}
+
+int
+start_file_structure_phase(struct apply_ctx *ctx, uint64_t end_file_count)
+{
+       return start_file_phase(ctx, end_file_count, WIMLIB_PROGRESS_MSG_EXTRACT_FILE_STRUCTURE);
+}
+
+int
+start_file_metadata_phase(struct apply_ctx *ctx, uint64_t end_file_count)
+{
+       return start_file_phase(ctx, end_file_count, WIMLIB_PROGRESS_MSG_EXTRACT_METADATA);
+}
+
+static int
+end_file_phase(struct apply_ctx *ctx, enum wimlib_progress_msg msg)
+{
+       ctx->progress.extract.current_file_count = ctx->progress.extract.end_file_count;
+       return do_file_extract_progress(ctx, msg);
+}
+
+int
+end_file_structure_phase(struct apply_ctx *ctx)
+{
+       return end_file_phase(ctx, WIMLIB_PROGRESS_MSG_EXTRACT_FILE_STRUCTURE);
+}
+
+int
+end_file_metadata_phase(struct apply_ctx *ctx)
+{
+       return end_file_phase(ctx, WIMLIB_PROGRESS_MSG_EXTRACT_METADATA);
+}
+
 /* Check whether the extraction of a dentry should be skipped completely.  */
 static bool
 dentry_is_supported(struct wim_dentry *dentry,
index 5f0e7c7..db28968 100644 (file)
@@ -675,11 +675,11 @@ ntfs_3g_create_nondirectories(struct list_head *dentry_list,
                inode = dentry->d_inode;
                if (inode->i_attributes & FILE_ATTRIBUTE_DIRECTORY)
                        continue;
-               if (dentry != inode_first_extraction_dentry(inode))
-                       continue;
-               ret = ntfs_3g_create_nondirectory(inode, ctx);
-               if (ret)
-                       return ret;
+               if (dentry == inode_first_extraction_dentry(inode)) {
+                       ret = ntfs_3g_create_nondirectory(inode, ctx);
+                       if (ret)
+                               return ret;
+               }
                ret = report_file_created(&ctx->common);
                if (ret)
                        return ret;
@@ -891,6 +891,25 @@ out:
        return ret;
 }
 
+static uint64_t
+ntfs_3g_count_dentries(const struct list_head *dentry_list)
+{
+       const struct wim_dentry *dentry;
+       uint64_t count = 0;
+
+       list_for_each_entry(dentry, dentry_list, d_extraction_list_node) {
+               count++;
+               if ((dentry->d_inode->i_attributes & FILE_ATTRIBUTE_DIRECTORY) &&
+                   dentry_has_short_name(dentry))
+               {
+                       count++;
+               }
+
+       }
+
+       return count;
+}
+
 static int
 ntfs_3g_extract(struct list_head *dentry_list, struct apply_ctx *_ctx)
 {
@@ -916,7 +935,10 @@ ntfs_3g_extract(struct list_head *dentry_list, struct apply_ctx *_ctx)
        /* Create all inodes and aliases, including short names, and set
         * metadata (attributes, security descriptors, and timestamps).  */
 
-       reset_file_progress(&ctx->common);
+       ret = start_file_structure_phase(&ctx->common,
+                                        ntfs_3g_count_dentries(dentry_list));
+       if (ret)
+               goto out_unmount;
 
        ret = ntfs_3g_create_directories(root, dentry_list, ctx);
        if (ret)
@@ -926,6 +948,10 @@ ntfs_3g_extract(struct list_head *dentry_list, struct apply_ctx *_ctx)
        if (ret)
                goto out_unmount;
 
+       ret = end_file_structure_phase(&ctx->common);
+       if (ret)
+               goto out_unmount;
+
        /* Extract streams.  */
        struct read_stream_list_callbacks cbs = {
                .begin_stream      = ntfs_3g_begin_extract_stream,
index 7ed1422..ff11977 100644 (file)
@@ -472,6 +472,29 @@ unix_create_dirs_and_empty_files(const struct list_head *dentry_list,
        return 0;
 }
 
+static void
+unix_count_dentries(const struct list_head *dentry_list,
+                   uint64_t *dir_count_ret, uint64_t *empty_file_count_ret)
+{
+       const struct wim_dentry *dentry;
+       uint64_t dir_count = 0;
+       uint64_t empty_file_count = 0;
+
+       list_for_each_entry(dentry, dentry_list, d_extraction_list_node) {
+
+               const struct wim_inode *inode = dentry->d_inode;
+
+               if (inode_is_directory(inode))
+                       dir_count++;
+               else if ((dentry == inode_first_extraction_dentry(inode)) &&
+                        !inode_unnamed_lte_resolved(inode))
+                       empty_file_count++;
+       }
+
+       *dir_count_ret = dir_count;
+       *empty_file_count_ret = empty_file_count;
+}
+
 static int
 unix_create_symlink(const struct wim_inode *inode, const char *path,
                    const u8 *rpdata, u16 rpdatalen, bool rpfix,
@@ -699,6 +722,8 @@ unix_extract(struct list_head *dentry_list, struct apply_ctx *_ctx)
        int ret;
        struct unix_apply_ctx *ctx = (struct unix_apply_ctx *)_ctx;
        size_t path_max;
+       uint64_t dir_count;
+       uint64_t empty_file_count;
 
        /* Compute the maximum path length that will be needed, then allocate
         * some path buffers.  */
@@ -720,11 +745,21 @@ unix_extract(struct list_head *dentry_list, struct apply_ctx *_ctx)
         * because we can't extract any other files until their directories
         * exist.  Empty files are needed because they don't have
         * representatives in the stream list.  */
-       reset_file_progress(&ctx->common);
+
+       unix_count_dentries(dentry_list, &dir_count, &empty_file_count);
+
+       ret = start_file_structure_phase(&ctx->common, dir_count + empty_file_count);
+       if (ret)
+               goto out;
+
        ret = unix_create_dirs_and_empty_files(dentry_list, ctx);
        if (ret)
                goto out;
 
+       ret = end_file_structure_phase(&ctx->common);
+       if (ret)
+               goto out;
+
        /* Get full path to target if needed for absolute symlink fixups.  */
        if ((ctx->common.extract_flags & WIMLIB_EXTRACT_FLAG_RPFIX) &&
            ctx->common.required_features.symlink_reparse_points)
@@ -754,10 +789,18 @@ unix_extract(struct list_head *dentry_list, struct apply_ctx *_ctx)
 
        /* Set directory metadata.  We do this last so that we get the right
         * directory timestamps.  */
-       reset_file_progress(&ctx->common);
+       ret = start_file_metadata_phase(&ctx->common, dir_count);
+       if (ret)
+               goto out;
+
        ret = unix_set_dir_metadata(dentry_list, ctx);
        if (ret)
                goto out;
+
+       ret = end_file_metadata_phase(&ctx->common);
+       if (ret)
+               goto out;
+
        if (ctx->num_special_files_ignored) {
                WARNING("%lu special files were not extracted due to EPERM!",
                        ctx->num_special_files_ignored);
index a6b37c4..133eeff 100644 (file)
@@ -1104,16 +1104,15 @@ create_directories(struct list_head *dentry_list,
 
                /* If the root dentry is being extracted, it was already done so
                 * in prepare_target().  */
-               if (dentry_is_root(dentry))
-                       continue;
-
-               ret = create_directory(dentry, ctx);
-               if (ret)
-                       return ret;
+               if (!dentry_is_root(dentry)) {
+                       ret = create_directory(dentry, ctx);
+                       if (ret)
+                               return ret;
 
-               ret = create_any_empty_ads(dentry, ctx);
-               if (ret)
-                       return ret;
+                       ret = create_any_empty_ads(dentry, ctx);
+                       if (ret)
+                               return ret;
+               }
 
                ret = report_file_created(&ctx->common);
                if (ret)
@@ -1360,11 +1359,11 @@ create_nondirectories(struct list_head *dentry_list, struct win32_apply_ctx *ctx
                if (inode->i_attributes & FILE_ATTRIBUTE_DIRECTORY)
                        continue;
                /* Call create_nondirectory() only once per inode  */
-               if (dentry != inode_first_extraction_dentry(inode))
-                       continue;
-               ret = create_nondirectory(inode, ctx);
-               if (ret)
-                       return ret;
+               if (dentry == inode_first_extraction_dentry(inode)) {
+                       ret = create_nondirectory(inode, ctx);
+                       if (ret)
+                               return ret;
+               }
                ret = report_file_created(&ctx->common);
                if (ret)
                        return ret;
@@ -2171,12 +2170,25 @@ do_warnings(const struct win32_apply_ctx *ctx)
        }
 }
 
+static uint64_t
+count_dentries(const struct list_head *dentry_list)
+{
+       const struct list_head *cur;
+       uint64_t count = 0;
+
+       list_for_each(cur, dentry_list)
+               count++;
+
+       return count;
+}
+
 /* Extract files from a WIM image to a directory on Windows  */
 static int
 win32_extract(struct list_head *dentry_list, struct apply_ctx *_ctx)
 {
        int ret;
        struct win32_apply_ctx *ctx = (struct win32_apply_ctx *)_ctx;
+       uint64_t dentry_count;
 
        ret = prepare_target(dentry_list, ctx);
        if (ret)
@@ -2188,7 +2200,11 @@ win32_extract(struct list_head *dentry_list, struct apply_ctx *_ctx)
                        goto out;
        }
 
-       reset_file_progress(&ctx->common);
+       dentry_count = count_dentries(dentry_list);
+
+       ret = start_file_structure_phase(&ctx->common, dentry_count);
+       if (ret)
+               goto out;
 
        ret = create_directories(dentry_list, ctx);
        if (ret)
@@ -2198,6 +2214,10 @@ win32_extract(struct list_head *dentry_list, struct apply_ctx *_ctx)
        if (ret)
                goto out;
 
+       ret = end_file_structure_phase(&ctx->common);
+       if (ret)
+               goto out;
+
        struct read_stream_list_callbacks cbs = {
                .begin_stream      = begin_extract_stream,
                .begin_stream_ctx  = ctx,
@@ -2210,12 +2230,18 @@ win32_extract(struct list_head *dentry_list, struct apply_ctx *_ctx)
        if (ret)
                goto out;
 
-       reset_file_progress(&ctx->common);
+       ret = start_file_metadata_phase(&ctx->common, dentry_count);
+       if (ret)
+               goto out;
 
        ret = apply_metadata(dentry_list, ctx);
        if (ret)
                goto out;
 
+       ret = end_file_metadata_phase(&ctx->common);
+       if (ret)
+               goto out;
+
        if (unlikely(ctx->common.extract_flags & WIMLIB_EXTRACT_FLAG_WIMBOOT)) {
                ret = end_wimboot_extraction(ctx);
                if (ret)