From e040fe071bd22f58716b851696a1b0a0139877c0 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Wed, 27 Aug 2014 21:41:39 -0500 Subject: [PATCH] Support file counts for extract file structure and metadata progress --- include/wimlib.h | 32 +++++++++++++++++++++-- include/wimlib/apply.h | 21 ++++++++++----- programs/imagex.c | 22 ++++++++++++++++ src/extract.c | 41 ++++++++++++++++++++++++++++- src/ntfs-3g_apply.c | 38 ++++++++++++++++++++++----- src/unix_apply.c | 47 ++++++++++++++++++++++++++++++++-- src/win32_apply.c | 58 ++++++++++++++++++++++++++++++------------ 7 files changed, 225 insertions(+), 34 deletions(-) diff --git a/include/wimlib.h b/include/wimlib.h index 6aa36f58..d28fd344 100644 --- a/include/wimlib.h +++ b/include/wimlib.h @@ -552,7 +552,9 @@ enum wimlib_progress_msg { /** This message may be sent periodically (not for every file) while * files or directories are being created, prior to data stream * extraction. @p info will point to ::wimlib_progress_info.extract. - */ + * In particular, the @p current_file_count and @p end_file_count + * members may be used to track the progress of this phase of + * extraction. */ WIMLIB_PROGRESS_MSG_EXTRACT_FILE_STRUCTURE = 3, /** File data is currently being extracted. @p info will point to @@ -567,7 +569,9 @@ enum wimlib_progress_msg { /** This message may be sent periodically (not for every file) while * file and directory metadata is being applied, following data stream * extraction. @p info will point to ::wimlib_progress_info.extract. - */ + * In particular, the @p current_file_count and @p end_file_count + * members may be used to track the progress of this phase of + * extraction. */ WIMLIB_PROGRESS_MSG_EXTRACT_METADATA = 6, /** Confirms that the image has been successfully extracted. @p info @@ -984,6 +988,30 @@ union wimlib_progress_info { /** Currently only used for * ::WIMLIB_PROGRESS_MSG_EXTRACT_SPWM_PART_BEGIN. */ uint8_t guid[WIMLIB_GUID_LEN]; + + /** For ::WIMLIB_PROGRESS_MSG_EXTRACT_FILE_STRUCTURE and + * ::WIMLIB_PROGRESS_MSG_EXTRACT_METADATA messages, this is the + * number of files that have been processed so far. Once the + * corresponding phase of extraction is complete, this value + * will be equal to @c end_file_count. */ + uint64_t current_file_count; + + /** For ::WIMLIB_PROGRESS_MSG_EXTRACT_FILE_STRUCTURE and + * ::WIMLIB_PROGRESS_MSG_EXTRACT_METADATA messages, this is + * total number of files that will be processed. + * + * This number is provided for informational purposes only. + * This number will not necessarily be equal to the number of + * files actually being extracted. This is because extraction + * backends are free to implement an extraction algorithm that + * might be more efficient than processing every file in the + * "extract file structure" and "extract metadata" phases. For + * example, the current implementation of the UNIX extraction + * backend will create files on-demand during the stream + * extraction phase. Therefore, when using that particular + * extraction backend, @p end_file_count will only include + * directories and empty files. */ + uint64_t end_file_count; } extract; /** Valid on messages ::WIMLIB_PROGRESS_MSG_RENAME. */ diff --git a/include/wimlib/apply.h b/include/wimlib/apply.h index 8a1bf87e..73ddd871 100644 --- a/include/wimlib/apply.h +++ b/include/wimlib/apply.h @@ -88,21 +88,22 @@ extract_progress(struct apply_ctx *ctx, enum wimlib_progress_msg msg) extern int do_file_extract_progress(struct apply_ctx *ctx, enum wimlib_progress_msg msg); +#define COUNT_PER_FILE_PROGRESS 256 + static inline int maybe_do_file_progress(struct apply_ctx *ctx, enum wimlib_progress_msg msg) { + ctx->progress.extract.current_file_count++; if (unlikely(!--ctx->count_until_file_progress)) return do_file_extract_progress(ctx, msg); return 0; } -/* Call this to reset the counter for report_file_created() and - * report_file_metadata_applied(). */ -static inline void -reset_file_progress(struct apply_ctx *ctx) -{ - ctx->count_until_file_progress = 1; -} +extern int +start_file_structure_phase(struct apply_ctx *ctx, uint64_t end_file_count); + +extern int +start_file_metadata_phase(struct apply_ctx *ctx, uint64_t end_file_count); /* Report that a file was created, prior to stream extraction. */ static inline int @@ -118,6 +119,12 @@ report_file_metadata_applied(struct apply_ctx *ctx) return maybe_do_file_progress(ctx, WIMLIB_PROGRESS_MSG_EXTRACT_METADATA); } +extern int +end_file_structure_phase(struct apply_ctx *ctx); + +extern int +end_file_metadata_phase(struct apply_ctx *ctx); + /* Returns any of the aliases of an inode that are being extracted. */ #define inode_first_extraction_dentry(inode) \ list_first_entry(&(inode)->i_extraction_aliases, \ diff --git a/programs/imagex.c b/programs/imagex.c index 60b533c3..82354feb 100644 --- a/programs/imagex.c +++ b/programs/imagex.c @@ -1160,6 +1160,17 @@ imagex_progress_func(enum wimlib_progress_msg msg, T("NTFS volume") : T("directory")), info->extract.target); break; + case WIMLIB_PROGRESS_MSG_EXTRACT_FILE_STRUCTURE: + if (info->extract.end_file_count >= 2000) { + percent_done = TO_PERCENT(info->extract.current_file_count, + info->extract.end_file_count); + imagex_printf(T("\rCreating files: %"PRIu64" of %"PRIu64" (%u%%) done"), + info->extract.current_file_count, + info->extract.end_file_count, percent_done); + if (info->extract.current_file_count == info->extract.end_file_count) + imagex_printf(T("\n")); + } + break; case WIMLIB_PROGRESS_MSG_EXTRACT_STREAMS: percent_done = TO_PERCENT(info->extract.completed_bytes, info->extract.total_bytes); @@ -1174,6 +1185,17 @@ imagex_progress_func(enum wimlib_progress_msg msg, if (info->extract.completed_bytes >= info->extract.total_bytes) imagex_printf(T("\n")); break; + case WIMLIB_PROGRESS_MSG_EXTRACT_METADATA: + if (info->extract.end_file_count >= 2000) { + percent_done = TO_PERCENT(info->extract.current_file_count, + info->extract.end_file_count); + imagex_printf(T("\rApplying metadata to files: %"PRIu64" of %"PRIu64" (%u%%) done"), + info->extract.current_file_count, + info->extract.end_file_count, percent_done); + if (info->extract.current_file_count == info->extract.end_file_count) + imagex_printf(T("\n")); + } + break; case WIMLIB_PROGRESS_MSG_EXTRACT_SPWM_PART_BEGIN: if (info->extract.total_parts != 1) { imagex_printf(T("\nReading split pipable WIM part %u of %u\n"), diff --git a/src/extract.c b/src/extract.c index 3b047041..8e35994a 100644 --- a/src/extract.c +++ b/src/extract.c @@ -95,10 +95,49 @@ int do_file_extract_progress(struct apply_ctx *ctx, enum wimlib_progress_msg msg) { - ctx->count_until_file_progress = 512; /* Arbitrary value to limit calls */ + ctx->count_until_file_progress = 500; /* Arbitrary value to limit calls */ return extract_progress(ctx, msg); } +static int +start_file_phase(struct apply_ctx *ctx, uint64_t end_file_count, enum wimlib_progress_msg msg) +{ + ctx->progress.extract.current_file_count = 0; + ctx->progress.extract.end_file_count = end_file_count; + return do_file_extract_progress(ctx, msg); +} + +int +start_file_structure_phase(struct apply_ctx *ctx, uint64_t end_file_count) +{ + return start_file_phase(ctx, end_file_count, WIMLIB_PROGRESS_MSG_EXTRACT_FILE_STRUCTURE); +} + +int +start_file_metadata_phase(struct apply_ctx *ctx, uint64_t end_file_count) +{ + return start_file_phase(ctx, end_file_count, WIMLIB_PROGRESS_MSG_EXTRACT_METADATA); +} + +static int +end_file_phase(struct apply_ctx *ctx, enum wimlib_progress_msg msg) +{ + ctx->progress.extract.current_file_count = ctx->progress.extract.end_file_count; + return do_file_extract_progress(ctx, msg); +} + +int +end_file_structure_phase(struct apply_ctx *ctx) +{ + return end_file_phase(ctx, WIMLIB_PROGRESS_MSG_EXTRACT_FILE_STRUCTURE); +} + +int +end_file_metadata_phase(struct apply_ctx *ctx) +{ + return end_file_phase(ctx, WIMLIB_PROGRESS_MSG_EXTRACT_METADATA); +} + /* Check whether the extraction of a dentry should be skipped completely. */ static bool dentry_is_supported(struct wim_dentry *dentry, diff --git a/src/ntfs-3g_apply.c b/src/ntfs-3g_apply.c index 5f0e7c7e..db289684 100644 --- a/src/ntfs-3g_apply.c +++ b/src/ntfs-3g_apply.c @@ -675,11 +675,11 @@ ntfs_3g_create_nondirectories(struct list_head *dentry_list, inode = dentry->d_inode; if (inode->i_attributes & FILE_ATTRIBUTE_DIRECTORY) continue; - if (dentry != inode_first_extraction_dentry(inode)) - continue; - ret = ntfs_3g_create_nondirectory(inode, ctx); - if (ret) - return ret; + if (dentry == inode_first_extraction_dentry(inode)) { + ret = ntfs_3g_create_nondirectory(inode, ctx); + if (ret) + return ret; + } ret = report_file_created(&ctx->common); if (ret) return ret; @@ -891,6 +891,25 @@ out: return ret; } +static uint64_t +ntfs_3g_count_dentries(const struct list_head *dentry_list) +{ + const struct wim_dentry *dentry; + uint64_t count = 0; + + list_for_each_entry(dentry, dentry_list, d_extraction_list_node) { + count++; + if ((dentry->d_inode->i_attributes & FILE_ATTRIBUTE_DIRECTORY) && + dentry_has_short_name(dentry)) + { + count++; + } + + } + + return count; +} + static int ntfs_3g_extract(struct list_head *dentry_list, struct apply_ctx *_ctx) { @@ -916,7 +935,10 @@ ntfs_3g_extract(struct list_head *dentry_list, struct apply_ctx *_ctx) /* Create all inodes and aliases, including short names, and set * metadata (attributes, security descriptors, and timestamps). */ - reset_file_progress(&ctx->common); + ret = start_file_structure_phase(&ctx->common, + ntfs_3g_count_dentries(dentry_list)); + if (ret) + goto out_unmount; ret = ntfs_3g_create_directories(root, dentry_list, ctx); if (ret) @@ -926,6 +948,10 @@ ntfs_3g_extract(struct list_head *dentry_list, struct apply_ctx *_ctx) if (ret) goto out_unmount; + ret = end_file_structure_phase(&ctx->common); + if (ret) + goto out_unmount; + /* Extract streams. */ struct read_stream_list_callbacks cbs = { .begin_stream = ntfs_3g_begin_extract_stream, diff --git a/src/unix_apply.c b/src/unix_apply.c index 7ed14225..ff11977c 100644 --- a/src/unix_apply.c +++ b/src/unix_apply.c @@ -472,6 +472,29 @@ unix_create_dirs_and_empty_files(const struct list_head *dentry_list, return 0; } +static void +unix_count_dentries(const struct list_head *dentry_list, + uint64_t *dir_count_ret, uint64_t *empty_file_count_ret) +{ + const struct wim_dentry *dentry; + uint64_t dir_count = 0; + uint64_t empty_file_count = 0; + + list_for_each_entry(dentry, dentry_list, d_extraction_list_node) { + + const struct wim_inode *inode = dentry->d_inode; + + if (inode_is_directory(inode)) + dir_count++; + else if ((dentry == inode_first_extraction_dentry(inode)) && + !inode_unnamed_lte_resolved(inode)) + empty_file_count++; + } + + *dir_count_ret = dir_count; + *empty_file_count_ret = empty_file_count; +} + static int unix_create_symlink(const struct wim_inode *inode, const char *path, const u8 *rpdata, u16 rpdatalen, bool rpfix, @@ -699,6 +722,8 @@ unix_extract(struct list_head *dentry_list, struct apply_ctx *_ctx) int ret; struct unix_apply_ctx *ctx = (struct unix_apply_ctx *)_ctx; size_t path_max; + uint64_t dir_count; + uint64_t empty_file_count; /* Compute the maximum path length that will be needed, then allocate * some path buffers. */ @@ -720,11 +745,21 @@ unix_extract(struct list_head *dentry_list, struct apply_ctx *_ctx) * because we can't extract any other files until their directories * exist. Empty files are needed because they don't have * representatives in the stream list. */ - reset_file_progress(&ctx->common); + + unix_count_dentries(dentry_list, &dir_count, &empty_file_count); + + ret = start_file_structure_phase(&ctx->common, dir_count + empty_file_count); + if (ret) + goto out; + ret = unix_create_dirs_and_empty_files(dentry_list, ctx); if (ret) goto out; + ret = end_file_structure_phase(&ctx->common); + if (ret) + goto out; + /* Get full path to target if needed for absolute symlink fixups. */ if ((ctx->common.extract_flags & WIMLIB_EXTRACT_FLAG_RPFIX) && ctx->common.required_features.symlink_reparse_points) @@ -754,10 +789,18 @@ unix_extract(struct list_head *dentry_list, struct apply_ctx *_ctx) /* Set directory metadata. We do this last so that we get the right * directory timestamps. */ - reset_file_progress(&ctx->common); + ret = start_file_metadata_phase(&ctx->common, dir_count); + if (ret) + goto out; + ret = unix_set_dir_metadata(dentry_list, ctx); if (ret) goto out; + + ret = end_file_metadata_phase(&ctx->common); + if (ret) + goto out; + if (ctx->num_special_files_ignored) { WARNING("%lu special files were not extracted due to EPERM!", ctx->num_special_files_ignored); diff --git a/src/win32_apply.c b/src/win32_apply.c index a6b37c4a..133eeff2 100644 --- a/src/win32_apply.c +++ b/src/win32_apply.c @@ -1104,16 +1104,15 @@ create_directories(struct list_head *dentry_list, /* If the root dentry is being extracted, it was already done so * in prepare_target(). */ - if (dentry_is_root(dentry)) - continue; - - ret = create_directory(dentry, ctx); - if (ret) - return ret; + if (!dentry_is_root(dentry)) { + ret = create_directory(dentry, ctx); + if (ret) + return ret; - ret = create_any_empty_ads(dentry, ctx); - if (ret) - return ret; + ret = create_any_empty_ads(dentry, ctx); + if (ret) + return ret; + } ret = report_file_created(&ctx->common); if (ret) @@ -1360,11 +1359,11 @@ create_nondirectories(struct list_head *dentry_list, struct win32_apply_ctx *ctx if (inode->i_attributes & FILE_ATTRIBUTE_DIRECTORY) continue; /* Call create_nondirectory() only once per inode */ - if (dentry != inode_first_extraction_dentry(inode)) - continue; - ret = create_nondirectory(inode, ctx); - if (ret) - return ret; + if (dentry == inode_first_extraction_dentry(inode)) { + ret = create_nondirectory(inode, ctx); + if (ret) + return ret; + } ret = report_file_created(&ctx->common); if (ret) return ret; @@ -2171,12 +2170,25 @@ do_warnings(const struct win32_apply_ctx *ctx) } } +static uint64_t +count_dentries(const struct list_head *dentry_list) +{ + const struct list_head *cur; + uint64_t count = 0; + + list_for_each(cur, dentry_list) + count++; + + return count; +} + /* Extract files from a WIM image to a directory on Windows */ static int win32_extract(struct list_head *dentry_list, struct apply_ctx *_ctx) { int ret; struct win32_apply_ctx *ctx = (struct win32_apply_ctx *)_ctx; + uint64_t dentry_count; ret = prepare_target(dentry_list, ctx); if (ret) @@ -2188,7 +2200,11 @@ win32_extract(struct list_head *dentry_list, struct apply_ctx *_ctx) goto out; } - reset_file_progress(&ctx->common); + dentry_count = count_dentries(dentry_list); + + ret = start_file_structure_phase(&ctx->common, dentry_count); + if (ret) + goto out; ret = create_directories(dentry_list, ctx); if (ret) @@ -2198,6 +2214,10 @@ win32_extract(struct list_head *dentry_list, struct apply_ctx *_ctx) if (ret) goto out; + ret = end_file_structure_phase(&ctx->common); + if (ret) + goto out; + struct read_stream_list_callbacks cbs = { .begin_stream = begin_extract_stream, .begin_stream_ctx = ctx, @@ -2210,12 +2230,18 @@ win32_extract(struct list_head *dentry_list, struct apply_ctx *_ctx) if (ret) goto out; - reset_file_progress(&ctx->common); + ret = start_file_metadata_phase(&ctx->common, dentry_count); + if (ret) + goto out; ret = apply_metadata(dentry_list, ctx); if (ret) goto out; + ret = end_file_metadata_phase(&ctx->common); + if (ret) + goto out; + if (unlikely(ctx->common.extract_flags & WIMLIB_EXTRACT_FLAG_WIMBOOT)) { ret = end_wimboot_extraction(ctx); if (ret) -- 2.43.0