From: Eric Biggers Date: Tue, 24 Dec 2013 16:36:53 +0000 (-0600) Subject: Scan progress: Add # of files, directories, and bytes scanned X-Git-Tag: v1.6.0~111 X-Git-Url: https://wimlib.net/git/?p=wimlib;a=commitdiff_plain;h=5816eba3f41b63910c5d932a8ed22a2c9f65c492 Scan progress: Add # of files, directories, and bytes scanned --- diff --git a/doc/imagex-capture.1.in b/doc/imagex-capture.1.in index cedebc44..6a0ed43d 100644 --- a/doc/imagex-capture.1.in +++ b/doc/imagex-capture.1.in @@ -218,9 +218,6 @@ optimize\fR(1). Specify a string to use in the element of the XML data for the new image. .TP -\fB--verbose\fR -Print the names of files and directories as they are captured. -.TP \fB--dereference\fR (UNIX-like systems only) Follow symbolic links and archive the files they point to, rather than archiving the links themselves. diff --git a/doc/imagex-extract.1.in b/doc/imagex-extract.1.in index 96e55e66..c3f01031 100644 --- a/doc/imagex-extract.1.in +++ b/doc/imagex-extract.1.in @@ -56,9 +56,6 @@ See \fBSPLIT_WIMS\fR. Note: since \fIGLOB\fR is listed in quotes because it is interpreted by \fB@IMAGEX_PROGNAME@\fR and may need to be quoted to protect against shell expansion. .TP -\fB--verbose\fR -This option no longer does anything but is reserved for future use. -.TP \fB--unix-data\fR See the documentation for this option in \fB@IMAGEX_PROGNAME@-apply\fR (1). .TP diff --git a/doc/imagex-update.1.in b/doc/imagex-update.1.in index b2745467..39349ac3 100644 --- a/doc/imagex-update.1.in +++ b/doc/imagex-update.1.in @@ -34,7 +34,7 @@ not exist in the WIM image, then any prerequisite directories are created as needed to add the \fISOURCE\fR at that location. .PP The \fBadd\fR command supports a subset of the options accepted by -\fB@IMAGEX_PROGNAME@-capture\fR; namely, \fB--verbose\fR, \fB--dereference\fR, +\fB@IMAGEX_PROGNAME@-capture\fR; namely, \fB--dereference\fR, \fB--unix-data\fR, \fB--no-acls\fR, and \fB--strict-acls\fR. See \fB@IMAGEX_PROGNAME@-capture\fR (1) for explanations of these options. .SS \fBdelete\fR [\fIOPTION\fR...] \fIPATH\fR @@ -63,9 +63,6 @@ There are no options available for the \fBrename\fR command. The following options are accepted on the command line by \fB@IMAGEX_PROGNAME@ update\fR itself: .TP 6 -\fB--verbose\fR -Use \fB--verbose\fR for all \fBadd\fR commands. -.TP \fB--dereference\fR Use \fB--dereference\fR for all \fBadd\fR commands. .TP diff --git a/include/wimlib.h b/include/wimlib.h index 1526a7a3..71ef0a8e 100644 --- a/include/wimlib.h +++ b/include/wimlib.h @@ -637,6 +637,21 @@ union wimlib_progress_info { * ::WIMLIB_PROGRESS_MSG_SCAN_BEGIN and * ::WIMLIB_PROGRESS_MSG_SCAN_END. */ const wimlib_tchar *wim_target_path; + + /** Number of directories scanned so far, including the root + * directory but excluding any unsupported/excluded directories. + * */ + uint64_t num_dirs_scanned; + + /** Number of non-directories scanned so far, excluding any + * unsupported/excluded files. */ + uint64_t num_nondirs_scanned; + + /** Number of bytes of file data that have been detected so far. + * This data may not actually have been read yet, and it will + * not actually be written to the WIM file until wimlib_write() + * or wimlib_overwrite() has been called. */ + uint64_t num_bytes_scanned; } scan; /** Valid on messages ::WIMLIB_PROGRESS_MSG_EXTRACT_IMAGE_BEGIN, diff --git a/include/wimlib/capture.h b/include/wimlib/capture.h index 50402cf3..9f9e4617 100644 --- a/include/wimlib/capture.h +++ b/include/wimlib/capture.h @@ -8,6 +8,7 @@ struct wim_lookup_table; struct wim_dentry; +struct wim_inode; /* Hash table to find inodes, given an inode number (in the case of reading * a WIM images), or both an inode number and a device number (in the case of @@ -70,7 +71,8 @@ struct add_image_params { /* capture_common.c */ extern void -do_capture_progress(struct add_image_params *params, int status); +do_capture_progress(struct add_image_params *params, int status, + const struct wim_inode *inode); extern bool exclude_path(const tchar *path, size_t path_len, diff --git a/programs/imagex.c b/programs/imagex.c index a9e6fa63..b2164402 100644 --- a/programs/imagex.c +++ b/programs/imagex.c @@ -1053,6 +1053,34 @@ get_unit(uint64_t total_bytes, const tchar **name_ret) } } +static struct wimlib_progress_info_scan last_scan_progress; + +static void +report_scan_progress(const struct wimlib_progress_info_scan *scan, bool done) +{ + uint64_t prev_count, cur_count; + + prev_count = last_scan_progress.num_nondirs_scanned + + last_scan_progress.num_dirs_scanned; + cur_count = scan->num_nondirs_scanned + scan->num_dirs_scanned; + + if (done || prev_count == 0 || cur_count >= prev_count + 100 || + cur_count % 128 == 0) + { + unsigned unit_shift; + const tchar *unit_name; + + unit_shift = get_unit(scan->num_bytes_scanned, &unit_name); + imagex_printf(T("\r%"PRIu64" %"TS" scanned (%"PRIu64" files, " + "%"PRIu64" directories)"), + scan->num_bytes_scanned >> unit_shift, + unit_name, + scan->num_nondirs_scanned, + scan->num_dirs_scanned); + last_scan_progress = *scan; + } +} + /* Progress callback function passed to various wimlib functions. */ static int imagex_progress_func(enum wimlib_progress_msg msg, @@ -1061,6 +1089,7 @@ imagex_progress_func(enum wimlib_progress_msg msg, unsigned percent_done; unsigned unit_shift; const tchar *unit_name; + if (imagex_be_quiet) return 0; switch (msg) { @@ -1107,23 +1136,28 @@ imagex_progress_func(enum wimlib_progress_msg msg, "\""WIMLIB_WIM_PATH_SEPARATOR_STRING"%"TS"\")...\n"), info->scan.wim_target_path); } else { - imagex_printf(T(" (loading as root of WIM image)...\n")); + imagex_printf(T("\n")); } + memset(&last_scan_progress, 0, sizeof(last_scan_progress)); break; case WIMLIB_PROGRESS_MSG_SCAN_DENTRY: switch (info->scan.status) { case WIMLIB_SCAN_DENTRY_OK: - imagex_printf(T("Scanning \"%"TS"\"\n"), info->scan.cur_path); + report_scan_progress(&info->scan, false); break; case WIMLIB_SCAN_DENTRY_EXCLUDED: - imagex_printf(T("Excluding \"%"TS"\" from capture\n"), info->scan.cur_path); + imagex_printf(T("\nExcluding \"%"TS"\" from capture\n"), info->scan.cur_path); break; case WIMLIB_SCAN_DENTRY_UNSUPPORTED: - imagex_printf(T("WARNING: Excluding unsupported file or directory\n" + imagex_printf(T("\nWARNING: Excluding unsupported file or directory\n" " \"%"TS"\" from capture\n"), info->scan.cur_path); break; } break; + case WIMLIB_PROGRESS_MSG_SCAN_END: + report_scan_progress(&info->scan, true); + imagex_printf(T("\n")); + break; case WIMLIB_PROGRESS_MSG_VERIFY_INTEGRITY: unit_shift = get_unit(info->integrity.total_bytes, &unit_name); percent_done = TO_PERCENT(info->integrity.completed_bytes, @@ -1516,7 +1550,7 @@ imagex_apply(int argc, tchar **argv, int cmd) extract_flags |= WIMLIB_EXTRACT_FLAG_SYMLINK; break; case IMAGEX_VERBOSE_OPTION: - extract_flags |= WIMLIB_EXTRACT_FLAG_VERBOSE; + /* No longer does anything. */ break; case IMAGEX_REF_OPTION: ret = string_set_append(&refglobs, optarg); @@ -1673,7 +1707,8 @@ imagex_capture_or_append(int argc, tchar **argv, int cmd) int c; int open_flags = WIMLIB_OPEN_FLAG_WRITE_ACCESS; int add_image_flags = WIMLIB_ADD_IMAGE_FLAG_EXCLUDE_VERBOSE | - WIMLIB_ADD_IMAGE_FLAG_WINCONFIG; + WIMLIB_ADD_IMAGE_FLAG_WINCONFIG | + WIMLIB_ADD_IMAGE_FLAG_VERBOSE; int write_flags = 0; int compression_type = WIMLIB_COMPRESSION_TYPE_INVALID; uint32_t chunk_size = UINT32_MAX; @@ -1752,7 +1787,7 @@ imagex_capture_or_append(int argc, tchar **argv, int cmd) add_image_flags |= WIMLIB_ADD_IMAGE_FLAG_DEREFERENCE; break; case IMAGEX_VERBOSE_OPTION: - add_image_flags |= WIMLIB_ADD_IMAGE_FLAG_VERBOSE; + /* No longer does anything. */ break; case IMAGEX_THREADS_OPTION: num_threads = parse_num_threads(optarg); @@ -2653,7 +2688,7 @@ imagex_extract(int argc, tchar **argv, int cmd) open_flags |= WIMLIB_OPEN_FLAG_CHECK_INTEGRITY; break; case IMAGEX_VERBOSE_OPTION: - extract_flags |= WIMLIB_EXTRACT_FLAG_VERBOSE; + /* No longer does anything. */ break; case IMAGEX_REF_OPTION: ret = string_set_append(&refglobs, optarg); @@ -3521,7 +3556,8 @@ imagex_update(int argc, tchar **argv, int cmd) int open_flags = WIMLIB_OPEN_FLAG_WRITE_ACCESS; int write_flags = 0; int update_flags = WIMLIB_UPDATE_FLAG_SEND_PROGRESS; - int default_add_flags = WIMLIB_ADD_FLAG_EXCLUDE_VERBOSE; + int default_add_flags = WIMLIB_ADD_FLAG_EXCLUDE_VERBOSE | + WIMLIB_ADD_FLAG_VERBOSE; int default_delete_flags = 0; unsigned num_threads = 0; int c; @@ -3580,7 +3616,7 @@ imagex_update(int argc, tchar **argv, int cmd) /* Default add options */ case IMAGEX_VERBOSE_OPTION: - default_add_flags |= WIMLIB_ADD_FLAG_VERBOSE; + /* No longer does anything. */ break; case IMAGEX_DEREFERENCE_OPTION: default_add_flags |= WIMLIB_ADD_FLAG_DEREFERENCE; @@ -3757,7 +3793,7 @@ static const tchar *usage_strings[] = { T( " %"TS" (DIRECTORY | NTFS_VOLUME) WIMFILE\n" " [IMAGE_NAME [IMAGE_DESCRIPTION]] [--boot]\n" -" [--check] [--nocheck] [--flags EDITION_ID] [--verbose]\n" +" [--check] [--nocheck] [--flags EDITION_ID]\n" " [--dereference] [--config=FILE] [--threads=NUM_THREADS]\n" " [--source-list] [--no-acls] [--strict-acls] [--rpfix]\n" " [--norpfix] [--unix-data] [--pipable]\n" @@ -3767,7 +3803,7 @@ T( T( " %"TS" WIMFILE [(IMAGE_NUM | IMAGE_NAME | all)]\n" " (DIRECTORY | NTFS_VOLUME) [--check] [--hardlink]\n" -" [--symlink] [--verbose] [--ref=\"GLOB\"] [--unix-data]\n" +" [--symlink] [--ref=\"GLOB\"] [--unix-data]\n" " [--no-acls] [--strict-acls] [--rpfix] [--norpfix]\n" " [--include-invalid-names]\n" ), @@ -3776,7 +3812,7 @@ T( " %"TS" (DIRECTORY | NTFS_VOLUME) WIMFILE\n" " [IMAGE_NAME [IMAGE_DESCRIPTION]] [--boot]\n" " [--check] [--nocheck] [--compress=TYPE]\n" -" [--flags EDITION_ID] [--verbose] [--dereference]\n" +" [--flags EDITION_ID] [--dereference]\n" " [--config=FILE] [--threads=NUM_THREADS] [--source-list]\n" " [--no-acls] [--strict-acls] [--rpfix] [--norpfix]\n" " [--unix-data] [--pipable] [--update-of=[WIMFILE:]IMAGE]\n" @@ -3802,7 +3838,7 @@ T( [CMD_EXTRACT] = T( " %"TS" WIMFILE (IMAGE_NUM | IMAGE_NAME) [PATH...]\n" -" [--check] [--ref=\"GLOB\"] [--verbose] [--unix-data]\n" +" [--check] [--ref=\"GLOB\"] [--unix-data]\n" " [--no-acls] [--strict-acls] [--to-stdout]\n" " [--dest-dir=CMD_DIR] [--include-invalid-names]\n" ), diff --git a/src/capture_common.c b/src/capture_common.c index 28a830fa..c45ae152 100644 --- a/src/capture_common.c +++ b/src/capture_common.c @@ -27,7 +27,9 @@ #include "wimlib/assert.h" #include "wimlib/capture.h" +#include "wimlib/dentry.h" #include "wimlib/error.h" +#include "wimlib/lookup_table.h" #include "wimlib/paths.h" #ifdef __WIN32__ @@ -183,7 +185,8 @@ match_pattern(const tchar *path, } void -do_capture_progress(struct add_image_params *params, int status) +do_capture_progress(struct add_image_params *params, int status, + const struct wim_inode *inode) { switch (status) { case WIMLIB_SCAN_DENTRY_OK: @@ -195,6 +198,18 @@ do_capture_progress(struct add_image_params *params, int status) return; } params->progress.scan.status = status; + if (status == WIMLIB_SCAN_DENTRY_OK && inode->i_nlink == 1) { + const struct wim_lookup_table_entry *lte; + for (unsigned i = 0; i <= inode->i_num_ads; i++) { + lte = inode_stream_lte_resolved(inode, i); + if (lte != NULL) + params->progress.scan.num_bytes_scanned += lte->size; + } + if (inode->i_attributes & FILE_ATTRIBUTE_DIRECTORY) + params->progress.scan.num_dirs_scanned++; + else + params->progress.scan.num_nondirs_scanned++; + } if (params->progress_func) { params->progress_func(WIMLIB_PROGRESS_MSG_SCAN_DENTRY, ¶ms->progress); diff --git a/src/ntfs-3g_capture.c b/src/ntfs-3g_capture.c index b6d92262..444cccbd 100644 --- a/src/ntfs-3g_capture.c +++ b/src/ntfs-3g_capture.c @@ -538,12 +538,11 @@ build_dentry_tree_ntfs_recursive(struct wim_dentry **root_ret, struct wim_inode *inode; ATTR_TYPES stream_type; - params->progress.scan.cur_path = path; - if (exclude_path(path, path_len, params->config, false)) { /* Exclude a file or directory tree based on the capture - * configuration file */ - do_capture_progress(params, WIMLIB_SCAN_DENTRY_EXCLUDED); + * configuration file. */ + params->progress.scan.cur_path = path; + do_capture_progress(params, WIMLIB_SCAN_DENTRY_EXCLUDED, NULL); ret = 0; goto out; } @@ -565,13 +564,12 @@ build_dentry_tree_ntfs_recursive(struct wim_dentry **root_ret, ret = WIMLIB_ERR_UNSUPPORTED_FILE; goto out; } - do_capture_progress(params, WIMLIB_SCAN_DENTRY_UNSUPPORTED); + params->progress.scan.cur_path = path; + do_capture_progress(params, WIMLIB_SCAN_DENTRY_UNSUPPORTED, NULL); ret = 0; goto out; } - do_capture_progress(params, WIMLIB_SCAN_DENTRY_OK); - /* Create a WIM dentry with an associated inode, which may be shared */ ret = inode_table_new_dentry(¶ms->inode_table, path_basename_with_len(path, path_len), @@ -587,7 +585,7 @@ build_dentry_tree_ntfs_recursive(struct wim_dentry **root_ret, if (inode->i_nlink > 1) { /* Shared inode; nothing more to do */ ret = 0; - goto out; + goto out_progress_ok; } inode->i_creation_time = le64_to_cpu(ni->creation_time); @@ -638,6 +636,7 @@ build_dentry_tree_ntfs_recursive(struct wim_dentry **root_ret, if (ret) goto out; } + path[path_len] = '\0'; /* Reparse-point fixups are a no-op because in NTFS-3g capture mode we * only allow capturing an entire volume. */ @@ -681,6 +680,10 @@ build_dentry_tree_ntfs_recursive(struct wim_dentry **root_ret, DEBUG("No security ID for `%s'", path); } } + +out_progress_ok: + params->progress.scan.cur_path = path; + do_capture_progress(params, WIMLIB_SCAN_DENTRY_OK, inode); out: if (ret == 0) *root_ret = root; diff --git a/src/unix_capture.c b/src/unix_capture.c index d16eae6d..114d03dd 100644 --- a/src/unix_capture.c +++ b/src/unix_capture.c @@ -128,6 +128,7 @@ unix_capture_directory(struct wim_dentry *dir_dentry, if (child) dentry_add_child(dir_dentry, child); } + path[path_len] = '\0'; closedir(dir); return ret; } @@ -203,44 +204,43 @@ unix_build_dentry_tree_recursive(struct wim_dentry **root_ret, struct wim_dentry *root = NULL; int ret; struct wim_inode *inode; - - params->progress.scan.cur_path = path; + struct stat stbuf; if (exclude_path(path, path_len, params->config, true)) { - do_capture_progress(params, WIMLIB_SCAN_DENTRY_EXCLUDED); + params->progress.scan.cur_path = path; + do_capture_progress(params, WIMLIB_SCAN_DENTRY_EXCLUDED, NULL); ret = 0; goto out; } - struct stat stbuf; - int (*stat_fn)(const char *restrict, struct stat *restrict); if ((params->add_flags & WIMLIB_ADD_FLAG_DEREFERENCE) || (params->add_flags & WIMLIB_ADD_FLAG_ROOT)) - stat_fn = stat; + ret = stat(path, &stbuf); else - stat_fn = lstat; + ret = lstat(path, &stbuf); - ret = (*stat_fn)(path, &stbuf); if (ret) { - ERROR_WITH_ERRNO("Failed to stat `%s'", path); + ERROR_WITH_ERRNO("Failed to stat \"%s\"", path); ret = WIMLIB_ERR_STAT; goto out; } - if (!S_ISREG(stbuf.st_mode) && !S_ISDIR(stbuf.st_mode) - && !S_ISLNK(stbuf.st_mode)) { + + if (!S_ISREG(stbuf.st_mode) && + !S_ISDIR(stbuf.st_mode) && + !S_ISLNK(stbuf.st_mode)) + { if (params->add_flags & WIMLIB_ADD_FLAG_NO_UNSUPPORTED_EXCLUDE) { ERROR("Can't archive unsupported file \"%s\"", path); ret = WIMLIB_ERR_UNSUPPORTED_FILE; goto out; } - do_capture_progress(params, WIMLIB_SCAN_DENTRY_UNSUPPORTED); + params->progress.scan.cur_path = path; + do_capture_progress(params, WIMLIB_SCAN_DENTRY_UNSUPPORTED, NULL); ret = 0; goto out; } - do_capture_progress(params, WIMLIB_SCAN_DENTRY_OK); - ret = inode_table_new_dentry(¶ms->inode_table, path_basename_with_len(path, path_len), stbuf.st_ino, stbuf.st_dev, false, &root); @@ -252,7 +252,7 @@ unix_build_dentry_tree_recursive(struct wim_dentry **root_ret, if (inode->i_nlink > 1) { /* Already captured this inode? */ ret = 0; - goto out; + goto out_progress_ok; } #ifdef HAVE_STAT_NANOSECOND_PRECISION @@ -286,6 +286,9 @@ unix_build_dentry_tree_recursive(struct wim_dentry **root_ret, if (ret) goto out; +out_progress_ok: + params->progress.scan.cur_path = path; + do_capture_progress(params, WIMLIB_SCAN_DENTRY_OK, inode); out: if (ret) free_dentry_tree(root, params->lookup_table); diff --git a/src/win32_capture.c b/src/win32_capture.c index 6e3552f5..97855b3b 100644 --- a/src/win32_capture.c +++ b/src/win32_capture.c @@ -1133,7 +1133,6 @@ win32_build_dentry_tree_recursive(struct wim_dentry **root_ret, HANDLE hFile; DWORD desiredAccess; - params->progress.scan.cur_path = path; if (exclude_path(path, path_num_chars, params->config, true)) { if (params->add_flags & WIMLIB_ADD_FLAG_ROOT) { @@ -1141,7 +1140,8 @@ win32_build_dentry_tree_recursive(struct wim_dentry **root_ret, ret = WIMLIB_ERR_INVALID_CAPTURE_CONFIG; goto out; } - do_capture_progress(params, WIMLIB_SCAN_DENTRY_EXCLUDED); + params->progress.scan.cur_path = path; + do_capture_progress(params, WIMLIB_SCAN_DENTRY_EXCLUDED, NULL); ret = 0; goto out; } @@ -1158,8 +1158,6 @@ win32_build_dentry_tree_recursive(struct wim_dentry **root_ret, } #endif - do_capture_progress(params, WIMLIB_SCAN_DENTRY_OK); - desiredAccess = FILE_READ_DATA | FILE_READ_ATTRIBUTES | READ_CONTROL | ACCESS_SYSTEM_SECURITY; again: @@ -1232,8 +1230,10 @@ again: inode = root->d_inode; - if (inode->i_nlink > 1) /* Shared inode; nothing more to do */ - goto out_close_handle; + if (inode->i_nlink > 1) { + /* Shared inode; nothing more to do */ + goto out_progress_ok; + } inode->i_attributes = file_info.dwFileAttributes; inode->i_creation_time = FILETIME_to_u64(&file_info.ftCreationTime); @@ -1299,6 +1299,10 @@ again: state, vol_flags); } + path[path_num_chars] = '\0'; +out_progress_ok: + params->progress.scan.cur_path = path; + do_capture_progress(params, WIMLIB_SCAN_DENTRY_OK, inode); out_close_handle: CloseHandle(hFile); out: