Scan progress: Add # of files, directories, and bytes scanned
authorEric Biggers <ebiggers3@gmail.com>
Tue, 24 Dec 2013 16:36:53 +0000 (10:36 -0600)
committerEric Biggers <ebiggers3@gmail.com>
Tue, 24 Dec 2013 16:36:53 +0000 (10:36 -0600)
doc/imagex-capture.1.in
doc/imagex-extract.1.in
doc/imagex-update.1.in
include/wimlib.h
include/wimlib/capture.h
programs/imagex.c
src/capture_common.c
src/ntfs-3g_capture.c
src/unix_capture.c
src/win32_capture.c

index cedebc4..6a0ed43 100644 (file)
@@ -218,9 +218,6 @@ optimize\fR(1).
 Specify a string to use in the <FLAGS> element of the XML data for the new
 image.
 .TP
-\fB--verbose\fR
-Print the names of files and directories as they are captured.
-.TP
 \fB--dereference\fR
 (UNIX-like systems only) Follow symbolic links and archive the files they point
 to, rather than archiving the links themselves.
index 96e55e6..c3f0103 100644 (file)
@@ -56,9 +56,6 @@ See \fBSPLIT_WIMS\fR.  Note: since \fIGLOB\fR is listed in quotes because it is
 interpreted by \fB@IMAGEX_PROGNAME@\fR and may need to be quoted to protect
 against shell expansion.
 .TP
-\fB--verbose\fR
-This option no longer does anything but is reserved for future use.
-.TP
 \fB--unix-data\fR
 See the documentation for this option in \fB@IMAGEX_PROGNAME@-apply\fR (1).
 .TP
index b274546..39349ac 100644 (file)
@@ -34,7 +34,7 @@ not exist in the WIM image, then any prerequisite directories are created as
 needed to add the \fISOURCE\fR at that location.
 .PP
 The \fBadd\fR command supports a subset of the options accepted by
-\fB@IMAGEX_PROGNAME@-capture\fR; namely, \fB--verbose\fR, \fB--dereference\fR,
+\fB@IMAGEX_PROGNAME@-capture\fR; namely, \fB--dereference\fR,
 \fB--unix-data\fR, \fB--no-acls\fR, and \fB--strict-acls\fR.  See
 \fB@IMAGEX_PROGNAME@-capture\fR (1) for explanations of these options.
 .SS \fBdelete\fR [\fIOPTION\fR...] \fIPATH\fR
@@ -63,9 +63,6 @@ There are no options available for the \fBrename\fR command.
 The following options are accepted on the command line by \fB@IMAGEX_PROGNAME@
 update\fR itself:
 .TP 6
-\fB--verbose\fR
-Use \fB--verbose\fR for all \fBadd\fR commands.
-.TP
 \fB--dereference\fR
 Use \fB--dereference\fR for all \fBadd\fR commands.
 .TP
index 1526a7a..71ef0a8 100644 (file)
@@ -637,6 +637,21 @@ union wimlib_progress_info {
                 * ::WIMLIB_PROGRESS_MSG_SCAN_BEGIN and
                 * ::WIMLIB_PROGRESS_MSG_SCAN_END. */
                const wimlib_tchar *wim_target_path;
+
+               /** Number of directories scanned so far, including the root
+                * directory but excluding any unsupported/excluded directories.
+                * */
+               uint64_t num_dirs_scanned;
+
+               /** Number of non-directories scanned so far, excluding any
+                * unsupported/excluded files.  */
+               uint64_t num_nondirs_scanned;
+
+               /** Number of bytes of file data that have been detected so far.
+                * This data may not actually have been read yet, and it will
+                * not actually be written to the WIM file until wimlib_write()
+                * or wimlib_overwrite() has been called.  */
+               uint64_t num_bytes_scanned;
        } scan;
 
        /** Valid on messages ::WIMLIB_PROGRESS_MSG_EXTRACT_IMAGE_BEGIN,
index 50402cf..9f9e461 100644 (file)
@@ -8,6 +8,7 @@
 
 struct wim_lookup_table;
 struct wim_dentry;
+struct wim_inode;
 
 /* Hash table to find inodes, given an inode number (in the case of reading
  * a WIM images), or both an inode number and a device number (in the case of
@@ -70,7 +71,8 @@ struct add_image_params {
 /* capture_common.c */
 
 extern void
-do_capture_progress(struct add_image_params *params, int status);
+do_capture_progress(struct add_image_params *params, int status,
+                   const struct wim_inode *inode);
 
 extern bool
 exclude_path(const tchar *path, size_t path_len,
index a9e6fa6..b216440 100644 (file)
@@ -1053,6 +1053,34 @@ get_unit(uint64_t total_bytes, const tchar **name_ret)
        }
 }
 
+static struct wimlib_progress_info_scan last_scan_progress;
+
+static void
+report_scan_progress(const struct wimlib_progress_info_scan *scan, bool done)
+{
+       uint64_t prev_count, cur_count;
+
+       prev_count = last_scan_progress.num_nondirs_scanned +
+                    last_scan_progress.num_dirs_scanned;
+       cur_count = scan->num_nondirs_scanned + scan->num_dirs_scanned;
+
+       if (done || prev_count == 0 || cur_count >= prev_count + 100 ||
+           cur_count % 128 == 0)
+       {
+               unsigned unit_shift;
+               const tchar *unit_name;
+
+               unit_shift = get_unit(scan->num_bytes_scanned, &unit_name);
+               imagex_printf(T("\r%"PRIu64" %"TS" scanned (%"PRIu64" files, "
+                               "%"PRIu64" directories)"),
+                             scan->num_bytes_scanned >> unit_shift,
+                             unit_name,
+                             scan->num_nondirs_scanned,
+                             scan->num_dirs_scanned);
+               last_scan_progress = *scan;
+       }
+}
+
 /* Progress callback function passed to various wimlib functions. */
 static int
 imagex_progress_func(enum wimlib_progress_msg msg,
@@ -1061,6 +1089,7 @@ imagex_progress_func(enum wimlib_progress_msg msg,
        unsigned percent_done;
        unsigned unit_shift;
        const tchar *unit_name;
+
        if (imagex_be_quiet)
                return 0;
        switch (msg) {
@@ -1107,23 +1136,28 @@ imagex_progress_func(enum wimlib_progress_msg msg,
                                  "\""WIMLIB_WIM_PATH_SEPARATOR_STRING"%"TS"\")...\n"),
                               info->scan.wim_target_path);
                } else {
-                       imagex_printf(T(" (loading as root of WIM image)...\n"));
+                       imagex_printf(T("\n"));
                }
+               memset(&last_scan_progress, 0, sizeof(last_scan_progress));
                break;
        case WIMLIB_PROGRESS_MSG_SCAN_DENTRY:
                switch (info->scan.status) {
                case WIMLIB_SCAN_DENTRY_OK:
-                       imagex_printf(T("Scanning \"%"TS"\"\n"), info->scan.cur_path);
+                       report_scan_progress(&info->scan, false);
                        break;
                case WIMLIB_SCAN_DENTRY_EXCLUDED:
-                       imagex_printf(T("Excluding \"%"TS"\" from capture\n"), info->scan.cur_path);
+                       imagex_printf(T("\nExcluding \"%"TS"\" from capture\n"), info->scan.cur_path);
                        break;
                case WIMLIB_SCAN_DENTRY_UNSUPPORTED:
-                       imagex_printf(T("WARNING: Excluding unsupported file or directory\n"
+                       imagex_printf(T("\nWARNING: Excluding unsupported file or directory\n"
                                        "         \"%"TS"\" from capture\n"), info->scan.cur_path);
                        break;
                }
                break;
+       case WIMLIB_PROGRESS_MSG_SCAN_END:
+               report_scan_progress(&info->scan, true);
+               imagex_printf(T("\n"));
+               break;
        case WIMLIB_PROGRESS_MSG_VERIFY_INTEGRITY:
                unit_shift = get_unit(info->integrity.total_bytes, &unit_name);
                percent_done = TO_PERCENT(info->integrity.completed_bytes,
@@ -1516,7 +1550,7 @@ imagex_apply(int argc, tchar **argv, int cmd)
                        extract_flags |= WIMLIB_EXTRACT_FLAG_SYMLINK;
                        break;
                case IMAGEX_VERBOSE_OPTION:
-                       extract_flags |= WIMLIB_EXTRACT_FLAG_VERBOSE;
+                       /* No longer does anything.  */
                        break;
                case IMAGEX_REF_OPTION:
                        ret = string_set_append(&refglobs, optarg);
@@ -1673,7 +1707,8 @@ imagex_capture_or_append(int argc, tchar **argv, int cmd)
        int c;
        int open_flags = WIMLIB_OPEN_FLAG_WRITE_ACCESS;
        int add_image_flags = WIMLIB_ADD_IMAGE_FLAG_EXCLUDE_VERBOSE |
-                             WIMLIB_ADD_IMAGE_FLAG_WINCONFIG;
+                             WIMLIB_ADD_IMAGE_FLAG_WINCONFIG |
+                             WIMLIB_ADD_IMAGE_FLAG_VERBOSE;
        int write_flags = 0;
        int compression_type = WIMLIB_COMPRESSION_TYPE_INVALID;
        uint32_t chunk_size = UINT32_MAX;
@@ -1752,7 +1787,7 @@ imagex_capture_or_append(int argc, tchar **argv, int cmd)
                        add_image_flags |= WIMLIB_ADD_IMAGE_FLAG_DEREFERENCE;
                        break;
                case IMAGEX_VERBOSE_OPTION:
-                       add_image_flags |= WIMLIB_ADD_IMAGE_FLAG_VERBOSE;
+                       /* No longer does anything.  */
                        break;
                case IMAGEX_THREADS_OPTION:
                        num_threads = parse_num_threads(optarg);
@@ -2653,7 +2688,7 @@ imagex_extract(int argc, tchar **argv, int cmd)
                        open_flags |= WIMLIB_OPEN_FLAG_CHECK_INTEGRITY;
                        break;
                case IMAGEX_VERBOSE_OPTION:
-                       extract_flags |= WIMLIB_EXTRACT_FLAG_VERBOSE;
+                       /* No longer does anything.  */
                        break;
                case IMAGEX_REF_OPTION:
                        ret = string_set_append(&refglobs, optarg);
@@ -3521,7 +3556,8 @@ imagex_update(int argc, tchar **argv, int cmd)
        int open_flags = WIMLIB_OPEN_FLAG_WRITE_ACCESS;
        int write_flags = 0;
        int update_flags = WIMLIB_UPDATE_FLAG_SEND_PROGRESS;
-       int default_add_flags = WIMLIB_ADD_FLAG_EXCLUDE_VERBOSE;
+       int default_add_flags = WIMLIB_ADD_FLAG_EXCLUDE_VERBOSE |
+                               WIMLIB_ADD_FLAG_VERBOSE;
        int default_delete_flags = 0;
        unsigned num_threads = 0;
        int c;
@@ -3580,7 +3616,7 @@ imagex_update(int argc, tchar **argv, int cmd)
 
                /* Default add options */
                case IMAGEX_VERBOSE_OPTION:
-                       default_add_flags |= WIMLIB_ADD_FLAG_VERBOSE;
+                       /* No longer does anything.  */
                        break;
                case IMAGEX_DEREFERENCE_OPTION:
                        default_add_flags |= WIMLIB_ADD_FLAG_DEREFERENCE;
@@ -3757,7 +3793,7 @@ static const tchar *usage_strings[] = {
 T(
 "    %"TS" (DIRECTORY | NTFS_VOLUME) WIMFILE\n"
 "                    [IMAGE_NAME [IMAGE_DESCRIPTION]] [--boot]\n"
-"                    [--check] [--nocheck] [--flags EDITION_ID] [--verbose]\n"
+"                    [--check] [--nocheck] [--flags EDITION_ID]\n"
 "                    [--dereference] [--config=FILE] [--threads=NUM_THREADS]\n"
 "                    [--source-list] [--no-acls] [--strict-acls] [--rpfix]\n"
 "                    [--norpfix] [--unix-data] [--pipable]\n"
@@ -3767,7 +3803,7 @@ T(
 T(
 "    %"TS" WIMFILE [(IMAGE_NUM | IMAGE_NAME | all)]\n"
 "                    (DIRECTORY | NTFS_VOLUME) [--check] [--hardlink]\n"
-"                    [--symlink] [--verbose] [--ref=\"GLOB\"] [--unix-data]\n"
+"                    [--symlink] [--ref=\"GLOB\"] [--unix-data]\n"
 "                    [--no-acls] [--strict-acls] [--rpfix] [--norpfix]\n"
 "                    [--include-invalid-names]\n"
 ),
@@ -3776,7 +3812,7 @@ T(
 "    %"TS" (DIRECTORY | NTFS_VOLUME) WIMFILE\n"
 "                   [IMAGE_NAME [IMAGE_DESCRIPTION]] [--boot]\n"
 "                    [--check] [--nocheck] [--compress=TYPE]\n"
-"                    [--flags EDITION_ID] [--verbose] [--dereference]\n"
+"                    [--flags EDITION_ID] [--dereference]\n"
 "                    [--config=FILE] [--threads=NUM_THREADS] [--source-list]\n"
 "                    [--no-acls] [--strict-acls] [--rpfix] [--norpfix]\n"
 "                    [--unix-data] [--pipable] [--update-of=[WIMFILE:]IMAGE]\n"
@@ -3802,7 +3838,7 @@ T(
 [CMD_EXTRACT] =
 T(
 "    %"TS" WIMFILE (IMAGE_NUM | IMAGE_NAME) [PATH...]\n"
-"                    [--check] [--ref=\"GLOB\"] [--verbose] [--unix-data]\n"
+"                    [--check] [--ref=\"GLOB\"] [--unix-data]\n"
 "                    [--no-acls] [--strict-acls] [--to-stdout]\n"
 "                    [--dest-dir=CMD_DIR] [--include-invalid-names]\n"
 ),
index 28a830f..c45ae15 100644 (file)
@@ -27,7 +27,9 @@
 
 #include "wimlib/assert.h"
 #include "wimlib/capture.h"
+#include "wimlib/dentry.h"
 #include "wimlib/error.h"
+#include "wimlib/lookup_table.h"
 #include "wimlib/paths.h"
 
 #ifdef __WIN32__
@@ -183,7 +185,8 @@ match_pattern(const tchar *path,
 }
 
 void
-do_capture_progress(struct add_image_params *params, int status)
+do_capture_progress(struct add_image_params *params, int status,
+                   const struct wim_inode *inode)
 {
        switch (status) {
        case WIMLIB_SCAN_DENTRY_OK:
@@ -195,6 +198,18 @@ do_capture_progress(struct add_image_params *params, int status)
                        return;
        }
        params->progress.scan.status = status;
+       if (status == WIMLIB_SCAN_DENTRY_OK && inode->i_nlink == 1) {
+               const struct wim_lookup_table_entry *lte;
+               for (unsigned i = 0; i <= inode->i_num_ads; i++) {
+                       lte = inode_stream_lte_resolved(inode, i);
+                       if (lte != NULL)
+                               params->progress.scan.num_bytes_scanned += lte->size;
+               }
+               if (inode->i_attributes & FILE_ATTRIBUTE_DIRECTORY)
+                       params->progress.scan.num_dirs_scanned++;
+               else
+                       params->progress.scan.num_nondirs_scanned++;
+       }
        if (params->progress_func) {
                params->progress_func(WIMLIB_PROGRESS_MSG_SCAN_DENTRY,
                                      &params->progress);
index b6d9226..444cccb 100644 (file)
@@ -538,12 +538,11 @@ build_dentry_tree_ntfs_recursive(struct wim_dentry **root_ret,
        struct wim_inode *inode;
        ATTR_TYPES stream_type;
 
-       params->progress.scan.cur_path = path;
-
        if (exclude_path(path, path_len, params->config, false)) {
                /* Exclude a file or directory tree based on the capture
-                * configuration file */
-               do_capture_progress(params, WIMLIB_SCAN_DENTRY_EXCLUDED);
+                * configuration file.  */
+               params->progress.scan.cur_path = path;
+               do_capture_progress(params, WIMLIB_SCAN_DENTRY_EXCLUDED, NULL);
                ret = 0;
                goto out;
        }
@@ -565,13 +564,12 @@ build_dentry_tree_ntfs_recursive(struct wim_dentry **root_ret,
                        ret = WIMLIB_ERR_UNSUPPORTED_FILE;
                        goto out;
                }
-               do_capture_progress(params, WIMLIB_SCAN_DENTRY_UNSUPPORTED);
+               params->progress.scan.cur_path = path;
+               do_capture_progress(params, WIMLIB_SCAN_DENTRY_UNSUPPORTED, NULL);
                ret = 0;
                goto out;
        }
 
-       do_capture_progress(params, WIMLIB_SCAN_DENTRY_OK);
-
        /* Create a WIM dentry with an associated inode, which may be shared */
        ret = inode_table_new_dentry(&params->inode_table,
                                     path_basename_with_len(path, path_len),
@@ -587,7 +585,7 @@ build_dentry_tree_ntfs_recursive(struct wim_dentry **root_ret,
        if (inode->i_nlink > 1) {
                /* Shared inode; nothing more to do */
                ret = 0;
-               goto out;
+               goto out_progress_ok;
        }
 
        inode->i_creation_time    = le64_to_cpu(ni->creation_time);
@@ -638,6 +636,7 @@ build_dentry_tree_ntfs_recursive(struct wim_dentry **root_ret,
                if (ret)
                        goto out;
        }
+       path[path_len] = '\0';
 
        /* Reparse-point fixups are a no-op because in NTFS-3g capture mode we
         * only allow capturing an entire volume. */
@@ -681,6 +680,10 @@ build_dentry_tree_ntfs_recursive(struct wim_dentry **root_ret,
                        DEBUG("No security ID for `%s'", path);
                }
        }
+
+out_progress_ok:
+       params->progress.scan.cur_path = path;
+       do_capture_progress(params, WIMLIB_SCAN_DENTRY_OK, inode);
 out:
        if (ret == 0)
                *root_ret = root;
index d16eae6..114d03d 100644 (file)
@@ -128,6 +128,7 @@ unix_capture_directory(struct wim_dentry *dir_dentry,
                if (child)
                        dentry_add_child(dir_dentry, child);
        }
+       path[path_len] = '\0';
        closedir(dir);
        return ret;
 }
@@ -203,44 +204,43 @@ unix_build_dentry_tree_recursive(struct wim_dentry **root_ret,
        struct wim_dentry *root = NULL;
        int ret;
        struct wim_inode *inode;
-
-       params->progress.scan.cur_path = path;
+       struct stat stbuf;
 
        if (exclude_path(path, path_len, params->config, true)) {
-               do_capture_progress(params, WIMLIB_SCAN_DENTRY_EXCLUDED);
+               params->progress.scan.cur_path = path;
+               do_capture_progress(params, WIMLIB_SCAN_DENTRY_EXCLUDED, NULL);
                ret = 0;
                goto out;
        }
 
-       struct stat stbuf;
-       int (*stat_fn)(const char *restrict, struct stat *restrict);
        if ((params->add_flags & WIMLIB_ADD_FLAG_DEREFERENCE) ||
            (params->add_flags & WIMLIB_ADD_FLAG_ROOT))
-               stat_fn = stat;
+               ret = stat(path, &stbuf);
        else
-               stat_fn = lstat;
+               ret = lstat(path, &stbuf);
 
-       ret = (*stat_fn)(path, &stbuf);
        if (ret) {
-               ERROR_WITH_ERRNO("Failed to stat `%s'", path);
+               ERROR_WITH_ERRNO("Failed to stat \"%s\"", path);
                ret = WIMLIB_ERR_STAT;
                goto out;
        }
-       if (!S_ISREG(stbuf.st_mode) && !S_ISDIR(stbuf.st_mode)
-           && !S_ISLNK(stbuf.st_mode)) {
+
+       if (!S_ISREG(stbuf.st_mode) &&
+           !S_ISDIR(stbuf.st_mode) &&
+           !S_ISLNK(stbuf.st_mode))
+       {
                if (params->add_flags & WIMLIB_ADD_FLAG_NO_UNSUPPORTED_EXCLUDE)
                {
                        ERROR("Can't archive unsupported file \"%s\"", path);
                        ret = WIMLIB_ERR_UNSUPPORTED_FILE;
                        goto out;
                }
-               do_capture_progress(params, WIMLIB_SCAN_DENTRY_UNSUPPORTED);
+               params->progress.scan.cur_path = path;
+               do_capture_progress(params, WIMLIB_SCAN_DENTRY_UNSUPPORTED, NULL);
                ret = 0;
                goto out;
        }
 
-       do_capture_progress(params, WIMLIB_SCAN_DENTRY_OK);
-
        ret = inode_table_new_dentry(&params->inode_table,
                                     path_basename_with_len(path, path_len),
                                     stbuf.st_ino, stbuf.st_dev, false, &root);
@@ -252,7 +252,7 @@ unix_build_dentry_tree_recursive(struct wim_dentry **root_ret,
        if (inode->i_nlink > 1) {
                /* Already captured this inode? */
                ret = 0;
-               goto out;
+               goto out_progress_ok;
        }
 
 #ifdef HAVE_STAT_NANOSECOND_PRECISION
@@ -286,6 +286,9 @@ unix_build_dentry_tree_recursive(struct wim_dentry **root_ret,
        if (ret)
                goto out;
 
+out_progress_ok:
+       params->progress.scan.cur_path = path;
+       do_capture_progress(params, WIMLIB_SCAN_DENTRY_OK, inode);
 out:
        if (ret)
                free_dentry_tree(root, params->lookup_table);
index 6e3552f..97855b3 100644 (file)
@@ -1133,7 +1133,6 @@ win32_build_dentry_tree_recursive(struct wim_dentry **root_ret,
        HANDLE hFile;
        DWORD desiredAccess;
 
-       params->progress.scan.cur_path = path;
 
        if (exclude_path(path, path_num_chars, params->config, true)) {
                if (params->add_flags & WIMLIB_ADD_FLAG_ROOT) {
@@ -1141,7 +1140,8 @@ win32_build_dentry_tree_recursive(struct wim_dentry **root_ret,
                        ret = WIMLIB_ERR_INVALID_CAPTURE_CONFIG;
                        goto out;
                }
-               do_capture_progress(params, WIMLIB_SCAN_DENTRY_EXCLUDED);
+               params->progress.scan.cur_path = path;
+               do_capture_progress(params, WIMLIB_SCAN_DENTRY_EXCLUDED, NULL);
                ret = 0;
                goto out;
        }
@@ -1158,8 +1158,6 @@ win32_build_dentry_tree_recursive(struct wim_dentry **root_ret,
        }
 #endif
 
-       do_capture_progress(params, WIMLIB_SCAN_DENTRY_OK);
-
        desiredAccess = FILE_READ_DATA | FILE_READ_ATTRIBUTES |
                        READ_CONTROL | ACCESS_SYSTEM_SECURITY;
 again:
@@ -1232,8 +1230,10 @@ again:
 
        inode = root->d_inode;
 
-       if (inode->i_nlink > 1) /* Shared inode; nothing more to do */
-               goto out_close_handle;
+       if (inode->i_nlink > 1) {
+               /* Shared inode; nothing more to do */
+               goto out_progress_ok;
+       }
 
        inode->i_attributes = file_info.dwFileAttributes;
        inode->i_creation_time = FILETIME_to_u64(&file_info.ftCreationTime);
@@ -1299,6 +1299,10 @@ again:
                                              state,
                                              vol_flags);
        }
+       path[path_num_chars] = '\0';
+out_progress_ok:
+       params->progress.scan.cur_path = path;
+       do_capture_progress(params, WIMLIB_SCAN_DENTRY_OK, inode);
 out_close_handle:
        CloseHandle(hFile);
 out: