Support for file exclusions via progress function
authorEric Biggers <ebiggers3@gmail.com>
Sat, 30 Aug 2014 17:05:30 +0000 (12:05 -0500)
committerEric Biggers <ebiggers3@gmail.com>
Sat, 30 Aug 2014 17:38:36 +0000 (12:38 -0500)
NEWS
include/wimlib.h
include/wimlib/capture.h
src/capture_common.c
src/ntfs-3g_capture.c
src/unix_capture.c
src/update_image.c
src/win32_capture.c

diff --git a/NEWS b/NEWS
index 2a89e4e..70ebfb4 100644 (file)
--- a/NEWS
+++ b/NEWS
@@ -5,13 +5,17 @@ Version 1.7.2-BETA:
 
        Faster "WIMBoot" extraction on Windows.
 
-       Added file count progress data for
-       WIMLIB_PROGRESS_MSG_EXTRACT_FILE_STRUCTURE and
-       WIMLIB_PROGRESS_MSG_EXTRACT_METADATA.
-
        Updated dependent DLLs in Windows release.  libxml2 is slimmed down.
 
-       Some documentation improvements.
+       Library changes:
+               Added file count progress data for
+               WIMLIB_PROGRESS_MSG_EXTRACT_FILE_STRUCTURE and
+               WIMLIB_PROGRESS_MSG_EXTRACT_METADATA.
+
+               Added support for testing file exclusions via the user-provided
+               progress function.
+
+               Some documentation improvements.
 
 Version 1.7.1:
        Made more improvements to the XPRESS, LZX, and LZMS compressors.
index d28fd34..76d63b2 100644 (file)
@@ -700,6 +700,20 @@ enum wimlib_progress_msg {
        /** wimlib_verify_wim() is verifying stream integrity.  @p info will
         * point to ::wimlib_progress_info.verify_streams.  */
        WIMLIB_PROGRESS_MSG_VERIFY_STREAMS = 29,
+
+       /**
+        * The progress function is being asked whether a file should be
+        * excluded from capture or not.  @p info will point to
+        * ::wimlib_progress_info.test_file_exclusion.  This is a bidirectional
+        * message that allows the progress function to set a flag if the file
+        * should be excluded.
+        *
+        * This message is only received if the flag
+        * ::WIMLIB_ADD_FLAG_TEST_FILE_EXCLUSION is used.  This method for file
+        * exclusions is independent of the "capture configuration file"
+        * mechanism.
+        */
+       WIMLIB_PROGRESS_MSG_TEST_FILE_EXCLUSION = 30,
 };
 
 /** Valid return values from user-provided progress functions
@@ -1170,6 +1184,32 @@ union wimlib_progress_info {
                uint64_t completed_streams;
                uint64_t completed_bytes;
        } verify_streams;
+
+       /** Valid on messages ::WIMLIB_PROGRESS_MSG_TEST_FILE_EXCLUSION.  */
+       struct wimlib_progress_info_test_file_exclusion {
+
+               /**
+                * Path to the file for which exclusion is being tested.
+                *
+                * UNIX capture mode:  The path will be a standard relative or
+                * absolute UNIX filesystem path.
+                *
+                * NTFS-3g capture mode:  The path will be given relative to the
+                * root of the NTFS volume, with a leading slash.
+                *
+                * Windows capture mode:  The path will be a Win32 namespace
+                * path to the file.
+                */
+               const wimlib_tchar *path;
+
+               /**
+                * Indicates whether the file or directory will be excluded from
+                * capture or not.  This will be <tt>false</tt> by default.  The
+                * progress function can set this to <tt>true</tt> if it decides
+                * that the file needs to be excluded.
+                */
+               bool will_exclude;
+       } test_file_exclusion;
 };
 
 /**
@@ -1682,6 +1722,15 @@ typedef int (*wimlib_iterate_lookup_table_callback_t)(const struct wimlib_resour
  */
 #define WIMLIB_ADD_FLAG_NO_REPLACE             0x00002000
 
+/**
+ * Send ::WIMLIB_PROGRESS_MSG_TEST_FILE_EXCLUSION messages to the progress
+ * function.
+ *
+ * Note: This method for file exclusions is independent from the capture
+ * configuration file mechanism.
+ */
+#define WIMLIB_ADD_FLAG_TEST_FILE_EXCLUSION 0x00004000
+
 #define WIMLIB_ADD_IMAGE_FLAG_NTFS             WIMLIB_ADD_FLAG_NTFS
 #define WIMLIB_ADD_IMAGE_FLAG_DEREFERENCE      WIMLIB_ADD_FLAG_DEREFERENCE
 #define WIMLIB_ADD_IMAGE_FLAG_VERBOSE          WIMLIB_ADD_FLAG_VERBOSE
index 58e0eac..b30688a 100644 (file)
@@ -52,10 +52,14 @@ struct add_image_params {
        /* Progress data.  */
        union wimlib_progress_info progress;
 
+       /* The capture implementation must set this to the number of characters
+        * that try_exclude() will strip from the path before testing exclusion
+        * patterns from the capture configuration file.  */
+       size_t capture_root_nchars;
+
        /* Can be used by the capture implementation.  */
        u64 capture_root_ino;
        u64 capture_root_dev;
-       size_t capture_root_nchars;
 };
 
 /* capture_common.c */
@@ -78,9 +82,9 @@ extern bool
 match_pattern_list(const tchar *path, size_t path_nchars,
                   const struct string_set *list);
 
-extern bool
-should_exclude_path(const tchar *path, size_t path_nchars,
-                   const struct capture_config *config);
+extern int
+try_exclude(const tchar *full_path, size_t full_path_nchars,
+           const struct add_image_params *params);
 
 typedef int (*capture_tree_t)(struct wim_dentry **, const tchar *,
                              struct add_image_params *);
index def3a13..a0d56bf 100644 (file)
@@ -242,7 +242,7 @@ match_pattern_list(const tchar *path, size_t path_nchars,
  * As a special case, the empty string will be interpreted as a single path
  * separator (which means the root of capture itself).
  */
-bool
+static bool
 should_exclude_path(const tchar *path, size_t path_nchars,
                    const struct capture_config *config)
 {
@@ -262,3 +262,71 @@ should_exclude_path(const tchar *path, size_t path_nchars,
              !match_pattern_list(path, path_nchars, &config->exclusion_exception_pats);
 
 }
+
+/*
+ * Determine if a file should be excluded from capture.
+ *
+ * This function tests exclusions from both of the two possible sources of
+ * exclusions:
+ *
+ *     (1) The capture configuration file
+ *     (2) The user-provided progress function
+ *
+ * The capture implementation must have set params->capture_root_nchars to an
+ * appropriate value.  Example for UNIX:  if the capture root directory is
+ * "foobar/subdir", then all paths will be provided starting with
+ * "foobar/subdir", so params->capture_root_nchars must be set to
+ * strlen("foobar/subdir") so that try_exclude() can use the appropriate suffix
+ * when it calls should_exclude_path().
+ *
+ *
+ * Returns:
+ *     < 0 if excluded
+ *     = 0 if not excluded and no error
+ *     > 0 (wimlib error code) if error
+ */
+int
+try_exclude(const tchar *full_path, size_t full_path_nchars,
+           const struct add_image_params *params)
+{
+       int ret;
+
+       if (should_exclude_path(full_path + params->capture_root_nchars,
+                               full_path_nchars - params->capture_root_nchars,
+                               params->config))
+               return -1;
+
+       if (unlikely(params->add_flags & WIMLIB_ADD_FLAG_TEST_FILE_EXCLUSION)) {
+               union wimlib_progress_info info;
+
+               info.test_file_exclusion.path = full_path;
+               info.test_file_exclusion.will_exclude = false;
+
+       #ifdef __WIN32__
+               /* Hack for Windows...  */
+
+               wchar_t *p_question_mark = NULL;
+
+               if (!wcsncmp(full_path, L"\\??\\", 4)) {
+                       /* Trivial transformation:  NT namespace => Win32 namespace  */
+                       p_question_mark = (wchar_t *)&full_path[1];
+                       *p_question_mark = L'\\';
+               }
+       #endif
+
+               ret = call_progress(params->progfunc, WIMLIB_PROGRESS_MSG_TEST_FILE_EXCLUSION,
+                                   &info, params->progctx);
+
+       #ifdef __WIN32__
+               if (p_question_mark)
+                       *p_question_mark = L'?';
+       #endif
+
+               if (ret)
+                       return ret;
+               if (info.test_file_exclusion.will_exclude)
+                       return -1;
+       }
+
+       return 0;
+}
index b8fb277..5bc9563 100644 (file)
@@ -341,7 +341,7 @@ insert_dos_name(struct dos_name_map *map, const ntfschar *dos_name,
        DEBUG("DOS name_len = %zu", name_nbytes);
        new_node = MALLOC(sizeof(struct dos_name_node));
        if (!new_node)
-               return -1;
+               return WIMLIB_ERR_NOMEM;
 
        /* DOS names are supposed to be 12 characters max (that's 24 bytes,
         * assuming 2-byte ntfs characters) */
@@ -363,7 +363,7 @@ insert_dos_name(struct dos_name_map *map, const ntfschar *dos_name,
                ERROR("NTFS inode %"PRIu64" has multiple DOS names",
                        le64_to_cpu(ntfs_ino));
                FREE(new_node);
-               return -1;
+               return WIMLIB_ERR_NOMEM;
        }
        DEBUG("Inserted DOS name for inode %"PRIu64, le64_to_cpu(ntfs_ino));
        return 0;
@@ -432,6 +432,7 @@ struct readdir_ctx {
        struct dos_name_map *dos_name_map;
        ntfs_volume *vol;
        struct add_image_params *params;
+       int ret;
 };
 
 static int
@@ -513,6 +514,7 @@ out_free_mbs_name:
        FREE(mbs_name);
 out:
        ctx->path[ctx->path_len] = '\0';
+       ctx->ret = ret;
        return ret;
 }
 
@@ -534,11 +536,11 @@ build_dentry_tree_ntfs_recursive(struct wim_dentry **root_ret,
        struct wim_dentry *root = NULL;
        struct wim_inode *inode = NULL;
 
-       if (should_exclude_path(path, path_len, params->config)) {
-               /* Exclude a file or directory tree based on the capture
-                * configuration file.  */
+       ret = try_exclude(path, path_len, params);
+       if (ret < 0) /* Excluded? */
                goto out_progress;
-       }
+       if (ret > 0) /* Error? */
+               goto out;
 
        /* Get file attributes */
        ret = ntfs_get_ntfs_attrib(ni, (char*)&attributes, sizeof(attributes));
@@ -624,11 +626,18 @@ build_dentry_tree_ntfs_recursive(struct wim_dentry **root_ret,
                        .dos_name_map    = &dos_name_map,
                        .vol             = vol,
                        .params          = params,
+                       .ret             = 0,
                };
                ret = ntfs_readdir(ni, &pos, &ctx, wim_ntfs_capture_filldir);
                if (ret) {
-                       ERROR_WITH_ERRNO("Error reading directory \"%s\"", path);
-                       ret = WIMLIB_ERR_NTFS_3G;
+                       if (ctx.ret) {
+                               /* wimlib error  */
+                               ret = ctx.ret;
+                       } else {
+                               /* error from ntfs_readdir() itself  */
+                               ERROR_WITH_ERRNO("Error reading directory \"%s\"", path);
+                               ret = WIMLIB_ERR_NTFS_3G;
+                       }
                } else {
                        struct wim_dentry *child;
 
index 15e1d15..ec2b379 100644 (file)
@@ -335,10 +335,11 @@ unix_build_dentry_tree_recursive(struct wim_dentry **tree_ret,
        struct stat stbuf;
        int stat_flags;
 
-       if (should_exclude_path(full_path + params->capture_root_nchars,
-                               full_path_len - params->capture_root_nchars,
-                               params->config))
+       ret = try_exclude(full_path, full_path_len, params);
+       if (ret < 0) /* Excluded? */
                goto out_progress;
+       if (ret > 0) /* Error? */
+               goto out;
 
        if (params->add_flags & (WIMLIB_ADD_FLAG_DEREFERENCE |
                                 WIMLIB_ADD_FLAG_ROOT))
index 2577e03..e2d33d5 100644 (file)
@@ -1224,7 +1224,8 @@ check_add_command(struct wimlib_update_command *cmd,
                          WIMLIB_ADD_FLAG_NO_UNSUPPORTED_EXCLUDE |
                          WIMLIB_ADD_FLAG_WINCONFIG |
                          WIMLIB_ADD_FLAG_WIMBOOT |
-                         WIMLIB_ADD_FLAG_NO_REPLACE))
+                         WIMLIB_ADD_FLAG_NO_REPLACE |
+                         WIMLIB_ADD_FLAG_TEST_FILE_EXCLUSION))
                return WIMLIB_ERR_INVALID_PARAM;
 
        bool is_entire_image = WIMLIB_IS_WIM_ROOT_PATH(cmd->add.wim_target_path);
index ad40d45..71a5936 100644 (file)
@@ -1097,10 +1097,11 @@ winnt_build_dentry_tree_recursive(struct wim_dentry **root_ret,
        u16 rpbuflen;
        u16 not_rpfixed;
 
-       if (should_exclude_path(full_path + params->capture_root_nchars,
-                               full_path_nchars - params->capture_root_nchars,
-                               params->config))
+       ret = try_exclude(full_path, full_path_nchars, params);
+       if (ret < 0) /* Excluded? */
                goto out_progress;
+       if (ret > 0) /* Error? */
+               goto out;
 
        /* Open the file.  */
        status = winnt_openat(cur_dir,