From c64549817d956075c0489661eac56128ee6fa630 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Sat, 30 Aug 2014 12:05:30 -0500 Subject: [PATCH] Support for file exclusions via progress function --- NEWS | 14 +++++--- include/wimlib.h | 49 ++++++++++++++++++++++++++++ include/wimlib/capture.h | 12 ++++--- src/capture_common.c | 70 +++++++++++++++++++++++++++++++++++++++- src/ntfs-3g_capture.c | 25 +++++++++----- src/unix_capture.c | 7 ++-- src/update_image.c | 3 +- src/win32_capture.c | 7 ++-- 8 files changed, 162 insertions(+), 25 deletions(-) diff --git a/NEWS b/NEWS index 2a89e4e4..70ebfb4f 100644 --- a/NEWS +++ b/NEWS @@ -5,13 +5,17 @@ Version 1.7.2-BETA: Faster "WIMBoot" extraction on Windows. - Added file count progress data for - WIMLIB_PROGRESS_MSG_EXTRACT_FILE_STRUCTURE and - WIMLIB_PROGRESS_MSG_EXTRACT_METADATA. - Updated dependent DLLs in Windows release. libxml2 is slimmed down. - Some documentation improvements. + Library changes: + Added file count progress data for + WIMLIB_PROGRESS_MSG_EXTRACT_FILE_STRUCTURE and + WIMLIB_PROGRESS_MSG_EXTRACT_METADATA. + + Added support for testing file exclusions via the user-provided + progress function. + + Some documentation improvements. Version 1.7.1: Made more improvements to the XPRESS, LZX, and LZMS compressors. diff --git a/include/wimlib.h b/include/wimlib.h index d28fd344..76d63b26 100644 --- a/include/wimlib.h +++ b/include/wimlib.h @@ -700,6 +700,20 @@ enum wimlib_progress_msg { /** wimlib_verify_wim() is verifying stream integrity. @p info will * point to ::wimlib_progress_info.verify_streams. */ WIMLIB_PROGRESS_MSG_VERIFY_STREAMS = 29, + + /** + * The progress function is being asked whether a file should be + * excluded from capture or not. @p info will point to + * ::wimlib_progress_info.test_file_exclusion. This is a bidirectional + * message that allows the progress function to set a flag if the file + * should be excluded. + * + * This message is only received if the flag + * ::WIMLIB_ADD_FLAG_TEST_FILE_EXCLUSION is used. This method for file + * exclusions is independent of the "capture configuration file" + * mechanism. + */ + WIMLIB_PROGRESS_MSG_TEST_FILE_EXCLUSION = 30, }; /** Valid return values from user-provided progress functions @@ -1170,6 +1184,32 @@ union wimlib_progress_info { uint64_t completed_streams; uint64_t completed_bytes; } verify_streams; + + /** Valid on messages ::WIMLIB_PROGRESS_MSG_TEST_FILE_EXCLUSION. */ + struct wimlib_progress_info_test_file_exclusion { + + /** + * Path to the file for which exclusion is being tested. + * + * UNIX capture mode: The path will be a standard relative or + * absolute UNIX filesystem path. + * + * NTFS-3g capture mode: The path will be given relative to the + * root of the NTFS volume, with a leading slash. + * + * Windows capture mode: The path will be a Win32 namespace + * path to the file. + */ + const wimlib_tchar *path; + + /** + * Indicates whether the file or directory will be excluded from + * capture or not. This will be false by default. The + * progress function can set this to true if it decides + * that the file needs to be excluded. + */ + bool will_exclude; + } test_file_exclusion; }; /** @@ -1682,6 +1722,15 @@ typedef int (*wimlib_iterate_lookup_table_callback_t)(const struct wimlib_resour */ #define WIMLIB_ADD_FLAG_NO_REPLACE 0x00002000 +/** + * Send ::WIMLIB_PROGRESS_MSG_TEST_FILE_EXCLUSION messages to the progress + * function. + * + * Note: This method for file exclusions is independent from the capture + * configuration file mechanism. + */ +#define WIMLIB_ADD_FLAG_TEST_FILE_EXCLUSION 0x00004000 + #define WIMLIB_ADD_IMAGE_FLAG_NTFS WIMLIB_ADD_FLAG_NTFS #define WIMLIB_ADD_IMAGE_FLAG_DEREFERENCE WIMLIB_ADD_FLAG_DEREFERENCE #define WIMLIB_ADD_IMAGE_FLAG_VERBOSE WIMLIB_ADD_FLAG_VERBOSE diff --git a/include/wimlib/capture.h b/include/wimlib/capture.h index 58e0eac3..b30688a4 100644 --- a/include/wimlib/capture.h +++ b/include/wimlib/capture.h @@ -52,10 +52,14 @@ struct add_image_params { /* Progress data. */ union wimlib_progress_info progress; + /* The capture implementation must set this to the number of characters + * that try_exclude() will strip from the path before testing exclusion + * patterns from the capture configuration file. */ + size_t capture_root_nchars; + /* Can be used by the capture implementation. */ u64 capture_root_ino; u64 capture_root_dev; - size_t capture_root_nchars; }; /* capture_common.c */ @@ -78,9 +82,9 @@ extern bool match_pattern_list(const tchar *path, size_t path_nchars, const struct string_set *list); -extern bool -should_exclude_path(const tchar *path, size_t path_nchars, - const struct capture_config *config); +extern int +try_exclude(const tchar *full_path, size_t full_path_nchars, + const struct add_image_params *params); typedef int (*capture_tree_t)(struct wim_dentry **, const tchar *, struct add_image_params *); diff --git a/src/capture_common.c b/src/capture_common.c index def3a13f..a0d56bfd 100644 --- a/src/capture_common.c +++ b/src/capture_common.c @@ -242,7 +242,7 @@ match_pattern_list(const tchar *path, size_t path_nchars, * As a special case, the empty string will be interpreted as a single path * separator (which means the root of capture itself). */ -bool +static bool should_exclude_path(const tchar *path, size_t path_nchars, const struct capture_config *config) { @@ -262,3 +262,71 @@ should_exclude_path(const tchar *path, size_t path_nchars, !match_pattern_list(path, path_nchars, &config->exclusion_exception_pats); } + +/* + * Determine if a file should be excluded from capture. + * + * This function tests exclusions from both of the two possible sources of + * exclusions: + * + * (1) The capture configuration file + * (2) The user-provided progress function + * + * The capture implementation must have set params->capture_root_nchars to an + * appropriate value. Example for UNIX: if the capture root directory is + * "foobar/subdir", then all paths will be provided starting with + * "foobar/subdir", so params->capture_root_nchars must be set to + * strlen("foobar/subdir") so that try_exclude() can use the appropriate suffix + * when it calls should_exclude_path(). + * + * + * Returns: + * < 0 if excluded + * = 0 if not excluded and no error + * > 0 (wimlib error code) if error + */ +int +try_exclude(const tchar *full_path, size_t full_path_nchars, + const struct add_image_params *params) +{ + int ret; + + if (should_exclude_path(full_path + params->capture_root_nchars, + full_path_nchars - params->capture_root_nchars, + params->config)) + return -1; + + if (unlikely(params->add_flags & WIMLIB_ADD_FLAG_TEST_FILE_EXCLUSION)) { + union wimlib_progress_info info; + + info.test_file_exclusion.path = full_path; + info.test_file_exclusion.will_exclude = false; + + #ifdef __WIN32__ + /* Hack for Windows... */ + + wchar_t *p_question_mark = NULL; + + if (!wcsncmp(full_path, L"\\??\\", 4)) { + /* Trivial transformation: NT namespace => Win32 namespace */ + p_question_mark = (wchar_t *)&full_path[1]; + *p_question_mark = L'\\'; + } + #endif + + ret = call_progress(params->progfunc, WIMLIB_PROGRESS_MSG_TEST_FILE_EXCLUSION, + &info, params->progctx); + + #ifdef __WIN32__ + if (p_question_mark) + *p_question_mark = L'?'; + #endif + + if (ret) + return ret; + if (info.test_file_exclusion.will_exclude) + return -1; + } + + return 0; +} diff --git a/src/ntfs-3g_capture.c b/src/ntfs-3g_capture.c index b8fb2779..5bc9563e 100644 --- a/src/ntfs-3g_capture.c +++ b/src/ntfs-3g_capture.c @@ -341,7 +341,7 @@ insert_dos_name(struct dos_name_map *map, const ntfschar *dos_name, DEBUG("DOS name_len = %zu", name_nbytes); new_node = MALLOC(sizeof(struct dos_name_node)); if (!new_node) - return -1; + return WIMLIB_ERR_NOMEM; /* DOS names are supposed to be 12 characters max (that's 24 bytes, * assuming 2-byte ntfs characters) */ @@ -363,7 +363,7 @@ insert_dos_name(struct dos_name_map *map, const ntfschar *dos_name, ERROR("NTFS inode %"PRIu64" has multiple DOS names", le64_to_cpu(ntfs_ino)); FREE(new_node); - return -1; + return WIMLIB_ERR_NOMEM; } DEBUG("Inserted DOS name for inode %"PRIu64, le64_to_cpu(ntfs_ino)); return 0; @@ -432,6 +432,7 @@ struct readdir_ctx { struct dos_name_map *dos_name_map; ntfs_volume *vol; struct add_image_params *params; + int ret; }; static int @@ -513,6 +514,7 @@ out_free_mbs_name: FREE(mbs_name); out: ctx->path[ctx->path_len] = '\0'; + ctx->ret = ret; return ret; } @@ -534,11 +536,11 @@ build_dentry_tree_ntfs_recursive(struct wim_dentry **root_ret, struct wim_dentry *root = NULL; struct wim_inode *inode = NULL; - if (should_exclude_path(path, path_len, params->config)) { - /* Exclude a file or directory tree based on the capture - * configuration file. */ + ret = try_exclude(path, path_len, params); + if (ret < 0) /* Excluded? */ goto out_progress; - } + if (ret > 0) /* Error? */ + goto out; /* Get file attributes */ ret = ntfs_get_ntfs_attrib(ni, (char*)&attributes, sizeof(attributes)); @@ -624,11 +626,18 @@ build_dentry_tree_ntfs_recursive(struct wim_dentry **root_ret, .dos_name_map = &dos_name_map, .vol = vol, .params = params, + .ret = 0, }; ret = ntfs_readdir(ni, &pos, &ctx, wim_ntfs_capture_filldir); if (ret) { - ERROR_WITH_ERRNO("Error reading directory \"%s\"", path); - ret = WIMLIB_ERR_NTFS_3G; + if (ctx.ret) { + /* wimlib error */ + ret = ctx.ret; + } else { + /* error from ntfs_readdir() itself */ + ERROR_WITH_ERRNO("Error reading directory \"%s\"", path); + ret = WIMLIB_ERR_NTFS_3G; + } } else { struct wim_dentry *child; diff --git a/src/unix_capture.c b/src/unix_capture.c index 15e1d153..ec2b3796 100644 --- a/src/unix_capture.c +++ b/src/unix_capture.c @@ -335,10 +335,11 @@ unix_build_dentry_tree_recursive(struct wim_dentry **tree_ret, struct stat stbuf; int stat_flags; - if (should_exclude_path(full_path + params->capture_root_nchars, - full_path_len - params->capture_root_nchars, - params->config)) + ret = try_exclude(full_path, full_path_len, params); + if (ret < 0) /* Excluded? */ goto out_progress; + if (ret > 0) /* Error? */ + goto out; if (params->add_flags & (WIMLIB_ADD_FLAG_DEREFERENCE | WIMLIB_ADD_FLAG_ROOT)) diff --git a/src/update_image.c b/src/update_image.c index 2577e037..e2d33d58 100644 --- a/src/update_image.c +++ b/src/update_image.c @@ -1224,7 +1224,8 @@ check_add_command(struct wimlib_update_command *cmd, WIMLIB_ADD_FLAG_NO_UNSUPPORTED_EXCLUDE | WIMLIB_ADD_FLAG_WINCONFIG | WIMLIB_ADD_FLAG_WIMBOOT | - WIMLIB_ADD_FLAG_NO_REPLACE)) + WIMLIB_ADD_FLAG_NO_REPLACE | + WIMLIB_ADD_FLAG_TEST_FILE_EXCLUSION)) return WIMLIB_ERR_INVALID_PARAM; bool is_entire_image = WIMLIB_IS_WIM_ROOT_PATH(cmd->add.wim_target_path); diff --git a/src/win32_capture.c b/src/win32_capture.c index ad40d452..71a59369 100644 --- a/src/win32_capture.c +++ b/src/win32_capture.c @@ -1097,10 +1097,11 @@ winnt_build_dentry_tree_recursive(struct wim_dentry **root_ret, u16 rpbuflen; u16 not_rpfixed; - if (should_exclude_path(full_path + params->capture_root_nchars, - full_path_nchars - params->capture_root_nchars, - params->config)) + ret = try_exclude(full_path, full_path_nchars, params); + if (ret < 0) /* Excluded? */ goto out_progress; + if (ret > 0) /* Error? */ + goto out; /* Open the file. */ status = winnt_openat(cur_dir, -- 2.43.0