From: Eric Biggers Date: Wed, 22 Jul 2015 01:28:59 +0000 (-0500) Subject: add System Compression support X-Git-Tag: v1.8.2~35 X-Git-Url: https://wimlib.net/git/?p=wimlib;a=commitdiff_plain;h=8b709192cd2811b83c248fbe61ca4f11ee9de797;hp=12e7f4e2f9e3254e97a67cfb2b77bbb0d166a2f2 add System Compression support --- diff --git a/NEWS b/NEWS index 0917d65e..3eb7fef9 100644 --- a/NEWS +++ b/NEWS @@ -1,6 +1,6 @@ Version 1.8.2-BETA: - This release contains various minor bug fixes and improvements, - including: + This release primarily contains various minor bug fixes and + improvements, including: Reduced the stack space used by directory tree scans. @@ -21,6 +21,11 @@ Version 1.8.2-BETA: Optimized memory usage of some data structures. + In addition, *experimental* support has been added for compressing + extracted files using System Compression on Windows 10. This + functionality is available through the new '--compact' option to + 'wimapply' and 'wimextract' as well as new library flags. + Version 1.8.1: Fixed a bug in the LZX decompressor: malicious input data could cause out of bounds writes to memory (since wimlib v1.2.2). diff --git a/doc/man1/wimlib-imagex-apply.1 b/doc/man1/wimlib-imagex-apply.1 index bab6702a..64067b25 100644 --- a/doc/man1/wimlib-imagex-apply.1 +++ b/doc/man1/wimlib-imagex-apply.1 @@ -387,6 +387,18 @@ WIM files that use XPRESS chunks of size 8192, 16384, and 32768, or LZX chunks of size 32768, in addition to the default XPRESS chunks of size 4096 that are created when \fBwimlib-imagex capture\fR is run with the \fB--wimboot\fR option. +.TP +\fB--compact\fR=\fIFORMAT\fR +EXPERIMENTAL and only works on Windows 10 or later: compress the extracted files +using the System Compression feature, when possible. System Compression is only +supported by Windows 10 or later. Several different compression formats may be +used with System Compression, and one must be specified as \fIFORMAT\fR. The +choices are: xpress4k, xpress8k, xpress16k, and lzx. +.IP "" +Exclusions are handled in the same way as with the \fB--wimboot\fR option. +That is: if it exists, the [PrepopulateList] section of the file +\\Windows\\System32\\WimBootCompress.ini in the WIM image will be read, and +files matching any of the patterns in this section will not be compressed. .SH NOTES \fIData integrity\fR: WIM files include SHA1 message digests for file data. \fBwimlib-imagex apply\fR calculates the SHA1 message digest of every file diff --git a/doc/man1/wimlib-imagex-extract.1 b/doc/man1/wimlib-imagex-extract.1 index e21f4e2a..d4648de3 100644 --- a/doc/man1/wimlib-imagex-extract.1 +++ b/doc/man1/wimlib-imagex-extract.1 @@ -151,6 +151,9 @@ command line. .TP \fB--wimboot\fR See the documentation for this option in \fBwimlib-imagex-apply\fR (1). +.TP +\fB--compact\fR=\fIFORMAT\fR +See the documentation for this option in \fBwimlib-imagex-apply\fR (1). .SH NOTES See the documentation \fBwimlib-imagex apply\fR (1) for documentation about what data and metadata are extracted on UNIX-like systems versus on Windows. diff --git a/include/wimlib.h b/include/wimlib.h index 7496973a..63cc3e20 100644 --- a/include/wimlib.h +++ b/include/wimlib.h @@ -1944,6 +1944,30 @@ typedef int (*wimlib_iterate_lookup_table_callback_t)(const struct wimlib_resour */ #define WIMLIB_EXTRACT_FLAG_WIMBOOT 0x00400000 +/** EXPERIMENTAL and only works on Windows 10 and later: compress the extracted + * files using the System Compression feature (when possible). System + * Compression is only supported by Windows 10 or later. Several different + * compression formats may be used with System Compression; this particular flag + * selects the XPRESS compression format with 4096 byte chunks. This flag is + * currently experimental and may be changed or removed in future releases of + * wimlib. */ +#define WIMLIB_EXTRACT_FLAG_COMPACT_XPRESS4K 0x01000000 + +/** Like ::WIMLIB_EXTRACT_FLAG_COMPACT_XPRESS4K, but use XPRESS compression with + * 8192 byte chunks. This flag is currently experimental and may be changed + * or removed in future releases of wimlib. */ +#define WIMLIB_EXTRACT_FLAG_COMPACT_XPRESS8K 0x02000000 + +/** Like ::WIMLIB_EXTRACT_FLAG_COMPACT_XPRESS4K, but use XPRESS compression with + * 16384 byte chunks. This flag is currently experimental and may be changed + * or removed in future releases of wimlib. */ +#define WIMLIB_EXTRACT_FLAG_COMPACT_XPRESS16K 0x04000000 + +/** Like ::WIMLIB_EXTRACT_FLAG_COMPACT_XPRESS4K, but use LZX compression with + * 32768 byte chunks. This flag is currently experimental and may be changed + * or removed in future releases of wimlib. */ +#define WIMLIB_EXTRACT_FLAG_COMPACT_LZX 0x08000000 + /** @} */ /** @addtogroup G_mounting_wim_images * @{ */ diff --git a/include/wimlib/apply.h b/include/wimlib/apply.h index 071f5ed4..f36f0b3f 100644 --- a/include/wimlib/apply.h +++ b/include/wimlib/apply.h @@ -220,9 +220,10 @@ struct apply_operations { /* * Query whether the unnamed data stream of the specified file will be - * extracted as "externally backed". If so, the extraction backend is - * assumed to handle this separately, and the common extraction code - * will not register a usage of the unnamed data stream's blob. + * extracted as "externally backed" from the WIM archive itself. If so, + * then the extraction backend is assumed to handle this separately, and + * the common extraction code will not register a usage of the unnamed + * data stream's blob. * * This routine is optional. * @@ -231,7 +232,7 @@ struct apply_operations { * = 0 if the file will be externally backed. * > 0 (wimlib error code) if another error occurred. */ - int (*will_externally_back)(struct wim_dentry *dentry, struct apply_ctx *ctx); + int (*will_back_from_wim)(struct wim_dentry *dentry, struct apply_ctx *ctx); /* * Size of the backend-specific extraction context. It must contain diff --git a/include/wimlib/wof.h b/include/wimlib/wof.h index 0715211d..71d97d80 100644 --- a/include/wimlib/wof.h +++ b/include/wimlib/wof.h @@ -1,9 +1,10 @@ /* * wof.h * - * Definitions for Windows Overlay File System Filter (WOF) ioctls. See + * Definitions for the Windows Overlay File System Filter (WOF) ioctls, as well + * some definitions for associated undocumented data structures. See * http://msdn.microsoft.com/en-us/library/windows/hardware/ff540367(v=vs.85).aspx - * for more information. + * for more information about the documented ioctls. * * The author dedicates this file to the public domain. * You can do whatever you want with this file. @@ -15,9 +16,55 @@ #include "wimlib/compiler.h" #include "wimlib/types.h" -#define WOF_CURRENT_VERSION 1 -#define WOF_PROVIDER_WIM 1 -#define WIM_PROVIDER_CURRENT_VERSION 1 +/* + * The Windows Overlay FileSystem Filter (WOF, a.k.a. wof.sys) is a filesystem + * filter driver, available in Windows 8.1 and later, which allows files to be + * "externally backed", meaning that their data is stored in another location, + * possibly in compressed form. + * + * WOF implements a plug-in mechanism by which a specific "provider" is + * responsible for actually externally backing a given file. The currently + * known providers are: + * + * - The WIM provider: allows a file to be externally backed by a + * compressed resource in a WIM archive + * - The file provider: allows a file to be "externally backed" by a named + * data stream stored with the file itself, where that named data stream + * has the format of a compressed WIM resource + * + * For both of these providers, externally backed files are effectively + * read-only. If you try to write to such a file, Windows automatically + * decompresses it and turns it into a regular, non-externally-backed file. + * + * WOF provides various ioctls that control its operation. For example, + * FSCTL_SET_EXTERNAL_BACKING sets up a file as externally backed. + * + * WOF external backings are implemented using reparse points. One consequence + * of this is that WOF external backings can only be set on files that do not + * already have a reparse point set. Another consequence of this is that it is + * possible to create a WOF external backing by manually creating the reparse + * point, although this requires dealing with undocumented data structures and + * it only works when the WOF driver is not currently attached to the volume. + * + * Note that only the unnamed data stream portion of a file can be externally + * backed. Other NTFS streams and metadata are not externally backed. + */ + + +/* Current version of the WOF driver/protocol */ +#define WOF_CURRENT_VERSION 1 + +/* Specifies the WIM backing provider */ +#define WOF_PROVIDER_WIM 1 + +/* Specifies the "file" backing provider (a.k.a. System Compression) */ +#define WOF_PROVIDER_FILE 2 + +/* The current version of the WIM backing provider */ +#define WIM_PROVIDER_CURRENT_VERSION 1 + +/* The current version of the file backing provider */ +#define FILE_PROVIDER_CURRENT_VERSION 1 /* Identifies a backing provider for a specific overlay service version. */ struct wof_external_info { @@ -31,25 +78,16 @@ struct wof_external_info { u32 provider; }; -/* On Windows, WOF reparse points can only be directly created when the WOF - * driver is NOT running on the volume. Otherwise, the WOF ioctl - * (FSCTL_SET_EXTERNAL_BACKING, FSCTL_GET_EXTERNAL_BACKING, - * FSCTL_DELETE_EXTERNAL_BACKING) must be used instead. */ /* - * Format of the reparse data of WoF (Windows Overlay File System Filter) - * reparse points. These include WIMBoot "pointer files". - * - * This is not documented by Microsoft!!! - * - * Notes: - * - 'struct wim_provider_rpdata' must be preceded by - * 'struct wof_external_info'. - * - Reparse tag is 0x80000017 - * - Don't make these if the file has no unnamed data stream, has an empty - * unnamed data stream, or already is a reparse point. - * - There is nowhere to put named data streams. They have to copied - * literally to the reparse point file. + * Format of the WIM provider reparse data. This is the data which follows the + * portion of the reparse point common to WOF. (The common portion consists of + * a reparse point header where the reparse tag is 0x80000017, then a 'struct + * wof_external_info' which specifies the provider.) + * + * Note that Microsoft does not document any of the reparse point formats for + * WOF, although they document the structures which must be passed into the + * ioctls, which are often similar. */ struct wim_provider_rpdata { /* Set to 2. Uncertain meaning. */ @@ -297,6 +335,18 @@ struct wim_provider_external_info { u8 unnamed_data_stream_hash[20]; }; +struct file_provider_external_info { + + /* Set to FILE_PROVIDER_CURRENT_VERSION. */ + u32 version; + + u32 compression_format; +#define FILE_PROVIDER_COMPRESSION_FORMAT_XPRESS4K 0 +#define FILE_PROVIDER_COMPRESSION_FORMAT_LZX 1 +#define FILE_PROVIDER_COMPRESSION_FORMAT_XPRESS8K 2 +#define FILE_PROVIDER_COMPRESSION_FORMAT_XPRESS16K 3 +}; + /***************************************************************************** * * --- FSCTL_GET_EXTERNAL_BACKING --- diff --git a/programs/imagex.c b/programs/imagex.c index 816ea4fb..556006f6 100644 --- a/programs/imagex.c +++ b/programs/imagex.c @@ -147,6 +147,7 @@ enum { IMAGEX_CHUNK_SIZE_OPTION, IMAGEX_COMMAND_OPTION, IMAGEX_COMMIT_OPTION, + IMAGEX_COMPACT_OPTION, IMAGEX_COMPRESS_OPTION, IMAGEX_COMPRESS_SLOW_OPTION, IMAGEX_CONFIG_OPTION, @@ -216,6 +217,7 @@ static const struct option apply_options[] = { /* --resume is undocumented for now as it needs improvement. */ {T("resume"), no_argument, NULL, IMAGEX_RESUME_OPTION}, {T("wimboot"), no_argument, NULL, IMAGEX_WIMBOOT_OPTION}, + {T("compact"), required_argument, NULL, IMAGEX_COMPACT_OPTION}, {NULL, 0, NULL, 0}, }; @@ -310,6 +312,7 @@ static const struct option extract_options[] = { {T("nullglob"), no_argument, NULL, IMAGEX_NULLGLOB_OPTION}, {T("preserve-dir-structure"), no_argument, NULL, IMAGEX_PRESERVE_DIR_STRUCTURE_OPTION}, {T("wimboot"), no_argument, NULL, IMAGEX_WIMBOOT_OPTION}, + {T("compact"), required_argument, NULL, IMAGEX_COMPACT_OPTION}, {NULL, 0, NULL, 0}, }; @@ -547,6 +550,37 @@ get_compression_type(tchar *optarg) return ctype; } +/* Parse the argument to --compact */ +static int +set_compact_mode(const tchar *arg, int *extract_flags) +{ + int flag = 0; + if (!tstrcasecmp(arg, T("xpress4k"))) + flag = WIMLIB_EXTRACT_FLAG_COMPACT_XPRESS4K; + else if (!tstrcasecmp(arg, T("xpress8k"))) + flag = WIMLIB_EXTRACT_FLAG_COMPACT_XPRESS8K; + else if (!tstrcasecmp(arg, T("xpress16k"))) + flag = WIMLIB_EXTRACT_FLAG_COMPACT_XPRESS16K; + else if (!tstrcasecmp(arg, T("lzx"))) + flag = WIMLIB_EXTRACT_FLAG_COMPACT_LZX; + + if (flag) { + *extract_flags |= flag; + return 0; + } + + imagex_error(T( +"\"%"TS"\" is not a recognized System Compression format. The options are:" +"\n" +" --compact=xpress4k\n" +" --compact=xpress8k\n" +" --compact=xpress16k\n" +" --compact=lzx\n" + ), arg); + return -1; +} + + static void set_compress_slow(void) { @@ -1611,6 +1645,11 @@ imagex_apply(int argc, tchar **argv, int cmd) case IMAGEX_WIMBOOT_OPTION: extract_flags |= WIMLIB_EXTRACT_FLAG_WIMBOOT; break; + case IMAGEX_COMPACT_OPTION: + ret = set_compact_mode(optarg, &extract_flags); + if (ret) + goto out_free_refglobs; + break; default: goto out_usage; } @@ -3029,6 +3068,11 @@ imagex_extract(int argc, tchar **argv, int cmd) case IMAGEX_WIMBOOT_OPTION: extract_flags |= WIMLIB_EXTRACT_FLAG_WIMBOOT; break; + case IMAGEX_COMPACT_OPTION: + ret = set_compact_mode(optarg, &extract_flags); + if (ret) + goto out_free_refglobs; + break; default: goto out_usage; } @@ -4212,6 +4256,7 @@ T( " [--check] [--ref=\"GLOB\"] [--no-acls] [--strict-acls]\n" " [--no-attributes] [--rpfix] [--norpfix]\n" " [--include-invalid-names] [--wimboot] [--unix-data]\n" +" [--compact=FORMAT]\n" ), [CMD_CAPTURE] = T( diff --git a/src/extract.c b/src/extract.c index 4f1c54a0..e80bbe94 100644 --- a/src/extract.c +++ b/src/extract.c @@ -84,7 +84,12 @@ WIMLIB_EXTRACT_FLAG_STRICT_GLOB | \ WIMLIB_EXTRACT_FLAG_NO_ATTRIBUTES | \ WIMLIB_EXTRACT_FLAG_NO_PRESERVE_DIR_STRUCTURE | \ - WIMLIB_EXTRACT_FLAG_WIMBOOT) + WIMLIB_EXTRACT_FLAG_WIMBOOT | \ + WIMLIB_EXTRACT_FLAG_COMPACT_XPRESS4K | \ + WIMLIB_EXTRACT_FLAG_COMPACT_XPRESS8K | \ + WIMLIB_EXTRACT_FLAG_COMPACT_XPRESS16K | \ + WIMLIB_EXTRACT_FLAG_COMPACT_LZX \ + ) /* Send WIMLIB_PROGRESS_MSG_EXTRACT_FILE_STRUCTURE or * WIMLIB_PROGRESS_MSG_EXTRACT_METADATA. */ @@ -973,10 +978,11 @@ ref_stream_if_needed(struct wim_dentry *dentry, struct wim_inode *inode, * - file is a directory * - file is encrypted * - backend needs to create the file as UNIX symlink - * - backend will extract the stream as externally backed + * - backend will extract the stream as externally + * backed from the WIM archive itself */ - if (ctx->apply_ops->will_externally_back) { - int ret = (*ctx->apply_ops->will_externally_back)(dentry, ctx); + if (ctx->apply_ops->will_back_from_wim) { + int ret = (*ctx->apply_ops->will_back_from_wim)(dentry, ctx); if (ret > 0) /* Error? */ return ret; if (ret < 0) /* Won't externally back? */ @@ -1479,6 +1485,34 @@ check_extract_flags(const WIMStruct *wim, int *extract_flags_p) #endif } + if (extract_flags & (WIMLIB_EXTRACT_FLAG_COMPACT_XPRESS4K | + WIMLIB_EXTRACT_FLAG_COMPACT_XPRESS8K | + WIMLIB_EXTRACT_FLAG_COMPACT_XPRESS16K | + WIMLIB_EXTRACT_FLAG_COMPACT_LZX)) + { + #ifdef __WIN32__ + int count = 0; + count += ((extract_flags & WIMLIB_EXTRACT_FLAG_COMPACT_XPRESS4K) != 0); + count += ((extract_flags & WIMLIB_EXTRACT_FLAG_COMPACT_XPRESS8K) != 0); + count += ((extract_flags & WIMLIB_EXTRACT_FLAG_COMPACT_XPRESS16K) != 0); + count += ((extract_flags & WIMLIB_EXTRACT_FLAG_COMPACT_LZX) != 0); + if (count != 1) { + ERROR("Only one compression format can be specified " + "for compact-mode extraction!"); + return WIMLIB_ERR_INVALID_PARAM; + } + if (extract_flags & WIMLIB_EXTRACT_FLAG_WIMBOOT) { + ERROR("Compact-mode extraction and WIMBoot-mode " + "extraction are mutually exclusive!"); + return WIMLIB_ERR_INVALID_PARAM; + } + #else + ERROR("Compact-mode extraction (System Compression) " + "is only supported on Windows!"); + return WIMLIB_ERR_UNSUPPORTED; + #endif + } + if ((extract_flags & (WIMLIB_EXTRACT_FLAG_RPFIX | WIMLIB_EXTRACT_FLAG_NORPFIX | diff --git a/src/win32_apply.c b/src/win32_apply.c index 49e25d4b..03677195 100644 --- a/src/win32_apply.c +++ b/src/win32_apply.c @@ -41,6 +41,7 @@ #include "wimlib/textfile.h" #include "wimlib/xml.h" #include "wimlib/wimboot.h" +#include "wimlib/wof.h" struct win32_apply_ctx { @@ -59,17 +60,18 @@ struct win32_apply_ctx { u8 blob_table_hash[SHA1_HASH_SIZE]; } *wims; size_t num_wims; - struct string_set *prepopulate_pats; - void *mem_prepopulate_pats; bool wof_running; - bool tried_to_load_prepopulate_list; - bool have_wrong_version_wims; bool have_uncompressed_wims; bool have_unsupported_compressed_resources; bool have_huge_resources; } wimboot; + /* External backing information */ + struct string_set *prepopulate_pats; + void *mem_prepopulate_pats; + bool tried_to_load_prepopulate_list; + /* Open handle to the target directory */ HANDLE h_target; @@ -146,6 +148,9 @@ struct win32_apply_ctx { /* Number of files for which we couldn't remove the short name. */ unsigned long num_remove_short_name_failures; + /* Number of files on which we couldn't set System Compression. */ + unsigned long num_system_compression_failures; + /* Have we tried to enable short name support on the target volume yet? */ bool tried_to_enable_short_names; @@ -282,8 +287,22 @@ win32_get_supported_features(const wchar_t *target, return 0; } -/* Load the patterns from [PrepopulateList] of WimBootCompress.ini in the WIM - * image being extracted. */ +#define COMPACT_FLAGS (WIMLIB_EXTRACT_FLAG_COMPACT_XPRESS4K | \ + WIMLIB_EXTRACT_FLAG_COMPACT_XPRESS8K | \ + WIMLIB_EXTRACT_FLAG_COMPACT_XPRESS16K | \ + WIMLIB_EXTRACT_FLAG_COMPACT_LZX) + + + +/* + * If not done already, load the patterns from the [PrepopulateList] section of + * WimBootCompress.ini in the WIM image being extracted. + * + * Note: WimBootCompress.ini applies to both types of "external backing": + * + * - WIM backing ("WIMBoot" - Windows 8.1 and later) + * - File backing ("System Compression" - Windows 10 and later) + */ static int load_prepopulate_pats(struct win32_apply_ctx *ctx) { @@ -296,7 +315,10 @@ load_prepopulate_pats(struct win32_apply_ctx *ctx) void *mem; struct text_file_section sec; - ctx->wimboot.tried_to_load_prepopulate_list = true; + if (ctx->tried_to_load_prepopulate_list) + return 0; + + ctx->tried_to_load_prepopulate_list = true; dentry = get_dentry(ctx->common.wim, path, WIMLIB_CASE_INSENSITIVE); if (!dentry || @@ -336,8 +358,8 @@ load_prepopulate_pats(struct win32_apply_ctx *ctx) FREE(s); return ret; } - ctx->wimboot.prepopulate_pats = s; - ctx->wimboot.mem_prepopulate_pats = mem; + ctx->prepopulate_pats = s; + ctx->mem_prepopulate_pats = mem; return 0; } @@ -348,8 +370,7 @@ can_externally_back_path(const wchar_t *path, const struct win32_apply_ctx *ctx) { /* Does the path match a pattern given in the [PrepopulateList] section * of WimBootCompress.ini? */ - if (ctx->wimboot.prepopulate_pats && - match_pattern_list(path, ctx->wimboot.prepopulate_pats)) + if (ctx->prepopulate_pats && match_pattern_list(path, ctx->prepopulate_pats)) return false; /* Since we attempt to modify the SYSTEM registry after it's extracted @@ -367,6 +388,8 @@ can_externally_back_path(const wchar_t *path, const struct win32_apply_ctx *ctx) return true; } +/* Can the specified WIM resource be used as the source of an external backing + * for the wof.sys WIM provider? */ static bool is_resource_valid_for_external_backing(const struct wim_resource_descriptor *rdesc, struct win32_apply_ctx *ctx) @@ -436,18 +459,38 @@ is_resource_valid_for_external_backing(const struct wim_resource_descriptor *rde return true; } -#define WIM_BACKING_NOT_ENABLED -1 -#define WIM_BACKING_NOT_POSSIBLE -2 -#define WIM_BACKING_EXCLUDED -3 +#define EXTERNAL_BACKING_NOT_ENABLED -1 +#define EXTERNAL_BACKING_NOT_POSSIBLE -2 +#define EXTERNAL_BACKING_EXCLUDED -3 +/* + * Determines whether the specified file will be externally backed. Returns a + * negative status code if no, 0 if yes, or a positive wimlib error code on + * error. If the file is excluded from external backing based on its path, then + * *excluded_dentry_ret is set to the dentry for the path that matched the + * exclusion rule. + * + * Note that this logic applies to both types of "external backing": + * + * - WIM backing ("WIMBoot" - Windows 8.1 and later) + * - File backing ("System Compression" - Windows 10 and later) + * + * However, in the case of WIM backing we also need to validate that the WIM + * resource that would be the source of the backing is supported by the wof.sys + * WIM provider. + */ static int will_externally_back_inode(struct wim_inode *inode, struct win32_apply_ctx *ctx, - const struct wim_dentry **excluded_dentry_ret) + const struct wim_dentry **excluded_dentry_ret, + bool wimboot_mode) { struct wim_dentry *dentry; struct blob_descriptor *blob; int ret; + if (load_prepopulate_pats(ctx) == WIMLIB_ERR_NOMEM) + return WIMLIB_ERR_NOMEM; + if (inode->i_can_externally_back) return 0; @@ -459,13 +502,17 @@ will_externally_back_inode(struct wim_inode *inode, struct win32_apply_ctx *ctx, if (inode->i_attributes & (FILE_ATTRIBUTE_DIRECTORY | FILE_ATTRIBUTE_REPARSE_POINT | FILE_ATTRIBUTE_ENCRYPTED)) - return WIM_BACKING_NOT_POSSIBLE; + return EXTERNAL_BACKING_NOT_POSSIBLE; blob = inode_get_blob_for_unnamed_data_stream_resolved(inode); - if (!blob || blob->blob_location != BLOB_IN_WIM || - !is_resource_valid_for_external_backing(blob->rdesc, ctx)) - return WIM_BACKING_NOT_POSSIBLE; + if (!blob) + return EXTERNAL_BACKING_NOT_POSSIBLE; + + if (wimboot_mode && + (blob->blob_location != BLOB_IN_WIM || + !is_resource_valid_for_external_backing(blob->rdesc, ctx))) + return EXTERNAL_BACKING_NOT_POSSIBLE; /* * We need to check the patterns in [PrepopulateList] against every name @@ -481,7 +528,7 @@ will_externally_back_inode(struct wim_inode *inode, struct win32_apply_ctx *ctx, if (!can_externally_back_path(dentry->d_full_path, ctx)) { if (excluded_dentry_ret) *excluded_dentry_ret = dentry; - return WIM_BACKING_EXCLUDED; + return EXTERNAL_BACKING_EXCLUDED; } } @@ -490,22 +537,19 @@ will_externally_back_inode(struct wim_inode *inode, struct win32_apply_ctx *ctx, } /* - * Determines if the unnamed data stream of a file will be created as an - * external backing, as opposed to a standard extraction. + * Determines if the unnamed data stream of a file will be created as a WIM + * external backing (a "WIMBoot pointer file"), as opposed to a standard + * extraction. */ static int -win32_will_externally_back(struct wim_dentry *dentry, struct apply_ctx *_ctx) +win32_will_back_from_wim(struct wim_dentry *dentry, struct apply_ctx *_ctx) { struct win32_apply_ctx *ctx = (struct win32_apply_ctx *)_ctx; if (!(ctx->common.extract_flags & WIMLIB_EXTRACT_FLAG_WIMBOOT)) - return WIM_BACKING_NOT_ENABLED; + return EXTERNAL_BACKING_NOT_ENABLED; - if (!ctx->wimboot.tried_to_load_prepopulate_list) - if (load_prepopulate_pats(ctx) == WIMLIB_ERR_NOMEM) - return WIMLIB_ERR_NOMEM; - - return will_externally_back_inode(dentry->d_inode, ctx, NULL); + return will_externally_back_inode(dentry->d_inode, ctx, NULL, true); } /* Find the WOF registration information for the specified WIM file. */ @@ -521,21 +565,21 @@ find_wimboot_wim(WIMStruct *wim_to_find, struct win32_apply_ctx *ctx) } static int -set_external_backing(HANDLE h, struct wim_inode *inode, struct win32_apply_ctx *ctx) +set_backed_from_wim(HANDLE h, struct wim_inode *inode, struct win32_apply_ctx *ctx) { int ret; const struct wim_dentry *excluded_dentry; const struct blob_descriptor *blob; const struct wimboot_wim *wimboot_wim; - ret = will_externally_back_inode(inode, ctx, &excluded_dentry); + ret = will_externally_back_inode(inode, ctx, &excluded_dentry, true); if (ret > 0) /* Error. */ return ret; - if (ret < 0 && ret != WIM_BACKING_EXCLUDED) + if (ret < 0 && ret != EXTERNAL_BACKING_EXCLUDED) return 0; /* Not externally backing, other than due to exclusion. */ - if (unlikely(ret == WIM_BACKING_EXCLUDED)) { + if (unlikely(ret == EXTERNAL_BACKING_EXCLUDED)) { /* Not externally backing due to exclusion. */ union wimlib_progress_info info; @@ -615,8 +659,7 @@ register_wim_with_wof(WIMStruct *wim, struct win32_apply_ctx *ctx) return 0; } -/* Prepare for doing a "WIMBoot" extraction by loading patterns from - * [PrepopulateList] of WimBootCompress.ini and registering each source WIM file +/* Prepare for doing a "WIMBoot" extraction by registering each source WIM file * with WOF on the target volume. */ static int start_wimboot_extraction(struct list_head *dentry_list, struct win32_apply_ctx *ctx) @@ -624,10 +667,6 @@ start_wimboot_extraction(struct list_head *dentry_list, struct win32_apply_ctx * int ret; struct wim_dentry *dentry; - if (!ctx->wimboot.tried_to_load_prepopulate_list) - if (load_prepopulate_pats(ctx) == WIMLIB_ERR_NOMEM) - return WIMLIB_ERR_NOMEM; - if (!wim_info_get_wimboot(ctx->common.wim->wim_info, ctx->common.wim->current_image)) WARNING("The WIM image is not marked as WIMBoot compatible. This usually\n" @@ -637,7 +676,7 @@ start_wimboot_extraction(struct list_head *dentry_list, struct win32_apply_ctx * list_for_each_entry(dentry, dentry_list, d_extraction_list_node) { struct blob_descriptor *blob; - ret = win32_will_externally_back(dentry, &ctx->common); + ret = win32_will_back_from_wim(dentry, &ctx->common); if (ret > 0) /* Error */ return ret; if (ret < 0) /* Won't externally back */ @@ -1780,7 +1819,7 @@ create_nondirectory(struct wim_inode *inode, struct win32_apply_ctx *ctx) /* "WIMBoot" extraction: set external backing by the WIM file if needed. */ if (!ret && unlikely(ctx->common.extract_flags & WIMLIB_EXTRACT_FLAG_WIMBOOT)) - ret = set_external_backing(h, inode, ctx); + ret = set_backed_from_wim(h, inode, ctx); (*func_NtClose)(h); return ret; @@ -2210,6 +2249,123 @@ extract_chunk(const void *chunk, size_t size, void *_ctx) return 0; } +static int +get_system_compression_format(int extract_flags) +{ + if (extract_flags & WIMLIB_EXTRACT_FLAG_COMPACT_XPRESS4K) + return FILE_PROVIDER_COMPRESSION_FORMAT_XPRESS4K; + + if (extract_flags & WIMLIB_EXTRACT_FLAG_COMPACT_XPRESS8K) + return FILE_PROVIDER_COMPRESSION_FORMAT_XPRESS8K; + + if (extract_flags & WIMLIB_EXTRACT_FLAG_COMPACT_XPRESS16K) + return FILE_PROVIDER_COMPRESSION_FORMAT_XPRESS16K; + + return FILE_PROVIDER_COMPRESSION_FORMAT_LZX; +} + +static DWORD +set_system_compression(HANDLE h, int format) +{ + DWORD bytes_returned; + DWORD err; + struct { + struct wof_external_info wof_info; + struct file_provider_external_info file_info; + } in = { + .wof_info = { + .version = WOF_CURRENT_VERSION, + .provider = WOF_PROVIDER_FILE, + }, + .file_info = { + .version = FILE_PROVIDER_CURRENT_VERSION, + .compression_format = format, + }, + }; + + if (DeviceIoControl(h, FSCTL_SET_EXTERNAL_BACKING, &in, sizeof(in), + NULL, 0, &bytes_returned, NULL)) + return 0; + + err = GetLastError(); + + if (err == 344) /* "Compressing this object would not save space." */ + return 0; + + return err; +} + +/* + * This function is called when doing a "compact-mode" extraction and we just + * finished extracting a blob to one or more locations. For each location that + * was the unnamed data stream of a file, this function compresses the + * corresponding file using System Compression, if allowed. + * + * Note: we're doing the compression immediately after extracting the data + * rather than during a separate compression pass. This way should be faster + * since the operating system should still have the file's data cached. + * + * Note: we're having the operating system do the compression, which is not + * ideal because wimlib could create the compressed data faster and more + * efficiently (the compressed data format is identical to a WIM resource). But + * we seemingly don't have a choice because WOF prevents applications from + * creating its reparse points. + */ +static void +handle_system_compression(struct blob_descriptor *blob, struct win32_apply_ctx *ctx) +{ + const struct blob_extraction_target *targets = blob_extraction_targets(blob); + + const int format = get_system_compression_format(ctx->common.extract_flags); + + for (u32 i = 0; i < blob->out_refcnt; i++) { + struct wim_inode *inode = targets[i].inode; + struct wim_inode_stream *strm = targets[i].stream; + HANDLE h; + NTSTATUS status; + DWORD err; + + if (!stream_is_unnamed_data_stream(strm)) + continue; + + if (will_externally_back_inode(inode, ctx, NULL, false) != 0) + continue; + + status = create_file(&h, GENERIC_READ | GENERIC_WRITE, NULL, + 0, FILE_OPEN, 0, + inode_first_extraction_dentry(inode), ctx); + + if (NT_SUCCESS(status)) { + err = set_system_compression(h, format); + (*func_NtClose)(h); + } else { + err = (*func_RtlNtStatusToDosError)(status); + } + + if (err == ERROR_INVALID_FUNCTION) { + WARNING( + "The request to compress the extracted files using System Compression\n" +" will not be honored because the operating system or target volume\n" +" does not support it. System Compression is only supported on\n" +" Windows 10 and later, and only on NTFS volumes."); + ctx->common.extract_flags &= ~COMPACT_FLAGS; + return; + } + + if (err) { + ctx->num_system_compression_failures++; + if (ctx->num_system_compression_failures < 10) { + win32_warning(err, L"\"%ls\": Failed to compress " + "extracted file using System Compression", + current_path(ctx)); + } else if (ctx->num_system_compression_failures == 10) { + WARNING("Suppressing further warnings about " + "System Compression failures."); + } + } + } +} + /* Called when a blob has been fully read for extraction on Windows */ static int end_extract_blob(struct blob_descriptor *blob, int status, void *_ctx) @@ -2223,6 +2379,9 @@ end_extract_blob(struct blob_descriptor *blob, int status, void *_ctx) if (status) return status; + if (unlikely(ctx->common.extract_flags & COMPACT_FLAGS)) + handle_system_compression(blob, ctx); + if (likely(!ctx->data_buffer_ptr)) return 0; @@ -2676,11 +2835,11 @@ out: FREE(ctx->pathbuf.Buffer); FREE(ctx->print_buffer); FREE(ctx->wimboot.wims); - if (ctx->wimboot.prepopulate_pats) { - FREE(ctx->wimboot.prepopulate_pats->strings); - FREE(ctx->wimboot.prepopulate_pats); + if (ctx->prepopulate_pats) { + FREE(ctx->prepopulate_pats->strings); + FREE(ctx->prepopulate_pats); } - FREE(ctx->wimboot.mem_prepopulate_pats); + FREE(ctx->mem_prepopulate_pats); FREE(ctx->data_buffer); return ret; } @@ -2689,7 +2848,7 @@ const struct apply_operations win32_apply_ops = { .name = "Windows", .get_supported_features = win32_get_supported_features, .extract = win32_extract, - .will_externally_back = win32_will_externally_back, + .will_back_from_wim = win32_will_back_from_wim, .context_size = sizeof(struct win32_apply_ctx), };