From 1c893605029270ae2b75ad1d90517aebdb78eb14 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Wed, 15 Jan 2014 14:18:44 -0600 Subject: [PATCH] wimlib-imagex: Add and document --pack-chunk-size option This allows users to change the pack chunk size to 67108864 rather than the default of 33554432 if they want to. --- doc/imagex-capture.1.in | 13 ++++++++++++- doc/imagex-export.1.in | 3 +++ doc/imagex-optimize.1.in | 3 +++ include/wimlib.h | 10 +++++++--- programs/imagex.c | 40 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 65 insertions(+), 4 deletions(-) diff --git a/doc/imagex-capture.1.in b/doc/imagex-capture.1.in index 76ef71f1..eecfb987 100644 --- a/doc/imagex-capture.1.in +++ b/doc/imagex-capture.1.in @@ -248,7 +248,18 @@ Packed resources use a compression type and chunk size that is independent of the WIM's "default compression type" and "default chunk size" (which may be adjusted by the \fB--compress\fR and \fB--chunk-size\fR options, respectively). For compatibility reasons, \fB@IMAGEX_PROGNAME@ capture\fR currently has no -option to change the compression type or chunk size used in packed resources. +option to change the compression type used in packed resources; however, the +\fB--pack-chunk-size\fR option may be used to set the chunk size. +.TP +\fB--pack-chunk-size\fR=\fISIZE\fR, \fB--solid-chunk-size\fR=\fISIZE\fR +Like \fB--chunk-size\fR, but set the chunk size used in packed resources. The +compression format is LZMS, so the chunk size can be any power of 2 between 2^15 +and 2^26, inclusively. WIMGAPI (Windows 8) appears to be compatible with all +these sizes, despite not being compatible with sizes greater than 2^20 in +non-packed resources. The default is currently 2^25 (33554432). Note: +currently, the LZMS compression algorithm uses about 15 times the chunk size in +memory per thread, which is about 500 MB per thread for the default pack chunk +size of 2^25 or 1 GB per thread if you change it to 2^26 (67108864). .TP \fB--threads\fR=\fINUM_THREADS\fR Number of threads to use for compressing data. Default: autodetect (number of diff --git a/doc/imagex-export.1.in b/doc/imagex-export.1.in index 32eea666..3330a231 100644 --- a/doc/imagex-export.1.in +++ b/doc/imagex-export.1.in @@ -85,6 +85,9 @@ result in a higher compression ratio, but has disadvantages such as reduced compatibility; see the documentation for this option to \fB@IMAGEX_PROGNAME@ capture\fR (1) for more details. .TP +\fB--pack-chunk-size\fR=\fISIZE\fR, \fB--solid-chunk-size\fR=\fISIZE\fR +Like \fB--chunk-size\fR, but set the chunk size used in packed resources. +.TP \fB--threads\fR=\fINUM_THREADS\fR Number of threads to use for compressing data. Default: autodetect (number of processors). Note: multiple compressor threads are not very useful when diff --git a/doc/imagex-optimize.1.in b/doc/imagex-optimize.1.in index 37eb71bb..479b6e46 100644 --- a/doc/imagex-optimize.1.in +++ b/doc/imagex-optimize.1.in @@ -66,6 +66,9 @@ result in a higher compression ratio, but has disadvantages such as reduced compatibility; see the documentation for this option to \fB@IMAGEX_PROGNAME@-capture\fR (1) for more details. .TP +\fB--pack-chunk-size\fR=\fISIZE\fR, \fB--solid-chunk-size\fR=\fISIZE\fR +Like \fB--chunk-size\fR, but set the chunk size used in packed resources. +.TP \fB--threads\fR=\fINUM_THREADS\fR Number of threads to use for compressing data. Default: autodetect (number of processors). This parameter is only meaningful when \fB--recompress\fR is also diff --git a/include/wimlib.h b/include/wimlib.h index 67b38379..de10307c 100644 --- a/include/wimlib.h +++ b/include/wimlib.h @@ -3414,8 +3414,7 @@ wimlib_set_output_chunk_size(WIMStruct *wim, uint32_t chunk_size); * @ingroup G_writing_and_overwriting_wims * * Similar to wimlib_set_output_chunk_size(), but set the chunk size for writing - * packed streams. For compatibility reasons, using this function is not - * generally recommended. + * packed streams. */ extern int wimlib_set_output_pack_chunk_size(WIMStruct *wim, uint32_t chunk_size); @@ -3446,7 +3445,12 @@ wimlib_set_output_compression_type(WIMStruct *wim, int ctype); * @ingroup G_writing_and_overwriting_wims * * Similar to wimlib_set_output_compression_type(), but set the compression type - * for writing packed streams. + * for writing packed streams (solid blocks). + * + * Note: based on testing, WIMGAPI is seemingly only compatible with LZMS + * compression in packed streams. Therefore the use of this function is not + * recommended. Also, with large chunk sizes, LZMS gives the best compression + * ratio among the alternatives anyway. */ extern int wimlib_set_output_pack_compression_type(WIMStruct *wim, int ctype); diff --git a/programs/imagex.c b/programs/imagex.c index 9ebe2771..5dd52322 100644 --- a/programs/imagex.c +++ b/programs/imagex.c @@ -144,6 +144,7 @@ enum { IMAGEX_NO_ACLS_OPTION, IMAGEX_NO_WILDCARDS_OPTION, IMAGEX_NOT_PIPABLE_OPTION, + IMAGEX_PACK_CHUNK_SIZE_OPTION, IMAGEX_PACK_STREAMS_OPTION, IMAGEX_PATH_OPTION, IMAGEX_PIPABLE_OPTION, @@ -195,6 +196,8 @@ static const struct option capture_or_append_options[] = { {T("compress"), required_argument, NULL, IMAGEX_COMPRESS_OPTION}, {T("compress-slow"), no_argument, NULL, IMAGEX_COMPRESS_SLOW_OPTION}, {T("chunk-size"), required_argument, NULL, IMAGEX_CHUNK_SIZE_OPTION}, + {T("pack-chunk-size"), required_argument, NULL, IMAGEX_PACK_CHUNK_SIZE_OPTION}, + {T("solid-chunk-size"),required_argument, NULL, IMAGEX_PACK_CHUNK_SIZE_OPTION}, {T("pack-streams"), no_argument, NULL, IMAGEX_PACK_STREAMS_OPTION}, {T("solid"), no_argument, NULL, IMAGEX_PACK_STREAMS_OPTION}, {T("config"), required_argument, NULL, IMAGEX_CONFIG_OPTION}, @@ -237,6 +240,8 @@ static const struct option export_options[] = { {T("pack-streams"),no_argument, NULL, IMAGEX_PACK_STREAMS_OPTION}, {T("solid"), no_argument, NULL, IMAGEX_PACK_STREAMS_OPTION}, {T("chunk-size"), required_argument, NULL, IMAGEX_CHUNK_SIZE_OPTION}, + {T("pack-chunk-size"), required_argument, NULL, IMAGEX_PACK_CHUNK_SIZE_OPTION}, + {T("solid-chunk-size"),required_argument, NULL, IMAGEX_PACK_CHUNK_SIZE_OPTION}, {T("ref"), required_argument, NULL, IMAGEX_REF_OPTION}, {T("threads"), required_argument, NULL, IMAGEX_THREADS_OPTION}, {T("rebuild"), no_argument, NULL, IMAGEX_REBUILD_OPTION}, @@ -299,6 +304,8 @@ static const struct option optimize_options[] = { {T("compress-slow"), no_argument, NULL, IMAGEX_COMPRESS_SLOW_OPTION}, {T("recompress-slow"), no_argument, NULL, IMAGEX_COMPRESS_SLOW_OPTION}, {T("chunk-size"), required_argument, NULL, IMAGEX_CHUNK_SIZE_OPTION}, + {T("pack-chunk-size"), required_argument, NULL, IMAGEX_PACK_CHUNK_SIZE_OPTION}, + {T("solid-chunk-size"),required_argument, NULL, IMAGEX_PACK_CHUNK_SIZE_OPTION}, {T("pack-streams"),no_argument, NULL, IMAGEX_PACK_STREAMS_OPTION}, {T("solid"), no_argument, NULL, IMAGEX_PACK_STREAMS_OPTION}, {T("threads"), required_argument, NULL, IMAGEX_THREADS_OPTION}, @@ -1751,6 +1758,7 @@ imagex_capture_or_append(int argc, tchar **argv, int cmd) int write_flags = 0; int compression_type = WIMLIB_COMPRESSION_TYPE_INVALID; uint32_t chunk_size = UINT32_MAX; + uint32_t pack_chunk_size = UINT32_MAX; const tchar *wimfile; int wim_fd; const tchar *name; @@ -1814,6 +1822,11 @@ imagex_capture_or_append(int argc, tchar **argv, int cmd) if (chunk_size == UINT32_MAX) goto out_err; break; + case IMAGEX_PACK_CHUNK_SIZE_OPTION: + pack_chunk_size = parse_chunk_size(optarg); + if (pack_chunk_size == UINT32_MAX) + goto out_err; + break; case IMAGEX_PACK_STREAMS_OPTION: write_flags |= WIMLIB_WRITE_FLAG_PACK_STREAMS; break; @@ -2061,6 +2074,11 @@ imagex_capture_or_append(int argc, tchar **argv, int cmd) if (ret) goto out_free_wim; } + if (pack_chunk_size != UINT32_MAX) { + ret = wimlib_set_output_pack_chunk_size(wim, pack_chunk_size); + if (ret) + goto out_free_wim; + } #ifndef __WIN32__ /* Detect if source is regular file or block device and set NTFS volume @@ -2454,6 +2472,7 @@ imagex_export(int argc, tchar **argv, int cmd) STRING_SET(refglobs); unsigned num_threads = 0; uint32_t chunk_size = UINT32_MAX; + uint32_t pack_chunk_size = UINT32_MAX; for_opt(c, export_options) { switch (c) { @@ -2480,6 +2499,11 @@ imagex_export(int argc, tchar **argv, int cmd) if (chunk_size == UINT32_MAX) goto out_err; break; + case IMAGEX_PACK_CHUNK_SIZE_OPTION: + pack_chunk_size = parse_chunk_size(optarg); + if (pack_chunk_size == UINT32_MAX) + goto out_err; + break; case IMAGEX_REF_OPTION: ret = string_set_append(&refglobs, optarg); if (ret) @@ -2609,6 +2633,11 @@ imagex_export(int argc, tchar **argv, int cmd) if (ret) goto out_free_dest_wim; } + if (pack_chunk_size != UINT32_MAX) { + ret = wimlib_set_output_pack_chunk_size(dest_wim, pack_chunk_size); + if (ret) + goto out_free_dest_wim; + } image = wimlib_resolve_image(src_wim, src_image_num_or_name); ret = verify_image_exists(image, src_image_num_or_name, src_wimfile); @@ -3408,6 +3437,7 @@ imagex_optimize(int argc, tchar **argv, int cmd) int write_flags = WIMLIB_WRITE_FLAG_REBUILD; int compression_type = WIMLIB_COMPRESSION_TYPE_INVALID; uint32_t chunk_size = UINT32_MAX; + uint32_t pack_chunk_size = UINT32_MAX; int ret; WIMStruct *wim; const tchar *wimfile; @@ -3442,6 +3472,11 @@ imagex_optimize(int argc, tchar **argv, int cmd) if (chunk_size == UINT32_MAX) goto out_err; break; + case IMAGEX_PACK_CHUNK_SIZE_OPTION: + pack_chunk_size = parse_chunk_size(optarg); + if (pack_chunk_size == UINT32_MAX) + goto out_err; + break; case IMAGEX_PACK_STREAMS_OPTION: write_flags |= WIMLIB_WRITE_FLAG_PACK_STREAMS; write_flags |= WIMLIB_WRITE_FLAG_RECOMPRESS; @@ -3486,6 +3521,11 @@ imagex_optimize(int argc, tchar **argv, int cmd) if (ret) goto out_wimlib_free; } + if (pack_chunk_size != UINT32_MAX) { + ret = wimlib_set_output_pack_chunk_size(wim, pack_chunk_size); + if (ret) + goto out_wimlib_free; + } old_size = file_get_size(wimfile); tprintf(T("\"%"TS"\" original size: "), wimfile); -- 2.43.0