wimlib-imagex: Add and document --pack-chunk-size option

This allows users to change the pack chunk size to 67108864 rather than
the default of 33554432 if they want to.

diff --git a/doc/imagex-capture.1.in b/doc/imagex-capture.1.in
index 76ef71f10187a6c90ef28598f9f093236af3122f..eecfb987c8d077fd9813fba22981f0bae31b7bfe 100644 (file)
--- a/doc/imagex-capture.1.in
+++ b/doc/imagex-capture.1.in
@@ -248,7 +248,18 @@ Packed resources use a compression type and chunk size that is independent of
 the WIM's "default compression type" and "default chunk size" (which may be
 adjusted by the \fB--compress\fR and \fB--chunk-size\fR options, respectively).
 For compatibility reasons, \fB@IMAGEX_PROGNAME@ capture\fR currently has no
-option to change the compression type or chunk size used in packed resources.
+option to change the compression type used in packed resources; however, the
+\fB--pack-chunk-size\fR option may be used to set the chunk size.
+.TP
+\fB--pack-chunk-size\fR=\fISIZE\fR, \fB--solid-chunk-size\fR=\fISIZE\fR
+Like \fB--chunk-size\fR, but set the chunk size used in packed resources.  The
+compression format is LZMS, so the chunk size can be any power of 2 between 2^15
+and 2^26, inclusively.  WIMGAPI (Windows 8) appears to be compatible with all
+these sizes, despite not being compatible with sizes greater than 2^20 in
+non-packed resources.  The default is currently 2^25 (33554432).  Note:
+currently, the LZMS compression algorithm uses about 15 times the chunk size in
+memory per thread, which is about 500 MB per thread for the default pack chunk
+size of 2^25 or 1 GB per thread if you change it to 2^26 (67108864).
 .TP
 \fB--threads\fR=\fINUM_THREADS\fR
 Number of threads to use for compressing data.  Default: autodetect (number of

diff --git a/doc/imagex-export.1.in b/doc/imagex-export.1.in
index 32eea666422bb263c001a9d867fe0dcd61a159e6..3330a231196c55d019960b59bc243fe258ba0e36 100644 (file)
--- a/doc/imagex-export.1.in
+++ b/doc/imagex-export.1.in
@@ -85,6 +85,9 @@ result in a higher compression ratio, but has disadvantages such as reduced
 compatibility; see the documentation for this option to
 \fB@IMAGEX_PROGNAME@ capture\fR (1) for more details.
 .TP
+\fB--pack-chunk-size\fR=\fISIZE\fR, \fB--solid-chunk-size\fR=\fISIZE\fR
+Like \fB--chunk-size\fR, but set the chunk size used in packed resources.
+.TP
 \fB--threads\fR=\fINUM_THREADS\fR
 Number of threads to use for compressing data.  Default: autodetect (number of
 processors).  Note: multiple compressor threads are not very useful when

diff --git a/doc/imagex-optimize.1.in b/doc/imagex-optimize.1.in
index 37eb71bb6a9dbcc53e64673b9c514a94e892854c..479b6e468bee35cafc488bb599f0eeedc6c11006 100644 (file)
--- a/doc/imagex-optimize.1.in
+++ b/doc/imagex-optimize.1.in
@@ -66,6 +66,9 @@ result in a higher compression ratio, but has disadvantages such as reduced
 compatibility; see the documentation for this option to
 \fB@IMAGEX_PROGNAME@-capture\fR (1) for more details.
 .TP
+\fB--pack-chunk-size\fR=\fISIZE\fR, \fB--solid-chunk-size\fR=\fISIZE\fR
+Like \fB--chunk-size\fR, but set the chunk size used in packed resources.
+.TP
 \fB--threads\fR=\fINUM_THREADS\fR
 Number of threads to use for compressing data.  Default: autodetect (number of
 processors).  This parameter is only meaningful when \fB--recompress\fR is also

diff --git a/include/wimlib.h b/include/wimlib.h
index 67b3837961bc333725677dbb2e37cb92b45ecd71..de10307c6f6a36ea40b336b8028cb7ba55aaaa09 100644 (file)
--- a/include/wimlib.h
+++ b/include/wimlib.h
@@ -3414,8 +3414,7 @@ wimlib_set_output_chunk_size(WIMStruct *wim, uint32_t chunk_size);
  * @ingroup G_writing_and_overwriting_wims
  *
  * Similar to wimlib_set_output_chunk_size(), but set the chunk size for writing
- * packed streams.  For compatibility reasons, using this function is not
- * generally recommended.
+ * packed streams.
  */
 extern int
 wimlib_set_output_pack_chunk_size(WIMStruct *wim, uint32_t chunk_size);
@@ -3446,7 +3445,12 @@ wimlib_set_output_compression_type(WIMStruct *wim, int ctype);
  * @ingroup G_writing_and_overwriting_wims
  *
  * Similar to wimlib_set_output_compression_type(), but set the compression type
- * for writing packed streams.
+ * for writing packed streams (solid blocks).
+ *
+ * Note: based on testing, WIMGAPI is seemingly only compatible with LZMS
+ * compression in packed streams.  Therefore the use of this function is not
+ * recommended.  Also, with large chunk sizes, LZMS gives the best compression
+ * ratio among the alternatives anyway.
  */
 extern int
 wimlib_set_output_pack_compression_type(WIMStruct *wim, int ctype);

diff --git a/programs/imagex.c b/programs/imagex.c
index 9ebe277181dee6b7eb559f26131214f51b177c7d..5dd523228ae7a53a808c36db7de3379d5e9a06c6 100644 (file)
--- a/programs/imagex.c
+++ b/programs/imagex.c
@@ -144,6 +144,7 @@ enum {
        IMAGEX_NO_ACLS_OPTION,
        IMAGEX_NO_WILDCARDS_OPTION,
        IMAGEX_NOT_PIPABLE_OPTION,
+       IMAGEX_PACK_CHUNK_SIZE_OPTION,
        IMAGEX_PACK_STREAMS_OPTION,
        IMAGEX_PATH_OPTION,
        IMAGEX_PIPABLE_OPTION,
@@ -195,6 +196,8 @@ static const struct option capture_or_append_options[] = {
        {T("compress"),    required_argument, NULL, IMAGEX_COMPRESS_OPTION},
        {T("compress-slow"), no_argument,     NULL, IMAGEX_COMPRESS_SLOW_OPTION},
        {T("chunk-size"),  required_argument, NULL, IMAGEX_CHUNK_SIZE_OPTION},
+       {T("pack-chunk-size"), required_argument, NULL, IMAGEX_PACK_CHUNK_SIZE_OPTION},
+       {T("solid-chunk-size"),required_argument, NULL, IMAGEX_PACK_CHUNK_SIZE_OPTION},
        {T("pack-streams"), no_argument,      NULL, IMAGEX_PACK_STREAMS_OPTION},
        {T("solid"),       no_argument,      NULL, IMAGEX_PACK_STREAMS_OPTION},
        {T("config"),      required_argument, NULL, IMAGEX_CONFIG_OPTION},
@@ -237,6 +240,8 @@ static const struct option export_options[] = {
        {T("pack-streams"),no_argument,       NULL, IMAGEX_PACK_STREAMS_OPTION},
        {T("solid"),       no_argument,       NULL, IMAGEX_PACK_STREAMS_OPTION},
        {T("chunk-size"),  required_argument, NULL, IMAGEX_CHUNK_SIZE_OPTION},
+       {T("pack-chunk-size"), required_argument, NULL, IMAGEX_PACK_CHUNK_SIZE_OPTION},
+       {T("solid-chunk-size"),required_argument, NULL, IMAGEX_PACK_CHUNK_SIZE_OPTION},
        {T("ref"),         required_argument, NULL, IMAGEX_REF_OPTION},
        {T("threads"),     required_argument, NULL, IMAGEX_THREADS_OPTION},
        {T("rebuild"),     no_argument,       NULL, IMAGEX_REBUILD_OPTION},
@@ -299,6 +304,8 @@ static const struct option optimize_options[] = {
        {T("compress-slow"), no_argument,     NULL, IMAGEX_COMPRESS_SLOW_OPTION},
        {T("recompress-slow"), no_argument,     NULL, IMAGEX_COMPRESS_SLOW_OPTION},
        {T("chunk-size"),  required_argument, NULL, IMAGEX_CHUNK_SIZE_OPTION},
+       {T("pack-chunk-size"), required_argument, NULL, IMAGEX_PACK_CHUNK_SIZE_OPTION},
+       {T("solid-chunk-size"),required_argument, NULL, IMAGEX_PACK_CHUNK_SIZE_OPTION},
        {T("pack-streams"),no_argument,       NULL, IMAGEX_PACK_STREAMS_OPTION},
        {T("solid"),       no_argument,       NULL, IMAGEX_PACK_STREAMS_OPTION},
        {T("threads"),     required_argument, NULL, IMAGEX_THREADS_OPTION},
@@ -1751,6 +1758,7 @@ imagex_capture_or_append(int argc, tchar **argv, int cmd)
        int write_flags = 0;
        int compression_type = WIMLIB_COMPRESSION_TYPE_INVALID;
        uint32_t chunk_size = UINT32_MAX;
+       uint32_t pack_chunk_size = UINT32_MAX;
        const tchar *wimfile;
        int wim_fd;
        const tchar *name;
@@ -1814,6 +1822,11 @@ imagex_capture_or_append(int argc, tchar **argv, int cmd)
                        if (chunk_size == UINT32_MAX)
                                goto out_err;
                        break;
+               case IMAGEX_PACK_CHUNK_SIZE_OPTION:
+                       pack_chunk_size = parse_chunk_size(optarg);
+                       if (pack_chunk_size == UINT32_MAX)
+                               goto out_err;
+                       break;
                case IMAGEX_PACK_STREAMS_OPTION:
                        write_flags |= WIMLIB_WRITE_FLAG_PACK_STREAMS;
                        break;
@@ -2061,6 +2074,11 @@ imagex_capture_or_append(int argc, tchar **argv, int cmd)
                if (ret)
                        goto out_free_wim;
        }
+       if (pack_chunk_size != UINT32_MAX) {
+               ret = wimlib_set_output_pack_chunk_size(wim, pack_chunk_size);
+               if (ret)
+                       goto out_free_wim;
+       }
 
 #ifndef __WIN32__
        /* Detect if source is regular file or block device and set NTFS volume
@@ -2454,6 +2472,7 @@ imagex_export(int argc, tchar **argv, int cmd)
        STRING_SET(refglobs);
        unsigned num_threads = 0;
        uint32_t chunk_size = UINT32_MAX;
+       uint32_t pack_chunk_size = UINT32_MAX;
 
        for_opt(c, export_options) {
                switch (c) {
@@ -2480,6 +2499,11 @@ imagex_export(int argc, tchar **argv, int cmd)
                        if (chunk_size == UINT32_MAX)
                                goto out_err;
                        break;
+               case IMAGEX_PACK_CHUNK_SIZE_OPTION:
+                       pack_chunk_size = parse_chunk_size(optarg);
+                       if (pack_chunk_size == UINT32_MAX)
+                               goto out_err;
+                       break;
                case IMAGEX_REF_OPTION:
                        ret = string_set_append(&refglobs, optarg);
                        if (ret)
@@ -2609,6 +2633,11 @@ imagex_export(int argc, tchar **argv, int cmd)
                if (ret)
                        goto out_free_dest_wim;
        }
+       if (pack_chunk_size != UINT32_MAX) {
+               ret = wimlib_set_output_pack_chunk_size(dest_wim, pack_chunk_size);
+               if (ret)
+                       goto out_free_dest_wim;
+       }
 
        image = wimlib_resolve_image(src_wim, src_image_num_or_name);
        ret = verify_image_exists(image, src_image_num_or_name, src_wimfile);
@@ -3408,6 +3437,7 @@ imagex_optimize(int argc, tchar **argv, int cmd)
        int write_flags = WIMLIB_WRITE_FLAG_REBUILD;
        int compression_type = WIMLIB_COMPRESSION_TYPE_INVALID;
        uint32_t chunk_size = UINT32_MAX;
+       uint32_t pack_chunk_size = UINT32_MAX;
        int ret;
        WIMStruct *wim;
        const tchar *wimfile;
@@ -3442,6 +3472,11 @@ imagex_optimize(int argc, tchar **argv, int cmd)
                        if (chunk_size == UINT32_MAX)
                                goto out_err;
                        break;
+               case IMAGEX_PACK_CHUNK_SIZE_OPTION:
+                       pack_chunk_size = parse_chunk_size(optarg);
+                       if (pack_chunk_size == UINT32_MAX)
+                               goto out_err;
+                       break;
                case IMAGEX_PACK_STREAMS_OPTION:
                        write_flags |= WIMLIB_WRITE_FLAG_PACK_STREAMS;
                        write_flags |= WIMLIB_WRITE_FLAG_RECOMPRESS;
@@ -3486,6 +3521,11 @@ imagex_optimize(int argc, tchar **argv, int cmd)
                if (ret)
                        goto out_wimlib_free;
        }
+       if (pack_chunk_size != UINT32_MAX) {
+               ret = wimlib_set_output_pack_chunk_size(wim, pack_chunk_size);
+               if (ret)
+                       goto out_wimlib_free;
+       }
 
        old_size = file_get_size(wimfile);
        tprintf(T("\"%"TS"\" original size: "), wimfile);