wimlib-imagex: Add and document --pack-chunk-size option
authorEric Biggers <ebiggers3@gmail.com>
Wed, 15 Jan 2014 20:18:44 +0000 (14:18 -0600)
committerEric Biggers <ebiggers3@gmail.com>
Wed, 15 Jan 2014 20:19:15 +0000 (14:19 -0600)
This allows users to change the pack chunk size to 67108864 rather than
the default of 33554432 if they want to.

doc/imagex-capture.1.in
doc/imagex-export.1.in
doc/imagex-optimize.1.in
include/wimlib.h
programs/imagex.c

index 76ef71f..eecfb98 100644 (file)
@@ -248,7 +248,18 @@ Packed resources use a compression type and chunk size that is independent of
 the WIM's "default compression type" and "default chunk size" (which may be
 adjusted by the \fB--compress\fR and \fB--chunk-size\fR options, respectively).
 For compatibility reasons, \fB@IMAGEX_PROGNAME@ capture\fR currently has no
-option to change the compression type or chunk size used in packed resources.
+option to change the compression type used in packed resources; however, the
+\fB--pack-chunk-size\fR option may be used to set the chunk size.
+.TP
+\fB--pack-chunk-size\fR=\fISIZE\fR, \fB--solid-chunk-size\fR=\fISIZE\fR
+Like \fB--chunk-size\fR, but set the chunk size used in packed resources.  The
+compression format is LZMS, so the chunk size can be any power of 2 between 2^15
+and 2^26, inclusively.  WIMGAPI (Windows 8) appears to be compatible with all
+these sizes, despite not being compatible with sizes greater than 2^20 in
+non-packed resources.  The default is currently 2^25 (33554432).  Note:
+currently, the LZMS compression algorithm uses about 15 times the chunk size in
+memory per thread, which is about 500 MB per thread for the default pack chunk
+size of 2^25 or 1 GB per thread if you change it to 2^26 (67108864).
 .TP
 \fB--threads\fR=\fINUM_THREADS\fR
 Number of threads to use for compressing data.  Default: autodetect (number of
index 32eea66..3330a23 100644 (file)
@@ -85,6 +85,9 @@ result in a higher compression ratio, but has disadvantages such as reduced
 compatibility; see the documentation for this option to
 \fB@IMAGEX_PROGNAME@ capture\fR (1) for more details.
 .TP
+\fB--pack-chunk-size\fR=\fISIZE\fR, \fB--solid-chunk-size\fR=\fISIZE\fR
+Like \fB--chunk-size\fR, but set the chunk size used in packed resources.
+.TP
 \fB--threads\fR=\fINUM_THREADS\fR
 Number of threads to use for compressing data.  Default: autodetect (number of
 processors).  Note: multiple compressor threads are not very useful when
index 37eb71b..479b6e4 100644 (file)
@@ -66,6 +66,9 @@ result in a higher compression ratio, but has disadvantages such as reduced
 compatibility; see the documentation for this option to
 \fB@IMAGEX_PROGNAME@-capture\fR (1) for more details.
 .TP
+\fB--pack-chunk-size\fR=\fISIZE\fR, \fB--solid-chunk-size\fR=\fISIZE\fR
+Like \fB--chunk-size\fR, but set the chunk size used in packed resources.
+.TP
 \fB--threads\fR=\fINUM_THREADS\fR
 Number of threads to use for compressing data.  Default: autodetect (number of
 processors).  This parameter is only meaningful when \fB--recompress\fR is also
index 67b3837..de10307 100644 (file)
@@ -3414,8 +3414,7 @@ wimlib_set_output_chunk_size(WIMStruct *wim, uint32_t chunk_size);
  * @ingroup G_writing_and_overwriting_wims
  *
  * Similar to wimlib_set_output_chunk_size(), but set the chunk size for writing
- * packed streams.  For compatibility reasons, using this function is not
- * generally recommended.
+ * packed streams.
  */
 extern int
 wimlib_set_output_pack_chunk_size(WIMStruct *wim, uint32_t chunk_size);
@@ -3446,7 +3445,12 @@ wimlib_set_output_compression_type(WIMStruct *wim, int ctype);
  * @ingroup G_writing_and_overwriting_wims
  *
  * Similar to wimlib_set_output_compression_type(), but set the compression type
- * for writing packed streams.
+ * for writing packed streams (solid blocks).
+ *
+ * Note: based on testing, WIMGAPI is seemingly only compatible with LZMS
+ * compression in packed streams.  Therefore the use of this function is not
+ * recommended.  Also, with large chunk sizes, LZMS gives the best compression
+ * ratio among the alternatives anyway.
  */
 extern int
 wimlib_set_output_pack_compression_type(WIMStruct *wim, int ctype);
index 9ebe277..5dd5232 100644 (file)
@@ -144,6 +144,7 @@ enum {
        IMAGEX_NO_ACLS_OPTION,
        IMAGEX_NO_WILDCARDS_OPTION,
        IMAGEX_NOT_PIPABLE_OPTION,
+       IMAGEX_PACK_CHUNK_SIZE_OPTION,
        IMAGEX_PACK_STREAMS_OPTION,
        IMAGEX_PATH_OPTION,
        IMAGEX_PIPABLE_OPTION,
@@ -195,6 +196,8 @@ static const struct option capture_or_append_options[] = {
        {T("compress"),    required_argument, NULL, IMAGEX_COMPRESS_OPTION},
        {T("compress-slow"), no_argument,     NULL, IMAGEX_COMPRESS_SLOW_OPTION},
        {T("chunk-size"),  required_argument, NULL, IMAGEX_CHUNK_SIZE_OPTION},
+       {T("pack-chunk-size"), required_argument, NULL, IMAGEX_PACK_CHUNK_SIZE_OPTION},
+       {T("solid-chunk-size"),required_argument, NULL, IMAGEX_PACK_CHUNK_SIZE_OPTION},
        {T("pack-streams"), no_argument,      NULL, IMAGEX_PACK_STREAMS_OPTION},
        {T("solid"),       no_argument,      NULL, IMAGEX_PACK_STREAMS_OPTION},
        {T("config"),      required_argument, NULL, IMAGEX_CONFIG_OPTION},
@@ -237,6 +240,8 @@ static const struct option export_options[] = {
        {T("pack-streams"),no_argument,       NULL, IMAGEX_PACK_STREAMS_OPTION},
        {T("solid"),       no_argument,       NULL, IMAGEX_PACK_STREAMS_OPTION},
        {T("chunk-size"),  required_argument, NULL, IMAGEX_CHUNK_SIZE_OPTION},
+       {T("pack-chunk-size"), required_argument, NULL, IMAGEX_PACK_CHUNK_SIZE_OPTION},
+       {T("solid-chunk-size"),required_argument, NULL, IMAGEX_PACK_CHUNK_SIZE_OPTION},
        {T("ref"),         required_argument, NULL, IMAGEX_REF_OPTION},
        {T("threads"),     required_argument, NULL, IMAGEX_THREADS_OPTION},
        {T("rebuild"),     no_argument,       NULL, IMAGEX_REBUILD_OPTION},
@@ -299,6 +304,8 @@ static const struct option optimize_options[] = {
        {T("compress-slow"), no_argument,     NULL, IMAGEX_COMPRESS_SLOW_OPTION},
        {T("recompress-slow"), no_argument,     NULL, IMAGEX_COMPRESS_SLOW_OPTION},
        {T("chunk-size"),  required_argument, NULL, IMAGEX_CHUNK_SIZE_OPTION},
+       {T("pack-chunk-size"), required_argument, NULL, IMAGEX_PACK_CHUNK_SIZE_OPTION},
+       {T("solid-chunk-size"),required_argument, NULL, IMAGEX_PACK_CHUNK_SIZE_OPTION},
        {T("pack-streams"),no_argument,       NULL, IMAGEX_PACK_STREAMS_OPTION},
        {T("solid"),       no_argument,       NULL, IMAGEX_PACK_STREAMS_OPTION},
        {T("threads"),     required_argument, NULL, IMAGEX_THREADS_OPTION},
@@ -1751,6 +1758,7 @@ imagex_capture_or_append(int argc, tchar **argv, int cmd)
        int write_flags = 0;
        int compression_type = WIMLIB_COMPRESSION_TYPE_INVALID;
        uint32_t chunk_size = UINT32_MAX;
+       uint32_t pack_chunk_size = UINT32_MAX;
        const tchar *wimfile;
        int wim_fd;
        const tchar *name;
@@ -1814,6 +1822,11 @@ imagex_capture_or_append(int argc, tchar **argv, int cmd)
                        if (chunk_size == UINT32_MAX)
                                goto out_err;
                        break;
+               case IMAGEX_PACK_CHUNK_SIZE_OPTION:
+                       pack_chunk_size = parse_chunk_size(optarg);
+                       if (pack_chunk_size == UINT32_MAX)
+                               goto out_err;
+                       break;
                case IMAGEX_PACK_STREAMS_OPTION:
                        write_flags |= WIMLIB_WRITE_FLAG_PACK_STREAMS;
                        break;
@@ -2061,6 +2074,11 @@ imagex_capture_or_append(int argc, tchar **argv, int cmd)
                if (ret)
                        goto out_free_wim;
        }
+       if (pack_chunk_size != UINT32_MAX) {
+               ret = wimlib_set_output_pack_chunk_size(wim, pack_chunk_size);
+               if (ret)
+                       goto out_free_wim;
+       }
 
 #ifndef __WIN32__
        /* Detect if source is regular file or block device and set NTFS volume
@@ -2454,6 +2472,7 @@ imagex_export(int argc, tchar **argv, int cmd)
        STRING_SET(refglobs);
        unsigned num_threads = 0;
        uint32_t chunk_size = UINT32_MAX;
+       uint32_t pack_chunk_size = UINT32_MAX;
 
        for_opt(c, export_options) {
                switch (c) {
@@ -2480,6 +2499,11 @@ imagex_export(int argc, tchar **argv, int cmd)
                        if (chunk_size == UINT32_MAX)
                                goto out_err;
                        break;
+               case IMAGEX_PACK_CHUNK_SIZE_OPTION:
+                       pack_chunk_size = parse_chunk_size(optarg);
+                       if (pack_chunk_size == UINT32_MAX)
+                               goto out_err;
+                       break;
                case IMAGEX_REF_OPTION:
                        ret = string_set_append(&refglobs, optarg);
                        if (ret)
@@ -2609,6 +2633,11 @@ imagex_export(int argc, tchar **argv, int cmd)
                if (ret)
                        goto out_free_dest_wim;
        }
+       if (pack_chunk_size != UINT32_MAX) {
+               ret = wimlib_set_output_pack_chunk_size(dest_wim, pack_chunk_size);
+               if (ret)
+                       goto out_free_dest_wim;
+       }
 
        image = wimlib_resolve_image(src_wim, src_image_num_or_name);
        ret = verify_image_exists(image, src_image_num_or_name, src_wimfile);
@@ -3408,6 +3437,7 @@ imagex_optimize(int argc, tchar **argv, int cmd)
        int write_flags = WIMLIB_WRITE_FLAG_REBUILD;
        int compression_type = WIMLIB_COMPRESSION_TYPE_INVALID;
        uint32_t chunk_size = UINT32_MAX;
+       uint32_t pack_chunk_size = UINT32_MAX;
        int ret;
        WIMStruct *wim;
        const tchar *wimfile;
@@ -3442,6 +3472,11 @@ imagex_optimize(int argc, tchar **argv, int cmd)
                        if (chunk_size == UINT32_MAX)
                                goto out_err;
                        break;
+               case IMAGEX_PACK_CHUNK_SIZE_OPTION:
+                       pack_chunk_size = parse_chunk_size(optarg);
+                       if (pack_chunk_size == UINT32_MAX)
+                               goto out_err;
+                       break;
                case IMAGEX_PACK_STREAMS_OPTION:
                        write_flags |= WIMLIB_WRITE_FLAG_PACK_STREAMS;
                        write_flags |= WIMLIB_WRITE_FLAG_RECOMPRESS;
@@ -3486,6 +3521,11 @@ imagex_optimize(int argc, tchar **argv, int cmd)
                if (ret)
                        goto out_wimlib_free;
        }
+       if (pack_chunk_size != UINT32_MAX) {
+               ret = wimlib_set_output_pack_chunk_size(wim, pack_chunk_size);
+               if (ret)
+                       goto out_wimlib_free;
+       }
 
        old_size = file_get_size(wimfile);
        tprintf(T("\"%"TS"\" original size: "), wimfile);