From dadccd777e56b473128d87e485fcf8565b8cb93b Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Sat, 14 Feb 2015 18:30:24 -0600 Subject: [PATCH] Support "destructive" compression to save memory --- include/wimlib.h | 11 +++++++++ include/wimlib/compressor_ops.h | 4 +++- src/compress.c | 18 +++++++++++++- src/compress_parallel.c | 3 ++- src/compress_serial.c | 3 ++- src/lzms_compress.c | 42 +++++++++++++++++++++++---------- src/lzx_compress.c | 36 +++++++++++++++++++++------- src/xpress_compress.c | 5 ++-- 8 files changed, 95 insertions(+), 27 deletions(-) diff --git a/include/wimlib.h b/include/wimlib.h index a7b18489..c2ecb48d 100644 --- a/include/wimlib.h +++ b/include/wimlib.h @@ -4624,6 +4624,8 @@ wimlib_get_compressor_needed_memory(enum wimlib_compression_type ctype, size_t max_block_size, unsigned int compression_level); +#define WIMLIB_COMPRESSOR_FLAG_DESTRUCTIVE 0x80000000 + /** * Allocate a compressor for the specified compression type using the specified * parameters. This function is part of wimlib's compression API; it is not @@ -4668,6 +4670,15 @@ wimlib_get_compressor_needed_memory(enum wimlib_compression_type ctype, * The compression level does not affect the format of the compressed data. * Therefore, it is a compressor-only parameter and does not need to be * passed to the decompressor. + *
+ * Since wimlib v1.7.5, this parameter can be OR-ed with the flag + * ::WIMLIB_COMPRESSOR_FLAG_DESTRUCTIVE. This creates the compressor in a + * mode where it is allowed to modify the input buffer. Specifically, in + * this mode, if compression succeeds, the input buffer may have been + * modified, whereas if compression does not succeed the input buffer still + * may have been written to but will have been restored exactly to its + * original state. This mode is designed to save some memory when using + * large buffer sizes. * @param compressor_ret * A location into which to return the pointer to the allocated compressor. * The allocated compressor can be used for any number of calls to diff --git a/include/wimlib/compressor_ops.h b/include/wimlib/compressor_ops.h index af81546c..bef5cf6d 100644 --- a/include/wimlib/compressor_ops.h +++ b/include/wimlib/compressor_ops.h @@ -12,10 +12,12 @@ struct compressor_ops { u64 (*get_needed_memory)(size_t max_block_size, - unsigned int compression_level); + unsigned int compression_level, + bool destructive); int (*create_compressor)(size_t max_block_size, unsigned int compression_level, + bool destructive, void **private_ret); size_t (*compress)(const void *uncompressed_data, diff --git a/src/compress.c b/src/compress.c index 47026e36..ff28ebc0 100644 --- a/src/compress.c +++ b/src/compress.c @@ -81,12 +81,19 @@ wimlib_get_compressor_needed_memory(enum wimlib_compression_type ctype, size_t max_block_size, unsigned int compression_level) { + bool destructive; const struct compressor_ops *ops; u64 size; + destructive = (compression_level & WIMLIB_COMPRESSOR_FLAG_DESTRUCTIVE); + compression_level &= ~WIMLIB_COMPRESSOR_FLAG_DESTRUCTIVE; + if (!compressor_ctype_valid(ctype)) return 0; + if (compression_level > 0xFFFFFF) + return 0; + if (max_block_size == 0) return 0; @@ -98,7 +105,8 @@ wimlib_get_compressor_needed_memory(enum wimlib_compression_type ctype, compression_level = DEFAULT_COMPRESSION_LEVEL; if (ops->get_needed_memory) { - size = ops->get_needed_memory(max_block_size, compression_level); + size = ops->get_needed_memory(max_block_size, compression_level, + destructive); /* 0 is never valid and indicates an invalid max_block_size. */ if (size == 0) @@ -115,11 +123,18 @@ wimlib_create_compressor(enum wimlib_compression_type ctype, unsigned int compression_level, struct wimlib_compressor **c_ret) { + bool destructive; struct wimlib_compressor *c; + destructive = (compression_level & WIMLIB_COMPRESSOR_FLAG_DESTRUCTIVE); + compression_level &= ~WIMLIB_COMPRESSOR_FLAG_DESTRUCTIVE; + if (!compressor_ctype_valid(ctype)) return WIMLIB_ERR_INVALID_COMPRESSION_TYPE; + if (compression_level > 0xFFFFFF) + return WIMLIB_ERR_INVALID_PARAM; + if (c_ret == NULL) return WIMLIB_ERR_INVALID_PARAM; @@ -143,6 +158,7 @@ wimlib_create_compressor(enum wimlib_compression_type ctype, ret = c->ops->create_compressor(max_block_size, compression_level, + destructive, &c->private); if (ret) { FREE(c); diff --git a/src/compress_parallel.c b/src/compress_parallel.c index b3377ffc..0569b176 100644 --- a/src/compress_parallel.c +++ b/src/compress_parallel.c @@ -523,7 +523,8 @@ new_parallel_chunk_compressor(int out_ctype, u32 out_chunk_size, dat->chunks_to_compress_queue = &ctx->chunks_to_compress_queue; dat->compressed_chunks_queue = &ctx->compressed_chunks_queue; - ret = wimlib_create_compressor(out_ctype, out_chunk_size, 0, + ret = wimlib_create_compressor(out_ctype, out_chunk_size, + WIMLIB_COMPRESSOR_FLAG_DESTRUCTIVE, &dat->compressor); if (ret) goto err; diff --git a/src/compress_serial.c b/src/compress_serial.c index 35470ec5..6bac63ec 100644 --- a/src/compress_serial.c +++ b/src/compress_serial.c @@ -127,7 +127,8 @@ new_serial_chunk_compressor(int out_ctype, u32 out_chunk_size, ctx->base.get_compression_result = serial_chunk_compressor_get_compression_result; ret = wimlib_create_compressor(out_ctype, out_chunk_size, - 0, &ctx->compressor); + WIMLIB_COMPRESSOR_FLAG_DESTRUCTIVE, + &ctx->compressor); if (ret) goto err; diff --git a/src/lzms_compress.c b/src/lzms_compress.c index abf39f5e..caa93130 100644 --- a/src/lzms_compress.c +++ b/src/lzms_compress.c @@ -287,6 +287,10 @@ struct lzms_compressor { */ bool use_delta_matches; + /* If true, the compressor need not preserve the input buffer if it + * compresses the data successfully. */ + bool destructive; + /* 'last_target_usages' is a large array that is only needed for * preprocessing, so it is in union with fields that don't need to be * initialized until after preprocessing. */ @@ -2100,7 +2104,8 @@ lzms_finalize(struct lzms_compressor *c) } static u64 -lzms_get_needed_memory(size_t max_bufsize, unsigned compression_level) +lzms_get_needed_memory(size_t max_bufsize, unsigned compression_level, + bool destructive) { u64 size = 0; @@ -2109,8 +2114,8 @@ lzms_get_needed_memory(size_t max_bufsize, unsigned compression_level) size += sizeof(struct lzms_compressor); - /* in_buffer */ - size += max_bufsize; + if (!destructive) + size += max_bufsize; /* in_buffer */ /* mf */ size += lcpit_matchfinder_get_needed_memory(max_bufsize); @@ -2120,7 +2125,7 @@ lzms_get_needed_memory(size_t max_bufsize, unsigned compression_level) static int lzms_create_compressor(size_t max_bufsize, unsigned compression_level, - void **c_ret) + bool destructive, void **c_ret) { struct lzms_compressor *c; u32 nice_match_len; @@ -2132,6 +2137,8 @@ lzms_create_compressor(size_t max_bufsize, unsigned compression_level, if (!c) goto oom0; + c->destructive = destructive; + /* Scale nice_match_len with the compression level. But to allow an * optimization for length cost calculations, don't allow nice_match_len * to exceed MAX_FAST_LENGTH. */ @@ -2142,9 +2149,11 @@ lzms_create_compressor(size_t max_bufsize, unsigned compression_level, c->try_lit_lzrep0 = (compression_level >= 60); c->try_lzrep_lit_lzrep0 = (compression_level >= 60); - c->in_buffer = MALLOC(max_bufsize); - if (!c->in_buffer) - goto oom1; + if (!c->destructive) { + c->in_buffer = MALLOC(max_bufsize); + if (!c->in_buffer) + goto oom1; + } if (!lcpit_matchfinder_init(&c->mf, max_bufsize, 2, nice_match_len)) goto oom2; @@ -2156,7 +2165,8 @@ lzms_create_compressor(size_t max_bufsize, unsigned compression_level, return 0; oom2: - FREE(c->in_buffer); + if (!c->destructive) + FREE(c->in_buffer); oom1: ALIGNED_FREE(c); oom0: @@ -2168,13 +2178,17 @@ lzms_compress(const void *in, size_t in_nbytes, void *out, size_t out_nbytes_avail, void *_c) { struct lzms_compressor *c = _c; + size_t result; /* Don't bother trying to compress extremely small inputs. */ if (in_nbytes < 4) return 0; /* Copy the input data into the internal buffer and preprocess it. */ - memcpy(c->in_buffer, in, in_nbytes); + if (c->destructive) + c->in_buffer = (void *)in; + else + memcpy(c->in_buffer, in, in_nbytes); c->in_nbytes = in_nbytes; lzms_x86_filter(c->in_buffer, in_nbytes, c->last_target_usages, false); @@ -2187,13 +2201,16 @@ lzms_compress(const void *in, size_t in_nbytes, lzms_range_encoder_init(&c->rc, out, out_nbytes_avail / sizeof(le16)); lzms_output_bitstream_init(&c->os, out, out_nbytes_avail / sizeof(le16)); lzms_init_states_and_probabilities(c); - lzms_init_huffman_codes(c, lzms_get_num_offset_slots(in_nbytes)); + lzms_init_huffman_codes(c, lzms_get_num_offset_slots(c->in_nbytes)); /* The main loop: parse and encode. */ lzms_near_optimal_parse(c); /* Return the compressed data size or 0. */ - return lzms_finalize(c); + result = lzms_finalize(c); + if (!result && c->destructive) + lzms_x86_filter(c->in_buffer, c->in_nbytes, c->last_target_usages, true); + return result; } static void @@ -2201,7 +2218,8 @@ lzms_free_compressor(void *_c) { struct lzms_compressor *c = _c; - FREE(c->in_buffer); + if (!c->destructive) + FREE(c->in_buffer); lcpit_matchfinder_destroy(&c->mf); ALIGNED_FREE(c); } diff --git a/src/lzx_compress.c b/src/lzx_compress.c index 075a65a1..1506b884 100644 --- a/src/lzx_compress.c +++ b/src/lzx_compress.c @@ -368,6 +368,10 @@ struct lzx_compressor { /* Pointer to the compress() implementation chosen at allocation time */ void (*impl)(struct lzx_compressor *, struct lzx_output_bitstream *); + /* If true, the compressor need not preserve the input buffer if it + * compresses the data successfully. */ + bool destructive; + /* The Huffman symbol frequency counters for the current block. */ struct lzx_freqs freqs; @@ -2010,7 +2014,8 @@ lzx_get_compressor_size(size_t max_bufsize, unsigned compression_level) } static u64 -lzx_get_needed_memory(size_t max_bufsize, unsigned compression_level) +lzx_get_needed_memory(size_t max_bufsize, unsigned compression_level, + bool destructive) { u64 size = 0; @@ -2018,13 +2023,14 @@ lzx_get_needed_memory(size_t max_bufsize, unsigned compression_level) return 0; size += lzx_get_compressor_size(max_bufsize, compression_level); - size += max_bufsize; /* in_buffer */ + if (!destructive) + size += max_bufsize; /* in_buffer */ return size; } static int lzx_create_compressor(size_t max_bufsize, unsigned compression_level, - void **c_ret) + bool destructive, void **c_ret) { unsigned window_order; struct lzx_compressor *c; @@ -2039,12 +2045,16 @@ lzx_create_compressor(size_t max_bufsize, unsigned compression_level, if (!c) goto oom0; + c->destructive = destructive; + c->num_main_syms = lzx_get_num_main_syms(window_order); c->window_order = window_order; - c->in_buffer = MALLOC(max_bufsize); - if (!c->in_buffer) - goto oom1; + if (!c->destructive) { + c->in_buffer = MALLOC(max_bufsize); + if (!c->in_buffer) + goto oom1; + } if (compression_level <= LZX_MAX_FAST_LEVEL) { @@ -2117,13 +2127,17 @@ lzx_compress(const void *in, size_t in_nbytes, { struct lzx_compressor *c = _c; struct lzx_output_bitstream os; + size_t result; /* Don't bother trying to compress very small inputs. */ if (in_nbytes < 100) return 0; /* Copy the input data into the internal buffer and preprocess it. */ - memcpy(c->in_buffer, in, in_nbytes); + if (c->destructive) + c->in_buffer = (void *)in; + else + memcpy(c->in_buffer, in, in_nbytes); c->in_nbytes = in_nbytes; lzx_do_e8_preprocessing(c->in_buffer, in_nbytes); @@ -2138,7 +2152,10 @@ lzx_compress(const void *in, size_t in_nbytes, (*c->impl)(c, &os); /* Flush the output bitstream and return the compressed size or 0. */ - return lzx_flush_output(&os); + result = lzx_flush_output(&os); + if (!result && c->destructive) + lzx_undo_e8_preprocessing(c->in_buffer, c->in_nbytes); + return result; } static void @@ -2146,7 +2163,8 @@ lzx_free_compressor(void *_c) { struct lzx_compressor *c = _c; - FREE(c->in_buffer); + if (!c->destructive) + FREE(c->in_buffer); ALIGNED_FREE(c); } diff --git a/src/xpress_compress.c b/src/xpress_compress.c index 1c824a57..cf29df99 100644 --- a/src/xpress_compress.c +++ b/src/xpress_compress.c @@ -1031,7 +1031,8 @@ xpress_get_compressor_size(size_t max_bufsize, unsigned compression_level) } static u64 -xpress_get_needed_memory(size_t max_bufsize, unsigned compression_level) +xpress_get_needed_memory(size_t max_bufsize, unsigned compression_level, + bool destructive) { u64 size = 0; @@ -1060,7 +1061,7 @@ xpress_get_needed_memory(size_t max_bufsize, unsigned compression_level) static int xpress_create_compressor(size_t max_bufsize, unsigned compression_level, - void **c_ret) + bool destructive, void **c_ret) { struct xpress_compressor *c; -- 2.43.0