From aae306916e3c97040650b669e21f30bf32b0839d Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Wed, 24 Sep 2014 23:21:13 -0500 Subject: [PATCH] compress_parallel.c: Smarter setting of chunks_per_msg Useful when compressing small chunks with many threads. --- src/compress_parallel.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/compress_parallel.c b/src/compress_parallel.c index e624819b..bba4e120 100644 --- a/src/compress_parallel.c +++ b/src/compress_parallel.c @@ -62,7 +62,7 @@ struct compressor_thread_data { struct wimlib_compressor *compressor; }; -#define MAX_CHUNKS_PER_MSG 2 +#define MAX_CHUNKS_PER_MSG 16 struct message { u8 *uncompressed_chunks[MAX_CHUNKS_PER_MSG]; @@ -433,7 +433,13 @@ new_parallel_chunk_compressor(int out_ctype, u32 out_chunk_size, desired_num_threads = num_threads; if (out_chunk_size < ((u32)1 << 23)) { - chunks_per_msg = MAX_CHUNKS_PER_MSG; + /* Relatively small chunks. Use 2 messages per thread, each + * with at least 2 chunks. Use more chunks per message if there + * are lots of threads and/or the chunks are very small. */ + chunks_per_msg = 2; + chunks_per_msg += num_threads * (65536 / out_chunk_size) / 16; + chunks_per_msg = max(chunks_per_msg, 2); + chunks_per_msg = min(chunks_per_msg, MAX_CHUNKS_PER_MSG); msgs_per_thread = 2; } else { /* Big chunks: Just have one buffer per thread --- more would -- 2.43.0