X-Git-Url: https://wimlib.net/git/?a=blobdiff_plain;f=src%2Fcompress_parallel.c;h=90b81dd8357ce5e2c02da6f165789e71f8c3bc0c;hb=a5f0f107247cc6400c0bd25f41e49d658fd2b7d7;hp=f339298abf51715e0cb52f43b4a266a1064b62eb;hpb=f675c45459508d058aeda4869e2928be4b7ffd99;p=wimlib

diff --git a/src/compress_parallel.c b/src/compress_parallel.c
index f339298a..90b81dd8 100644
--- a/src/compress_parallel.c
+++ b/src/compress_parallel.c
@@ -7,19 +7,18 @@
 /*
  * Copyright (C) 2013 Eric Biggers
  *
- * This file is part of wimlib, a library for working with WIM files.
+ * This file is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU Lesser General Public License as published by the Free
+ * Software Foundation; either version 3 of the License, or (at your option) any
+ * later version.
  *
- * wimlib is free software; you can redistribute it and/or modify it under the
- * terms of the GNU General Public License as published by the Free Software
- * Foundation; either version 3 of the License, or (at your option) any later
- * version.
+ * This file is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
+ * details.
  *
- * wimlib is distributed in the hope that it will be useful, but WITHOUT ANY
- * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
- * A PARTICULAR PURPOSE. See the GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * wimlib; if not, see http://www.gnu.org/licenses/.
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this file; if not, see http://www.gnu.org/licenses/.
  */
 
 #ifdef HAVE_CONFIG_H
@@ -62,7 +61,7 @@ struct compressor_thread_data {
 	struct wimlib_compressor *compressor;
 };
 
-#define MAX_CHUNKS_PER_MSG 2
+#define MAX_CHUNKS_PER_MSG 16
 
 struct message {
 	u8 *uncompressed_chunks[MAX_CHUNKS_PER_MSG];
@@ -82,7 +81,7 @@ struct parallel_chunk_compressor {
 	struct message_queue chunks_to_compress_queue;
 	struct message_queue compressed_chunks_queue;
 	struct compressor_thread_data *thread_data;
-	unsigned num_threads;
+	unsigned num_thread_data;
 	unsigned num_started_threads;
 
 	struct message *msgs;
@@ -136,7 +135,7 @@ get_avail_memory(void)
 
 default_size:
 	WARNING("Failed to determine available memory; assuming 1 GiB");
-	return 1U << 30;
+	return 1ULL << 30;
 }
 
 static int
@@ -309,7 +308,7 @@ parallel_chunk_compressor_destroy(struct chunk_compressor *_ctx)
 	message_queue_destroy(&ctx->compressed_chunks_queue);
 
 	if (ctx->thread_data != NULL)
-		for (i = 0; i < ctx->num_threads; i++)
+		for (i = 0; i < ctx->num_thread_data; i++)
 			wimlib_free_compressor(ctx->thread_data[i].compressor);
 
 	FREE(ctx->thread_data);
@@ -332,7 +331,7 @@ submit_compression_msg(struct parallel_chunk_compressor *ctx)
 
 static bool
 parallel_chunk_compressor_submit_chunk(struct chunk_compressor *_ctx,
-				       const void *chunk, size_t size)
+				       const void *chunk, u32 size)
 {
 	struct parallel_chunk_compressor *ctx = (struct parallel_chunk_compressor *)_ctx;
 	struct message *msg;
@@ -433,7 +432,13 @@ new_parallel_chunk_compressor(int out_ctype, u32 out_chunk_size,
 	desired_num_threads = num_threads;
 
 	if (out_chunk_size < ((u32)1 << 23)) {
-		chunks_per_msg = MAX_CHUNKS_PER_MSG;
+		/* Relatively small chunks.  Use 2 messages per thread, each
+		 * with at least 2 chunks.  Use more chunks per message if there
+		 * are lots of threads and/or the chunks are very small.  */
+		chunks_per_msg = 2;
+		chunks_per_msg += num_threads * (65536 / out_chunk_size) / 16;
+		chunks_per_msg = max(chunks_per_msg, 2);
+		chunks_per_msg = min(chunks_per_msg, MAX_CHUNKS_PER_MSG);
 		msgs_per_thread = 2;
 	} else {
 		/* Big chunks: Just have one buffer per thread --- more would
@@ -451,7 +456,7 @@ new_parallel_chunk_compressor(int out_ctype, u32 out_chunk_size,
 			+ 1000000
 			+ num_threads * wimlib_get_compressor_needed_memory(out_ctype,
 									    out_chunk_size,
-									    NULL);
+									    0);
 		if (approx_mem_required <= max_memory)
 			break;
 
@@ -484,12 +489,11 @@ new_parallel_chunk_compressor(int out_ctype, u32 out_chunk_size,
 
 	ctx->base.out_ctype = out_ctype;
 	ctx->base.out_chunk_size = out_chunk_size;
-	ctx->base.num_threads = num_threads;
 	ctx->base.destroy = parallel_chunk_compressor_destroy;
 	ctx->base.submit_chunk = parallel_chunk_compressor_submit_chunk;
 	ctx->base.get_chunk = parallel_chunk_compressor_get_chunk;
 
-	ctx->num_threads = num_threads;
+	ctx->num_thread_data = num_threads;
 
 	ret = message_queue_init(&ctx->chunks_to_compress_queue);
 	if (ret)
@@ -511,8 +515,8 @@ new_parallel_chunk_compressor(int out_ctype, u32 out_chunk_size,
 
 		dat->chunks_to_compress_queue = &ctx->chunks_to_compress_queue;
 		dat->compressed_chunks_queue = &ctx->compressed_chunks_queue;
-		ret = wimlib_create_compressor(out_ctype, out_chunk_size,
-					       NULL, &dat->compressor);
+		ret = wimlib_create_compressor(out_ctype, out_chunk_size, 0,
+					       &dat->compressor);
 		if (ret)
 			goto err;
 	}
@@ -529,16 +533,20 @@ new_parallel_chunk_compressor(int out_ctype, u32 out_chunk_size,
 				     &ctx->thread_data[ctx->num_started_threads]);
 		if (ret) {
 			errno = ret;
-			ret = WIMLIB_ERR_NOMEM;
 			WARNING_WITH_ERRNO("Failed to create compressor thread %u of %u",
 					   ctx->num_started_threads + 1,
 					   num_threads);
+			ret = WIMLIB_ERR_NOMEM;
+			if (ctx->num_started_threads >= 2)
+				break;
 			goto err;
 		}
 	}
 
+	ctx->base.num_threads = ctx->num_started_threads;
+
 	ret = WIMLIB_ERR_NOMEM;
-	ctx->num_messages = num_threads * msgs_per_thread;
+	ctx->num_messages = ctx->num_started_threads * msgs_per_thread;
 	ctx->msgs = allocate_messages(ctx->num_messages,
 				      chunks_per_msg, out_chunk_size);
 	if (ctx->msgs == NULL)