X-Git-Url: https://wimlib.net/git/?a=blobdiff_plain;f=src%2Fcompress_parallel.c;h=f339298abf51715e0cb52f43b4a266a1064b62eb;hb=4a21a385ee1d7789fa871785033a877b1ac94eed;hp=d0ce5266464cb4021797cbd2a81c4ed5c5c652c3;hpb=883833a4b3dabec325edf1ca938000f91d587c00;p=wimlib

diff --git a/src/compress_parallel.c b/src/compress_parallel.c
index d0ce5266..f339298a 100644
--- a/src/compress_parallel.c
+++ b/src/compress_parallel.c
@@ -26,6 +26,8 @@
 #  include "config.h"
 #endif
 
+#ifdef ENABLE_MULTITHREADED_COMPRESSION
+
 #include "wimlib/assert.h"
 #include "wimlib/chunk_compressor.h"
 #include "wimlib/error.h"
@@ -41,6 +43,9 @@
 #include <stdlib.h>
 #include <string.h>
 #include <unistd.h>
+#ifdef HAVE_SYS_SYSCTL_H
+#  include <sys/sysctl.h>
+#endif
 
 struct message_queue {
 	struct list_head list;
@@ -62,8 +67,8 @@ struct compressor_thread_data {
 struct message {
 	u8 *uncompressed_chunks[MAX_CHUNKS_PER_MSG];
 	u8 *compressed_chunks[MAX_CHUNKS_PER_MSG];
-	unsigned uncompressed_chunk_sizes[MAX_CHUNKS_PER_MSG];
-	unsigned compressed_chunk_sizes[MAX_CHUNKS_PER_MSG];
+	u32 uncompressed_chunk_sizes[MAX_CHUNKS_PER_MSG];
+	u32 compressed_chunk_sizes[MAX_CHUNKS_PER_MSG];
 	size_t num_filled_chunks;
 	size_t num_alloc_chunks;
 	struct list_head list;
@@ -114,12 +119,19 @@ get_avail_memory(void)
 	if (phys_bytes == 0)
 		goto default_size;
 	return phys_bytes;
-#else
+#elif defined(_SC_PAGESIZE) && defined(_SC_PHYS_PAGES)
 	long page_size = sysconf(_SC_PAGESIZE);
 	long num_pages = sysconf(_SC_PHYS_PAGES);
 	if (page_size <= 0 || num_pages <= 0)
 		goto default_size;
 	return ((u64)page_size * (u64)num_pages);
+#else
+	int mib[2] = {CTL_HW, HW_MEMSIZE};
+	u64 memsize;
+	size_t len = sizeof(memsize);
+	if (sysctl(mib, ARRAY_LEN(mib), &memsize, &len, NULL, 0) < 0 || len != 8)
+		goto default_size;
+	return memsize;
 #endif
 
 default_size:
@@ -349,8 +361,8 @@ parallel_chunk_compressor_submit_chunk(struct chunk_compressor *_ctx,
 
 static bool
 parallel_chunk_compressor_get_chunk(struct chunk_compressor *_ctx,
-				    const void **cdata_ret, unsigned *csize_ret,
-				    unsigned *usize_ret)
+				    const void **cdata_ret, u32 *csize_ret,
+				    u32 *usize_ret)
 {
 	struct parallel_chunk_compressor *ctx = (struct parallel_chunk_compressor *)_ctx;
 	struct message *msg;
@@ -405,7 +417,6 @@ new_parallel_chunk_compressor(int out_ctype, u32 out_chunk_size,
 	unsigned desired_num_threads;
 
 	wimlib_assert(out_chunk_size > 0);
-	wimlib_assert(out_ctype != WIMLIB_COMPRESSION_TYPE_NONE);
 
 	if (num_threads == 0)
 		num_threads = get_default_num_threads();
@@ -421,16 +432,26 @@ new_parallel_chunk_compressor(int out_ctype, u32 out_chunk_size,
 
 	desired_num_threads = num_threads;
 
-	chunks_per_msg = MAX_CHUNKS_PER_MSG;
-	msgs_per_thread = 2;
+	if (out_chunk_size < ((u32)1 << 23)) {
+		chunks_per_msg = MAX_CHUNKS_PER_MSG;
+		msgs_per_thread = 2;
+	} else {
+		/* Big chunks: Just have one buffer per thread --- more would
+		 * just waste memory.  */
+		chunks_per_msg = 1;
+		msgs_per_thread = 1;
+	}
 	for (;;) {
 		approx_mem_required =
 			(u64)chunks_per_msg *
 			(u64)msgs_per_thread *
 			(u64)num_threads *
 			(u64)out_chunk_size
+			+ out_chunk_size
 			+ 1000000
-			+ (out_chunk_size * num_threads * 4);
+			+ num_threads * wimlib_get_compressor_needed_memory(out_ctype,
+									    out_chunk_size,
+									    NULL);
 		if (approx_mem_required <= max_memory)
 			break;
 
@@ -509,7 +530,9 @@ new_parallel_chunk_compressor(int out_ctype, u32 out_chunk_size,
 		if (ret) {
 			errno = ret;
 			ret = WIMLIB_ERR_NOMEM;
-			WARNING_WITH_ERRNO("Failed to create compressor thread %u of %u");
+			WARNING_WITH_ERRNO("Failed to create compressor thread %u of %u",
+					   ctx->num_started_threads + 1,
+					   num_threads);
 			goto err;
 		}
 	}
@@ -534,3 +557,5 @@ err:
 	parallel_chunk_compressor_destroy(&ctx->base);
 	return ret;
 }
+
+#endif /* ENABLE_MULTITHREADED_COMPRESSION */