X-Git-Url: https://wimlib.net/git/?p=wimlib;a=blobdiff_plain;f=src%2Fcompress_parallel.c;h=e624819be77bb432a6c685ece3e2e039537c56b7;hp=376e59d6a2fdca8a05fae9486d58fc0f51bba08d;hb=4dd45340f9fe3a533e0f1a9d6b79f8118e45ca2a;hpb=41f15b937564a3ae58f199c27e8290a1b1a40856

diff --git a/src/compress_parallel.c b/src/compress_parallel.c
index 376e59d6..e624819b 100644
--- a/src/compress_parallel.c
+++ b/src/compress_parallel.c
@@ -43,6 +43,9 @@
 #include <stdlib.h>
 #include <string.h>
 #include <unistd.h>
+#ifdef HAVE_SYS_SYSCTL_H
+#  include <sys/sysctl.h>
+#endif
 
 struct message_queue {
 	struct list_head list;
@@ -64,8 +67,8 @@ struct compressor_thread_data {
 struct message {
 	u8 *uncompressed_chunks[MAX_CHUNKS_PER_MSG];
 	u8 *compressed_chunks[MAX_CHUNKS_PER_MSG];
-	unsigned uncompressed_chunk_sizes[MAX_CHUNKS_PER_MSG];
-	unsigned compressed_chunk_sizes[MAX_CHUNKS_PER_MSG];
+	u32 uncompressed_chunk_sizes[MAX_CHUNKS_PER_MSG];
+	u32 compressed_chunk_sizes[MAX_CHUNKS_PER_MSG];
 	size_t num_filled_chunks;
 	size_t num_alloc_chunks;
 	struct list_head list;
@@ -79,7 +82,7 @@ struct parallel_chunk_compressor {
 	struct message_queue chunks_to_compress_queue;
 	struct message_queue compressed_chunks_queue;
 	struct compressor_thread_data *thread_data;
-	unsigned num_threads;
+	unsigned num_thread_data;
 	unsigned num_started_threads;
 
 	struct message *msgs;
@@ -116,17 +119,24 @@ get_avail_memory(void)
 	if (phys_bytes == 0)
 		goto default_size;
 	return phys_bytes;
-#else
+#elif defined(_SC_PAGESIZE) && defined(_SC_PHYS_PAGES)
 	long page_size = sysconf(_SC_PAGESIZE);
 	long num_pages = sysconf(_SC_PHYS_PAGES);
 	if (page_size <= 0 || num_pages <= 0)
 		goto default_size;
 	return ((u64)page_size * (u64)num_pages);
+#else
+	int mib[2] = {CTL_HW, HW_MEMSIZE};
+	u64 memsize;
+	size_t len = sizeof(memsize);
+	if (sysctl(mib, ARRAY_LEN(mib), &memsize, &len, NULL, 0) < 0 || len != 8)
+		goto default_size;
+	return memsize;
 #endif
 
 default_size:
 	WARNING("Failed to determine available memory; assuming 1 GiB");
-	return 1U << 30;
+	return 1ULL << 30;
 }
 
 static int
@@ -299,7 +309,7 @@ parallel_chunk_compressor_destroy(struct chunk_compressor *_ctx)
 	message_queue_destroy(&ctx->compressed_chunks_queue);
 
 	if (ctx->thread_data != NULL)
-		for (i = 0; i < ctx->num_threads; i++)
+		for (i = 0; i < ctx->num_thread_data; i++)
 			wimlib_free_compressor(ctx->thread_data[i].compressor);
 
 	FREE(ctx->thread_data);
@@ -322,7 +332,7 @@ submit_compression_msg(struct parallel_chunk_compressor *ctx)
 
 static bool
 parallel_chunk_compressor_submit_chunk(struct chunk_compressor *_ctx,
-				       const void *chunk, size_t size)
+				       const void *chunk, u32 size)
 {
 	struct parallel_chunk_compressor *ctx = (struct parallel_chunk_compressor *)_ctx;
 	struct message *msg;
@@ -351,8 +361,8 @@ parallel_chunk_compressor_submit_chunk(struct chunk_compressor *_ctx,
 
 static bool
 parallel_chunk_compressor_get_chunk(struct chunk_compressor *_ctx,
-				    const void **cdata_ret, unsigned *csize_ret,
-				    unsigned *usize_ret)
+				    const void **cdata_ret, u32 *csize_ret,
+				    u32 *usize_ret)
 {
 	struct parallel_chunk_compressor *ctx = (struct parallel_chunk_compressor *)_ctx;
 	struct message *msg;
@@ -407,7 +417,6 @@ new_parallel_chunk_compressor(int out_ctype, u32 out_chunk_size,
 	unsigned desired_num_threads;
 
 	wimlib_assert(out_chunk_size > 0);
-	wimlib_assert(out_ctype != WIMLIB_COMPRESSION_TYPE_NONE);
 
 	if (num_threads == 0)
 		num_threads = get_default_num_threads();
@@ -442,7 +451,7 @@ new_parallel_chunk_compressor(int out_ctype, u32 out_chunk_size,
 			+ 1000000
 			+ num_threads * wimlib_get_compressor_needed_memory(out_ctype,
 									    out_chunk_size,
-									    NULL);
+									    0);
 		if (approx_mem_required <= max_memory)
 			break;
 
@@ -475,12 +484,11 @@ new_parallel_chunk_compressor(int out_ctype, u32 out_chunk_size,
 
 	ctx->base.out_ctype = out_ctype;
 	ctx->base.out_chunk_size = out_chunk_size;
-	ctx->base.num_threads = num_threads;
 	ctx->base.destroy = parallel_chunk_compressor_destroy;
 	ctx->base.submit_chunk = parallel_chunk_compressor_submit_chunk;
 	ctx->base.get_chunk = parallel_chunk_compressor_get_chunk;
 
-	ctx->num_threads = num_threads;
+	ctx->num_thread_data = num_threads;
 
 	ret = message_queue_init(&ctx->chunks_to_compress_queue);
 	if (ret)
@@ -502,8 +510,8 @@ new_parallel_chunk_compressor(int out_ctype, u32 out_chunk_size,
 
 		dat->chunks_to_compress_queue = &ctx->chunks_to_compress_queue;
 		dat->compressed_chunks_queue = &ctx->compressed_chunks_queue;
-		ret = wimlib_create_compressor(out_ctype, out_chunk_size,
-					       NULL, &dat->compressor);
+		ret = wimlib_create_compressor(out_ctype, out_chunk_size, 0,
+					       &dat->compressor);
 		if (ret)
 			goto err;
 	}
@@ -520,14 +528,20 @@ new_parallel_chunk_compressor(int out_ctype, u32 out_chunk_size,
 				     &ctx->thread_data[ctx->num_started_threads]);
 		if (ret) {
 			errno = ret;
+			WARNING_WITH_ERRNO("Failed to create compressor thread %u of %u",
+					   ctx->num_started_threads + 1,
+					   num_threads);
 			ret = WIMLIB_ERR_NOMEM;
-			WARNING_WITH_ERRNO("Failed to create compressor thread %u of %u");
+			if (ctx->num_started_threads >= 2)
+				break;
 			goto err;
 		}
 	}
 
+	ctx->base.num_threads = ctx->num_started_threads;
+
 	ret = WIMLIB_ERR_NOMEM;
-	ctx->num_messages = num_threads * msgs_per_thread;
+	ctx->num_messages = ctx->num_started_threads * msgs_per_thread;
 	ctx->msgs = allocate_messages(ctx->num_messages,
 				      chunks_per_msg, out_chunk_size);
 	if (ctx->msgs == NULL)