/*
* Copyright (C) 2013 Eric Biggers
*
- * This file is part of wimlib, a library for working with WIM files.
+ * This file is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU Lesser General Public License as published by the Free
+ * Software Foundation; either version 3 of the License, or (at your option) any
+ * later version.
*
- * wimlib is free software; you can redistribute it and/or modify it under the
- * terms of the GNU General Public License as published by the Free Software
- * Foundation; either version 3 of the License, or (at your option) any later
- * version.
+ * This file is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
+ * details.
*
- * wimlib is distributed in the hope that it will be useful, but WITHOUT ANY
- * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
- * A PARTICULAR PURPOSE. See the GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * wimlib; if not, see http://www.gnu.org/licenses/.
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this file; if not, see http://www.gnu.org/licenses/.
*/
#ifdef HAVE_CONFIG_H
#ifdef ENABLE_MULTITHREADED_COMPRESSION
-#include "wimlib/assert.h"
-#include "wimlib/chunk_compressor.h"
-#include "wimlib/error.h"
-#include "wimlib/list.h"
-#include "wimlib/util.h"
-#ifdef __WIN32__
-# include "wimlib/win32.h" /* win32_get_number_of_processors() */
-#endif
-
#include <errno.h>
#include <limits.h>
#include <pthread.h>
# include <sys/sysctl.h>
#endif
+#include "wimlib/assert.h"
+#include "wimlib/chunk_compressor.h"
+#include "wimlib/error.h"
+#include "wimlib/list.h"
+#include "wimlib/util.h"
+#ifdef __WIN32__
+# include "wimlib/win32.h" /* win32_get_number_of_processors() */
+#endif
+
struct message_queue {
struct list_head list;
pthread_mutex_t lock;
struct wimlib_compressor *compressor;
};
-#define MAX_CHUNKS_PER_MSG 2
+#define MAX_CHUNKS_PER_MSG 16
struct message {
u8 *uncompressed_chunks[MAX_CHUNKS_PER_MSG];
struct message_queue chunks_to_compress_queue;
struct message_queue compressed_chunks_queue;
struct compressor_thread_data *thread_data;
- unsigned num_threads;
+ unsigned num_thread_data;
unsigned num_started_threads;
struct message *msgs;
message_queue_destroy(&ctx->compressed_chunks_queue);
if (ctx->thread_data != NULL)
- for (i = 0; i < ctx->num_threads; i++)
+ for (i = 0; i < ctx->num_thread_data; i++)
wimlib_free_compressor(ctx->thread_data[i].compressor);
FREE(ctx->thread_data);
desired_num_threads = num_threads;
if (out_chunk_size < ((u32)1 << 23)) {
- chunks_per_msg = MAX_CHUNKS_PER_MSG;
+ /* Relatively small chunks. Use 2 messages per thread, each
+ * with at least 2 chunks. Use more chunks per message if there
+ * are lots of threads and/or the chunks are very small. */
+ chunks_per_msg = 2;
+ chunks_per_msg += num_threads * (65536 / out_chunk_size) / 16;
+ chunks_per_msg = max(chunks_per_msg, 2);
+ chunks_per_msg = min(chunks_per_msg, MAX_CHUNKS_PER_MSG);
msgs_per_thread = 2;
} else {
/* Big chunks: Just have one buffer per thread --- more would
+ 1000000
+ num_threads * wimlib_get_compressor_needed_memory(out_ctype,
out_chunk_size,
- NULL);
+ 0);
if (approx_mem_required <= max_memory)
break;
ctx->base.out_ctype = out_ctype;
ctx->base.out_chunk_size = out_chunk_size;
- ctx->base.num_threads = num_threads;
ctx->base.destroy = parallel_chunk_compressor_destroy;
ctx->base.submit_chunk = parallel_chunk_compressor_submit_chunk;
ctx->base.get_chunk = parallel_chunk_compressor_get_chunk;
- ctx->num_threads = num_threads;
+ ctx->num_thread_data = num_threads;
ret = message_queue_init(&ctx->chunks_to_compress_queue);
if (ret)
dat->chunks_to_compress_queue = &ctx->chunks_to_compress_queue;
dat->compressed_chunks_queue = &ctx->compressed_chunks_queue;
- ret = wimlib_create_compressor(out_ctype, out_chunk_size,
- NULL, &dat->compressor);
+ ret = wimlib_create_compressor(out_ctype, out_chunk_size, 0,
+ &dat->compressor);
if (ret)
goto err;
}
&ctx->thread_data[ctx->num_started_threads]);
if (ret) {
errno = ret;
- ret = WIMLIB_ERR_NOMEM;
WARNING_WITH_ERRNO("Failed to create compressor thread %u of %u",
ctx->num_started_threads + 1,
num_threads);
+ ret = WIMLIB_ERR_NOMEM;
+ if (ctx->num_started_threads >= 2)
+ break;
goto err;
}
}
+ ctx->base.num_threads = ctx->num_started_threads;
+
ret = WIMLIB_ERR_NOMEM;
- ctx->num_messages = num_threads * msgs_per_thread;
+ ctx->num_messages = ctx->num_started_threads * msgs_per_thread;
ctx->msgs = allocate_messages(ctx->num_messages,
chunks_per_msg, out_chunk_size);
if (ctx->msgs == NULL)
git://wimlib.net / wimlib / blobdiff