From: Eric Biggers Date: Sun, 18 Nov 2012 18:46:29 +0000 (-0600) Subject: --disable-multithreaded-compression option X-Git-Tag: v1.1.0~10 X-Git-Url: https://wimlib.net/git/?p=wimlib;a=commitdiff_plain;h=eff67c0d712100642ae6dce8262366eb0c3b911b;ds=sidebyside --disable-multithreaded-compression option --- diff --git a/README b/README index e74c23f6..ef387b27 100644 --- a/README +++ b/README @@ -169,6 +169,11 @@ wimlib's `configure' script: the setxattr() function and the attr/xattr.h header be available. The default is to autodetect whether support is possible. +--disable-multithreaded-compression + By default, data will be compressed using multiple threads when writing + a WIM, unless only 1 processor is detected. Specify this option to + disable support for this. + --enable-ssse3-sha1 Use a very fast assembly language implementation of SHA1 from Intel. Only use this if the build target supports the SSSE3 instructions. diff --git a/config.h.in b/config.h.in index c67c3226..16781521 100644 --- a/config.h.in +++ b/config.h.in @@ -112,6 +112,9 @@ */ #undef LT_OBJDIR +/* Define to 1 if including support for multithreaded compression */ +#undef MULTITHREADED_COMPRESSION + /* Define to 1 if your C compiler doesn't accept -c and -o together. */ #undef NO_MINUS_C_MINUS_O diff --git a/configure.ac b/configure.ac index 0bed8cd5..cc96b932 100644 --- a/configure.ac +++ b/configure.ac @@ -135,6 +135,20 @@ if test "x$ENABLE_VERIFY_COMPRESSION" = "xyes"; then compressed data]) fi +AC_MSG_CHECKING([whether to include support for multi-threaded compression]) +AC_ARG_ENABLE([multithreaded-compression], + AS_HELP_STRING([--disable-multithreaded-compression], + [do not compile in the code for multi-threaded compression]), + [ENABLE_MULTITHREADED_COMPRESSION=$enableval], + [ENABLE_MULTITHREADED_COMPRESSION=yes] + ) +AC_MSG_RESULT([$ENABLE_MULTITHREADED_COMPRESSION]) +if test "x$ENABLE_MULTITHREADED_COMPRESSION" = "xyes"; then + AC_DEFINE([MULTITHREADED_COMPRESSION], [1], + [Define to 1 if including support for multithreaded + compression]) +fi + AC_MSG_CHECKING([whether to include support for ntfs-3g]) AC_ARG_WITH([ntfs-3g], AS_HELP_STRING([--without-ntfs-3g], [build without NTFS-3g. diff --git a/src/wimlib.h b/src/wimlib.h index a12a05f4..c83a36dc 100644 --- a/src/wimlib.h +++ b/src/wimlib.h @@ -280,7 +280,7 @@ enum wim_compression_type { #define WIMLIB_ADD_IMAGE_FLAG_DEREFERENCE 0x00000004 /** Show progress information when scanning a directory tree */ -#define WIMLIB_ADD_IMAGE_FLAG_SHOW_PROGRESS 0x00000004 +#define WIMLIB_ADD_IMAGE_FLAG_SHOW_PROGRESS 0x00000008 /** See documentation for wimlib_export_image(). */ #define WIMLIB_EXPORT_FLAG_BOOT 0x00000001 diff --git a/src/write.c b/src/write.c index a84c16b0..ef1e02db 100644 --- a/src/write.c +++ b/src/write.c @@ -33,9 +33,12 @@ #include "lzx.h" #include "xpress.h" #include + +#ifdef ENABLE_MULTITHREADED_COMPRESSION #include #include #include +#endif #ifdef WITH_NTFS_3G #include @@ -130,7 +133,7 @@ WIMLIBAPI int wimlib_overwrite(WIMStruct *w, int write_flags, return WIMLIB_ERR_RENAME; } - if (write_flags & WIMLIB_WRITE_FLAG_VERBOSE) + if (write_flags & WIMLIB_WRITE_FLAG_SHOW_PROGRESS) printf("Successfully renamed `%s' to `%s'\n", tmpfile, wimfile_name); return 0; @@ -784,6 +787,7 @@ out: } +#ifdef ENABLE_MULTITHREADED_COMPRESSION struct shared_queue { sem_t filled_slots; sem_t empty_slots; @@ -903,6 +907,7 @@ static void *compressor_thread_proc(void *arg) } DEBUG("Compressor thread terminating"); } +#endif static void show_stream_write_progress(u64 *cur_size, u64 *next_size, u64 total_size, u64 one_percent, @@ -957,6 +962,7 @@ static int write_stream_list_serial(struct list_head *stream_list, return 0; } +#ifdef ENABLE_MULTITHREADED_COMPRESSION static int write_wim_chunks(struct message *msg, FILE *out_fp, struct chunk_table *chunk_tab) { @@ -979,7 +985,6 @@ static int write_wim_chunks(struct message *msg, FILE *out_fp, return 0; } - /* * This function is executed by the main thread when the resources are being * compressed in parallel. The main thread is in change of all reading of the @@ -1430,6 +1435,7 @@ static int write_stream_list_parallel(struct list_head *stream_list, int ret; struct shared_queue res_to_compress_queue; struct shared_queue compressed_res_queue; + pthread_t *compressor_threads = NULL; if (num_threads == 0) { long nthreads = sysconf(_SC_NPROCESSORS_ONLN); @@ -1443,64 +1449,65 @@ static int write_stream_list_parallel(struct list_head *stream_list, wimlib_assert(stream_list->next != stream_list); - { - pthread_t compressor_threads[num_threads]; - static const double MESSAGES_PER_THREAD = 2.0; - size_t queue_size = (size_t)(num_threads * MESSAGES_PER_THREAD); + static const double MESSAGES_PER_THREAD = 2.0; + size_t queue_size = (size_t)(num_threads * MESSAGES_PER_THREAD); - DEBUG("Initializing shared queues (queue_size=%zu)", queue_size); + DEBUG("Initializing shared queues (queue_size=%zu)", queue_size); - ret = shared_queue_init(&res_to_compress_queue, queue_size); - if (ret != 0) - goto out_serial; + ret = shared_queue_init(&res_to_compress_queue, queue_size); + if (ret != 0) + goto out_serial; - ret = shared_queue_init(&compressed_res_queue, queue_size); - if (ret != 0) - goto out_destroy_res_to_compress_queue; - - struct compressor_thread_params params; - params.res_to_compress_queue = &res_to_compress_queue; - params.compressed_res_queue = &compressed_res_queue; - params.compress = get_compress_func(out_ctype); - - for (unsigned i = 0; i < num_threads; i++) { - DEBUG("pthread_create thread %u", i); - ret = pthread_create(&compressor_threads[i], NULL, - compressor_thread_proc, ¶ms); - if (ret != 0) { - ERROR_WITH_ERRNO("Failed to create compressor " - "thread %u", i); - num_threads = i; - goto out_join; - } + ret = shared_queue_init(&compressed_res_queue, queue_size); + if (ret != 0) + goto out_destroy_res_to_compress_queue; + + struct compressor_thread_params params; + params.res_to_compress_queue = &res_to_compress_queue; + params.compressed_res_queue = &compressed_res_queue; + params.compress = get_compress_func(out_ctype); + + compressor_threads = MALLOC(num_threads * sizeof(pthread_t)); + + for (unsigned i = 0; i < num_threads; i++) { + DEBUG("pthread_create thread %u", i); + ret = pthread_create(&compressor_threads[i], NULL, + compressor_thread_proc, ¶ms); + if (ret != 0) { + ret = -1; + ERROR_WITH_ERRNO("Failed to create compressor " + "thread %u", i); + num_threads = i; + goto out_join; } + } - if (write_flags & WIMLIB_WRITE_FLAG_SHOW_PROGRESS) { - printf("Writing compressed data using %u threads...\n", - num_threads); - } + if (write_flags & WIMLIB_WRITE_FLAG_SHOW_PROGRESS) { + printf("Writing compressed data using %u threads...\n", + num_threads); + } - ret = main_writer_thread_proc(stream_list, - out_fp, - out_ctype, - &res_to_compress_queue, - &compressed_res_queue, - queue_size, - write_flags, - total_size); - - out_join: - for (unsigned i = 0; i < num_threads; i++) - shared_queue_put(&res_to_compress_queue, NULL); - - for (unsigned i = 0; i < num_threads; i++) { - if (pthread_join(compressor_threads[i], NULL)) { - WARNING("Failed to join compressor thread %u: %s", - i, strerror(errno)); - } + ret = main_writer_thread_proc(stream_list, + out_fp, + out_ctype, + &res_to_compress_queue, + &compressed_res_queue, + queue_size, + write_flags, + total_size); + +out_join: + for (unsigned i = 0; i < num_threads; i++) + shared_queue_put(&res_to_compress_queue, NULL); + + for (unsigned i = 0; i < num_threads; i++) { + if (pthread_join(compressor_threads[i], NULL)) { + WARNING("Failed to join compressor thread %u: %s", + i, strerror(errno)); } } + FREE(compressor_threads); shared_queue_destroy(&compressed_res_queue); out_destroy_res_to_compress_queue: shared_queue_destroy(&res_to_compress_queue); @@ -1511,6 +1518,7 @@ out_serial: return write_stream_list_serial(stream_list, out_fp, out_ctype, write_flags, total_size); } +#endif static int write_stream_list(struct list_head *stream_list, FILE *out_fp, int out_ctype, int write_flags, @@ -1540,14 +1548,18 @@ static int write_stream_list(struct list_head *stream_list, FILE *out_fp, wimlib_get_compression_type_string(out_ctype)); } +#ifdef ENABLE_MULTITHREADED_COMPRESSION if (compression_needed && total_size >= 1000000 && num_threads != 1) { return write_stream_list_parallel(stream_list, out_fp, out_ctype, write_flags, total_size, num_threads); - } else { + } + else +#endif + { if (write_flags & WIMLIB_WRITE_FLAG_SHOW_PROGRESS) { const char *reason = ""; - if (num_threads != 1) + if (!compression_needed) reason = " (no compression needed)"; printf("Writing data using 1 thread%s\n", reason); } @@ -1791,6 +1803,9 @@ WIMLIBAPI int wimlib_write(WIMStruct *w, const char *path, return ret; } + if (write_flags & WIMLIB_WRITE_FLAG_SHOW_PROGRESS) + printf("Writing image metadata...\n"); + ret = for_image(w, image, write_metadata_resource); if (ret != 0) { @@ -1802,7 +1817,7 @@ WIMLIBAPI int wimlib_write(WIMStruct *w, const char *path, if (ret != 0) return ret; - if (write_flags & WIMLIB_WRITE_FLAG_VERBOSE) + if (write_flags & WIMLIB_WRITE_FLAG_SHOW_PROGRESS) printf("Successfully wrote `%s'\n", path); return 0; }