--disable-multithreaded-compression option

diff --git a/README b/README
index e74c23f603b1c4a07dc205a9dcfb780aa8ae1e00..ef387b273890a21b2e9013512cc5bcb353e3c63b 100644 (file)
--- a/README
+++ b/README
@@ -169,6 +169,11 @@ wimlib's `configure' script:
        the setxattr() function and the attr/xattr.h header be available.  The
        default is to autodetect whether support is possible.
 
        the setxattr() function and the attr/xattr.h header be available.  The
        default is to autodetect whether support is possible.
 

+--disable-multithreaded-compression
+       By default, data will be compressed using multiple threads when writing
+       a WIM, unless only 1 processor is detected.  Specify this option to
+       disable support for this.
+

 --enable-ssse3-sha1
        Use a very fast assembly language implementation of SHA1 from Intel.
        Only use this if the build target supports the SSSE3 instructions.
 --enable-ssse3-sha1
        Use a very fast assembly language implementation of SHA1 from Intel.
        Only use this if the build target supports the SSSE3 instructions.

diff --git a/config.h.in b/config.h.in
index c67c3226180f12511a481da8b2e17f65897c2e62..16781521529cad0f9f1918776bce69eeaed2e43a 100644 (file)
--- a/config.h.in
+++ b/config.h.in
@@ -112,6 +112,9 @@
    */
 #undef LT_OBJDIR
 
    */
 #undef LT_OBJDIR
 

+/* Define to 1 if including support for multithreaded compression */
+#undef MULTITHREADED_COMPRESSION
+

 /* Define to 1 if your C compiler doesn't accept -c and -o together. */
 #undef NO_MINUS_C_MINUS_O
 
 /* Define to 1 if your C compiler doesn't accept -c and -o together. */
 #undef NO_MINUS_C_MINUS_O
 

diff --git a/configure.ac b/configure.ac
index 0bed8cd51e8d65b33409628d3ed201e4bfc22db8..cc96b9321bfd2de1e22ecffae16fe1dd1aaa6ff5 100644 (file)
--- a/configure.ac
+++ b/configure.ac
@@ -135,6 +135,20 @@ if test "x$ENABLE_VERIFY_COMPRESSION" = "xyes"; then
                   compressed data])
 fi
 
                   compressed data])
 fi
 

+AC_MSG_CHECKING([whether to include support for multi-threaded compression])
+AC_ARG_ENABLE([multithreaded-compression],
+       AS_HELP_STRING([--disable-multithreaded-compression],
+                       [do not compile in the code for multi-threaded compression]),
+       [ENABLE_MULTITHREADED_COMPRESSION=$enableval],
+       [ENABLE_MULTITHREADED_COMPRESSION=yes]
+       )
+AC_MSG_RESULT([$ENABLE_MULTITHREADED_COMPRESSION])
+if test "x$ENABLE_MULTITHREADED_COMPRESSION" = "xyes"; then
+       AC_DEFINE([MULTITHREADED_COMPRESSION], [1],
+                       [Define to 1 if including support for multithreaded
+                        compression])
+fi
+

 AC_MSG_CHECKING([whether to include support for ntfs-3g])
 AC_ARG_WITH([ntfs-3g],
              AS_HELP_STRING([--without-ntfs-3g], [build without NTFS-3g.
 AC_MSG_CHECKING([whether to include support for ntfs-3g])
 AC_ARG_WITH([ntfs-3g],
              AS_HELP_STRING([--without-ntfs-3g], [build without NTFS-3g.

diff --git a/src/wimlib.h b/src/wimlib.h
index a12a05f435c3e7125c1bcfdf8750ea9dd40845e8..c83a36dce53b6c0e5ce756aab02e89c18f377704 100644 (file)
--- a/src/wimlib.h
+++ b/src/wimlib.h
@@ -280,7 +280,7 @@ enum wim_compression_type {
 #define WIMLIB_ADD_IMAGE_FLAG_DEREFERENCE      0x00000004
 
 /** Show progress information when scanning a directory tree */
 #define WIMLIB_ADD_IMAGE_FLAG_DEREFERENCE      0x00000004
 
 /** Show progress information when scanning a directory tree */

-#define WIMLIB_ADD_IMAGE_FLAG_SHOW_PROGRESS    0x00000004
+#define WIMLIB_ADD_IMAGE_FLAG_SHOW_PROGRESS    0x00000008

 
 /** See documentation for wimlib_export_image(). */
 #define WIMLIB_EXPORT_FLAG_BOOT                        0x00000001
 
 /** See documentation for wimlib_export_image(). */
 #define WIMLIB_EXPORT_FLAG_BOOT                        0x00000001

diff --git a/src/write.c b/src/write.c
index a84c16b0c91e560ded0b13f6ea95b812899e9e70..ef1e02dba39effd838e25408d0d1e2e6b60b6fa3 100644 (file)
--- a/src/write.c
+++ b/src/write.c
@@ -33,9 +33,12 @@
 #include "lzx.h"
 #include "xpress.h"
 #include <unistd.h>
 #include "lzx.h"
 #include "xpress.h"
 #include <unistd.h>

+
+#ifdef ENABLE_MULTITHREADED_COMPRESSION

 #include <semaphore.h>
 #include <pthread.h>
 #include <errno.h>
 #include <semaphore.h>
 #include <pthread.h>
 #include <errno.h>

+#endif

 
 #ifdef WITH_NTFS_3G
 #include <time.h>
 
 #ifdef WITH_NTFS_3G
 #include <time.h>


@@ -130,7 +133,7 @@ WIMLIBAPI int wimlib_overwrite(WIMStruct *w, int write_flags,
                return WIMLIB_ERR_RENAME;
        }
 
                return WIMLIB_ERR_RENAME;
        }
 

-       if (write_flags & WIMLIB_WRITE_FLAG_VERBOSE)
+       if (write_flags & WIMLIB_WRITE_FLAG_SHOW_PROGRESS)

                printf("Successfully renamed `%s' to `%s'\n", tmpfile, wimfile_name);
 
        return 0;
                printf("Successfully renamed `%s' to `%s'\n", tmpfile, wimfile_name);
 
        return 0;


@@ -784,6 +787,7 @@ out:
 }
 
 
 }
 
 

+#ifdef ENABLE_MULTITHREADED_COMPRESSION

 struct shared_queue {
        sem_t filled_slots;
        sem_t empty_slots;
 struct shared_queue {
        sem_t filled_slots;
        sem_t empty_slots;


@@ -903,6 +907,7 @@ static void *compressor_thread_proc(void *arg)
        }
        DEBUG("Compressor thread terminating");
 }
        }
        DEBUG("Compressor thread terminating");
 }

+#endif

 
 static void show_stream_write_progress(u64 *cur_size, u64 *next_size,
                                       u64 total_size, u64 one_percent,
 
 static void show_stream_write_progress(u64 *cur_size, u64 *next_size,
                                       u64 total_size, u64 one_percent,


@@ -957,6 +962,7 @@ static int write_stream_list_serial(struct list_head *stream_list,
        return 0;
 }
 
        return 0;
 }
 

+#ifdef ENABLE_MULTITHREADED_COMPRESSION

 static int write_wim_chunks(struct message *msg, FILE *out_fp,
                            struct chunk_table *chunk_tab)
 {
 static int write_wim_chunks(struct message *msg, FILE *out_fp,
                            struct chunk_table *chunk_tab)
 {


@@ -979,7 +985,6 @@ static int write_wim_chunks(struct message *msg, FILE *out_fp,
        return 0;
 }
 
        return 0;
 }
 

-

 /*
  * This function is executed by the main thread when the resources are being
  * compressed in parallel.  The main thread is in change of all reading of the
 /*
  * This function is executed by the main thread when the resources are being
  * compressed in parallel.  The main thread is in change of all reading of the


@@ -1430,6 +1435,7 @@ static int write_stream_list_parallel(struct list_head *stream_list,
        int ret;
        struct shared_queue res_to_compress_queue;
        struct shared_queue compressed_res_queue;
        int ret;
        struct shared_queue res_to_compress_queue;
        struct shared_queue compressed_res_queue;

+       pthread_t *compressor_threads = NULL;

 
        if (num_threads == 0) {
                long nthreads = sysconf(_SC_NPROCESSORS_ONLN);
 
        if (num_threads == 0) {
                long nthreads = sysconf(_SC_NPROCESSORS_ONLN);


@@ -1443,64 +1449,65 @@ static int write_stream_list_parallel(struct list_head *stream_list,
 
        wimlib_assert(stream_list->next != stream_list);
 
 
        wimlib_assert(stream_list->next != stream_list);
 

-       {
-               pthread_t compressor_threads[num_threads];

 
 

-               static const double MESSAGES_PER_THREAD = 2.0;
-               size_t queue_size = (size_t)(num_threads * MESSAGES_PER_THREAD);
+       static const double MESSAGES_PER_THREAD = 2.0;
+       size_t queue_size = (size_t)(num_threads * MESSAGES_PER_THREAD);

 
 

-               DEBUG("Initializing shared queues (queue_size=%zu)", queue_size);
+       DEBUG("Initializing shared queues (queue_size=%zu)", queue_size);

 
 

-               ret = shared_queue_init(&res_to_compress_queue, queue_size);
-               if (ret != 0)
-                       goto out_serial;
+       ret = shared_queue_init(&res_to_compress_queue, queue_size);
+       if (ret != 0)
+               goto out_serial;

 
 

-               ret = shared_queue_init(&compressed_res_queue, queue_size);
-               if (ret != 0)
-                       goto out_destroy_res_to_compress_queue;
-
-               struct compressor_thread_params params;
-               params.res_to_compress_queue = &res_to_compress_queue;
-               params.compressed_res_queue = &compressed_res_queue;
-               params.compress = get_compress_func(out_ctype);
-
-               for (unsigned i = 0; i < num_threads; i++) {
-                       DEBUG("pthread_create thread %u", i);
-                       ret = pthread_create(&compressor_threads[i], NULL,
-                                            compressor_thread_proc, &params);
-                       if (ret != 0) {
-                               ERROR_WITH_ERRNO("Failed to create compressor "
-                                                "thread %u", i);
-                               num_threads = i;
-                               goto out_join;
-                       }
+       ret = shared_queue_init(&compressed_res_queue, queue_size);
+       if (ret != 0)
+               goto out_destroy_res_to_compress_queue;
+
+       struct compressor_thread_params params;
+       params.res_to_compress_queue = &res_to_compress_queue;
+       params.compressed_res_queue = &compressed_res_queue;
+       params.compress = get_compress_func(out_ctype);
+
+       compressor_threads = MALLOC(num_threads * sizeof(pthread_t));
+
+       for (unsigned i = 0; i < num_threads; i++) {
+               DEBUG("pthread_create thread %u", i);
+               ret = pthread_create(&compressor_threads[i], NULL,
+                                    compressor_thread_proc, &params);
+               if (ret != 0) {
+                       ret = -1;
+                       ERROR_WITH_ERRNO("Failed to create compressor "
+                                        "thread %u", i);
+                       num_threads = i;
+                       goto out_join;

                }
                }

+       }

 
 

-               if (write_flags & WIMLIB_WRITE_FLAG_SHOW_PROGRESS) {
-                       printf("Writing compressed data using %u threads...\n",
-                              num_threads);
-               }
+       if (write_flags & WIMLIB_WRITE_FLAG_SHOW_PROGRESS) {
+               printf("Writing compressed data using %u threads...\n",
+                      num_threads);
+       }

 
 

-               ret = main_writer_thread_proc(stream_list,
-                                             out_fp,
-                                             out_ctype,
-                                             &res_to_compress_queue,
-                                             &compressed_res_queue,
-                                             queue_size,
-                                             write_flags,
-                                             total_size);
-
-       out_join:
-               for (unsigned i = 0; i < num_threads; i++)
-                       shared_queue_put(&res_to_compress_queue, NULL);
-
-               for (unsigned i = 0; i < num_threads; i++) {
-                       if (pthread_join(compressor_threads[i], NULL)) {
-                               WARNING("Failed to join compressor thread %u: %s",
-                                       i, strerror(errno));
-                       }
+       ret = main_writer_thread_proc(stream_list,
+                                     out_fp,
+                                     out_ctype,
+                                     &res_to_compress_queue,
+                                     &compressed_res_queue,
+                                     queue_size,
+                                     write_flags,
+                                     total_size);
+
+out_join:
+       for (unsigned i = 0; i < num_threads; i++)
+               shared_queue_put(&res_to_compress_queue, NULL);
+
+       for (unsigned i = 0; i < num_threads; i++) {
+               if (pthread_join(compressor_threads[i], NULL)) {
+                       WARNING("Failed to join compressor thread %u: %s",
+                               i, strerror(errno));

                }
        }
                }
        }

+       FREE(compressor_threads);

        shared_queue_destroy(&compressed_res_queue);
 out_destroy_res_to_compress_queue:
        shared_queue_destroy(&res_to_compress_queue);
        shared_queue_destroy(&compressed_res_queue);
 out_destroy_res_to_compress_queue:
        shared_queue_destroy(&res_to_compress_queue);


@@ -1511,6 +1518,7 @@ out_serial:
        return write_stream_list_serial(stream_list, out_fp,
                                        out_ctype, write_flags, total_size);
 }
        return write_stream_list_serial(stream_list, out_fp,
                                        out_ctype, write_flags, total_size);
 }

+#endif

 
 static int write_stream_list(struct list_head *stream_list, FILE *out_fp,
                             int out_ctype, int write_flags,
 
 static int write_stream_list(struct list_head *stream_list, FILE *out_fp,
                             int out_ctype, int write_flags,


@@ -1540,14 +1548,18 @@ static int write_stream_list(struct list_head *stream_list, FILE *out_fp,
                       wimlib_get_compression_type_string(out_ctype));
        }
 
                       wimlib_get_compression_type_string(out_ctype));
        }
 

+#ifdef ENABLE_MULTITHREADED_COMPRESSION

        if (compression_needed && total_size >= 1000000 && num_threads != 1) {
                return write_stream_list_parallel(stream_list, out_fp,
                                                  out_ctype, write_flags,
                                                  total_size, num_threads);
        if (compression_needed && total_size >= 1000000 && num_threads != 1) {
                return write_stream_list_parallel(stream_list, out_fp,
                                                  out_ctype, write_flags,
                                                  total_size, num_threads);

-       } else {
+       }
+       else
+#endif
+       {

                if (write_flags & WIMLIB_WRITE_FLAG_SHOW_PROGRESS) {
                        const char *reason = "";
                if (write_flags & WIMLIB_WRITE_FLAG_SHOW_PROGRESS) {
                        const char *reason = "";

-                       if (num_threads != 1)
+                       if (!compression_needed)

                                reason = " (no compression needed)";
                        printf("Writing data using 1 thread%s\n", reason);
                }
                                reason = " (no compression needed)";
                        printf("Writing data using 1 thread%s\n", reason);
                }


@@ -1791,6 +1803,9 @@ WIMLIBAPI int wimlib_write(WIMStruct *w, const char *path,
                return ret;
        }
 
                return ret;
        }
 

+       if (write_flags & WIMLIB_WRITE_FLAG_SHOW_PROGRESS)
+               printf("Writing image metadata...\n");
+

        ret = for_image(w, image, write_metadata_resource);
 
        if (ret != 0) {
        ret = for_image(w, image, write_metadata_resource);
 
        if (ret != 0) {


@@ -1802,7 +1817,7 @@ WIMLIBAPI int wimlib_write(WIMStruct *w, const char *path,
        if (ret != 0)
                return ret;
 
        if (ret != 0)
                return ret;
 

-       if (write_flags & WIMLIB_WRITE_FLAG_VERBOSE)
+       if (write_flags & WIMLIB_WRITE_FLAG_SHOW_PROGRESS)

                printf("Successfully wrote `%s'\n", path);
        return 0;
 }
                printf("Successfully wrote `%s'\n", path);
        return 0;
 }