WIM chunk size: Calculate with existing macros when possible

[wimlib] / src / write.c

diff --git a/src/write.c b/src/write.c
index d184a64a23485689dc27ad3704b3c4b79da6875d..fde7e939ed57a83d4ef053ff0d69db755d96750d 100644 (file)
--- a/src/write.c
+++ b/src/write.c
@@ -24,25 +24,31 @@
  * along with wimlib; if not, see http://www.gnu.org/licenses/.
  */
 
-#include "config.h"
+#ifdef HAVE_CONFIG_H
+#  include "config.h"
+#endif
 
 #if defined(HAVE_SYS_FILE_H) && defined(HAVE_FLOCK)
-/* On BSD, this should be included before "list.h" so that "list.h" can
+/* On BSD, this should be included before "wimlib/list.h" so that "wimlib/list.h" can
  * overwrite the LIST_HEAD macro. */
 #  include <sys/file.h>
 #endif
 
+#include "wimlib/endianness.h"
+#include "wimlib/error.h"
+#include "wimlib/file_io.h"
+#include "wimlib/header.h"
+#include "wimlib/integrity.h"
+#include "wimlib/lookup_table.h"
+#include "wimlib/metadata.h"
+#include "wimlib/resource.h"
+#include "wimlib/write.h"
+#include "wimlib/xml.h"
+
 #ifdef __WIN32__
-#  include "win32.h"
+#  include "wimlib/win32.h" /* win32_get_number_of_processors() */
 #endif
 
-#include "list.h"
-#include "wimlib_internal.h"
-#include "buffer_io.h"
-#include "dentry.h"
-#include "lookup_table.h"
-#include "xml.h"
-
 #ifdef ENABLE_MULTITHREADED_COMPRESSION
 #  include <pthread.h>
 #endif
@@ -81,7 +87,10 @@ struct chunk_table {
        u64 table_disk_size;
        u64 cur_offset;
        u64 *cur_offset_p;
-       u64 offsets[0];
+       union {
+               u64 offsets[0];
+               u32 u32_offsets[0];
+       };
 };
 
 /*
@@ -95,7 +104,7 @@ begin_wim_resource_chunk_tab(const struct wim_lookup_table_entry *lte,
                             struct chunk_table **chunk_tab_ret)
 {
        u64 size = wim_resource_size(lte);
-       u64 num_chunks = (size + WIM_CHUNK_SIZE - 1) / WIM_CHUNK_SIZE;
+       u64 num_chunks = wim_resource_chunks(lte);
        size_t alloc_size = sizeof(struct chunk_table) + num_chunks * sizeof(u64);
        struct chunk_table *chunk_tab = CALLOC(1, alloc_size);
 
@@ -109,7 +118,7 @@ begin_wim_resource_chunk_tab(const struct wim_lookup_table_entry *lte,
        chunk_tab->file_offset = file_offset;
        chunk_tab->num_chunks = num_chunks;
        chunk_tab->original_resource_size = size;
-       chunk_tab->bytes_per_chunk_entry = (size >= (1ULL << 32)) ? 8 : 4;
+       chunk_tab->bytes_per_chunk_entry = (size > (1ULL << 32)) ? 8 : 4;
        chunk_tab->table_disk_size = chunk_tab->bytes_per_chunk_entry *
                                     (num_chunks - 1);
        chunk_tab->cur_offset = 0;
@@ -227,8 +236,7 @@ finish_wim_resource_chunk_tab(struct chunk_table *chunk_tab,
                array_cpu_to_le64(chunk_tab->offsets, chunk_tab->num_chunks);
        } else {
                for (u64 i = 0; i < chunk_tab->num_chunks; i++)
-                       ((u32*)chunk_tab->offsets)[i] =
-                               cpu_to_le32(chunk_tab->offsets[i]);
+                       chunk_tab->u32_offsets[i] = cpu_to_le32(chunk_tab->offsets[i]);
        }
        bytes_written = full_pwrite(out_fd,
                                    (u8*)chunk_tab->offsets + chunk_tab->bytes_per_chunk_entry,
@@ -599,9 +607,19 @@ compressor_thread_proc(void *arg)
 static void
 do_write_streams_progress(union wimlib_progress_info *progress,
                          wimlib_progress_func_t progress_func,
-                         uint64_t size_added)
+                         uint64_t size_added,
+                         bool stream_discarded)
 {
-       progress->write_streams.completed_bytes += size_added;
+       if (stream_discarded) {
+               progress->write_streams.total_bytes -= size_added;
+               if (progress->write_streams._private != ~(uint64_t)0 &&
+                   progress->write_streams._private > progress->write_streams.total_bytes)
+               {
+                       progress->write_streams._private = progress->write_streams.total_bytes;
+               }
+       } else {
+               progress->write_streams.completed_bytes += size_added;
+       }
        progress->write_streams.completed_streams++;
        if (progress_func &&
            progress->write_streams.completed_bytes >= progress->write_streams._private)
@@ -609,7 +627,7 @@ do_write_streams_progress(union wimlib_progress_info *progress,
                progress_func(WIMLIB_PROGRESS_MSG_WRITE_STREAMS,
                              progress);
                if (progress->write_streams._private == progress->write_streams.total_bytes) {
-                       progress->write_streams._private = ~0;
+                       progress->write_streams._private = ~(uint64_t)0;
                } else {
                        progress->write_streams._private =
                                min(progress->write_streams.total_bytes,
@@ -648,9 +666,11 @@ do_write_stream_list(struct list_head *stream_list,
 {
        int ret = 0;
        struct wim_lookup_table_entry *lte;
+       bool stream_discarded;
 
        /* For each stream in @stream_list ... */
        while (!list_empty(stream_list)) {
+               stream_discarded = false;
                lte = container_of(stream_list->next,
                                   struct wim_lookup_table_entry,
                                   write_streams_list);
@@ -675,6 +695,7 @@ do_write_stream_list(struct list_head *stream_list,
                                        DEBUG("Discarding duplicate stream of length %"PRIu64,
                                              wim_resource_size(lte));
                                        lte->no_progress = 0;
+                                       stream_discarded = true;
                                        goto skip_to_progress;
                                }
                        }
@@ -706,7 +727,8 @@ do_write_stream_list(struct list_head *stream_list,
                if (!lte->no_progress) {
                        do_write_streams_progress(progress,
                                                  progress_func,
-                                                 wim_resource_size(lte));
+                                                 wim_resource_size(lte),
+                                                 stream_discarded);
                }
        }
        return ret;
@@ -1016,7 +1038,8 @@ receive_compressed_chunks(struct main_writer_thread_ctx *ctx)
 
                        do_write_streams_progress(ctx->progress,
                                                  ctx->progress_func,
-                                                 wim_resource_size(cur_lte));
+                                                 wim_resource_size(cur_lte),
+                                                 false);
 
                        /* Since we just finished writing a stream, write any
                         * streams that have been added to the serial_streams
@@ -1185,7 +1208,7 @@ main_thread_process_next_stream(struct wim_lookup_table_entry *lte, void *_ctx)
 }
 
 static long
-get_default_num_threads()
+get_default_num_threads(void)
 {
 #ifdef __WIN32__
        return win32_get_number_of_processors();
@@ -1400,7 +1423,7 @@ write_stream_list(struct list_head *stream_list,
        progress.write_streams._private          = 0;
 
 #ifdef ENABLE_MULTITHREADED_COMPRESSION
-       if (total_compression_bytes >= 1000000 && num_threads != 1)
+       if (total_compression_bytes >= 2000000 && num_threads != 1)
                ret = write_stream_list_parallel(stream_list,
                                                 lookup_table,
                                                 out_fd,
@@ -1689,9 +1712,6 @@ write_wim_streams(WIMStruct *wim, int image, int write_flags,
  *     (public)  WIMLIB_WRITE_FLAG_CHECK_INTEGRITY:
  *             Include an integrity table.
  *
- *     (public)  WIMLIB_WRITE_FLAG_SHOW_PROGRESS:
- *             Show progress information when (if) writing the integrity table.
- *
  *     (private) WIMLIB_WRITE_FLAG_NO_LOOKUP_TABLE:
  *             Don't write the lookup table.
  *