From: Eric Biggers Date: Thu, 15 Aug 2013 19:08:00 +0000 (-0500) Subject: Improve write streams performance and handling of joins X-Git-Tag: v1.5.0~73 X-Git-Url: https://wimlib.net/git/?p=wimlib;a=commitdiff_plain;h=b348831df3fcf7d8eb66d35e4d0cf8434e788473;ds=sidebyside Improve write streams performance and handling of joins Provide the number of completed WIM parts and total WIM parts in WIMLIB_PROGRESS_MSG_WRITE_STREAMS, to replace WIMLIB_PROGRESS_MSG_JOIN. Furthermore, write streams in a more intelligent order; e.g. sorting by WIM part number and file offset when relevant. --- diff --git a/NEWS b/NEWS index 23b79dbc..e8281cc2 100644 --- a/NEWS +++ b/NEWS @@ -37,8 +37,11 @@ Version 1.5.0: The test suite no longer fails when run in a locale where the decimal separator is not a period. - WIMLIB_PROGRESS_MSG_EXTRACT_DENTRY has been removed and - WIMLIB_EXTRACT_FLAG_VERBOSE re-reserved for future use. + From the library, WIMLIB_PROGRESS_MSG_EXTRACT_DENTRY has been removed + and WIMLIB_EXTRACT_FLAG_VERBOSE re-reserved for future use. Also, + WIMLIB_PROGRESS_MSG_JOIN_STREAMS has been removed, but + WIMLIB_PROGRESS_MSG_WRITE_STREAMS will be received instead and now + provides WIM part numbers. The extraction code has been rewritten and it will now be easier to support new features on all supported backends (currently Win32, UNIX, diff --git a/include/wimlib.h b/include/wimlib.h index a7e068ae..6ab50ce4 100644 --- a/include/wimlib.h +++ b/include/wimlib.h @@ -327,9 +327,10 @@ enum wimlib_progress_msg { * ::wimlib_progress_info.integrity. */ WIMLIB_PROGRESS_MSG_CALC_INTEGRITY, - /** A wimlib_join() operation is in progress. @a info will point to - * ::wimlib_progress_info.join. */ - WIMLIB_PROGRESS_MSG_JOIN_STREAMS, + /** Reserved. (Previously used for WIMLIB_PROGRESS_MSG_JOIN_STREAMS, + * but in wimlib v1.5.0 this was removed to simplify the code and now + * you'll get ::WIMLIB_PROGRESS_MSG_WRITE_STREAMS messages instead.) */ + WIMLIB_PROGRESS_MSG_RESERVED, /** A wimlib_split() operation is in progress, and a new split part is * about to be started. @a info will point to @@ -371,6 +372,7 @@ union wimlib_progress_info { * (The actual number of bytes will be less if the data is being * written compressed.) */ uint64_t total_bytes; + /** Number of streams that are going to be written. */ uint64_t total_streams; @@ -384,7 +386,7 @@ union wimlib_progress_info { uint64_t completed_streams; /** Number of threads that are being used to compress resources - * (if applicable). */ + * (if applicable). */ unsigned num_threads; /** The compression type being used to write the streams; either @@ -393,8 +395,13 @@ union wimlib_progress_info { * ::WIMLIB_COMPRESSION_TYPE_LZX. */ int compression_type; - /** Library internal use only. */ - uint64_t _private; + /** Number of split WIM parts from which streams are being + * written (may be 0 if irrelevant). */ + unsigned total_parts; + + /** Number of split WIM parts from which streams have been + * written (may be 0 if irrelevant). */ + unsigned completed_parts; } write_streams; /** Valid on messages ::WIMLIB_PROGRESS_MSG_SCAN_BEGIN and @@ -518,25 +525,6 @@ union wimlib_progress_info { const wimlib_tchar *filename; } integrity; - /** Valid on messages ::WIMLIB_PROGRESS_MSG_JOIN_STREAMS. */ - struct wimlib_progress_info_join { - /** Total number of bytes of compressed data contained in all - * the split WIM part's file and metadata resources. */ - uint64_t total_bytes; - - /** Number of bytes that have been copied to the joined WIM so - * far. Will be 0 initially, and equal to @a total_bytes at the - * end. */ - uint64_t completed_bytes; - - /** Number of split WIM parts that have had all their file and - * metadata resources copied over to the joined WIM so far. */ - unsigned completed_parts; - - /** Number of split WIM parts. */ - unsigned total_parts; - } join; - /** Valid on messages ::WIMLIB_PROGRESS_MSG_SPLIT_BEGIN_PART and * ::WIMLIB_PROGRESS_MSG_SPLIT_END_PART. */ struct wimlib_progress_info_split { diff --git a/include/wimlib/lookup_table.h b/include/wimlib/lookup_table.h index 28862d5a..481794da 100644 --- a/include/wimlib/lookup_table.h +++ b/include/wimlib/lookup_table.h @@ -375,7 +375,8 @@ for_lookup_table_entry(struct wim_lookup_table *table, void *arg); extern int -cmp_streams_by_wim_position(const void *p1, const void *p2); +sort_stream_list_by_sequential_order(struct list_head *stream_list, + size_t list_head_offset); extern int for_lookup_table_entry_pos_sorted(struct wim_lookup_table *table, diff --git a/programs/imagex.c b/programs/imagex.c index e99202b1..767b6724 100644 --- a/programs/imagex.c +++ b/programs/imagex.c @@ -985,6 +985,7 @@ imagex_progress_func(enum wimlib_progress_msg msg, unit_shift = get_unit(info->write_streams.total_bytes, &unit_name); percent_done = TO_PERCENT(info->write_streams.completed_bytes, info->write_streams.total_bytes); + if (info->write_streams.completed_streams == 0) { const tchar *data_type; @@ -993,13 +994,28 @@ imagex_progress_func(enum wimlib_progress_msg msg, data_type, info->write_streams.num_threads, (info->write_streams.num_threads == 1) ? T("") : T("s")); } - imagex_printf(T("\r%"PRIu64" %"TS" of %"PRIu64" %"TS" (uncompressed) " - "written (%u%% done)"), - info->write_streams.completed_bytes >> unit_shift, - unit_name, - info->write_streams.total_bytes >> unit_shift, - unit_name, - percent_done); + if (info->write_streams.total_parts <= 1) { + imagex_printf(T("\r%"PRIu64" %"TS" of %"PRIu64" %"TS" (uncompressed) " + "written (%u%% done)"), + info->write_streams.completed_bytes >> unit_shift, + unit_name, + info->write_streams.total_bytes >> unit_shift, + unit_name, + percent_done); + } else { + imagex_printf(T("\rWriting resources from part %u of %u: " + "%"PRIu64 " %"TS" of %"PRIu64" %"TS" (%u%%) written"), + (info->write_streams.completed_parts == + info->write_streams.total_parts) ? + info->write_streams.completed_parts : + info->write_streams.completed_parts + 1, + info->write_streams.total_parts, + info->write_streams.completed_bytes >> unit_shift, + unit_name, + info->write_streams.total_bytes >> unit_shift, + unit_name, + percent_done); + } if (info->write_streams.completed_bytes >= info->write_streams.total_bytes) imagex_printf(T("\n")); break; @@ -1019,8 +1035,6 @@ imagex_progress_func(enum wimlib_progress_msg msg, else imagex_printf(T("Scanning \"%"TS"\"\n"), info->scan.cur_path); break; - /*case WIMLIB_PROGRESS_MSG_SCAN_END:*/ - /*break;*/ case WIMLIB_PROGRESS_MSG_VERIFY_INTEGRITY: unit_shift = get_unit(info->integrity.total_bytes, &unit_name); percent_done = TO_PERCENT(info->integrity.completed_bytes, @@ -1070,10 +1084,6 @@ imagex_progress_func(enum wimlib_progress_msg msg, info->extract.wimfile_name, info->extract.target); break; - /*case WIMLIB_PROGRESS_MSG_EXTRACT_DIR_STRUCTURE_BEGIN:*/ - /*imagex_printf(T("Applying directory structure to %"TS"\n"),*/ - /*info->extract.target);*/ - /*break;*/ case WIMLIB_PROGRESS_MSG_EXTRACT_STREAMS: percent_done = TO_PERCENT(info->extract.completed_bytes, info->extract.total_bytes); @@ -1098,21 +1108,6 @@ imagex_progress_func(enum wimlib_progress_msg msg, info->extract.target); } break; - case WIMLIB_PROGRESS_MSG_JOIN_STREAMS: - percent_done = TO_PERCENT(info->join.completed_bytes, - info->join.total_bytes); - unit_shift = get_unit(info->join.total_bytes, &unit_name); - imagex_printf(T("Writing resources from part %u of %u: " - "%"PRIu64 " %"TS" of %"PRIu64" %"TS" (%u%%) written\n"), - (info->join.completed_parts == info->join.total_parts) ? - info->join.completed_parts : info->join.completed_parts + 1, - info->join.total_parts, - info->join.completed_bytes >> unit_shift, - unit_name, - info->join.total_bytes >> unit_shift, - unit_name, - percent_done); - break; case WIMLIB_PROGRESS_MSG_SPLIT_BEGIN_PART: percent_done = TO_PERCENT(info->split.completed_bytes, info->split.total_bytes); diff --git a/src/extract.c b/src/extract.c index b96367b6..d9e03333 100644 --- a/src/extract.c +++ b/src/extract.c @@ -1337,42 +1337,6 @@ dentry_extract_final(struct wim_dentry *dentry, void *_ctx) return extract_timestamps(path, ctx, dentry); } -/* Sorts a list of streams in ascending order of their offset in the WIM file in - * order to prepare for sequential extraction. */ -static int -sort_stream_list_by_wim_position(struct list_head *stream_list) -{ - struct list_head *cur; - size_t num_streams; - struct wim_lookup_table_entry **array; - size_t i; - size_t array_size; - - num_streams = 0; - list_for_each(cur, stream_list) - num_streams++; - array_size = num_streams * sizeof(array[0]); - array = MALLOC(array_size); - if (!array) { - ERROR("Failed to allocate %zu bytes to sort stream entries", - array_size); - return WIMLIB_ERR_NOMEM; - } - cur = stream_list->next; - for (i = 0; i < num_streams; i++) { - array[i] = container_of(cur, struct wim_lookup_table_entry, extraction_list); - cur = cur->next; - } - - qsort(array, num_streams, sizeof(array[0]), cmp_streams_by_wim_position); - - INIT_LIST_HEAD(stream_list); - for (i = 0; i < num_streams; i++) - list_add_tail(&array[i]->extraction_list, stream_list); - FREE(array); - return 0; -} - /* * Extract a WIM dentry to standard output. * @@ -2033,7 +1997,10 @@ extract_tree(WIMStruct *wim, const tchar *wim_source_path, const tchar *target, WIMLIB_EXTRACT_FLAG_FROM_PIPE)) == WIMLIB_EXTRACT_FLAG_SEQUENTIAL) { - ret = sort_stream_list_by_wim_position(&ctx.stream_list); + ret = sort_stream_list_by_sequential_order( + &ctx.stream_list, + offsetof(struct wim_lookup_table_entry, + extraction_list)); if (ret) goto out_teardown_stream_list; } diff --git a/src/join.c b/src/join.c index 5cda7739..d606b7be 100644 --- a/src/join.c +++ b/src/join.c @@ -82,10 +82,8 @@ wimlib_join(const tchar * const *swm_names, if (ret) goto out_free_swms; - ret = wimlib_export_image(swm0, WIMLIB_ALL_IMAGES, - wim, NULL, NULL, 0, - swms, num_additional_swms, - progress_func); + ret = wimlib_export_image(swm0, WIMLIB_ALL_IMAGES, wim, NULL, NULL, 0, + swms, num_additional_swms, progress_func); if (ret) goto out_free_wim; diff --git a/src/lookup_table.c b/src/lookup_table.c index 3b98553e..15f5c083 100644 --- a/src/lookup_table.c +++ b/src/lookup_table.c @@ -288,20 +288,107 @@ for_lookup_table_entry(struct wim_lookup_table *table, return 0; } -int -cmp_streams_by_wim_position(const void *p1, const void *p2) +/* qsort() callback that sorts streams (represented by `struct + * wim_lookup_table_entry's) into an order optimized for reading and writing. + * + * Sorting is done primarily by resource location, then secondarily by a + * per-resource location order. For example, resources in WIM files are sorted + * primarily by part number, then secondarily by offset, as to implement optimal + * reading of either a standalone or split WIM. */ +static int +cmp_streams_by_sequential_order(const void *p1, const void *p2) { const struct wim_lookup_table_entry *lte1, *lte2; + int v; + lte1 = *(const struct wim_lookup_table_entry**)p1; lte2 = *(const struct wim_lookup_table_entry**)p2; - if (lte1->resource_entry.offset < lte2->resource_entry.offset) - return -1; - else if (lte1->resource_entry.offset > lte2->resource_entry.offset) - return 1; - else + + v = (int)lte1->resource_location - (int)lte2->resource_location; + + /* Different resource locations? */ + if (v) + return v; + + switch (lte1->resource_location) { + case RESOURCE_IN_WIM: + + /* Different (possibly split) WIMs? */ + if (lte1->wim != lte2->wim) { + v = memcmp(lte1->wim->hdr.guid, lte2->wim->hdr.guid, + WIM_GID_LEN); + if (v) + return v; + } + + /* Different part numbers in the same WIM? */ + v = (int)lte1->wim->hdr.part_number - (int)lte2->wim->hdr.part_number; + if (v) + return v; + + /* Compare by offset. */ + if (lte1->resource_entry.offset < lte2->resource_entry.offset) + return -1; + else if (lte1->resource_entry.offset > lte2->resource_entry.offset) + return 1; return 0; + case RESOURCE_IN_FILE_ON_DISK: +#ifdef __WIN32__ + case RESOURCE_WIN32_ENCRYPTED: +#endif + /* Compare files by path: just a heuristic that will place files + * in the same directory next to each other. */ + return tstrcmp(lte1->file_on_disk, lte2->file_on_disk); +#ifdef WITH_NTFS_3G + case RESOURCE_IN_NTFS_VOLUME: + return tstrcmp(lte1->ntfs_loc->path, lte2->ntfs_loc->path); +#endif + default: + /* No additional sorting order defined for this resource + * location (e.g. RESOURCE_IN_ATTACHED_BUFFER); simply compare + * everything equal to each other. */ + return 0; + } +} + +int +sort_stream_list_by_sequential_order(struct list_head *stream_list, + size_t list_head_offset) +{ + struct list_head *cur; + struct wim_lookup_table_entry **array; + size_t i; + size_t array_size; + size_t num_streams = 0; + + list_for_each(cur, stream_list) + num_streams++; + + array_size = num_streams * sizeof(array[0]); + array = MALLOC(array_size); + if (!array) + return WIMLIB_ERR_NOMEM; + cur = stream_list->next; + for (i = 0; i < num_streams; i++) { + array[i] = (struct wim_lookup_table_entry*)((u8*)cur - + list_head_offset); + cur = cur->next; + } + + qsort(array, num_streams, sizeof(array[0]), + cmp_streams_by_sequential_order); + + INIT_LIST_HEAD(stream_list); + for (i = 0; i < num_streams; i++) { + list_add_tail((struct list_head*) + ((u8*)array[i] + list_head_offset), + stream_list); + } + FREE(array); + return 0; } + static int add_lte_to_array(struct wim_lookup_table_entry *lte, void *_pp) @@ -333,7 +420,7 @@ for_lookup_table_entry_pos_sorted(struct wim_lookup_table *table, wimlib_assert(p == lte_array + num_streams); qsort(lte_array, num_streams, sizeof(lte_array[0]), - cmp_streams_by_wim_position); + cmp_streams_by_sequential_order); ret = 0; for (size_t i = 0; i < num_streams; i++) { ret = visitor(lte_array[i], arg); diff --git a/src/write.c b/src/write.c index 3cac5a28..e4a0f3ad 100644 --- a/src/write.c +++ b/src/write.c @@ -53,17 +53,16 @@ # include #endif -#include -#include #include +#include +#include +#include +#include #ifdef HAVE_ALLOCA_H # include -#else -# include #endif -#include #ifndef __WIN32__ # include /* for `struct iovec' */ @@ -683,7 +682,6 @@ struct message { u8 *compressed_chunks[MAX_CHUNKS_PER_MSG]; unsigned uncompressed_chunk_sizes[MAX_CHUNKS_PER_MSG]; struct iovec out_chunks[MAX_CHUNKS_PER_MSG]; - size_t total_out_bytes; unsigned num_chunks; struct list_head list; bool complete; @@ -693,7 +691,6 @@ struct message { static void compress_chunks(struct message *msg, compress_func_t compress) { - msg->total_out_bytes = 0; for (unsigned i = 0; i < msg->num_chunks; i++) { unsigned len = compress(msg->uncompressed_chunks[i], msg->uncompressed_chunk_sizes[i], @@ -711,7 +708,6 @@ compress_chunks(struct message *msg, compress_func_t compress) } msg->out_chunks[i].iov_base = out_chunk; msg->out_chunks[i].iov_len = out_len; - msg->total_out_bytes += out_len; } } @@ -739,32 +735,52 @@ compressor_thread_proc(void *arg) } #endif /* ENABLE_MULTITHREADED_COMPRESSION */ +struct write_streams_progress_data { + wimlib_progress_func_t progress_func; + union wimlib_progress_info progress; + uint64_t next_progress; + WIMStruct *prev_wim_part; +}; + static void -do_write_streams_progress(union wimlib_progress_info *progress, - wimlib_progress_func_t progress_func, - uint64_t size_added, +do_write_streams_progress(struct write_streams_progress_data *progress_data, + struct wim_lookup_table_entry *lte, bool stream_discarded) { + union wimlib_progress_info *progress = &progress_data->progress; + bool new_wim_part; + if (stream_discarded) { - progress->write_streams.total_bytes -= size_added; - if (progress->write_streams._private != ~(uint64_t)0 && - progress->write_streams._private > progress->write_streams.total_bytes) + progress->write_streams.total_bytes -= wim_resource_size(lte); + if (progress_data->next_progress != ~(uint64_t)0 && + progress_data->next_progress > progress->write_streams.total_bytes) { - progress->write_streams._private = progress->write_streams.total_bytes; + progress_data->next_progress = progress->write_streams.total_bytes; } } else { - progress->write_streams.completed_bytes += size_added; + progress->write_streams.completed_bytes += wim_resource_size(lte); + } + new_wim_part = false; + if (lte->resource_location == RESOURCE_IN_WIM && + lte->wim != progress_data->prev_wim_part) + { + if (progress_data->prev_wim_part) { + new_wim_part = true; + progress->write_streams.completed_parts++; + } + progress_data->prev_wim_part = lte->wim; } progress->write_streams.completed_streams++; - if (progress_func && - progress->write_streams.completed_bytes >= progress->write_streams._private) + if (progress_data->progress_func + && (progress->write_streams.completed_bytes >= progress_data->next_progress + || new_wim_part)) { - progress_func(WIMLIB_PROGRESS_MSG_WRITE_STREAMS, - progress); - if (progress->write_streams._private == progress->write_streams.total_bytes) { - progress->write_streams._private = ~(uint64_t)0; + progress_data->progress_func(WIMLIB_PROGRESS_MSG_WRITE_STREAMS, + progress); + if (progress_data->next_progress == progress->write_streams.total_bytes) { + progress_data->next_progress = ~(uint64_t)0; } else { - progress->write_streams._private = + progress_data->next_progress = min(progress->write_streams.total_bytes, progress->write_streams.completed_bytes + progress->write_streams.total_bytes / 100); @@ -787,6 +803,7 @@ serial_write_stream(struct wim_lookup_table_entry *lte, void *_ctx) ctx->write_resource_flags); } + /* Write a list of streams, taking into account that some streams may be * duplicates that are checksummed and discarded on the fly, and also delegating * the actual writing of a stream to a function @write_stream_cb, which is @@ -796,8 +813,7 @@ do_write_stream_list(struct list_head *stream_list, struct wim_lookup_table *lookup_table, int (*write_stream_cb)(struct wim_lookup_table_entry *, void *), void *write_stream_ctx, - wimlib_progress_func_t progress_func, - union wimlib_progress_info *progress) + struct write_streams_progress_data *progress_data) { int ret = 0; struct wim_lookup_table_entry *lte; @@ -860,10 +876,8 @@ do_write_stream_list(struct list_head *stream_list, } skip_to_progress: if (!lte->no_progress) { - do_write_streams_progress(progress, - progress_func, - wim_resource_size(lte), - stream_discarded); + do_write_streams_progress(progress_data, + lte, stream_discarded); } } return ret; @@ -875,8 +889,7 @@ do_write_stream_list_serial(struct list_head *stream_list, struct filedes *out_fd, int out_ctype, int write_resource_flags, - wimlib_progress_func_t progress_func, - union wimlib_progress_info *progress) + struct write_streams_progress_data *progress_data) { struct serial_write_stream_ctx ctx = { .out_fd = out_fd, @@ -887,8 +900,7 @@ do_write_stream_list_serial(struct list_head *stream_list, lookup_table, serial_write_stream, &ctx, - progress_func, - progress); + progress_data); } static inline int @@ -909,21 +921,22 @@ write_stream_list_serial(struct list_head *stream_list, struct filedes *out_fd, int out_ctype, int write_resource_flags, - wimlib_progress_func_t progress_func, - union wimlib_progress_info *progress) + struct write_streams_progress_data *progress_data) { + union wimlib_progress_info *progress = &progress_data->progress; DEBUG("Writing stream list of size %"PRIu64" (serial version)", progress->write_streams.total_streams); progress->write_streams.num_threads = 1; - if (progress_func) - progress_func(WIMLIB_PROGRESS_MSG_WRITE_STREAMS, progress); + if (progress_data->progress_func) { + progress_data->progress_func(WIMLIB_PROGRESS_MSG_WRITE_STREAMS, + progress); + } return do_write_stream_list_serial(stream_list, lookup_table, out_fd, out_ctype, write_resource_flags, - progress_func, - progress); + progress_data); } #ifdef ENABLE_MULTITHREADED_COMPRESSION @@ -935,7 +948,6 @@ write_wim_chunks(struct message *msg, struct filedes *out_fd, struct iovec *vecs; struct pwm_chunk_hdr *chunk_hdrs; unsigned nvecs; - size_t nbytes; int ret; for (unsigned i = 0; i < msg->num_chunks; i++) @@ -944,7 +956,6 @@ write_wim_chunks(struct message *msg, struct filedes *out_fd, if (!(write_resource_flags & WIMLIB_WRITE_RESOURCE_FLAG_PIPABLE)) { nvecs = msg->num_chunks; vecs = msg->out_chunks; - nbytes = msg->total_out_bytes; } else { /* Special case: If writing a compressed resource to a pipable * WIM, prefix each compressed chunk with a header that gives @@ -960,7 +971,6 @@ write_wim_chunks(struct message *msg, struct filedes *out_fd, vecs[i * 2 + 1].iov_base = msg->out_chunks[i].iov_base; vecs[i * 2 + 1].iov_len = msg->out_chunks[i].iov_len; } - nbytes = msg->total_out_bytes + msg->num_chunks * sizeof(chunk_hdrs[0]); } ret = full_writev(out_fd, vecs, nvecs); if (ret) @@ -978,8 +988,7 @@ struct main_writer_thread_ctx { struct shared_queue *res_to_compress_queue; struct shared_queue *compressed_res_queue; size_t num_messages; - wimlib_progress_func_t progress_func; - union wimlib_progress_info *progress; + struct write_streams_progress_data *progress_data; struct list_head available_msgs; struct list_head outstanding_streams; @@ -1221,10 +1230,8 @@ receive_compressed_chunks(struct main_writer_thread_ctx *ctx) cur_lte->output_resource_entry.flags); } - do_write_streams_progress(ctx->progress, - ctx->progress_func, - wim_resource_size(cur_lte), - false); + do_write_streams_progress(ctx->progress_data, + cur_lte, false); /* Since we just finished writing a stream, write any * streams that have been added to the serial_streams @@ -1238,8 +1245,7 @@ receive_compressed_chunks(struct main_writer_thread_ctx *ctx) ctx->out_fd, ctx->out_ctype, ctx->write_resource_flags, - ctx->progress_func, - ctx->progress); + ctx->progress_data); if (ret) return ret; } @@ -1337,8 +1343,7 @@ main_writer_thread_finish(void *_ctx) ctx->out_fd, ctx->out_ctype, ctx->write_resource_flags, - ctx->progress_func, - ctx->progress); + ctx->progress_data); } static int @@ -1433,14 +1438,14 @@ write_stream_list_parallel(struct list_head *stream_list, struct filedes *out_fd, int out_ctype, int write_resource_flags, - wimlib_progress_func_t progress_func, - union wimlib_progress_info *progress, + struct write_streams_progress_data *progress_data, unsigned num_threads) { int ret; struct shared_queue res_to_compress_queue; struct shared_queue compressed_res_queue; pthread_t *compressor_threads = NULL; + union wimlib_progress_info *progress = &progress_data->progress; if (num_threads == 0) { long nthreads = get_default_num_threads(); @@ -1498,8 +1503,10 @@ write_stream_list_parallel(struct list_head *stream_list, } } - if (progress_func) - progress_func(WIMLIB_PROGRESS_MSG_WRITE_STREAMS, progress); + if (progress_data->progress_func) { + progress_data->progress_func(WIMLIB_PROGRESS_MSG_WRITE_STREAMS, + progress); + } struct main_writer_thread_ctx ctx; ctx.stream_list = stream_list; @@ -1510,14 +1517,13 @@ write_stream_list_parallel(struct list_head *stream_list, ctx.compressed_res_queue = &compressed_res_queue; ctx.num_messages = queue_size; ctx.write_resource_flags = write_resource_flags; - ctx.progress_func = progress_func; - ctx.progress = progress; + ctx.progress_data = progress_data; ret = main_writer_thread_init_ctx(&ctx); if (ret) goto out_join; ret = do_write_stream_list(stream_list, lookup_table, main_thread_process_next_stream, - &ctx, progress_func, progress); + &ctx, progress_data); if (ret) goto out_destroy_ctx; @@ -1556,8 +1562,7 @@ out_serial_quiet: out_fd, out_ctype, write_resource_flags, - progress_func, - progress); + progress_data); } #endif @@ -1576,9 +1581,11 @@ write_stream_list(struct list_head *stream_list, size_t num_streams = 0; u64 total_bytes = 0; u64 total_compression_bytes = 0; - union wimlib_progress_info progress; + struct write_streams_progress_data progress_data; int ret; int write_resource_flags; + unsigned total_parts = 0; + WIMStruct *prev_wim_part = NULL; if (list_empty(stream_list)) return 0; @@ -1587,6 +1594,10 @@ write_stream_list(struct list_head *stream_list, DEBUG("write_resource_flags=0x%08x", write_resource_flags); + sort_stream_list_by_sequential_order(stream_list, + offsetof(struct wim_lookup_table_entry, + write_streams_list)); + /* Calculate the total size of the streams to be written. Note: this * will be the uncompressed size, as we may not know the compressed size * yet, and also this will assume that every unhashed stream will be @@ -1600,14 +1611,29 @@ write_stream_list(struct list_head *stream_list, { total_compression_bytes += wim_resource_size(lte); } + if (lte->resource_location == RESOURCE_IN_WIM) { + if (prev_wim_part != lte->wim) { + prev_wim_part = lte->wim; + total_parts++; + } + } + } - progress.write_streams.total_bytes = total_bytes; - progress.write_streams.total_streams = num_streams; - progress.write_streams.completed_bytes = 0; - progress.write_streams.completed_streams = 0; - progress.write_streams.num_threads = num_threads; - progress.write_streams.compression_type = out_ctype; - progress.write_streams._private = 0; + + memset(&progress_data, 0, sizeof(progress_data)); + progress_data.progress_func = progress_func; + + progress_data.progress.write_streams.total_bytes = total_bytes; + progress_data.progress.write_streams.total_streams = num_streams; + progress_data.progress.write_streams.completed_bytes = 0; + progress_data.progress.write_streams.completed_streams = 0; + progress_data.progress.write_streams.num_threads = num_threads; + progress_data.progress.write_streams.compression_type = out_ctype; + progress_data.progress.write_streams.total_parts = total_parts; + progress_data.progress.write_streams.completed_parts = 0; + + progress_data.next_progress = 0; + progress_data.prev_wim_part = NULL; #ifdef ENABLE_MULTITHREADED_COMPRESSION if (total_compression_bytes >= 2000000 && num_threads != 1) @@ -1616,8 +1642,7 @@ write_stream_list(struct list_head *stream_list, out_fd, out_ctype, write_resource_flags, - progress_func, - &progress, + &progress_data, num_threads); else #endif @@ -1626,8 +1651,7 @@ write_stream_list(struct list_head *stream_list, out_fd, out_ctype, write_resource_flags, - progress_func, - &progress); + &progress_data); if (ret == 0) DEBUG("Successfully wrote stream list."); else