X-Git-Url: https://wimlib.net/git/?p=wimlib;a=blobdiff_plain;f=src%2Fwrite.c;h=5bca038a61afd909b5d868391ed21ebf37d49a5c;hp=7e597e18bf2410526a3225bb47946834fecd1f35;hb=b4f0ab9002559b774407fb692bcc7e83a6c82ab7;hpb=0ecf748e0db6bb4d9a02388b4ea925d8742848b1 diff --git a/src/write.c b/src/write.c index 7e597e18..5bca038a 100644 --- a/src/write.c +++ b/src/write.c @@ -39,9 +39,11 @@ #include "wimlib/error.h" #include "wimlib/file_io.h" #include "wimlib/header.h" +#include "wimlib/inode.h" #include "wimlib/integrity.h" #include "wimlib/lookup_table.h" #include "wimlib/metadata.h" +#include "wimlib/progress.h" #include "wimlib/resource.h" #ifdef __WIN32__ # include "wimlib/win32.h" /* win32_rename_replacement() */ @@ -168,23 +170,19 @@ can_raw_copy(const struct wim_lookup_table_entry *lte, if (rspec->is_pipable != !!(write_resource_flags & WRITE_RESOURCE_FLAG_PIPABLE)) return false; - if (rspec->flags & WIM_RESHDR_FLAG_COMPRESSED) { /* Normal compressed resource: Must use same compression type * and chunk size. */ - return (rspec->wim->compression_type == out_ctype && - rspec->wim->chunk_size == out_chunk_size); + return (rspec->compression_type == out_ctype && + rspec->chunk_size == out_chunk_size); } - /* XXX: For compatibility, we can't allow multiple packed resources per - * WIM. */ -#if 0 if ((rspec->flags & WIM_RESHDR_FLAG_PACKED_STREAMS) && (write_resource_flags & WRITE_RESOURCE_FLAG_PACK_STREAMS)) { /* Packed resource: Such resources may contain multiple streams, * and in general only a subset of them need to be written. As - * a heuristic, re-use the raw data if at least half the + * a heuristic, re-use the raw data if more than two-thirds the * uncompressed size is being written. */ /* Note: packed resources contain a header that specifies the @@ -199,9 +197,8 @@ can_raw_copy(const struct wim_lookup_table_entry *lte, if (res_stream->will_be_in_output_wim) write_size += res_stream->size; - return (write_size > rspec->uncompressed_size / 2); + return (write_size > rspec->uncompressed_size * 2 / 3); } -#endif return false; } @@ -233,7 +230,7 @@ stream_set_out_reshdr_for_reuse(struct wim_lookup_table_entry *lte) lte->out_res_offset_in_wim = rspec->offset_in_wim; lte->out_res_size_in_wim = rspec->size_in_wim; - /*lte->out_res_uncompressed_size = rspec->uncompressed_size;*/ + lte->out_res_uncompressed_size = rspec->uncompressed_size; } else { wimlib_assert(!(lte->flags & WIM_RESHDR_FLAG_PACKED_STREAMS)); @@ -274,13 +271,13 @@ write_pwm_stream_header(const struct wim_lookup_table_entry *lte, } struct write_streams_progress_data { - wimlib_progress_func_t progress_func; + wimlib_progress_func_t progfunc; + void *progctx; union wimlib_progress_info progress; uint64_t next_progress; - WIMStruct *prev_wim_part; }; -static void +static int do_write_streams_progress(struct write_streams_progress_data *progress_data, struct wim_lookup_table_entry *cur_stream, u64 complete_size, @@ -288,7 +285,7 @@ do_write_streams_progress(struct write_streams_progress_data *progress_data, bool discarded) { union wimlib_progress_info *progress = &progress_data->progress; - bool new_wim_part; + int ret; if (discarded) { progress->write_streams.total_bytes -= complete_size; @@ -303,23 +300,15 @@ do_write_streams_progress(struct write_streams_progress_data *progress_data, progress->write_streams.completed_streams += complete_count; } - new_wim_part = false; - if (cur_stream->resource_location == RESOURCE_IN_WIM && - cur_stream->rspec->wim != progress_data->prev_wim_part) + if (progress->write_streams.completed_bytes >= progress_data->next_progress) { - if (progress_data->prev_wim_part) { - new_wim_part = true; - progress->write_streams.completed_parts++; - } - progress_data->prev_wim_part = cur_stream->rspec->wim; - } + ret = call_progress(progress_data->progfunc, + WIMLIB_PROGRESS_MSG_WRITE_STREAMS, + progress, + progress_data->progctx); + if (ret) + return ret; - if (progress_data->progress_func - && (progress->write_streams.completed_bytes >= progress_data->next_progress - || new_wim_part)) - { - progress_data->progress_func(WIMLIB_PROGRESS_MSG_WRITE_STREAMS, - progress); if (progress_data->next_progress == progress->write_streams.total_bytes) { progress_data->next_progress = ~(uint64_t)0; } else { @@ -329,6 +318,7 @@ do_write_streams_progress(struct write_streams_progress_data *progress_data, progress->write_streams.total_bytes / 100); } } + return 0; } struct write_streams_ctx { @@ -409,16 +399,6 @@ struct write_streams_ctx { u64 chunks_start_offset; }; -static u64 -get_chunk_entry_size(u64 res_size, int write_resource_flags) -{ - if (res_size <= UINT32_MAX || - (write_resource_flags & WIM_RESHDR_FLAG_PACKED_STREAMS)) - return 4; - else - return 8; -} - /* Reserve space for the chunk table and prepare to accumulate the chunk table * in memory. */ static int @@ -470,7 +450,8 @@ begin_chunk_table(struct write_streams_ctx *ctx, u64 res_expected_size) * are unknown. */ reserve_size = expected_num_chunk_entries * get_chunk_entry_size(res_expected_size, - ctx->write_resource_flags); + 0 != (ctx->write_resource_flags & + WRITE_RESOURCE_FLAG_PACK_STREAMS)); if (ctx->write_resource_flags & WRITE_RESOURCE_FLAG_PACK_STREAMS) reserve_size += sizeof(struct alt_chunk_table_header_disk); memset(ctx->chunk_csizes, 0, reserve_size); @@ -517,7 +498,8 @@ end_chunk_table(struct write_streams_ctx *ctx, u64 res_actual_size, actual_num_chunk_entries--; chunk_entry_size = get_chunk_entry_size(res_actual_size, - ctx->write_resource_flags); + 0 != (ctx->write_resource_flags & + WRITE_RESOURCE_FLAG_PACK_STREAMS)); typedef le64 __attribute__((may_alias)) aliased_le64_t; typedef le32 __attribute__((may_alias)) aliased_le32_t; @@ -576,8 +558,8 @@ end_chunk_table(struct write_streams_ctx *ctx, u64 res_actual_size, hdr.chunk_size = cpu_to_le32(ctx->out_chunk_size); hdr.compression_format = cpu_to_le32(ctx->out_ctype); - BUILD_BUG_ON(WIMLIB_COMPRESSION_TYPE_LZX != 1); - BUILD_BUG_ON(WIMLIB_COMPRESSION_TYPE_XPRESS != 2); + BUILD_BUG_ON(WIMLIB_COMPRESSION_TYPE_XPRESS != 1); + BUILD_BUG_ON(WIMLIB_COMPRESSION_TYPE_LZX != 2); BUILD_BUG_ON(WIMLIB_COMPRESSION_TYPE_LZMS != 3); ret = full_pwrite(ctx->out_fd, &hdr, sizeof(hdr), @@ -639,7 +621,7 @@ end_write_resource(struct write_streams_ctx *ctx, struct wim_reshdr *out_reshdr) /* Begin processing a stream for writing. */ static int write_stream_begin_read(struct wim_lookup_table_entry *lte, - bool is_partial_res, void *_ctx) + u32 flags, void *_ctx) { struct write_streams_ctx *ctx = _ctx; int ret; @@ -663,7 +645,7 @@ write_stream_begin_read(struct wim_lookup_table_entry *lte, ctx->stream_was_duplicate = false; if (ctx->lookup_table != NULL && lte->unhashed && !lte->unique_size) { - wimlib_assert(!is_partial_res); + wimlib_assert(!(flags & BEGIN_STREAM_FLAG_PARTIAL_RESOURCE)); struct wim_lookup_table_entry *lte_new; @@ -684,9 +666,9 @@ write_stream_begin_read(struct wim_lookup_table_entry *lte, * duplicate stream in the former case. */ DEBUG("Discarding duplicate stream of " "length %"PRIu64, lte->size); - do_write_streams_progress(&ctx->progress_data, - lte, lte->size, - 1, true); + ret = do_write_streams_progress(&ctx->progress_data, + lte, lte->size, + 1, true); list_del(<e->write_streams_list); list_del(<e->lookup_table_list); if (lte_new->will_be_in_output_wim) @@ -694,6 +676,8 @@ write_stream_begin_read(struct wim_lookup_table_entry *lte, if (ctx->write_resource_flags & WRITE_RESOURCE_FLAG_PACK_STREAMS) ctx->cur_write_res_size -= lte->size; free_lookup_table_entry(lte); + if (ret) + return ret; return BEGIN_STREAM_STATUS_SKIP_STREAM; } else { /* The duplicate stream can validly be written, @@ -902,11 +886,9 @@ write_chunk(struct write_streams_ctx *ctx, const void *cchunk, } } - do_write_streams_progress(&ctx->progress_data, lte, - completed_size, completed_stream_count, - false); - - return 0; + return do_write_streams_progress(&ctx->progress_data, lte, + completed_size, completed_stream_count, + false); error: ERROR_WITH_ERRNO("Write error"); @@ -922,8 +904,8 @@ submit_chunk_for_compression(struct write_streams_ctx *ctx, * compressed chunk. */ while (!ctx->compressor->submit_chunk(ctx->compressor, chunk, size)) { const void *cchunk; - unsigned csize; - unsigned usize; + u32 csize; + u32 usize; bool bret; int ret; @@ -1068,24 +1050,23 @@ compute_stream_list_stats(struct list_head *stream_list, ctx->progress_data.progress.write_streams.total_parts = total_parts; ctx->progress_data.progress.write_streams.completed_parts = 0; ctx->progress_data.next_progress = 0; - ctx->progress_data.prev_wim_part = NULL; } /* Find streams in @stream_list that can be copied to the output WIM in raw form - * rather than compressed. Delete these streams from @stream_list, and move one - * per resource to @raw_copy_resources. Return the total uncompressed size of - * the streams that need to be compressed. */ + * rather than compressed. Delete these streams from @stream_list and move them + * to @raw_copy_streams. Return the total uncompressed size of the streams that + * need to be compressed. */ static u64 -find_raw_copy_resources(struct list_head *stream_list, - int write_resource_flags, - int out_ctype, - u32 out_chunk_size, - struct list_head *raw_copy_resources) +find_raw_copy_streams(struct list_head *stream_list, + int write_resource_flags, + int out_ctype, + u32 out_chunk_size, + struct list_head *raw_copy_streams) { struct wim_lookup_table_entry *lte, *tmp; u64 num_bytes_to_compress = 0; - INIT_LIST_HEAD(raw_copy_resources); + INIT_LIST_HEAD(raw_copy_streams); /* Initialize temporary raw_copy_ok flag. */ list_for_each_entry(lte, stream_list, write_streams_list) @@ -1096,13 +1077,14 @@ find_raw_copy_resources(struct list_head *stream_list, if (lte->resource_location == RESOURCE_IN_WIM && lte->rspec->raw_copy_ok) { - list_del(<e->write_streams_list); + list_move_tail(<e->write_streams_list, + raw_copy_streams); } else if (can_raw_copy(lte, write_resource_flags, out_ctype, out_chunk_size)) { lte->rspec->raw_copy_ok = 1; list_move_tail(<e->write_streams_list, - raw_copy_resources); + raw_copy_streams); } else { num_bytes_to_compress += lte->size; } @@ -1173,22 +1155,32 @@ write_raw_copy_resource(struct wim_resource_spec *in_rspec, return 0; } -/* Copy a list of raw compressed resources located other WIM file(s) to the WIM - * file being written. */ +/* Copy a list of raw compressed resources located in other WIM file(s) to the + * WIM file being written. */ static int -write_raw_copy_resources(struct list_head *raw_copy_resources, +write_raw_copy_resources(struct list_head *raw_copy_streams, struct filedes *out_fd, struct write_streams_progress_data *progress_data) { struct wim_lookup_table_entry *lte; int ret; - list_for_each_entry(lte, raw_copy_resources, write_streams_list) { - ret = write_raw_copy_resource(lte->rspec, out_fd); + list_for_each_entry(lte, raw_copy_streams, write_streams_list) + lte->rspec->raw_copy_ok = 1; + + list_for_each_entry(lte, raw_copy_streams, write_streams_list) { + if (lte->rspec->raw_copy_ok) { + /* Write each packed resource only one time, no matter + * how many streams reference it. */ + ret = write_raw_copy_resource(lte->rspec, out_fd); + if (ret) + return ret; + lte->rspec->raw_copy_ok = 0; + } + ret = do_write_streams_progress(progress_data, lte, lte->size, + 1, false); if (ret) return ret; - do_write_streams_progress(progress_data, lte, lte->size, - 1, false); } return 0; } @@ -1198,8 +1190,8 @@ static int finish_remaining_chunks(struct write_streams_ctx *ctx) { const void *cdata; - unsigned csize; - unsigned usize; + u32 csize; + u32 usize; int ret; if (ctx->compressor == NULL) @@ -1241,7 +1233,7 @@ remove_zero_length_streams(struct list_head *stream_list) * Write a list of streams to the output WIM file. * * @stream_list - * The list of streams to write, specifies a list of `struct + * The list of streams to write, specified by a list of `struct * wim_lookup_table_entry's linked by the 'write_streams_list' member. * * @out_fd @@ -1252,26 +1244,25 @@ remove_zero_length_streams(struct list_head *stream_list) * * WRITE_RESOURCE_FLAG_RECOMPRESS: * Force compression of all resources, even if they could otherwise - * be re-used by caping the raw data, due to being located in a WIM + * be re-used by copying the raw data, due to being located in a WIM * file with compatible compression parameters. * * WRITE_RESOURCE_FLAG_PIPABLE: * Write the resources in the wimlib-specific pipable format, and * furthermore do so in such a way that no seeking backwards in - * @out_fd will be performed (so it may be a pipe, contrary to the - * default behavior). + * @out_fd will be performed (so it may be a pipe). * * WRITE_RESOURCE_FLAG_PACK_STREAMS: * Pack all the streams into a single resource rather than writing - * them in separate resources. This format is only valid if the - * WIM version number is WIM_VERSION_PACKED_STREAMS. This flag - * currently may not be combined with WRITE_RESOURCE_FLAG_PIPABLE. + * them in separate resources. This flag is only valid if the WIM + * version number has been, or will be, set to + * WIM_VERSION_PACKED_STREAMS. This flag may not be combined with + * WRITE_RESOURCE_FLAG_PIPABLE. * * @out_ctype * Compression format to use to write the output streams, specified as one - * of the WIMLIB_COMPRESSION_TYPE_* constants, excepting - * WIMLIB_COMPRESSION_TYPE_INVALID but including - * WIMLIB_COMPRESSION_TYPE_NONE. + * of the WIMLIB_COMPRESSION_TYPE_* constants. + * WIMLIB_COMPRESSION_TYPE_NONE is allowed. * * @out_chunk_size * Chunk size to use to write the streams. It must be a valid chunk size @@ -1296,27 +1287,21 @@ remove_zero_length_streams(struct list_head *stream_list) * no streams are hard-filtered or no streams are unhashed, this parameter * can be NULL. * - * @progress_func - * If non-NULL, a progress function that will be called periodically with - * WIMLIB_PROGRESS_MSG_WRITE_STREAMS messages. Note that on-the-fly - * deduplication of unhashed streams may result in the total bytes provided - * in the progress data to decrease from one message to the next. - * * This function will write the streams in @stream_list to resources in * consecutive positions in the output WIM file, or to a single packed resource * if WRITE_RESOURCE_FLAG_PACK_STREAMS was specified in @write_resource_flags. * In both cases, the @out_reshdr of the `struct wim_lookup_table_entry' for * each stream written will be updated to specify its location, size, and flags * in the output WIM. In the packed resource case, - * WIM_RESHDR_FLAG_PACKED_STREAMS shall be set in the @flags field of the - * @out_reshdr, and @out_res_offset_in_wim and @out_res_size_in_wim will also - * be set to the offset and size, respectively, in the output WIM of the full - * packed resource containing the corresponding stream. + * WIM_RESHDR_FLAG_PACKED_STREAMS will be set in the @flags field of each + * @out_reshdr, and furthermore @out_res_offset_in_wim and @out_res_size_in_wim + * of each @out_reshdr will be set to the offset and size, respectively, in the + * output WIM of the packed resource containing the corresponding stream. * * Each of the streams to write may be in any location supported by the * resource-handling code (specifically, read_stream_list()), such as the * contents of external file that has been logically added to the output WIM, or - * a stream in another WIM file that has been imported, or even stream in the + * a stream in another WIM file that has been imported, or even a stream in the * "same" WIM file of which a modified copy is being written. In the case that * a stream is already in a WIM file and uses compatible compression parameters, * by default this function will re-use the raw data instead of decompressing @@ -1324,32 +1309,31 @@ remove_zero_length_streams(struct list_head *stream_list) * specified in @write_resource_flags, this is not done. * * As a further requirement, this function requires that the - * @will_be_in_output_wim member be set on all streams in @stream_list as well - * as any other streams not in @stream_list that will be in the output WIM file, - * but not on any other streams in the output WIM's lookup table or sharing a - * packed resource with a stream in @stream_list. Still furthermore, if - * on-the-fly deduplication of streams is possible, then all streams in + * @will_be_in_output_wim member be set to 1 on all streams in @stream_list as + * well as any other streams not in @stream_list that will be in the output WIM + * file, but set to 0 on any other streams in the output WIM's lookup table or + * sharing a packed resource with a stream in @stream_list. Still furthermore, + * if on-the-fly deduplication of streams is possible, then all streams in * @stream_list must also be linked by @lookup_table_list along with any other * streams that have @will_be_in_output_wim set. * * This function handles on-the-fly deduplication of streams for which SHA1 - * message digests have not yet been calculated and it is therefore known - * whether such streams are already in @stream_list or in the WIM's lookup table - * at all. If @lookup_table is non-NULL, then each stream in @stream_list that - * has @unhashed set but not @unique_size set is checksummed immediately before - * it would otherwise be read for writing in order to determine if it is - * identical to another stream already being written or one that would be - * filtered out of the output WIM using stream_filtered() with the context - * @filter_ctx. Each such duplicate stream will be removed from @stream_list, its - * reference count transfered to the pre-existing duplicate stream, its memory - * freed, and will not be written. Alternatively, if a stream in @stream_list - * is a duplicate with any stream in @lookup_table that has not been marked for - * writing or would not be hard-filtered, it is freed and the pre-existing - * duplicate is written instead, taking ownership of the reference count and - * slot in the @lookup_table_list. - * - * Returns 0 if all streams were written successfully (or did not need to be - * written); otherwise a non-zero error code. + * message digests have not yet been calculated. Such streams may or may not + * need to be written. If @lookup_table is non-NULL, then each stream in + * @stream_list that has @unhashed set but not @unique_size set is checksummed + * immediately before it would otherwise be read for writing in order to + * determine if it is identical to another stream already being written or one + * that would be filtered out of the output WIM using stream_filtered() with the + * context @filter_ctx. Each such duplicate stream will be removed from + * @stream_list, its reference count transfered to the pre-existing duplicate + * stream, its memory freed, and will not be written. Alternatively, if a + * stream in @stream_list is a duplicate with any stream in @lookup_table that + * has not been marked for writing or would not be hard-filtered, it is freed + * and the pre-existing duplicate is written instead, taking ownership of the + * reference count and slot in the @lookup_table_list. + * + * Returns 0 if every stream was either written successfully or did not need to + * be written; otherwise returns a non-zero error code. */ static int write_stream_list(struct list_head *stream_list, @@ -1360,11 +1344,12 @@ write_stream_list(struct list_head *stream_list, unsigned num_threads, struct wim_lookup_table *lookup_table, struct filter_context *filter_ctx, - wimlib_progress_func_t progress_func) + wimlib_progress_func_t progfunc, + void *progctx) { int ret; struct write_streams_ctx ctx; - struct list_head raw_copy_resources; + struct list_head raw_copy_streams; wimlib_assert((write_resource_flags & (WRITE_RESOURCE_FLAG_PACK_STREAMS | @@ -1412,13 +1397,14 @@ write_stream_list(struct list_head *stream_list, compute_stream_list_stats(stream_list, &ctx); - ctx.progress_data.progress_func = progress_func; + ctx.progress_data.progfunc = progfunc; + ctx.progress_data.progctx = progctx; - ctx.num_bytes_to_compress = find_raw_copy_resources(stream_list, - write_resource_flags, - out_ctype, - out_chunk_size, - &raw_copy_resources); + ctx.num_bytes_to_compress = find_raw_copy_streams(stream_list, + write_resource_flags, + out_ctype, + out_chunk_size, + &raw_copy_streams); DEBUG("Writing stream list " "(offset = %"PRIu64", write_resource_flags=0x%08x, " @@ -1438,11 +1424,11 @@ write_stream_list(struct list_head *stream_list, * to do compression. There are serial and parallel implementations of * the chunk_compressor interface. We default to parallel using the * specified number of threads, unless the upper bound on the number - * bytes needing to be compressed is less 2000000 (heuristic value). */ + * bytes needing to be compressed is less than a heuristic value. */ if (out_ctype != WIMLIB_COMPRESSION_TYPE_NONE) { #ifdef ENABLE_MULTITHREADED_COMPRESSION - if (ctx.num_bytes_to_compress >= 2000000) { + if (ctx.num_bytes_to_compress > max(2000000, out_chunk_size)) { ret = new_parallel_chunk_compressor(out_ctype, out_chunk_size, num_threads, 0, @@ -1473,10 +1459,12 @@ write_stream_list(struct list_head *stream_list, INIT_LIST_HEAD(&ctx.pending_streams); INIT_LIST_HEAD(&ctx.pack_streams); - if (ctx.progress_data.progress_func) { - (*ctx.progress_data.progress_func)(WIMLIB_PROGRESS_MSG_WRITE_STREAMS, - &ctx.progress_data.progress); - } + ret = call_progress(ctx.progress_data.progfunc, + WIMLIB_PROGRESS_MSG_WRITE_STREAMS, + &ctx.progress_data.progress, + ctx.progress_data.progctx); + if (ret) + goto out_destroy_context; if (write_resource_flags & WRITE_RESOURCE_FLAG_PACK_STREAMS) { ret = begin_write_resource(&ctx, ctx.num_bytes_to_compress); @@ -1533,7 +1521,7 @@ write_stream_list(struct list_head *stream_list, lte->out_reshdr.offset_in_wim = offset_in_res; lte->out_res_offset_in_wim = reshdr.offset_in_wim; lte->out_res_size_in_wim = reshdr.size_in_wim; - /*lte->out_res_uncompressed_size = reshdr.uncompressed_size;*/ + lte->out_res_uncompressed_size = reshdr.uncompressed_size; offset_in_res += lte->size; } wimlib_assert(offset_in_res == reshdr.uncompressed_size); @@ -1542,7 +1530,7 @@ write_stream_list(struct list_head *stream_list, out_write_raw_copy_resources: /* Copy any compressed resources for which the raw data can be reused * without decompression. */ - ret = write_raw_copy_resources(&raw_copy_resources, ctx.out_fd, + ret = write_raw_copy_resources(&raw_copy_streams, ctx.out_fd, &ctx.progress_data); out_destroy_context: @@ -1555,13 +1543,24 @@ out_destroy_context: return ret; } +static int +is_stream_packed(struct wim_lookup_table_entry *lte, void *_ignore) +{ + return lte_is_partial(lte); +} + +static bool +wim_has_packed_streams(WIMStruct *wim) +{ + return for_lookup_table_entry(wim->lookup_table, is_stream_packed, NULL); +} + static int wim_write_stream_list(WIMStruct *wim, struct list_head *stream_list, int write_flags, unsigned num_threads, - struct filter_context *filter_ctx, - wimlib_progress_func_t progress_func) + struct filter_context *filter_ctx) { int out_ctype; u32 out_chunk_size; @@ -1569,6 +1568,16 @@ wim_write_stream_list(WIMStruct *wim, write_resource_flags = write_flags_to_resource_flags(write_flags); + /* wimlib v1.7.0: pack streams by default if the WIM version has been + * set to WIM_VERSION_PACKED_STREAMS and at least one stream in the + * WIM's lookup table is located in a packed resource (may be the same + * WIM, or a different one in the case of export). */ + if (wim->hdr.wim_version == WIM_VERSION_PACKED_STREAMS && + wim_has_packed_streams(wim)) + { + write_resource_flags |= WRITE_RESOURCE_FLAG_PACK_STREAMS; + } + if (write_resource_flags & WRITE_RESOURCE_FLAG_PACK_STREAMS) { out_chunk_size = wim->out_pack_chunk_size; out_ctype = wim->out_pack_compression_type; @@ -1585,7 +1594,8 @@ wim_write_stream_list(WIMStruct *wim, num_threads, wim->lookup_table, filter_ctx, - progress_func); + wim->progfunc, + wim->progctx); } static int @@ -1606,6 +1616,7 @@ write_wim_resource(struct wim_lookup_table_entry *lte, 1, NULL, NULL, + NULL, NULL); } @@ -1851,7 +1862,7 @@ determine_stream_size_uniquity(struct list_head *stream_list, struct stream_size_table tab; struct wim_lookup_table_entry *lte; - ret = init_stream_size_table(&tab, lt->capacity); + ret = init_stream_size_table(&tab, 9001); if (ret) return ret; @@ -1922,7 +1933,7 @@ filter_stream_list_for_write(struct list_head *stream_list, * @stream_list_ret. * * SKIP_EXTERNAL_WIMS: Streams already present in a WIM file, but not - * @wim, shall be be returned in neither @stream_list_ret nor + * @wim, shall be returned in neither @stream_list_ret nor * @lookup_table_list_ret. * * @stream_list_ret @@ -2003,7 +2014,6 @@ prepare_stream_list_for_write(WIMStruct *wim, int image, static int write_wim_streams(WIMStruct *wim, int image, int write_flags, unsigned num_threads, - wimlib_progress_func_t progress_func, struct list_head *stream_list_override, struct list_head *lookup_table_list_ret) { @@ -2044,13 +2054,11 @@ write_wim_streams(WIMStruct *wim, int image, int write_flags, stream_list, write_flags, num_threads, - filter_ctx, - progress_func); + filter_ctx); } static int -write_wim_metadata_resources(WIMStruct *wim, int image, int write_flags, - wimlib_progress_func_t progress_func) +write_wim_metadata_resources(WIMStruct *wim, int image, int write_flags) { int ret; int start_image; @@ -2069,8 +2077,11 @@ write_wim_metadata_resources(WIMStruct *wim, int image, int write_flags, DEBUG("Writing metadata resources (offset=%"PRIu64")", wim->out_fd.offset); - if (progress_func) - progress_func(WIMLIB_PROGRESS_MSG_WRITE_METADATA_BEGIN, NULL); + ret = call_progress(wim->progfunc, + WIMLIB_PROGRESS_MSG_WRITE_METADATA_BEGIN, + NULL, wim->progctx); + if (ret) + return ret; if (image == WIMLIB_ALL_IMAGES) { start_image = 1; @@ -2109,9 +2120,10 @@ write_wim_metadata_resources(WIMStruct *wim, int image, int write_flags, if (ret) return ret; } - if (progress_func) - progress_func(WIMLIB_PROGRESS_MSG_WRITE_METADATA_END, NULL); - return 0; + + return call_progress(wim->progfunc, + WIMLIB_PROGRESS_MSG_WRITE_METADATA_END, + NULL, wim->progctx); } static int @@ -2275,7 +2287,6 @@ write_wim_lookup_table(WIMStruct *wim, int image, int write_flags, */ static int finish_write(WIMStruct *wim, int image, int write_flags, - wimlib_progress_func_t progress_func, struct list_head *lookup_table_list) { int ret; @@ -2344,8 +2355,7 @@ finish_write(WIMStruct *wim, int image, int write_flags, ret = write_integrity_table(wim, new_lookup_table_end, - old_lookup_table_end, - progress_func); + old_lookup_table_end); if (ret) return ret; } else { @@ -2389,28 +2399,30 @@ finish_write(WIMStruct *wim, int image, int write_flags, } #if defined(HAVE_SYS_FILE_H) && defined(HAVE_FLOCK) + +/* Set advisory lock on WIM file (if not already done so) */ int -lock_wim(WIMStruct *wim, int fd) +lock_wim_for_append(WIMStruct *wim) { - int ret = 0; - if (fd != -1 && !wim->wim_locked) { - ret = flock(fd, LOCK_EX | LOCK_NB); - if (ret != 0) { - if (errno == EWOULDBLOCK) { - ERROR("`%"TS"' is already being modified or has been " - "mounted read-write\n" - " by another process!", wim->filename); - ret = WIMLIB_ERR_ALREADY_LOCKED; - } else { - WARNING_WITH_ERRNO("Failed to lock `%"TS"'", - wim->filename); - ret = 0; - } - } else { - wim->wim_locked = 1; - } + if (wim->locked_for_append) + return 0; + if (!flock(wim->in_fd.fd, LOCK_EX | LOCK_NB)) { + wim->locked_for_append = 1; + return 0; + } + if (errno != EWOULDBLOCK) + return 0; + return WIMLIB_ERR_ALREADY_LOCKED; +} + +/* Remove advisory lock on WIM file (if present) */ +void +unlock_wim_for_append(WIMStruct *wim) +{ + if (wim->locked_for_append) { + flock(wim->in_fd.fd, LOCK_UN); + wim->locked_for_append = 0; } - return ret; } #endif @@ -2494,7 +2506,7 @@ lock_wim(WIMStruct *wim, int fd) */ static int write_pipable_wim(WIMStruct *wim, int image, int write_flags, - unsigned num_threads, wimlib_progress_func_t progress_func, + unsigned num_threads, struct list_head *stream_list_override, struct list_head *lookup_table_list_ret) { @@ -2529,16 +2541,14 @@ write_pipable_wim(WIMStruct *wim, int image, int write_flags, /* Write metadata resources for the image(s) being included in the * output WIM. */ - ret = write_wim_metadata_resources(wim, image, write_flags, - progress_func); + ret = write_wim_metadata_resources(wim, image, write_flags); if (ret) return ret; /* Write streams needed for the image(s) being included in the output * WIM, or streams needed for the split WIM part. */ return write_wim_streams(wim, image, write_flags, num_threads, - progress_func, stream_list_override, - lookup_table_list_ret); + stream_list_override, lookup_table_list_ret); /* The lookup table, XML data, and header at end are handled by * finish_write(). */ @@ -2552,7 +2562,6 @@ write_wim_part(WIMStruct *wim, int image, int write_flags, unsigned num_threads, - wimlib_progress_func_t progress_func, unsigned part_number, unsigned total_parts, struct list_head *stream_list_override, @@ -2579,8 +2588,14 @@ write_wim_part(WIMStruct *wim, if (write_flags & WIMLIB_WRITE_FLAG_CHECK_INTEGRITY) DEBUG("\tCHECK_INTEGRITY"); - if (write_flags & WIMLIB_WRITE_FLAG_REBUILD) - DEBUG("\tREBUILD"); + if (write_flags & WIMLIB_WRITE_FLAG_NO_CHECK_INTEGRITY) + DEBUG("\tNO_CHECK_INTEGRITY"); + + if (write_flags & WIMLIB_WRITE_FLAG_PIPABLE) + DEBUG("\tPIPABLE"); + + if (write_flags & WIMLIB_WRITE_FLAG_NOT_PIPABLE) + DEBUG("\tNOT_PIPABLE"); if (write_flags & WIMLIB_WRITE_FLAG_RECOMPRESS) DEBUG("\tRECOMPRESS"); @@ -2588,17 +2603,23 @@ write_wim_part(WIMStruct *wim, if (write_flags & WIMLIB_WRITE_FLAG_FSYNC) DEBUG("\tFSYNC"); + if (write_flags & WIMLIB_WRITE_FLAG_REBUILD) + DEBUG("\tREBUILD"); + if (write_flags & WIMLIB_WRITE_FLAG_SOFT_DELETE) - DEBUG("\tFSYNC"); + DEBUG("\tSOFT_DELETE"); if (write_flags & WIMLIB_WRITE_FLAG_IGNORE_READONLY_FLAG) DEBUG("\tIGNORE_READONLY_FLAG"); - if (write_flags & WIMLIB_WRITE_FLAG_PIPABLE) - DEBUG("\tPIPABLE"); + if (write_flags & WIMLIB_WRITE_FLAG_SKIP_EXTERNAL_WIMS) + DEBUG("\tSKIP_EXTERNAL_WIMS"); - if (write_flags & WIMLIB_WRITE_FLAG_NOT_PIPABLE) - DEBUG("\tNOT_PIPABLE"); + if (write_flags & WIMLIB_WRITE_FLAG_STREAMS_OK) + DEBUG("\tSTREAMS_OK"); + + if (write_flags & WIMLIB_WRITE_FLAG_RETAIN_GUID) + DEBUG("\tRETAIN_GUID"); if (write_flags & WIMLIB_WRITE_FLAG_PACK_STREAMS) DEBUG("\tPACK_STREAMS"); @@ -2616,10 +2637,11 @@ write_wim_part(WIMStruct *wim, DEBUG("Number of threads: autodetect"); else DEBUG("Number of threads: %u", num_threads); - DEBUG("Progress function: %s", (progress_func ? "yes" : "no")); + DEBUG("Progress function: %s", (wim->progfunc ? "yes" : "no")); DEBUG("Stream list: %s", (stream_list_override ? "specified" : "autodetect")); - DEBUG("GUID: %s", ((guid || wim->guid_set_explicitly) ? - "specified" : "generate new")); + DEBUG("GUID: %s", (write_flags & + WIMLIB_WRITE_FLAG_RETAIN_GUID) ? "retain" + : guid ? "explicit" : "generate new"); /* Internally, this is always called with a valid part number and total * parts. */ @@ -2715,11 +2737,13 @@ write_wim_part(WIMStruct *wim, /* Set chunk size if different. */ wim->hdr.chunk_size = wim->out_chunk_size; - /* Use GUID if specified; otherwise generate a new one. */ - if (guid) - memcpy(wim->hdr.guid, guid, WIMLIB_GUID_LEN); - else if (!wim->guid_set_explicitly) - randomize_byte_array(wim->hdr.guid, WIMLIB_GUID_LEN); + /* Set GUID. */ + if (!(write_flags & WIMLIB_WRITE_FLAG_RETAIN_GUID)) { + if (guid) + memcpy(wim->hdr.guid, guid, WIMLIB_GUID_LEN); + else + randomize_byte_array(wim->hdr.guid, WIMLIB_GUID_LEN); + } /* Clear references to resources that have not been written yet. */ zero_reshdr(&wim->hdr.lookup_table_reshdr); @@ -2781,19 +2805,18 @@ write_wim_part(WIMStruct *wim, if (!(write_flags & WIMLIB_WRITE_FLAG_PIPABLE)) { /* Default case: create a normal (non-pipable) WIM. */ ret = write_wim_streams(wim, image, write_flags, num_threads, - progress_func, stream_list_override, + stream_list_override, &lookup_table_list); if (ret) goto out_restore_hdr; - ret = write_wim_metadata_resources(wim, image, write_flags, - progress_func); + ret = write_wim_metadata_resources(wim, image, write_flags); if (ret) goto out_restore_hdr; } else { /* Non-default case: create pipable WIM. */ ret = write_pipable_wim(wim, image, write_flags, num_threads, - progress_func, stream_list_override, + stream_list_override, &lookup_table_list); if (ret) goto out_restore_hdr; @@ -2802,8 +2825,7 @@ write_wim_part(WIMStruct *wim, /* Write lookup table, XML data, and (optional) integrity table. */ - ret = finish_write(wim, image, write_flags, progress_func, - &lookup_table_list); + ret = finish_write(wim, image, write_flags, &lookup_table_list); out_restore_hdr: memcpy(&wim->hdr, &hdr_save, sizeof(struct wim_header)); (void)close_wim_writable(wim, write_flags); @@ -2814,42 +2836,40 @@ out_restore_hdr: /* Write a standalone WIM to a file or file descriptor. */ static int write_standalone_wim(WIMStruct *wim, const void *path_or_fd, - int image, int write_flags, unsigned num_threads, - wimlib_progress_func_t progress_func) + int image, int write_flags, unsigned num_threads) { return write_wim_part(wim, path_or_fd, image, write_flags, - num_threads, progress_func, 1, 1, NULL, NULL); + num_threads, 1, 1, NULL, NULL); } /* API function documented in wimlib.h */ WIMLIBAPI int wimlib_write(WIMStruct *wim, const tchar *path, - int image, int write_flags, unsigned num_threads, - wimlib_progress_func_t progress_func) + int image, int write_flags, unsigned num_threads) { - if (!path) + if (write_flags & ~WIMLIB_WRITE_MASK_PUBLIC) return WIMLIB_ERR_INVALID_PARAM; - write_flags &= WIMLIB_WRITE_MASK_PUBLIC; + if (path == NULL || path[0] == T('\0')) + return WIMLIB_ERR_INVALID_PARAM; - return write_standalone_wim(wim, path, image, write_flags, - num_threads, progress_func); + return write_standalone_wim(wim, path, image, write_flags, num_threads); } /* API function documented in wimlib.h */ WIMLIBAPI int wimlib_write_to_fd(WIMStruct *wim, int fd, - int image, int write_flags, unsigned num_threads, - wimlib_progress_func_t progress_func) + int image, int write_flags, unsigned num_threads) { + if (write_flags & ~WIMLIB_WRITE_MASK_PUBLIC) + return WIMLIB_ERR_INVALID_PARAM; + if (fd < 0) return WIMLIB_ERR_INVALID_PARAM; - write_flags &= WIMLIB_WRITE_MASK_PUBLIC; write_flags |= WIMLIB_WRITE_FLAG_FILE_DESCRIPTOR; - return write_standalone_wim(wim, &fd, image, write_flags, - num_threads, progress_func); + return write_standalone_wim(wim, &fd, image, write_flags, num_threads); } static bool @@ -2947,15 +2967,13 @@ check_resource_offsets(WIMStruct *wim, off_t end_offset) * Integrity table (optional) (variable size) * * This method allows an image to be appended to a large WIM very quickly, and - * is is crash-safe except in the case of write re-ordering, but the + * is crash-safe except in the case of write re-ordering, but the * disadvantage is that a small hole is left in the WIM where the old lookup * table, xml data, and integrity table were. (These usually only take up a * small amount of space compared to the streams, however.) */ static int -overwrite_wim_inplace(WIMStruct *wim, int write_flags, - unsigned num_threads, - wimlib_progress_func_t progress_func) +overwrite_wim_inplace(WIMStruct *wim, int write_flags, unsigned num_threads) { int ret; off_t old_wim_end; @@ -3045,7 +3063,7 @@ overwrite_wim_inplace(WIMStruct *wim, int write_flags, if (ret) goto out_restore_memory_hdr; - ret = lock_wim(wim, wim->out_fd.fd); + ret = lock_wim_for_append(wim); if (ret) goto out_close_wim; @@ -3067,23 +3085,21 @@ overwrite_wim_inplace(WIMStruct *wim, int write_flags, &stream_list, write_flags, num_threads, - &filter_ctx, - progress_func); + &filter_ctx); if (ret) goto out_truncate; - ret = write_wim_metadata_resources(wim, WIMLIB_ALL_IMAGES, - write_flags, progress_func); + ret = write_wim_metadata_resources(wim, WIMLIB_ALL_IMAGES, write_flags); if (ret) goto out_truncate; write_flags |= WIMLIB_WRITE_FLAG_REUSE_INTEGRITY_TABLE; ret = finish_write(wim, WIMLIB_ALL_IMAGES, write_flags, - progress_func, &lookup_table_list); + &lookup_table_list); if (ret) goto out_truncate; - wim->wim_locked = 0; + unlock_wim_for_append(wim); return 0; out_truncate: @@ -3097,7 +3113,7 @@ out_truncate: out_restore_physical_hdr: (void)write_wim_header_flags(hdr_save.flags, &wim->out_fd); out_unlock_wim: - wim->wim_locked = 0; + unlock_wim_for_append(wim); out_close_wim: (void)close_wim_writable(wim, write_flags); out_restore_memory_hdr: @@ -3106,9 +3122,7 @@ out_restore_memory_hdr: } static int -overwrite_wim_via_tmpfile(WIMStruct *wim, int write_flags, - unsigned num_threads, - wimlib_progress_func_t progress_func) +overwrite_wim_via_tmpfile(WIMStruct *wim, int write_flags, unsigned num_threads) { size_t wim_name_len; int ret; @@ -3124,14 +3138,19 @@ overwrite_wim_via_tmpfile(WIMStruct *wim, int write_flags, tmpfile[wim_name_len + 9] = T('\0'); ret = wimlib_write(wim, tmpfile, WIMLIB_ALL_IMAGES, - write_flags | WIMLIB_WRITE_FLAG_FSYNC, - num_threads, progress_func); + write_flags | + WIMLIB_WRITE_FLAG_FSYNC | + WIMLIB_WRITE_FLAG_RETAIN_GUID, + num_threads); if (ret) { tunlink(tmpfile); return ret; } - close_wim(wim); + if (filedes_valid(&wim->in_fd)) { + filedes_close(&wim->in_fd); + filedes_invalidate(&wim->in_fd); + } /* Rename the new WIM file to the original WIM file. Note: on Windows * this actually calls win32_rename_replacement(), not _wrename(), so @@ -3150,13 +3169,11 @@ overwrite_wim_via_tmpfile(WIMStruct *wim, int write_flags, return WIMLIB_ERR_RENAME; } - if (progress_func) { - union wimlib_progress_info progress; - progress.rename.from = tmpfile; - progress.rename.to = wim->filename; - progress_func(WIMLIB_PROGRESS_MSG_RENAME, &progress); - } - return 0; + union wimlib_progress_info progress; + progress.rename.from = tmpfile; + progress.rename.to = wim->filename; + return call_progress(wim->progfunc, WIMLIB_PROGRESS_MSG_RENAME, + &progress, wim->progctx); } /* Determine if the specified WIM file may be updated by appending in-place @@ -3177,14 +3194,6 @@ can_overwrite_wim_inplace(const WIMStruct *wim, int write_flags) if (wim_is_pipable(wim) || (write_flags & WIMLIB_WRITE_FLAG_PIPABLE)) return false; - /* wimlib allows multiple packs in a single WIM, but they don't seem to - * be compatible with WIMGAPI, so force all streams to be repacked if - * the WIM already may have contained a pack and PACK_STREAMS was - * requested. */ - if (write_flags & WIMLIB_WRITE_FLAG_PACK_STREAMS && - wim->hdr.wim_version == WIM_VERSION_PACKED_STREAMS) - return false; - /* The default compression type and compression chunk size selected for * the output WIM must be the same as those currently used for the WIM. */ @@ -3198,16 +3207,12 @@ can_overwrite_wim_inplace(const WIMStruct *wim, int write_flags) /* API function documented in wimlib.h */ WIMLIBAPI int -wimlib_overwrite(WIMStruct *wim, int write_flags, - unsigned num_threads, - wimlib_progress_func_t progress_func) +wimlib_overwrite(WIMStruct *wim, int write_flags, unsigned num_threads) { int ret; u32 orig_hdr_flags; - write_flags &= WIMLIB_WRITE_MASK_PUBLIC; - - if (write_flags & WIMLIB_WRITE_FLAG_FILE_DESCRIPTOR) + if (write_flags & ~WIMLIB_WRITE_MASK_PUBLIC) return WIMLIB_ERR_INVALID_PARAM; if (!wim->filename) @@ -3222,12 +3227,10 @@ wimlib_overwrite(WIMStruct *wim, int write_flags, return ret; if (can_overwrite_wim_inplace(wim, write_flags)) { - ret = overwrite_wim_inplace(wim, write_flags, num_threads, - progress_func); + ret = overwrite_wim_inplace(wim, write_flags, num_threads); if (ret != WIMLIB_ERR_RESOURCE_ORDER) return ret; WARNING("Falling back to re-building entire WIM"); } - return overwrite_wim_via_tmpfile(wim, write_flags, num_threads, - progress_func); + return overwrite_wim_via_tmpfile(wim, write_flags, num_threads); }