X-Git-Url: https://wimlib.net/git/?p=wimlib;a=blobdiff_plain;f=src%2Fwrite.c;h=f397cbb280df33b2e1fb515b6f35094cd8662a1e;hp=22b1c9682c08e2509d072e907fa8abc0c6e522d2;hb=f675c45459508d058aeda4869e2928be4b7ffd99;hpb=4d42493a46a61df6f58b154a53af53884263ba64 diff --git a/src/write.c b/src/write.c index 22b1c968..f397cbb2 100644 --- a/src/write.c +++ b/src/write.c @@ -39,6 +39,7 @@ #include "wimlib/error.h" #include "wimlib/file_io.h" #include "wimlib/header.h" +#include "wimlib/inode.h" #include "wimlib/integrity.h" #include "wimlib/lookup_table.h" #include "wimlib/metadata.h" @@ -168,23 +169,19 @@ can_raw_copy(const struct wim_lookup_table_entry *lte, if (rspec->is_pipable != !!(write_resource_flags & WRITE_RESOURCE_FLAG_PIPABLE)) return false; - if (rspec->flags & WIM_RESHDR_FLAG_COMPRESSED) { /* Normal compressed resource: Must use same compression type * and chunk size. */ - return (rspec->wim->compression_type == out_ctype && - rspec->wim->chunk_size == out_chunk_size); + return (rspec->compression_type == out_ctype && + rspec->chunk_size == out_chunk_size); } - /* XXX: For compatibility, we can't allow multiple packed resources per - * WIM. */ -#if 0 if ((rspec->flags & WIM_RESHDR_FLAG_PACKED_STREAMS) && (write_resource_flags & WRITE_RESOURCE_FLAG_PACK_STREAMS)) { /* Packed resource: Such resources may contain multiple streams, * and in general only a subset of them need to be written. As - * a heuristic, re-use the raw data if at least half the + * a heuristic, re-use the raw data if more than two-thirds the * uncompressed size is being written. */ /* Note: packed resources contain a header that specifies the @@ -199,9 +196,8 @@ can_raw_copy(const struct wim_lookup_table_entry *lte, if (res_stream->will_be_in_output_wim) write_size += res_stream->size; - return (write_size > rspec->uncompressed_size / 2); + return (write_size > rspec->uncompressed_size * 2 / 3); } -#endif return false; } @@ -233,7 +229,7 @@ stream_set_out_reshdr_for_reuse(struct wim_lookup_table_entry *lte) lte->out_res_offset_in_wim = rspec->offset_in_wim; lte->out_res_size_in_wim = rspec->size_in_wim; - /*lte->out_res_uncompressed_size = rspec->uncompressed_size;*/ + lte->out_res_uncompressed_size = rspec->uncompressed_size; } else { wimlib_assert(!(lte->flags & WIM_RESHDR_FLAG_PACKED_STREAMS)); @@ -277,42 +273,32 @@ struct write_streams_progress_data { wimlib_progress_func_t progress_func; union wimlib_progress_info progress; uint64_t next_progress; - WIMStruct *prev_wim_part; }; static void do_write_streams_progress(struct write_streams_progress_data *progress_data, - u64 size, - bool discarded, - struct wim_lookup_table_entry *cur_stream) + struct wim_lookup_table_entry *cur_stream, + u64 complete_size, + u32 complete_count, + bool discarded) { union wimlib_progress_info *progress = &progress_data->progress; - bool new_wim_part; if (discarded) { - progress->write_streams.total_bytes -= size; + progress->write_streams.total_bytes -= complete_size; + progress->write_streams.total_streams -= complete_count; if (progress_data->next_progress != ~(uint64_t)0 && progress_data->next_progress > progress->write_streams.total_bytes) { progress_data->next_progress = progress->write_streams.total_bytes; } } else { - progress->write_streams.completed_bytes += size; - } - new_wim_part = false; - if (cur_stream->resource_location == RESOURCE_IN_WIM && - cur_stream->rspec->wim != progress_data->prev_wim_part) - { - if (progress_data->prev_wim_part) { - new_wim_part = true; - progress->write_streams.completed_parts++; - } - progress_data->prev_wim_part = cur_stream->rspec->wim; + progress->write_streams.completed_bytes += complete_size; + progress->write_streams.completed_streams += complete_count; } - progress->write_streams.completed_streams++; + if (progress_data->progress_func - && (progress->write_streams.completed_bytes >= progress_data->next_progress - || new_wim_part)) + && (progress->write_streams.completed_bytes >= progress_data->next_progress)) { progress_data->progress_func(WIMLIB_PROGRESS_MSG_WRITE_STREAMS, progress); @@ -368,20 +354,24 @@ struct write_streams_ctx { /* List of streams that currently have chunks being compressed. */ struct list_head pending_streams; + /* List of streams in the resource pack. Streams are moved here after + * @pending_streams only when writing a packed resource. */ + struct list_head pack_streams; + /* Set to true if the stream currently being read was a duplicate, and * therefore the corresponding stream entry needs to be freed once the * read finishes. (In this case we add the duplicate entry to * pending_streams rather than the entry being read.) */ bool stream_was_duplicate; - /* Current uncompressed offset in the resource being read. */ - u64 cur_read_res_offset; + /* Current uncompressed offset in the stream being read. */ + u64 cur_read_stream_offset; - /* Uncompressed size of the resource currently being read. */ - u64 cur_read_res_size; + /* Uncompressed size of the stream currently being read. */ + u64 cur_read_stream_size; - /* Current uncompressed offset in the resource being written. */ - u64 cur_write_res_offset; + /* Current uncompressed offset in the stream being written. */ + u64 cur_write_stream_offset; /* Uncompressed size of resource currently being written. */ u64 cur_write_res_size; @@ -401,16 +391,6 @@ struct write_streams_ctx { u64 chunks_start_offset; }; -static u64 -get_chunk_entry_size(u64 res_size, int write_resource_flags) -{ - if (res_size <= UINT32_MAX || - (write_resource_flags & WIM_RESHDR_FLAG_PACKED_STREAMS)) - return 4; - else - return 8; -} - /* Reserve space for the chunk table and prepare to accumulate the chunk table * in memory. */ static int @@ -462,7 +442,8 @@ begin_chunk_table(struct write_streams_ctx *ctx, u64 res_expected_size) * are unknown. */ reserve_size = expected_num_chunk_entries * get_chunk_entry_size(res_expected_size, - ctx->write_resource_flags); + 0 != (ctx->write_resource_flags & + WIM_RESHDR_FLAG_PACKED_STREAMS)); if (ctx->write_resource_flags & WRITE_RESOURCE_FLAG_PACK_STREAMS) reserve_size += sizeof(struct alt_chunk_table_header_disk); memset(ctx->chunk_csizes, 0, reserve_size); @@ -489,7 +470,7 @@ begin_write_resource(struct write_streams_ctx *ctx, u64 res_expected_size) /* Output file descriptor is now positioned at the offset at which to * write the first chunk of the resource. */ ctx->chunks_start_offset = ctx->out_fd->offset; - ctx->cur_write_res_offset = 0; + ctx->cur_write_stream_offset = 0; ctx->cur_write_res_size = res_expected_size; return 0; } @@ -509,7 +490,8 @@ end_chunk_table(struct write_streams_ctx *ctx, u64 res_actual_size, actual_num_chunk_entries--; chunk_entry_size = get_chunk_entry_size(res_actual_size, - ctx->write_resource_flags); + 0 != (ctx->write_resource_flags & + WIM_RESHDR_FLAG_PACKED_STREAMS)); typedef le64 __attribute__((may_alias)) aliased_le64_t; typedef le32 __attribute__((may_alias)) aliased_le32_t; @@ -568,8 +550,8 @@ end_chunk_table(struct write_streams_ctx *ctx, u64 res_actual_size, hdr.chunk_size = cpu_to_le32(ctx->out_chunk_size); hdr.compression_format = cpu_to_le32(ctx->out_ctype); - BUILD_BUG_ON(WIMLIB_COMPRESSION_TYPE_LZX != 1); - BUILD_BUG_ON(WIMLIB_COMPRESSION_TYPE_XPRESS != 2); + BUILD_BUG_ON(WIMLIB_COMPRESSION_TYPE_XPRESS != 1); + BUILD_BUG_ON(WIMLIB_COMPRESSION_TYPE_LZX != 2); BUILD_BUG_ON(WIMLIB_COMPRESSION_TYPE_LZMS != 3); ret = full_pwrite(ctx->out_fd, &hdr, sizeof(hdr), @@ -607,7 +589,8 @@ end_write_resource(struct write_streams_ctx *ctx, struct wim_reshdr *out_reshdr) u64 res_uncompressed_size; u64 res_offset_in_wim; - wimlib_assert(ctx->cur_write_res_size == ctx->cur_write_res_offset); + wimlib_assert(ctx->cur_write_stream_offset == ctx->cur_write_res_size || + (ctx->write_resource_flags & WRITE_RESOURCE_FLAG_PACK_STREAMS)); res_uncompressed_size = ctx->cur_write_res_size; if (ctx->compressor) { @@ -630,15 +613,15 @@ end_write_resource(struct write_streams_ctx *ctx, struct wim_reshdr *out_reshdr) /* Begin processing a stream for writing. */ static int write_stream_begin_read(struct wim_lookup_table_entry *lte, - bool is_partial_res, void *_ctx) + u32 flags, void *_ctx) { struct write_streams_ctx *ctx = _ctx; int ret; wimlib_assert(lte->size > 0); - ctx->cur_read_res_offset = 0; - ctx->cur_read_res_size = lte->size; + ctx->cur_read_stream_offset = 0; + ctx->cur_read_stream_size = lte->size; /* As an optimization, we allow some streams to be "unhashed", meaning * their SHA1 message digests are unknown. This is the case with @@ -654,7 +637,7 @@ write_stream_begin_read(struct wim_lookup_table_entry *lte, ctx->stream_was_duplicate = false; if (ctx->lookup_table != NULL && lte->unhashed && !lte->unique_size) { - wimlib_assert(!is_partial_res); + wimlib_assert(!(flags & BEGIN_STREAM_FLAG_PARTIAL_RESOURCE)); struct wim_lookup_table_entry *lte_new; @@ -676,7 +659,8 @@ write_stream_begin_read(struct wim_lookup_table_entry *lte, DEBUG("Discarding duplicate stream of " "length %"PRIu64, lte->size); do_write_streams_progress(&ctx->progress_data, - lte->size, true, lte); + lte, lte->size, + 1, true); list_del(<e->write_streams_list); list_del(<e->lookup_table_list); if (lte_new->will_be_in_output_wim) @@ -709,27 +693,48 @@ write_stream_begin_read(struct wim_lookup_table_entry *lte, return 0; } +/* Rewrite a stream that was just written compressed as uncompressed instead. + * This function is optional, but if a stream did not compress to less than its + * original size, it might as well be written uncompressed. */ static int write_stream_uncompressed(struct wim_lookup_table_entry *lte, struct filedes *out_fd) { int ret; + u64 begin_offset = lte->out_reshdr.offset_in_wim; + u64 end_offset = out_fd->offset; + + if (filedes_seek(out_fd, begin_offset) == -1) + return 0; + + ret = extract_full_stream_to_fd(lte, out_fd); + if (ret) { + /* Error reading the uncompressed data. */ + if (out_fd->offset == begin_offset && + filedes_seek(out_fd, end_offset) != -1) + { + /* Nothing was actually written yet, and we successfully + * seeked to the end of the compressed resource, so + * don't issue a hard error; just keep the compressed + * resource instead. */ + WARNING("Recovered compressed stream of " + "size %"PRIu64", continuing on.", + lte->size); + return 0; + } + return ret; + } - if (-1 == lseek(out_fd->fd, lte->out_reshdr.offset_in_wim, SEEK_SET) || - 0 != ftruncate(out_fd->fd, lte->out_reshdr.offset_in_wim)) + wimlib_assert(out_fd->offset - begin_offset == lte->size); + + if (out_fd->offset < end_offset && + 0 != ftruncate(out_fd->fd, out_fd->offset)) { ERROR_WITH_ERRNO("Can't truncate output file to " - "offset %"PRIu64, lte->out_reshdr.offset_in_wim); + "offset %"PRIu64, out_fd->offset); return WIMLIB_ERR_WRITE; } - out_fd->offset = lte->out_reshdr.offset_in_wim; - - ret = extract_stream_to_fd(lte, out_fd, lte->size); - if (ret) - return ret; - - wimlib_assert(out_fd->offset - lte->out_reshdr.offset_in_wim == lte->size); lte->out_reshdr.size_in_wim = lte->size; lte->out_reshdr.flags &= ~(WIM_RESHDR_FLAG_COMPRESSED | WIM_RESHDR_FLAG_PACKED_STREAMS); @@ -745,11 +750,13 @@ write_chunk(struct write_streams_ctx *ctx, const void *cchunk, int ret; struct wim_lookup_table_entry *lte; + u32 completed_stream_count; + u32 completed_size; lte = list_entry(ctx->pending_streams.next, struct wim_lookup_table_entry, write_streams_list); - if (ctx->cur_write_res_offset == 0 && + if (ctx->cur_write_stream_offset == 0 && !(ctx->write_resource_flags & WRITE_RESOURCE_FLAG_PACK_STREAMS)) { /* Starting to write a new stream in non-packed mode. */ @@ -796,52 +803,83 @@ write_chunk(struct write_streams_ctx *ctx, const void *cchunk, if (ret) goto error; - ctx->cur_write_res_offset += usize; + ctx->cur_write_stream_offset += usize; - do_write_streams_progress(&ctx->progress_data, - usize, false, lte); + completed_size = usize; + completed_stream_count = 0; + if (ctx->write_resource_flags & WRITE_RESOURCE_FLAG_PACK_STREAMS) { + /* Wrote chunk in packed mode. It may have finished multiple + * streams. */ + while (ctx->cur_write_stream_offset > lte->size) { + struct wim_lookup_table_entry *next; - if (ctx->cur_write_res_offset == ctx->cur_write_res_size && - !(ctx->write_resource_flags & WRITE_RESOURCE_FLAG_PACK_STREAMS)) - { - wimlib_assert(ctx->cur_write_res_offset == lte->size); + ctx->cur_write_stream_offset -= lte->size; - /* Finished writing a stream in non-packed mode. */ + wimlib_assert(!list_is_singular(&ctx->pending_streams) && + !list_empty(&ctx->pending_streams)); + next = list_entry(lte->write_streams_list.next, + struct wim_lookup_table_entry, + write_streams_list); + list_move_tail(<e->write_streams_list, + &ctx->pack_streams); + lte = next; + completed_stream_count++; + } + if (ctx->cur_write_stream_offset == lte->size) { + ctx->cur_write_stream_offset = 0; + list_move_tail(<e->write_streams_list, + &ctx->pack_streams); + completed_stream_count++; + } + } else { + /* Wrote chunk in non-packed mode. It may have finished a + * stream. */ + if (ctx->cur_write_stream_offset == lte->size) { - ret = end_write_resource(ctx, <e->out_reshdr); - if (ret) - return ret; + completed_stream_count++; - lte->out_reshdr.flags = filter_resource_flags(lte->flags); - if (ctx->compressor != NULL) - lte->out_reshdr.flags |= WIM_RESHDR_FLAG_COMPRESSED; + list_del(<e->write_streams_list); - if (ctx->compressor != NULL && - lte->out_reshdr.size_in_wim >= lte->out_reshdr.uncompressed_size && - !(ctx->write_resource_flags & WRITE_RESOURCE_FLAG_PIPABLE) && - !(lte->flags & WIM_RESHDR_FLAG_PACKED_STREAMS)) - { - /* Stream did not compress to less than its original - * size. If we're not writing a pipable WIM (which - * could mean the output file descriptor is - * non-seekable), and the stream isn't located in a - * resource pack (which would make reading it again - * costly), truncate the file to the start of the stream - * and write it uncompressed instead. */ - DEBUG("Stream of size %"PRIu64" did not compress to " - "less than original size; writing uncompressed.", - lte->size); - ret = write_stream_uncompressed(lte, ctx->out_fd); + wimlib_assert(ctx->cur_write_stream_offset == + ctx->cur_write_res_size); + + ret = end_write_resource(ctx, <e->out_reshdr); if (ret) return ret; - } - wimlib_assert(lte->out_reshdr.uncompressed_size == lte->size); + lte->out_reshdr.flags = filter_resource_flags(lte->flags); + if (ctx->compressor != NULL) + lte->out_reshdr.flags |= WIM_RESHDR_FLAG_COMPRESSED; + + if (ctx->compressor != NULL && + lte->out_reshdr.size_in_wim >= lte->out_reshdr.uncompressed_size && + !(ctx->write_resource_flags & WRITE_RESOURCE_FLAG_PIPABLE) && + !(lte->flags & WIM_RESHDR_FLAG_PACKED_STREAMS)) + { + /* Stream did not compress to less than its original + * size. If we're not writing a pipable WIM (which + * could mean the output file descriptor is + * non-seekable), and the stream isn't located in a + * resource pack (which would make reading it again + * costly), truncate the file to the start of the stream + * and write it uncompressed instead. */ + DEBUG("Stream of size %"PRIu64" did not compress to " + "less than original size; writing uncompressed.", + lte->size); + ret = write_stream_uncompressed(lte, ctx->out_fd); + if (ret) + return ret; + } + wimlib_assert(lte->out_reshdr.uncompressed_size == lte->size); - list_del(<e->write_streams_list); - ctx->cur_write_res_offset = 0; + ctx->cur_write_stream_offset = 0; + } } + do_write_streams_progress(&ctx->progress_data, lte, + completed_size, completed_stream_count, + false); + return 0; error: @@ -858,8 +896,8 @@ submit_chunk_for_compression(struct write_streams_ctx *ctx, * compressed chunk. */ while (!ctx->compressor->submit_chunk(ctx->compressor, chunk, size)) { const void *cchunk; - unsigned csize; - unsigned usize; + u32 csize; + u32 usize; bool bret; int ret; @@ -890,7 +928,7 @@ write_stream_process_chunk(const void *chunk, size_t size, void *_ctx) ret = write_chunk(ctx, chunk, size, size); if (ret) return ret; - ctx->cur_read_res_offset += size; + ctx->cur_read_stream_offset += size; return 0; } @@ -908,8 +946,8 @@ write_stream_process_chunk(const void *chunk, size_t size, void *_ctx) } else { u64 res_bytes_remaining; - res_bytes_remaining = ctx->cur_read_res_size - - ctx->cur_read_res_offset; + res_bytes_remaining = ctx->cur_read_stream_size - + ctx->cur_read_stream_offset; needed_chunk_size = min(ctx->out_chunk_size, ctx->chunk_buf_filled + res_bytes_remaining); @@ -921,7 +959,7 @@ write_stream_process_chunk(const void *chunk, size_t size, void *_ctx) /* No intermediate buffering needed. */ resized_chunk = chunkptr; chunkptr += needed_chunk_size; - ctx->cur_read_res_offset += needed_chunk_size; + ctx->cur_read_stream_offset += needed_chunk_size; } else { /* Intermediate buffering needed. */ size_t bytes_consumed; @@ -932,10 +970,8 @@ write_stream_process_chunk(const void *chunk, size_t size, void *_ctx) memcpy(&ctx->chunk_buf[ctx->chunk_buf_filled], chunkptr, bytes_consumed); - resized_chunk = ctx->chunk_buf; - chunkptr += bytes_consumed; - ctx->cur_read_res_offset += bytes_consumed; + ctx->cur_read_stream_offset += bytes_consumed; ctx->chunk_buf_filled += bytes_consumed; if (ctx->chunk_buf_filled == needed_chunk_size) { resized_chunk = ctx->chunk_buf; @@ -963,7 +999,7 @@ write_stream_end_read(struct wim_lookup_table_entry *lte, int status, void *_ctx { struct write_streams_ctx *ctx = _ctx; if (status == 0) - wimlib_assert(ctx->cur_read_res_offset == ctx->cur_read_res_size); + wimlib_assert(ctx->cur_read_stream_offset == ctx->cur_read_stream_size); if (ctx->stream_was_duplicate) { free_lookup_table_entry(lte); } else if (lte->unhashed && ctx->lookup_table != NULL) { @@ -1006,24 +1042,23 @@ compute_stream_list_stats(struct list_head *stream_list, ctx->progress_data.progress.write_streams.total_parts = total_parts; ctx->progress_data.progress.write_streams.completed_parts = 0; ctx->progress_data.next_progress = 0; - ctx->progress_data.prev_wim_part = NULL; } /* Find streams in @stream_list that can be copied to the output WIM in raw form - * rather than compressed. Delete these streams from @stream_list, and move one - * per resource to @raw_copy_resources. Return the total uncompressed size of - * the streams that need to be compressed. */ + * rather than compressed. Delete these streams from @stream_list and move them + * to @raw_copy_streams. Return the total uncompressed size of the streams that + * need to be compressed. */ static u64 -find_raw_copy_resources(struct list_head *stream_list, - int write_resource_flags, - int out_ctype, - u32 out_chunk_size, - struct list_head *raw_copy_resources) +find_raw_copy_streams(struct list_head *stream_list, + int write_resource_flags, + int out_ctype, + u32 out_chunk_size, + struct list_head *raw_copy_streams) { struct wim_lookup_table_entry *lte, *tmp; u64 num_bytes_to_compress = 0; - INIT_LIST_HEAD(raw_copy_resources); + INIT_LIST_HEAD(raw_copy_streams); /* Initialize temporary raw_copy_ok flag. */ list_for_each_entry(lte, stream_list, write_streams_list) @@ -1034,13 +1069,14 @@ find_raw_copy_resources(struct list_head *stream_list, if (lte->resource_location == RESOURCE_IN_WIM && lte->rspec->raw_copy_ok) { - list_del(<e->write_streams_list); + list_move_tail(<e->write_streams_list, + raw_copy_streams); } else if (can_raw_copy(lte, write_resource_flags, out_ctype, out_chunk_size)) { lte->rspec->raw_copy_ok = 1; list_move_tail(<e->write_streams_list, - raw_copy_resources); + raw_copy_streams); } else { num_bytes_to_compress += lte->size; } @@ -1111,21 +1147,30 @@ write_raw_copy_resource(struct wim_resource_spec *in_rspec, return 0; } -/* Copy a list of raw compressed resources located other WIM file(s) to the WIM - * file being written. */ +/* Copy a list of raw compressed resources located in other WIM file(s) to the + * WIM file being written. */ static int -write_raw_copy_resources(struct list_head *raw_copy_resources, +write_raw_copy_resources(struct list_head *raw_copy_streams, struct filedes *out_fd, struct write_streams_progress_data *progress_data) { struct wim_lookup_table_entry *lte; int ret; - list_for_each_entry(lte, raw_copy_resources, write_streams_list) { - ret = write_raw_copy_resource(lte->rspec, out_fd); - if (ret) - return ret; - do_write_streams_progress(progress_data, lte->size, false, lte); + list_for_each_entry(lte, raw_copy_streams, write_streams_list) + lte->rspec->raw_copy_ok = 1; + + list_for_each_entry(lte, raw_copy_streams, write_streams_list) { + if (lte->rspec->raw_copy_ok) { + /* Write each packed resource only one time, no matter + * how many streams reference it. */ + ret = write_raw_copy_resource(lte->rspec, out_fd); + if (ret) + return ret; + lte->rspec->raw_copy_ok = 0; + } + do_write_streams_progress(progress_data, lte, lte->size, + 1, false); } return 0; } @@ -1135,8 +1180,8 @@ static int finish_remaining_chunks(struct write_streams_ctx *ctx) { const void *cdata; - unsigned csize; - unsigned usize; + u32 csize; + u32 usize; int ret; if (ctx->compressor == NULL) @@ -1178,7 +1223,7 @@ remove_zero_length_streams(struct list_head *stream_list) * Write a list of streams to the output WIM file. * * @stream_list - * The list of streams to write, specifies a list of `struct + * The list of streams to write, specified by a list of `struct * wim_lookup_table_entry's linked by the 'write_streams_list' member. * * @out_fd @@ -1189,20 +1234,20 @@ remove_zero_length_streams(struct list_head *stream_list) * * WRITE_RESOURCE_FLAG_RECOMPRESS: * Force compression of all resources, even if they could otherwise - * be re-used by caping the raw data, due to being located in a WIM + * be re-used by copying the raw data, due to being located in a WIM * file with compatible compression parameters. * * WRITE_RESOURCE_FLAG_PIPABLE: * Write the resources in the wimlib-specific pipable format, and * furthermore do so in such a way that no seeking backwards in - * @out_fd will be performed (so it may be a pipe, contrary to the - * default behavior). + * @out_fd will be performed (so it may be a pipe). * * WRITE_RESOURCE_FLAG_PACK_STREAMS: * Pack all the streams into a single resource rather than writing - * them in separate resources. This format is only valid if the - * WIM version number is WIM_VERSION_PACKED_STREAMS. This flag - * currently may not be combined with WRITE_RESOURCE_FLAG_PIPABLE. + * them in separate resources. This flag is only valid if the WIM + * version number has been, or will be, set to + * WIM_VERSION_PACKED_STREAMS. This flag may not be combined with + * WRITE_RESOURCE_FLAG_PIPABLE. * * @out_ctype * Compression format to use to write the output streams, specified as one @@ -1245,15 +1290,15 @@ remove_zero_length_streams(struct list_head *stream_list) * In both cases, the @out_reshdr of the `struct wim_lookup_table_entry' for * each stream written will be updated to specify its location, size, and flags * in the output WIM. In the packed resource case, - * WIM_RESHDR_FLAG_PACKED_STREAMS shall be set in the @flags field of the - * @out_reshdr, and @out_res_offset_in_wim and @out_res_size_in_wim will also - * be set to the offset and size, respectively, in the output WIM of the full - * packed resource containing the corresponding stream. + * WIM_RESHDR_FLAG_PACKED_STREAMS will be set in the @flags field of each + * @out_reshdr, and furthermore @out_res_offset_in_wim and @out_res_size_in_wim + * of each @out_reshdr will be set to the offset and size, respectively, in the + * output WIM of the packed resource containing the corresponding stream. * * Each of the streams to write may be in any location supported by the * resource-handling code (specifically, read_stream_list()), such as the * contents of external file that has been logically added to the output WIM, or - * a stream in another WIM file that has been imported, or even stream in the + * a stream in another WIM file that has been imported, or even a stream in the * "same" WIM file of which a modified copy is being written. In the case that * a stream is already in a WIM file and uses compatible compression parameters, * by default this function will re-use the raw data instead of decompressing @@ -1261,32 +1306,31 @@ remove_zero_length_streams(struct list_head *stream_list) * specified in @write_resource_flags, this is not done. * * As a further requirement, this function requires that the - * @will_be_in_output_wim member be set on all streams in @stream_list as well - * as any other streams not in @stream_list that will be in the output WIM file, - * but not on any other streams in the output WIM's lookup table or sharing a - * packed resource with a stream in @stream_list. Still furthermore, if - * on-the-fly deduplication of streams is possible, then all streams in + * @will_be_in_output_wim member be set to 1 on all streams in @stream_list as + * well as any other streams not in @stream_list that will be in the output WIM + * file, but set to 0 on any other streams in the output WIM's lookup table or + * sharing a packed resource with a stream in @stream_list. Still furthermore, + * if on-the-fly deduplication of streams is possible, then all streams in * @stream_list must also be linked by @lookup_table_list along with any other * streams that have @will_be_in_output_wim set. * * This function handles on-the-fly deduplication of streams for which SHA1 - * message digests have not yet been calculated and it is therefore known - * whether such streams are already in @stream_list or in the WIM's lookup table - * at all. If @lookup_table is non-NULL, then each stream in @stream_list that - * has @unhashed set but not @unique_size set is checksummed immediately before - * it would otherwise be read for writing in order to determine if it is - * identical to another stream already being written or one that would be - * filtered out of the output WIM using stream_filtered() with the context - * @filter_ctx. Each such duplicate stream will be removed from @stream_list, its - * reference count transfered to the pre-existing duplicate stream, its memory - * freed, and will not be written. Alternatively, if a stream in @stream_list - * is a duplicate with any stream in @lookup_table that has not been marked for - * writing or would not be hard-filtered, it is freed and the pre-existing - * duplicate is written instead, taking ownership of the reference count and - * slot in the @lookup_table_list. - * - * Returns 0 if all streams were written successfully (or did not need to be - * written); otherwise a non-zero error code. + * message digests have not yet been calculated. Such streams may or may not + * need to be written. If @lookup_table is non-NULL, then each stream in + * @stream_list that has @unhashed set but not @unique_size set is checksummed + * immediately before it would otherwise be read for writing in order to + * determine if it is identical to another stream already being written or one + * that would be filtered out of the output WIM using stream_filtered() with the + * context @filter_ctx. Each such duplicate stream will be removed from + * @stream_list, its reference count transfered to the pre-existing duplicate + * stream, its memory freed, and will not be written. Alternatively, if a + * stream in @stream_list is a duplicate with any stream in @lookup_table that + * has not been marked for writing or would not be hard-filtered, it is freed + * and the pre-existing duplicate is written instead, taking ownership of the + * reference count and slot in the @lookup_table_list. + * + * Returns 0 if every stream was either written successfully or did not need to + * be written; otherwise returns a non-zero error code. */ static int write_stream_list(struct list_head *stream_list, @@ -1301,7 +1345,7 @@ write_stream_list(struct list_head *stream_list, { int ret; struct write_streams_ctx ctx; - struct list_head raw_copy_resources; + struct list_head raw_copy_streams; wimlib_assert((write_resource_flags & (WRITE_RESOURCE_FLAG_PACK_STREAMS | @@ -1333,7 +1377,8 @@ write_stream_list(struct list_head *stream_list, ctx.write_resource_flags = write_resource_flags; ctx.filter_ctx = filter_ctx; - if (out_chunk_size != 0) { + if (out_ctype != WIMLIB_COMPRESSION_TYPE_NONE) { + wimlib_assert(out_chunk_size != 0); if (out_chunk_size <= STACK_MAX) { ctx.chunk_buf = alloca(out_chunk_size); } else { @@ -1350,11 +1395,11 @@ write_stream_list(struct list_head *stream_list, ctx.progress_data.progress_func = progress_func; - ctx.num_bytes_to_compress = find_raw_copy_resources(stream_list, - write_resource_flags, - out_ctype, - out_chunk_size, - &raw_copy_resources); + ctx.num_bytes_to_compress = find_raw_copy_streams(stream_list, + write_resource_flags, + out_ctype, + out_chunk_size, + &raw_copy_streams); DEBUG("Writing stream list " "(offset = %"PRIu64", write_resource_flags=0x%08x, " @@ -1374,11 +1419,11 @@ write_stream_list(struct list_head *stream_list, * to do compression. There are serial and parallel implementations of * the chunk_compressor interface. We default to parallel using the * specified number of threads, unless the upper bound on the number - * bytes needing to be compressed is less 2000000 (heuristic value). */ + * bytes needing to be compressed is less than a heuristic value. */ if (out_ctype != WIMLIB_COMPRESSION_TYPE_NONE) { #ifdef ENABLE_MULTITHREADED_COMPRESSION - if (ctx.num_bytes_to_compress >= 2000000) { + if (ctx.num_bytes_to_compress > max(2000000, out_chunk_size)) { ret = new_parallel_chunk_compressor(out_ctype, out_chunk_size, num_threads, 0, @@ -1407,6 +1452,7 @@ write_stream_list(struct list_head *stream_list, ctx.progress_data.progress.write_streams.num_threads); INIT_LIST_HEAD(&ctx.pending_streams); + INIT_LIST_HEAD(&ctx.pack_streams); if (ctx.progress_data.progress_func) { (*ctx.progress_data.progress_func)(WIMLIB_PROGRESS_MSG_WRITE_STREAMS, @@ -1460,7 +1506,7 @@ write_stream_list(struct list_head *stream_list, reshdr.uncompressed_size); offset_in_res = 0; - list_for_each_entry(lte, &ctx.pending_streams, write_streams_list) { + list_for_each_entry(lte, &ctx.pack_streams, write_streams_list) { lte->out_reshdr.size_in_wim = lte->size; lte->out_reshdr.flags = filter_resource_flags(lte->flags); lte->out_reshdr.flags |= WIM_RESHDR_FLAG_PACKED_STREAMS; @@ -1468,7 +1514,7 @@ write_stream_list(struct list_head *stream_list, lte->out_reshdr.offset_in_wim = offset_in_res; lte->out_res_offset_in_wim = reshdr.offset_in_wim; lte->out_res_size_in_wim = reshdr.size_in_wim; - /*lte->out_res_uncompressed_size = reshdr.uncompressed_size;*/ + lte->out_res_uncompressed_size = reshdr.uncompressed_size; offset_in_res += lte->size; } wimlib_assert(offset_in_res == reshdr.uncompressed_size); @@ -1477,11 +1523,11 @@ write_stream_list(struct list_head *stream_list, out_write_raw_copy_resources: /* Copy any compressed resources for which the raw data can be reused * without decompression. */ - ret = write_raw_copy_resources(&raw_copy_resources, ctx.out_fd, + ret = write_raw_copy_resources(&raw_copy_streams, ctx.out_fd, &ctx.progress_data); out_destroy_context: - if (out_chunk_size > STACK_MAX) + if (out_ctype != WIMLIB_COMPRESSION_TYPE_NONE && out_chunk_size > STACK_MAX) FREE(ctx.chunk_buf); FREE(ctx.chunk_csizes); if (ctx.compressor) @@ -1504,6 +1550,11 @@ wim_write_stream_list(WIMStruct *wim, write_resource_flags = write_flags_to_resource_flags(write_flags); + /* wimlib v1.6.3: pack streams by default if the WIM version number is + * that usually used in solid archives. */ + if (wim->hdr.wim_version == WIM_VERSION_PACKED_STREAMS) + write_resource_flags |= WRITE_RESOURCE_FLAG_PACK_STREAMS; + if (write_resource_flags & WRITE_RESOURCE_FLAG_PACK_STREAMS) { out_chunk_size = wim->out_pack_chunk_size; out_ctype = wim->out_pack_compression_type; @@ -1786,7 +1837,7 @@ determine_stream_size_uniquity(struct list_head *stream_list, struct stream_size_table tab; struct wim_lookup_table_entry *lte; - ret = init_stream_size_table(&tab, lt->capacity); + ret = init_stream_size_table(&tab, 9001); if (ret) return ret; @@ -2762,10 +2813,11 @@ wimlib_write(WIMStruct *wim, const tchar *path, int image, int write_flags, unsigned num_threads, wimlib_progress_func_t progress_func) { - if (!path) + if (write_flags & ~WIMLIB_WRITE_MASK_PUBLIC) return WIMLIB_ERR_INVALID_PARAM; - write_flags &= WIMLIB_WRITE_MASK_PUBLIC; + if (path == NULL || path[0] == T('\0')) + return WIMLIB_ERR_INVALID_PARAM; return write_standalone_wim(wim, path, image, write_flags, num_threads, progress_func); @@ -2777,10 +2829,12 @@ wimlib_write_to_fd(WIMStruct *wim, int fd, int image, int write_flags, unsigned num_threads, wimlib_progress_func_t progress_func) { + if (write_flags & ~WIMLIB_WRITE_MASK_PUBLIC) + return WIMLIB_ERR_INVALID_PARAM; + if (fd < 0) return WIMLIB_ERR_INVALID_PARAM; - write_flags &= WIMLIB_WRITE_MASK_PUBLIC; write_flags |= WIMLIB_WRITE_FLAG_FILE_DESCRIPTOR; return write_standalone_wim(wim, &fd, image, write_flags, @@ -3066,7 +3120,10 @@ overwrite_wim_via_tmpfile(WIMStruct *wim, int write_flags, return ret; } - close_wim(wim); + if (filedes_valid(&wim->in_fd)) { + filedes_close(&wim->in_fd); + filedes_invalidate(&wim->in_fd); + } /* Rename the new WIM file to the original WIM file. Note: on Windows * this actually calls win32_rename_replacement(), not _wrename(), so @@ -3112,14 +3169,6 @@ can_overwrite_wim_inplace(const WIMStruct *wim, int write_flags) if (wim_is_pipable(wim) || (write_flags & WIMLIB_WRITE_FLAG_PIPABLE)) return false; - /* wimlib allows multiple packs in a single WIM, but they don't seem to - * be compatible with WIMGAPI, so force all streams to be repacked if - * the WIM already may have contained a pack and PACK_STREAMS was - * requested. */ - if (write_flags & WIMLIB_WRITE_FLAG_PACK_STREAMS && - wim->hdr.wim_version == WIM_VERSION_PACKED_STREAMS) - return false; - /* The default compression type and compression chunk size selected for * the output WIM must be the same as those currently used for the WIM. */ @@ -3140,9 +3189,7 @@ wimlib_overwrite(WIMStruct *wim, int write_flags, int ret; u32 orig_hdr_flags; - write_flags &= WIMLIB_WRITE_MASK_PUBLIC; - - if (write_flags & WIMLIB_WRITE_FLAG_FILE_DESCRIPTOR) + if (write_flags & ~WIMLIB_WRITE_MASK_PUBLIC) return WIMLIB_ERR_INVALID_PARAM; if (!wim->filename)