X-Git-Url: https://wimlib.net/git/?p=wimlib;a=blobdiff_plain;f=src%2Fwrite.c;h=cc569395d19c12f3f499b5a68f8523f33e5055bd;hp=ef78f9cadfc93e5df4fbb029d1f8e27270b2af3f;hb=832b0b2adaf6369d4d686006962206e867ac2a11;hpb=f8223526610a6ed9410ed6ce2e704ef15ab8cade diff --git a/src/write.c b/src/write.c index ef78f9ca..cc569395 100644 --- a/src/write.c +++ b/src/write.c @@ -28,7 +28,7 @@ #if defined(HAVE_SYS_FILE_H) && defined(HAVE_FLOCK) /* On BSD, this should be included before "wimlib/list.h" so that "wimlib/list.h" can - * overwrite the LIST_HEAD macro. */ + * override the LIST_HEAD macro. */ # include #endif @@ -52,9 +52,7 @@ #include "wimlib/progress.h" #include "wimlib/resource.h" #include "wimlib/solid.h" -#ifdef __WIN32__ -# include "wimlib/win32.h" /* win32_rename_replacement() */ -#endif +#include "wimlib/win32.h" /* win32_rename_replacement() */ #include "wimlib/write.h" #include "wimlib/xml.h" @@ -66,23 +64,28 @@ #define WRITE_RESOURCE_FLAG_SEND_DONE_WITH_FILE 0x00000008 #define WRITE_RESOURCE_FLAG_SOLID_SORT 0x00000010 -static inline int +static int write_flags_to_resource_flags(int write_flags) { int write_resource_flags = 0; if (write_flags & WIMLIB_WRITE_FLAG_RECOMPRESS) write_resource_flags |= WRITE_RESOURCE_FLAG_RECOMPRESS; + if (write_flags & WIMLIB_WRITE_FLAG_PIPABLE) write_resource_flags |= WRITE_RESOURCE_FLAG_PIPABLE; + if (write_flags & WIMLIB_WRITE_FLAG_SOLID) write_resource_flags |= WRITE_RESOURCE_FLAG_SOLID; + if (write_flags & WIMLIB_WRITE_FLAG_SEND_DONE_WITH_FILE_MESSAGES) write_resource_flags |= WRITE_RESOURCE_FLAG_SEND_DONE_WITH_FILE; + if ((write_flags & (WIMLIB_WRITE_FLAG_SOLID | WIMLIB_WRITE_FLAG_NO_SOLID_SORT)) == WIMLIB_WRITE_FLAG_SOLID) write_resource_flags |= WRITE_RESOURCE_FLAG_SOLID_SORT; + return write_resource_flags; } @@ -115,7 +118,7 @@ blob_filtered(const struct blob_descriptor *blob, write_flags = ctx->write_flags; wim = ctx->wim; - if (write_flags & WIMLIB_WRITE_FLAG_OVERWRITE && + if (write_flags & WIMLIB_WRITE_FLAG_APPEND && blob->blob_location == BLOB_IN_WIM && blob->rdesc->wim == wim) return 1; @@ -135,60 +138,77 @@ blob_hard_filtered(const struct blob_descriptor *blob, return blob_filtered(blob, ctx) < 0; } -static inline int +static inline bool may_soft_filter_blobs(const struct filter_context *ctx) { - if (ctx == NULL) - return 0; - return ctx->write_flags & WIMLIB_WRITE_FLAG_OVERWRITE; + return ctx && (ctx->write_flags & WIMLIB_WRITE_FLAG_APPEND); } -static inline int +static inline bool may_hard_filter_blobs(const struct filter_context *ctx) { - if (ctx == NULL) - return 0; - return ctx->write_flags & WIMLIB_WRITE_FLAG_SKIP_EXTERNAL_WIMS; + return ctx && (ctx->write_flags & WIMLIB_WRITE_FLAG_SKIP_EXTERNAL_WIMS); } -static inline int +static inline bool may_filter_blobs(const struct filter_context *ctx) { return (may_soft_filter_blobs(ctx) || may_hard_filter_blobs(ctx)); } -/* Return true if the specified resource is compressed and the compressed data - * can be reused with the specified output parameters. */ +/* Return true if the specified blob is located in a WIM resource which can be + * reused in the output WIM file, without being recompressed. */ static bool -can_raw_copy(const struct blob_descriptor *blob, - int write_resource_flags, int out_ctype, u32 out_chunk_size) +can_raw_copy(const struct blob_descriptor *blob, int write_resource_flags, + int out_ctype, u32 out_chunk_size) { const struct wim_resource_descriptor *rdesc; + /* Recompress everything if requested. */ if (write_resource_flags & WRITE_RESOURCE_FLAG_RECOMPRESS) return false; - if (out_ctype == WIMLIB_COMPRESSION_TYPE_NONE) - return false; - + /* A blob not located in a WIM resource cannot be reused. */ if (blob->blob_location != BLOB_IN_WIM) return false; rdesc = blob->rdesc; - if (rdesc->is_pipable != !!(write_resource_flags & WRITE_RESOURCE_FLAG_PIPABLE)) + /* In the case of an in-place compaction, always reuse resources located + * in the WIM being compacted. */ + if (rdesc->wim->being_compacted) + return true; + + /* Otherwise, only reuse compressed resources. */ + if (out_ctype == WIMLIB_COMPRESSION_TYPE_NONE || + !(rdesc->flags & (WIM_RESHDR_FLAG_COMPRESSED | + WIM_RESHDR_FLAG_SOLID))) + return false; + + /* When writing a pipable WIM, we can only reuse pipable resources; and + * when writing a non-pipable WIM, we can only reuse non-pipable + * resources. */ + if (rdesc->is_pipable != + !!(write_resource_flags & WRITE_RESOURCE_FLAG_PIPABLE)) return false; + /* When writing a solid WIM, we can only reuse solid resources; and when + * writing a non-solid WIM, we can only reuse non-solid resources. */ + if (!!(rdesc->flags & WIM_RESHDR_FLAG_SOLID) != + !!(write_resource_flags & WRITE_RESOURCE_FLAG_SOLID)) + return false; + + /* Note: it is theoretically possible to copy chunks of compressed data + * between non-solid, solid, and pipable resources. However, we don't + * currently implement this optimization because it would be complex and + * would usually go unused. */ + if (rdesc->flags & WIM_RESHDR_FLAG_COMPRESSED) { - /* Normal compressed resource: Must use same compression type - * and chunk size. */ + /* To re-use a non-solid resource, it must use the desired + * compression type and chunk size. */ return (rdesc->compression_type == out_ctype && rdesc->chunk_size == out_chunk_size); - } - - if ((rdesc->flags & WIM_RESHDR_FLAG_SOLID) && - (write_resource_flags & WRITE_RESOURCE_FLAG_SOLID)) - { + } else { /* Solid resource: Such resources may contain multiple blobs, * and in general only a subset of them need to be written. As * a heuristic, re-use the raw data if more than two-thirds the @@ -199,6 +219,10 @@ can_raw_copy(const struct blob_descriptor *blob, * check if they are compatible with @out_ctype and * @out_chunk_size. */ + /* Did we already decide to reuse the resource? */ + if (rdesc->raw_copy_ok) + return true; + struct blob_descriptor *res_blob; u64 write_size = 0; @@ -208,8 +232,6 @@ can_raw_copy(const struct blob_descriptor *blob, return (write_size > rdesc->uncompressed_size * 2 / 3); } - - return false; } static u32 @@ -274,7 +296,7 @@ struct write_blobs_progress_data { wimlib_progress_func_t progfunc; void *progctx; union wimlib_progress_info progress; - uint64_t next_progress; + u64 next_progress; }; static int @@ -287,7 +309,7 @@ do_write_blobs_progress(struct write_blobs_progress_data *progress_data, if (discarded) { progress->write_streams.total_bytes -= complete_size; progress->write_streams.total_streams -= complete_count; - if (progress_data->next_progress != ~(uint64_t)0 && + if (progress_data->next_progress != ~(u64)0 && progress_data->next_progress > progress->write_streams.total_bytes) { progress_data->next_progress = progress->write_streams.total_bytes; @@ -297,8 +319,8 @@ do_write_blobs_progress(struct write_blobs_progress_data *progress_data, progress->write_streams.completed_streams += complete_count; } - if (progress->write_streams.completed_bytes >= progress_data->next_progress) - { + if (progress->write_streams.completed_bytes >= progress_data->next_progress) { + ret = call_progress(progress_data->progfunc, WIMLIB_PROGRESS_MSG_WRITE_STREAMS, progress, @@ -306,32 +328,9 @@ do_write_blobs_progress(struct write_blobs_progress_data *progress_data, if (ret) return ret; - if (progress_data->next_progress == progress->write_streams.total_bytes) { - progress_data->next_progress = ~(uint64_t)0; - } else { - /* Handle rate-limiting of messages */ - - /* Send new message as soon as another 1/128 of the - * total has been written. (Arbitrary number.) */ - progress_data->next_progress = - progress->write_streams.completed_bytes + - progress->write_streams.total_bytes / 128; - - /* ... Unless that would be more than 5000000 bytes, in - * which case send the next after the next 5000000 - * bytes. (Another arbitrary number.) */ - if (progress->write_streams.completed_bytes + 5000000 < - progress_data->next_progress) - progress_data->next_progress = - progress->write_streams.completed_bytes + 5000000; - - /* ... But always send a message as soon as we're - * completely done. */ - if (progress->write_streams.total_bytes < - progress_data->next_progress) - progress_data->next_progress = - progress->write_streams.total_bytes; - } + set_next_progress(progress->write_streams.completed_bytes, + progress->write_streams.total_bytes, + &progress_data->next_progress); } return 0; } @@ -358,10 +357,6 @@ struct write_blobs_ctx { struct filter_context *filter_ctx; - /* Upper bound on the total number of bytes that need to be compressed. - * */ - u64 num_bytes_to_compress; - /* Pointer to the chunk_compressor implementation being used for * compressing chunks of data, or NULL if chunks are being written * uncompressed. */ @@ -568,9 +563,9 @@ end_chunk_table(struct write_blobs_ctx *ctx, u64 res_actual_size, hdr.chunk_size = cpu_to_le32(ctx->out_chunk_size); hdr.compression_format = cpu_to_le32(ctx->out_ctype); - BUILD_BUG_ON(WIMLIB_COMPRESSION_TYPE_XPRESS != 1); - BUILD_BUG_ON(WIMLIB_COMPRESSION_TYPE_LZX != 2); - BUILD_BUG_ON(WIMLIB_COMPRESSION_TYPE_LZMS != 3); + STATIC_ASSERT(WIMLIB_COMPRESSION_TYPE_XPRESS == 1); + STATIC_ASSERT(WIMLIB_COMPRESSION_TYPE_LZX == 2); + STATIC_ASSERT(WIMLIB_COMPRESSION_TYPE_LZMS == 3); ret = full_pwrite(ctx->out_fd, &hdr, sizeof(hdr), chunk_table_offset - sizeof(hdr)); @@ -623,8 +618,6 @@ end_write_resource(struct write_blobs_ctx *ctx, struct wim_reshdr *out_reshdr) out_reshdr->uncompressed_size = res_uncompressed_size; out_reshdr->size_in_wim = res_size_in_wim; out_reshdr->offset_in_wim = res_offset_in_wim; - DEBUG("Finished writing resource: %"PRIu64" => %"PRIu64" @ %"PRIu64"", - res_uncompressed_size, res_size_in_wim, res_offset_in_wim); return 0; } @@ -646,6 +639,9 @@ do_done_with_blob(struct blob_descriptor *blob, { int ret; struct wim_inode *inode; + const tchar *path; + tchar *cookie1; + tchar *cookie2; if (!blob->may_send_done_with_file) return 0; @@ -653,42 +649,20 @@ do_done_with_blob(struct blob_descriptor *blob, inode = blob->file_inode; wimlib_assert(inode != NULL); - wimlib_assert(inode->num_remaining_streams > 0); - if (--inode->num_remaining_streams > 0) + wimlib_assert(inode->i_num_remaining_streams > 0); + if (--inode->i_num_remaining_streams > 0) return 0; -#ifdef __WIN32__ - /* XXX: This logic really should be somewhere else. */ + path = blob_file_path(blob); - /* We want the path to the file, but blob->file_on_disk might actually - * refer to a named data stream. Temporarily strip the named data - * stream from the path. */ - wchar_t *p_colon = NULL; - wchar_t *p_question_mark = NULL; - const wchar_t *p_stream_name; + cookie1 = progress_get_streamless_path(path); + cookie2 = progress_get_win32_path(path); - p_stream_name = path_stream_name(blob->file_on_disk); - if (unlikely(p_stream_name)) { - p_colon = (wchar_t *)(p_stream_name - 1); - wimlib_assert(*p_colon == L':'); - *p_colon = L'\0'; - } + ret = done_with_file(path, progfunc, progctx); - /* We also should use a fake Win32 path instead of a NT path */ - if (!wcsncmp(blob->file_on_disk, L"\\??\\", 4)) { - p_question_mark = &blob->file_on_disk[1]; - *p_question_mark = L'\\'; - } -#endif - - ret = done_with_file(blob->file_on_disk, progfunc, progctx); + progress_put_win32_path(cookie2); + progress_put_streamless_path(cookie1); -#ifdef __WIN32__ - if (p_colon) - *p_colon = L':'; - if (p_question_mark) - *p_question_mark = L'?'; -#endif return ret; } @@ -744,8 +718,6 @@ write_blob_begin_read(struct blob_descriptor *blob, void *_ctx) * (and reading it again) entirely, passing its * output reference count to the duplicate blob * in the former case. */ - DEBUG("Discarding duplicate blob of " - "length %"PRIu64, blob->size); ret = do_write_blobs_progress(&ctx->progress_data, blob->size, 1, true); list_del(&blob->write_blobs_list); @@ -768,8 +740,6 @@ write_blob_begin_read(struct blob_descriptor *blob, void *_ctx) * blob descriptor must wait until * read_blob_list() has finished reading its * data. */ - DEBUG("Blob duplicate, but not already " - "selected for writing."); list_replace(&blob->write_blobs_list, &new_blob->write_blobs_list); list_replace(&blob->blob_table_list, @@ -786,8 +756,8 @@ write_blob_begin_read(struct blob_descriptor *blob, void *_ctx) return 0; } -/* Rewrite a blob that was just written compressed as uncompressed instead. - */ +/* Rewrite a blob that was just written compressed (as a non-solid WIM resource) + * as uncompressed instead. */ static int write_blob_uncompressed(struct blob_descriptor *blob, struct filedes *out_fd) { @@ -798,7 +768,7 @@ write_blob_uncompressed(struct blob_descriptor *blob, struct filedes *out_fd) if (filedes_seek(out_fd, begin_offset) == -1) return 0; - ret = extract_full_blob_to_fd(blob, out_fd); + ret = extract_blob_to_fd(blob, out_fd); if (ret) { /* Error reading the uncompressed data. */ if (out_fd->offset == begin_offset && @@ -808,7 +778,7 @@ write_blob_uncompressed(struct blob_descriptor *blob, struct filedes *out_fd) * seeked to the end of the compressed resource, so * don't issue a hard error; just keep the compressed * resource instead. */ - WARNING("Recovered compressed blob of " + WARNING("Recovered compressed resource of " "size %"PRIu64", continuing on.", blob->size); return 0; } @@ -817,13 +787,9 @@ write_blob_uncompressed(struct blob_descriptor *blob, struct filedes *out_fd) wimlib_assert(out_fd->offset - begin_offset == blob->size); - if (out_fd->offset < end_offset && - 0 != ftruncate(out_fd->fd, out_fd->offset)) - { - ERROR_WITH_ERRNO("Can't truncate output file to " - "offset %"PRIu64, out_fd->offset); - return WIMLIB_ERR_WRITE; - } + /* We could ftruncate() the file to 'out_fd->offset' here, but there + * isn't much point. Usually we will only be truncating by a few bytes + * and will just overwrite the data immediately. */ blob->out_reshdr.size_in_wim = blob->size; blob->out_reshdr.flags &= ~(WIM_RESHDR_FLAG_COMPRESSED | @@ -919,8 +885,6 @@ write_chunk(struct write_blobs_ctx *ctx, const void *cchunk, /* Starting to write a new blob in non-solid mode. */ if (ctx->write_resource_flags & WRITE_RESOURCE_FLAG_PIPABLE) { - DEBUG("Writing pipable WIM blob header " - "(offset=%"PRIu64")", ctx->out_fd->offset); ret = write_pwm_blob_header(blob, ctx->out_fd, ctx->compressor != NULL); if (ret) @@ -1148,11 +1112,16 @@ write_blob_end_read(struct blob_descriptor *blob, int status, void *_ctx) return status; } -/* Compute statistics about a list of blobs that will be written. +/* + * Compute statistics about a list of blobs that will be written. * * Assumes the blobs are sorted such that all blobs located in each distinct WIM - * (specified by WIMStruct) are together. */ -static void + * (specified by WIMStruct) are together. + * + * For compactions, also verify that there are no overlapping resources. This + * really should be checked earlier, but for now it's easiest to check here. + */ +static int compute_blob_list_stats(struct list_head *blob_list, struct write_blobs_ctx *ctx) { @@ -1161,15 +1130,32 @@ compute_blob_list_stats(struct list_head *blob_list, u64 num_blobs = 0; u64 total_parts = 0; WIMStruct *prev_wim_part = NULL; + const struct wim_resource_descriptor *prev_rdesc = NULL; list_for_each_entry(blob, blob_list, write_blobs_list) { num_blobs++; total_bytes += blob->size; if (blob->blob_location == BLOB_IN_WIM) { - if (prev_wim_part != blob->rdesc->wim) { - prev_wim_part = blob->rdesc->wim; + const struct wim_resource_descriptor *rdesc = blob->rdesc; + WIMStruct *wim = rdesc->wim; + + if (prev_wim_part != wim) { + prev_wim_part = wim; total_parts++; } + if (unlikely(wim->being_compacted) && rdesc != prev_rdesc) { + if (prev_rdesc != NULL && + rdesc->offset_in_wim < + prev_rdesc->offset_in_wim + + prev_rdesc->size_in_wim) + { + WARNING("WIM file contains overlapping " + "resources! Compaction is not " + "possible."); + return WIMLIB_ERR_RESOURCE_ORDER; + } + prev_rdesc = rdesc; + } } } ctx->progress_data.progress.write_streams.total_bytes = total_bytes; @@ -1180,6 +1166,7 @@ compute_blob_list_stats(struct list_head *blob_list, ctx->progress_data.progress.write_streams.total_parts = total_parts; ctx->progress_data.progress.write_streams.completed_parts = 0; ctx->progress_data.next_progress = 0; + return 0; } /* Find blobs in @blob_list that can be copied to the output WIM in raw form @@ -1187,14 +1174,12 @@ compute_blob_list_stats(struct list_head *blob_list, * @raw_copy_blobs. Return the total uncompressed size of the blobs that need * to be compressed. */ static u64 -find_raw_copy_blobs(struct list_head *blob_list, - int write_resource_flags, - int out_ctype, - u32 out_chunk_size, +find_raw_copy_blobs(struct list_head *blob_list, int write_resource_flags, + int out_ctype, u32 out_chunk_size, struct list_head *raw_copy_blobs) { struct blob_descriptor *blob, *tmp; - u64 num_bytes_to_compress = 0; + u64 num_nonraw_bytes = 0; INIT_LIST_HEAD(raw_copy_blobs); @@ -1204,23 +1189,17 @@ find_raw_copy_blobs(struct list_head *blob_list, blob->rdesc->raw_copy_ok = 0; list_for_each_entry_safe(blob, tmp, blob_list, write_blobs_list) { - if (blob->blob_location == BLOB_IN_WIM && - blob->rdesc->raw_copy_ok) - { - list_move_tail(&blob->write_blobs_list, - raw_copy_blobs); - } else if (can_raw_copy(blob, write_resource_flags, - out_ctype, out_chunk_size)) + if (can_raw_copy(blob, write_resource_flags, + out_ctype, out_chunk_size)) { blob->rdesc->raw_copy_ok = 1; - list_move_tail(&blob->write_blobs_list, - raw_copy_blobs); + list_move_tail(&blob->write_blobs_list, raw_copy_blobs); } else { - num_bytes_to_compress += blob->size; + num_nonraw_bytes += blob->size; } } - return num_bytes_to_compress; + return num_nonraw_bytes; } /* Copy a raw compressed resource located in another WIM file to the WIM file @@ -1238,10 +1217,6 @@ write_raw_copy_resource(struct wim_resource_descriptor *in_rdesc, struct blob_descriptor *blob; u64 out_offset_in_wim; - DEBUG("Copying raw compressed data (size_in_wim=%"PRIu64", " - "uncompressed_size=%"PRIu64")", - in_rdesc->size_in_wim, in_rdesc->uncompressed_size); - /* Copy the raw data. */ cur_read_offset = in_rdesc->offset_in_wim; end_read_offset = cur_read_offset + in_rdesc->size_in_wim; @@ -1256,21 +1231,37 @@ write_raw_copy_resource(struct wim_resource_descriptor *in_rdesc, } in_fd = &in_rdesc->wim->in_fd; wimlib_assert(cur_read_offset != end_read_offset); - do { - bytes_to_read = min(sizeof(buf), end_read_offset - cur_read_offset); + if (likely(!in_rdesc->wim->being_compacted) || + in_rdesc->offset_in_wim > out_fd->offset) { + do { + bytes_to_read = min(sizeof(buf), + end_read_offset - cur_read_offset); - ret = full_pread(in_fd, buf, bytes_to_read, cur_read_offset); - if (ret) - return ret; + ret = full_pread(in_fd, buf, bytes_to_read, + cur_read_offset); + if (ret) + return ret; - ret = full_write(out_fd, buf, bytes_to_read); - if (ret) - return ret; + ret = full_write(out_fd, buf, bytes_to_read); + if (ret) + return ret; + + cur_read_offset += bytes_to_read; - cur_read_offset += bytes_to_read; + } while (cur_read_offset != end_read_offset); + } else { + /* Optimization: the WIM file is being compacted and the + * resource being written is already in the desired location. + * Skip over the data instead of re-writing it. */ + + /* Due the earlier check for overlapping resources, it should + * never be the case that we already overwrote the resource. */ + wimlib_assert(!(in_rdesc->offset_in_wim < out_fd->offset)); - } while (cur_read_offset != end_read_offset); + if (-1 == filedes_seek(out_fd, out_fd->offset + in_rdesc->size_in_wim)) + return WIMLIB_ERR_WRITE; + } list_for_each_entry(blob, &in_rdesc->blob_list, rdesc_node) { if (blob->will_be_in_output_wim) { @@ -1342,33 +1333,16 @@ finish_remaining_chunks(struct write_blobs_ctx *ctx) } static void -remove_empty_blobs(struct list_head *blob_list) +validate_blob_list(struct list_head *blob_list) { - struct blob_descriptor *blob, *tmp; + struct blob_descriptor *blob; - list_for_each_entry_safe(blob, tmp, blob_list, write_blobs_list) { + list_for_each_entry(blob, blob_list, write_blobs_list) { wimlib_assert(blob->will_be_in_output_wim); - if (blob->size == 0) { - list_del(&blob->write_blobs_list); - blob->out_reshdr.offset_in_wim = 0; - blob->out_reshdr.size_in_wim = 0; - blob->out_reshdr.uncompressed_size = 0; - blob->out_reshdr.flags = reshdr_flags_for_blob(blob); - } + wimlib_assert(blob->size != 0); } } -static inline bool -blob_is_in_file(const struct blob_descriptor *blob) -{ - return blob->blob_location == BLOB_IN_FILE_ON_DISK -#ifdef __WIN32__ - || blob->blob_location == BLOB_IN_WINNT_FILE_ON_DISK - || blob->blob_location == BLOB_WIN32_ENCRYPTED -#endif - ; -} - static void init_done_with_file_info(struct list_head *blob_list) { @@ -1376,7 +1350,7 @@ init_done_with_file_info(struct list_head *blob_list) list_for_each_entry(blob, blob_list, write_blobs_list) { if (blob_is_in_file(blob)) { - blob->file_inode->num_remaining_streams = 0; + blob->file_inode->i_num_remaining_streams = 0; blob->may_send_done_with_file = 1; } else { blob->may_send_done_with_file = 0; @@ -1385,7 +1359,7 @@ init_done_with_file_info(struct list_head *blob_list) list_for_each_entry(blob, blob_list, write_blobs_list) if (blob->may_send_done_with_file) - blob->file_inode->num_remaining_streams++; + blob->file_inode->i_num_remaining_streams++; } /* @@ -1482,9 +1456,9 @@ init_done_with_file_info(struct list_head *blob_list) * identical to another blob already being written or one that would be filtered * out of the output WIM using blob_filtered() with the context @filter_ctx. * Each such duplicate blob will be removed from @blob_list, its reference count - * transfered to the pre-existing duplicate blob, its memory freed, and will not - * be written. Alternatively, if a blob in @blob_list is a duplicate with any - * blob in @blob_table that has not been marked for writing or would not be + * transferred to the pre-existing duplicate blob, its memory freed, and will + * not be written. Alternatively, if a blob in @blob_list is a duplicate with + * any blob in @blob_table that has not been marked for writing or would not be * hard-filtered, it is freed and the pre-existing duplicate is written instead, * taking ownership of the reference count and slot in the @blob_table_list. * @@ -1506,6 +1480,7 @@ write_blob_list(struct list_head *blob_list, int ret; struct write_blobs_ctx ctx; struct list_head raw_copy_blobs; + u64 num_nonraw_bytes; wimlib_assert((write_resource_flags & (WRITE_RESOURCE_FLAG_SOLID | @@ -1513,12 +1488,10 @@ write_blob_list(struct list_head *blob_list, (WRITE_RESOURCE_FLAG_SOLID | WRITE_RESOURCE_FLAG_PIPABLE)); - remove_empty_blobs(blob_list); + validate_blob_list(blob_list); - if (list_empty(blob_list)) { - DEBUG("No blobs to write."); + if (list_empty(blob_list)) return 0; - } /* If needed, set auxiliary information so that we can detect when the * library has finished using each external file. */ @@ -1553,7 +1526,9 @@ write_blob_list(struct list_head *blob_list, if (ret) return ret; - compute_blob_list_stats(blob_list, &ctx); + ret = compute_blob_list_stats(blob_list, &ctx); + if (ret) + return ret; if (write_resource_flags & WRITE_RESOURCE_FLAG_SOLID_SORT) { ret = sort_blob_list_for_solid_compression(blob_list); @@ -1564,35 +1539,18 @@ write_blob_list(struct list_head *blob_list, ctx.progress_data.progfunc = progfunc; ctx.progress_data.progctx = progctx; - ctx.num_bytes_to_compress = find_raw_copy_blobs(blob_list, - write_resource_flags, - out_ctype, - out_chunk_size, - &raw_copy_blobs); - - DEBUG("Writing blob list " - "(offset = %"PRIu64", write_resource_flags=0x%08x, " - "out_ctype=%d, out_chunk_size=%u, num_threads=%u, " - "total_bytes=%"PRIu64", num_bytes_to_compress=%"PRIu64")", - out_fd->offset, write_resource_flags, - out_ctype, out_chunk_size, num_threads, - ctx.progress_data.progress.write_streams.total_bytes, - ctx.num_bytes_to_compress); - - if (ctx.num_bytes_to_compress == 0) { - DEBUG("No compression needed; skipping to raw copy!"); - goto out_write_raw_copy_resources; - } - - /* Unless uncompressed output was required, allocate a chunk_compressor - * to do compression. There are serial and parallel implementations of - * the chunk_compressor interface. We default to parallel using the + num_nonraw_bytes = find_raw_copy_blobs(blob_list, write_resource_flags, + out_ctype, out_chunk_size, + &raw_copy_blobs); + + /* Unless no data needs to be compressed, allocate a chunk_compressor to + * do compression. There are serial and parallel implementations of the + * chunk_compressor interface. We default to parallel using the * specified number of threads, unless the upper bound on the number * bytes needing to be compressed is less than a heuristic value. */ - if (out_ctype != WIMLIB_COMPRESSION_TYPE_NONE) { - + if (num_nonraw_bytes != 0 && out_ctype != WIMLIB_COMPRESSION_TYPE_NONE) { #ifdef ENABLE_MULTITHREADED_COMPRESSION - if (ctx.num_bytes_to_compress > max(2000000, out_chunk_size)) { + if (num_nonraw_bytes > max(2000000, out_chunk_size)) { ret = new_parallel_chunk_compressor(out_ctype, out_chunk_size, num_threads, 0, @@ -1618,12 +1576,6 @@ write_blob_list(struct list_head *blob_list, else ctx.progress_data.progress.write_streams.num_threads = 1; - DEBUG("Actually using %u threads", - ctx.progress_data.progress.write_streams.num_threads); - - INIT_LIST_HEAD(&ctx.blobs_being_compressed); - INIT_LIST_HEAD(&ctx.blobs_in_solid_resource); - ret = call_progress(ctx.progress_data.progfunc, WIMLIB_PROGRESS_MSG_WRITE_STREAMS, &ctx.progress_data.progress, @@ -1631,8 +1583,21 @@ write_blob_list(struct list_head *blob_list, if (ret) goto out_destroy_context; + /* Copy any compressed resources for which the raw data can be reused + * without decompression. */ + ret = write_raw_copy_resources(&raw_copy_blobs, ctx.out_fd, + &ctx.progress_data); + + if (ret || num_nonraw_bytes == 0) + goto out_destroy_context; + + INIT_LIST_HEAD(&ctx.blobs_being_compressed); + if (write_resource_flags & WRITE_RESOURCE_FLAG_SOLID) { - ret = begin_write_resource(&ctx, ctx.num_bytes_to_compress); + + INIT_LIST_HEAD(&ctx.blobs_in_solid_resource); + + ret = begin_write_resource(&ctx, num_nonraw_bytes); if (ret) goto out_destroy_context; } @@ -1640,13 +1605,11 @@ write_blob_list(struct list_head *blob_list, /* Read the list of blobs needing to be compressed, using the specified * callbacks to execute processing of the data. */ - struct read_blob_list_callbacks cbs = { - .begin_blob = write_blob_begin_read, - .begin_blob_ctx = &ctx, - .consume_chunk = write_blob_process_chunk, - .consume_chunk_ctx = &ctx, - .end_blob = write_blob_end_read, - .end_blob_ctx = &ctx, + struct read_blob_callbacks cbs = { + .begin_blob = write_blob_begin_read, + .consume_chunk = write_blob_process_chunk, + .end_blob = write_blob_end_read, + .ctx = &ctx, }; ret = read_blob_list(blob_list, @@ -1672,11 +1635,6 @@ write_blob_list(struct list_head *blob_list, if (ret) goto out_destroy_context; - DEBUG("Ending solid resource: %lu %lu %lu.", - reshdr.offset_in_wim, - reshdr.size_in_wim, - reshdr.uncompressed_size); - offset_in_res = 0; list_for_each_entry(blob, &ctx.blobs_in_solid_resource, write_blobs_list) { blob->out_reshdr.size_in_wim = blob->size; @@ -1692,27 +1650,20 @@ write_blob_list(struct list_head *blob_list, wimlib_assert(offset_in_res == reshdr.uncompressed_size); } -out_write_raw_copy_resources: - /* Copy any compressed resources for which the raw data can be reused - * without decompression. */ - ret = write_raw_copy_resources(&raw_copy_blobs, ctx.out_fd, - &ctx.progress_data); - out_destroy_context: FREE(ctx.chunk_csizes); if (ctx.compressor) ctx.compressor->destroy(ctx.compressor); - DEBUG("Done (ret=%d)", ret); return ret; } static int -wim_write_blob_list(WIMStruct *wim, - struct list_head *blob_list, - int write_flags, - unsigned num_threads, - struct filter_context *filter_ctx) +write_file_data_blobs(WIMStruct *wim, + struct list_head *blob_list, + int write_flags, + unsigned num_threads, + struct filter_context *filter_ctx) { int out_ctype; u32 out_chunk_size; @@ -1720,16 +1671,6 @@ wim_write_blob_list(WIMStruct *wim, write_resource_flags = write_flags_to_resource_flags(write_flags); - /* wimlib v1.7.0: create a solid WIM file by default if the WIM version - * has been set to WIM_VERSION_SOLID and at least one blob in the WIM's - * blob table is located in a solid resource (may be the same WIM, or a - * different one in the case of export). */ - if (wim->hdr.wim_version == WIM_VERSION_SOLID && - wim_has_solid_resources(wim)) - { - write_resource_flags |= WRITE_RESOURCE_FLAG_SOLID; - } - if (write_resource_flags & WRITE_RESOURCE_FLAG_SOLID) { out_chunk_size = wim->out_solid_chunk_size; out_ctype = wim->out_solid_compression_type; @@ -1788,6 +1729,13 @@ write_wim_resource_from_buffer(const void *buf, int ret; struct blob_descriptor blob; + if (unlikely(buf_size == 0)) { + zero_reshdr(out_reshdr); + if (hash_ret) + copy_hash(hash_ret, zero_hash); + return 0; + } + blob_set_is_located_in_attached_buffer(&blob, (void *)buf, buf_size); sha1_buffer(buf, buf_size, blob.hash); blob.unhashed = 0; @@ -1887,12 +1835,16 @@ inode_find_blobs_to_reference(const struct wim_inode *inode, for (unsigned i = 0; i < inode->i_num_streams; i++) { struct blob_descriptor *blob; + const u8 *hash; blob = stream_blob(&inode->i_streams[i], table); - if (blob) + if (blob) { reference_blob_for_write(blob, blob_list, inode->i_nlink); - else if (!is_zero_hash(stream_hash(&inode->i_streams[i]))) - return WIMLIB_ERR_RESOURCE_NOT_FOUND; + } else { + hash = stream_hash(&inode->i_streams[i]); + if (!is_zero_hash(hash)) + return blob_not_found_error(inode, hash); + } } return 0; } @@ -2058,9 +2010,9 @@ filter_blob_list_for_write(struct list_head *blob_list, * STREAMS_OK: For writes of all images, assume that all blobs in the blob * table of @wim and the per-image lists of unhashed blobs should be taken * as-is, and image metadata should not be searched for references. This - * does not exclude filtering with OVERWRITE and SKIP_EXTERNAL_WIMS, below. + * does not exclude filtering with APPEND and SKIP_EXTERNAL_WIMS, below. * - * OVERWRITE: Blobs already present in @wim shall not be returned in + * APPEND: Blobs already present in @wim shall not be returned in * @blob_list_ret. * * SKIP_EXTERNAL_WIMS: Blobs already present in a WIM file, but not @wim, @@ -2080,9 +2032,9 @@ filter_blob_list_for_write(struct list_head *blob_list, * the blobs in @blob_list_ret. * * This list will be a proper superset of @blob_list_ret if and only if - * WIMLIB_WRITE_FLAG_OVERWRITE was specified in @write_flags and some of - * the blobs that would otherwise need to be written were already located - * in the WIM file. + * WIMLIB_WRITE_FLAG_APPEND was specified in @write_flags and some of the + * blobs that would otherwise need to be written were already located in + * the WIM file. * * All blobs in this list will have @out_refcnt set to the number of * references to the blob in the output WIM. If @@ -2142,10 +2094,10 @@ prepare_blob_list_for_write(WIMStruct *wim, int image, } static int -write_file_blobs(WIMStruct *wim, int image, int write_flags, - unsigned num_threads, - struct list_head *blob_list_override, - struct list_head *blob_table_list_ret) +write_file_data(WIMStruct *wim, int image, int write_flags, + unsigned num_threads, + struct list_head *blob_list_override, + struct list_head *blob_table_list_ret) { int ret; struct list_head _blob_list; @@ -2180,11 +2132,11 @@ write_file_blobs(WIMStruct *wim, int image, int write_flags, } } - return wim_write_blob_list(wim, - blob_list, - write_flags, - num_threads, - filter_ctx); + return write_file_data_blobs(wim, + blob_list, + write_flags, + num_threads, + filter_ctx); } static int @@ -2195,18 +2147,13 @@ write_metadata_resources(WIMStruct *wim, int image, int write_flags) int end_image; int write_resource_flags; - if (write_flags & WIMLIB_WRITE_FLAG_NO_METADATA) { - DEBUG("Not writing any metadata resources."); + if (write_flags & WIMLIB_WRITE_FLAG_NO_METADATA) return 0; - } write_resource_flags = write_flags_to_resource_flags(write_flags); write_resource_flags &= ~WRITE_RESOURCE_FLAG_SOLID; - DEBUG("Writing metadata resources (offset=%"PRIu64")", - wim->out_fd.offset); - ret = call_progress(wim->progfunc, WIMLIB_PROGRESS_MSG_WRITE_METADATA_BEGIN, NULL, wim->progctx); @@ -2225,22 +2172,28 @@ write_metadata_resources(WIMStruct *wim, int image, int write_flags) struct wim_image_metadata *imd; imd = wim->image_metadata[i - 1]; - /* Build a new metadata resource only if image was modified from - * the original (or was newly added). Otherwise just copy the - * existing one. */ - if (imd->modified) { - DEBUG("Image %u was modified; building and writing new " - "metadata resource", i); + if (is_image_dirty(imd)) { + /* The image was modified from the original, or was + * newly added, so we have to build and write a new + * metadata resource. */ ret = write_metadata_resource(wim, i, write_resource_flags); - } else if (write_flags & WIMLIB_WRITE_FLAG_OVERWRITE) { - DEBUG("Image %u was not modified; re-using existing " - "metadata resource.", i); - blob_set_out_reshdr_for_reuse(imd->metadata_blob); + } else if (is_image_unchanged_from_wim(imd, wim) && + (write_flags & (WIMLIB_WRITE_FLAG_UNSAFE_COMPACT | + WIMLIB_WRITE_FLAG_APPEND))) + { + /* The metadata resource is already in the WIM file. + * For appends, we don't need to write it at all. For + * compactions, we re-write existing metadata resources + * along with the existing file resources, not here. */ + if (write_flags & WIMLIB_WRITE_FLAG_APPEND) + blob_set_out_reshdr_for_reuse(imd->metadata_blob); ret = 0; } else { - DEBUG("Image %u was not modified; copying existing " - "metadata resource.", i); + /* The metadata resource is in a WIM file other than the + * one being written to. We need to rewrite it, + * possibly compressed differently; but rebuilding the + * metadata itself isn't necessary. */ ret = write_wim_resource(imd->metadata_blob, &wim->out_fd, wim->out_compression_type, @@ -2259,10 +2212,7 @@ write_metadata_resources(WIMStruct *wim, int image, int write_flags) static int open_wim_writable(WIMStruct *wim, const tchar *path, int open_flags) { - int raw_fd; - DEBUG("Opening \"%"TS"\" for writing.", path); - - raw_fd = topen(path, open_flags | O_BINARY, 0644); + int raw_fd = topen(path, open_flags | O_BINARY, 0644); if (raw_fd < 0) { ERROR_WITH_ERRNO("Failed to open \"%"TS"\" for writing", path); return WIMLIB_ERR_OPEN; @@ -2276,12 +2226,10 @@ close_wim_writable(WIMStruct *wim, int write_flags) { int ret = 0; - if (!(write_flags & WIMLIB_WRITE_FLAG_FILE_DESCRIPTOR)) { - DEBUG("Closing WIM file."); + if (!(write_flags & WIMLIB_WRITE_FLAG_FILE_DESCRIPTOR)) if (filedes_valid(&wim->out_fd)) if (filedes_close(&wim->out_fd)) ret = WIMLIB_ERR_WRITE; - } filedes_invalidate(&wim->out_fd); return ret; } @@ -2312,13 +2260,12 @@ cmp_blobs_by_out_rdesc(const void *p1, const void *p2) static int write_blob_table(WIMStruct *wim, int image, int write_flags, - struct wim_reshdr *out_reshdr, struct list_head *blob_table_list) { int ret; /* Set output resource metadata for blobs already present in WIM. */ - if (write_flags & WIMLIB_WRITE_FLAG_OVERWRITE) { + if (write_flags & WIMLIB_WRITE_FLAG_APPEND) { struct blob_descriptor *blob; list_for_each_entry(blob, blob_table_list, blob_table_list) { if (blob->blob_location == BLOB_IN_WIM && @@ -2363,68 +2310,28 @@ write_blob_table(WIMStruct *wim, int image, int write_flags, return write_blob_table_from_blob_list(blob_table_list, &wim->out_fd, - wim->hdr.part_number, - out_reshdr, + wim->out_hdr.part_number, + &wim->out_hdr.blob_table_reshdr, write_flags_to_resource_flags(write_flags)); } /* - * finish_write(): - * * Finish writing a WIM file: write the blob table, xml data, and integrity - * table, then overwrite the WIM header. By default, closes the WIM file - * descriptor (@wim->out_fd) if successful. - * - * write_flags is a bitwise OR of the following: - * - * (public) WIMLIB_WRITE_FLAG_CHECK_INTEGRITY: - * Include an integrity table. - * - * (public) WIMLIB_WRITE_FLAG_FSYNC: - * fsync() the output file before closing it. - * - * (public) WIMLIB_WRITE_FLAG_PIPABLE: - * Writing a pipable WIM, possibly to a pipe; include pipable WIM - * blob headers before the blob table and XML data, and also write - * the WIM header at the end instead of seeking to the beginning. - * Can't be combined with WIMLIB_WRITE_FLAG_CHECK_INTEGRITY. - * - * (private) WIMLIB_WRITE_FLAG_NO_BLOB_TABLE: - * Don't write the blob table. - * - * (private) WIMLIB_WRITE_FLAG_CHECKPOINT_AFTER_XML: - * After writing the XML data but before writing the integrity - * table, write a temporary WIM header and flush the file - * descriptor so that the WIM is less likely to become corrupted - * upon abrupt program termination. - * (private) WIMLIB_WRITE_FLAG_HEADER_AT_END: - * Instead of overwriting the WIM header at the beginning of the - * file, simply append it to the end of the file. (Used when - * writing to pipe.) - * (private) WIMLIB_WRITE_FLAG_FILE_DESCRIPTOR: - * Do not close the file descriptor @wim->out_fd on either success - * on failure. - * (private) WIMLIB_WRITE_FLAG_USE_EXISTING_TOTALBYTES: - * Use the existing stored in the in-memory XML - * information, rather than setting it to the offset of the XML - * data being written. - * (private) WIMLIB_WRITE_FLAG_OVERWRITE - * The existing WIM file is being updated in-place. The entries - * from its integrity table may be re-used. + * table, then overwrite the WIM header. + * + * The output file descriptor is closed on success, except when writing to a + * user-specified file descriptor (WIMLIB_WRITE_FLAG_FILE_DESCRIPTOR set). */ static int finish_write(WIMStruct *wim, int image, int write_flags, struct list_head *blob_table_list) { - int ret; - off_t hdr_offset; int write_resource_flags; off_t old_blob_table_end = 0; + struct integrity_table *old_integrity_table = NULL; off_t new_blob_table_end; u64 xml_totalbytes; - struct integrity_table *old_integrity_table = NULL; - - DEBUG("image=%d, write_flags=%08x", image, write_flags); + int ret; write_resource_flags = write_flags_to_resource_flags(write_flags); @@ -2432,28 +2339,27 @@ finish_write(WIMStruct *wim, int image, int write_flags, * metadata resource labeled as the "boot metadata". This entry should * be zeroed out if there is no bootable image (boot_idx 0). Otherwise, * it should be a copy of the resource entry for the image that is - * marked as bootable. This is not well documented... */ - if (wim->hdr.boot_idx == 0) { - zero_reshdr(&wim->hdr.boot_metadata_reshdr); + * marked as bootable. */ + if (wim->out_hdr.boot_idx == 0) { + zero_reshdr(&wim->out_hdr.boot_metadata_reshdr); } else { - copy_reshdr(&wim->hdr.boot_metadata_reshdr, + copy_reshdr(&wim->out_hdr.boot_metadata_reshdr, &wim->image_metadata[ - wim->hdr.boot_idx - 1]->metadata_blob->out_reshdr); + wim->out_hdr.boot_idx - 1]->metadata_blob->out_reshdr); } - /* If overwriting the WIM file containing an integrity table in-place, - * we'd like to re-use the information in the old integrity table - * instead of recalculating it. But we might overwrite the old - * integrity table when we expand the XML data. Read it into memory - * just in case. */ - if ((write_flags & (WIMLIB_WRITE_FLAG_OVERWRITE | + /* If appending to a WIM file containing an integrity table, we'd like + * to re-use the information in the old integrity table instead of + * recalculating it. But we might overwrite the old integrity table + * when we expand the XML data. Read it into memory just in case. */ + if ((write_flags & (WIMLIB_WRITE_FLAG_APPEND | WIMLIB_WRITE_FLAG_CHECK_INTEGRITY)) == - (WIMLIB_WRITE_FLAG_OVERWRITE | + (WIMLIB_WRITE_FLAG_APPEND | WIMLIB_WRITE_FLAG_CHECK_INTEGRITY) && wim_has_integrity_table(wim)) { old_blob_table_end = wim->hdr.blob_table_reshdr.offset_in_wim + - wim->hdr.blob_table_reshdr.size_in_wim; + wim->hdr.blob_table_reshdr.size_in_wim; (void)read_integrity_table(wim, old_blob_table_end - WIM_HEADER_DISK_SIZE, &old_integrity_table); @@ -2462,15 +2368,12 @@ finish_write(WIMStruct *wim, int image, int write_flags, * ignoring of the return value. */ } - /* Write blob table. */ - if (!(write_flags & WIMLIB_WRITE_FLAG_NO_BLOB_TABLE)) { + /* Write blob table if needed. */ + if (!(write_flags & WIMLIB_WRITE_FLAG_NO_NEW_BLOBS)) { ret = write_blob_table(wim, image, write_flags, - &wim->hdr.blob_table_reshdr, blob_table_list); - if (ret) { - free_integrity_table(old_integrity_table); - return ret; - } + if (ret) + goto out; } /* Write XML data. */ @@ -2478,55 +2381,66 @@ finish_write(WIMStruct *wim, int image, int write_flags, if (write_flags & WIMLIB_WRITE_FLAG_USE_EXISTING_TOTALBYTES) xml_totalbytes = WIM_TOTALBYTES_USE_EXISTING; ret = write_wim_xml_data(wim, image, xml_totalbytes, - &wim->hdr.xml_data_reshdr, + &wim->out_hdr.xml_data_reshdr, write_resource_flags); - if (ret) { - free_integrity_table(old_integrity_table); - return ret; - } + if (ret) + goto out; - /* Write integrity table (optional). */ - if (write_flags & WIMLIB_WRITE_FLAG_CHECK_INTEGRITY) { - if (write_flags & WIMLIB_WRITE_FLAG_CHECKPOINT_AFTER_XML) { + /* Write integrity table if needed. */ + if ((write_flags & WIMLIB_WRITE_FLAG_CHECK_INTEGRITY) && + wim->out_hdr.blob_table_reshdr.offset_in_wim != 0) + { + if (write_flags & WIMLIB_WRITE_FLAG_NO_NEW_BLOBS) { + /* The XML data we wrote may have overwritten part of + * the old integrity table, so while calculating the new + * integrity table we should temporarily update the WIM + * header to remove the integrity table reference. */ struct wim_header checkpoint_hdr; - memcpy(&checkpoint_hdr, &wim->hdr, sizeof(struct wim_header)); + memcpy(&checkpoint_hdr, &wim->out_hdr, sizeof(struct wim_header)); zero_reshdr(&checkpoint_hdr.integrity_table_reshdr); checkpoint_hdr.flags |= WIM_HDR_FLAG_WRITE_IN_PROGRESS; - ret = write_wim_header_at_offset(&checkpoint_hdr, - &wim->out_fd, 0); - if (ret) { - free_integrity_table(old_integrity_table); - return ret; - } + ret = write_wim_header(&checkpoint_hdr, &wim->out_fd, 0); + if (ret) + goto out; } - new_blob_table_end = wim->hdr.blob_table_reshdr.offset_in_wim + - wim->hdr.blob_table_reshdr.size_in_wim; + new_blob_table_end = wim->out_hdr.blob_table_reshdr.offset_in_wim + + wim->out_hdr.blob_table_reshdr.size_in_wim; ret = write_integrity_table(wim, new_blob_table_end, old_blob_table_end, old_integrity_table); - free_integrity_table(old_integrity_table); if (ret) - return ret; + goto out; } else { /* No integrity table. */ - zero_reshdr(&wim->hdr.integrity_table_reshdr); + zero_reshdr(&wim->out_hdr.integrity_table_reshdr); } /* Now that all information in the WIM header has been determined, the * preliminary header written earlier can be overwritten, the header of * the existing WIM file can be overwritten, or the final header can be * written to the end of the pipable WIM. */ - wim->hdr.flags &= ~WIM_HDR_FLAG_WRITE_IN_PROGRESS; - hdr_offset = 0; - if (write_flags & WIMLIB_WRITE_FLAG_HEADER_AT_END) - hdr_offset = wim->out_fd.offset; - DEBUG("Writing new header @ %"PRIu64".", hdr_offset); - ret = write_wim_header_at_offset(&wim->hdr, &wim->out_fd, hdr_offset); + wim->out_hdr.flags &= ~WIM_HDR_FLAG_WRITE_IN_PROGRESS; + if (write_flags & WIMLIB_WRITE_FLAG_PIPABLE) + ret = write_wim_header(&wim->out_hdr, &wim->out_fd, wim->out_fd.offset); + else + ret = write_wim_header(&wim->out_hdr, &wim->out_fd, 0); if (ret) - return ret; + goto out; + + ret = WIMLIB_ERR_WRITE; + if (unlikely(write_flags & WIMLIB_WRITE_FLAG_UNSAFE_COMPACT)) { + /* Truncate any data the compaction freed up. */ + if (ftruncate(wim->out_fd.fd, wim->out_fd.offset) && + errno != EINVAL) /* allow compaction on untruncatable files, + e.g. block devices */ + { + ERROR_WITH_ERRNO("Failed to truncate the output WIM file"); + goto out; + } + } /* Possibly sync file data to disk before closing. On POSIX systems, it * is necessary to do this before using rename() to overwrite an @@ -2534,20 +2448,24 @@ finish_write(WIMStruct *wim, int image, int write_flags, * the system is abruptly terminated when the metadata for the rename * operation has been written to disk, but the new file data has not. */ + ret = WIMLIB_ERR_WRITE; if (write_flags & WIMLIB_WRITE_FLAG_FSYNC) { - DEBUG("Syncing WIM file."); if (fsync(wim->out_fd.fd)) { ERROR_WITH_ERRNO("Error syncing data to WIM file"); - return WIMLIB_ERR_WRITE; + goto out; } } + ret = WIMLIB_ERR_WRITE; if (close_wim_writable(wim, write_flags)) { ERROR_WITH_ERRNO("Failed to close the output WIM file"); - return WIMLIB_ERR_WRITE; + goto out; } - return 0; + ret = 0; +out: + free_integrity_table(old_integrity_table); + return ret; } #if defined(HAVE_SYS_FILE_H) && defined(HAVE_FLOCK) @@ -2606,6 +2524,9 @@ unlock_wim_for_append(WIMStruct *wim) * reading the WIM from a pipe. This copy of the XML data is ignored if the * WIM is read from a seekable file (not a pipe). * + * - Solid resources are not allowed. Each blob is always stored in its own + * resource. + * * - The format of resources, or blobs, has been modified to allow them to be * used before the "blob table" has been read. Each blob is prefixed with a * `struct pwm_blob_hdr' that is basically an abbreviated form of `struct @@ -2667,13 +2588,13 @@ write_pipable_wim(WIMStruct *wim, int image, int write_flags, WARNING("Creating a pipable WIM, which will " "be incompatible\n" - " with Microsoft's software (wimgapi/imagex/Dism)."); + " with Microsoft's software (WIMGAPI/ImageX/DISM)."); /* At this point, the header at the beginning of the file has already * been written. */ /* For efficiency, when wimlib adds an image to the WIM with - * wimlib_add_image(), the SHA-1 message digests of files is not + * wimlib_add_image(), the SHA-1 message digests of files are not * calculated; instead, they are calculated while the files are being * written. However, this does not work when writing a pipable WIM, * since when writing a blob to a pipable WIM, its SHA-1 message digest @@ -2696,16 +2617,44 @@ write_pipable_wim(WIMStruct *wim, int image, int write_flags, if (ret) return ret; - /* Write blobs needed for the image(s) being included in the output WIM, - * or blobs needed for the split WIM part. */ - return write_file_blobs(wim, image, write_flags, - num_threads, blob_list_override, - blob_table_list_ret); + /* Write file data needed for the image(s) being included in the output + * WIM, or file data needed for the split WIM part. */ + return write_file_data(wim, image, write_flags, + num_threads, blob_list_override, + blob_table_list_ret); /* The blob table, XML data, and header at end are handled by * finish_write(). */ } +static bool +should_default_to_solid_compression(WIMStruct *wim, int write_flags) +{ + return wim->out_hdr.wim_version == WIM_VERSION_SOLID && + !(write_flags & (WIMLIB_WRITE_FLAG_SOLID | + WIMLIB_WRITE_FLAG_PIPABLE)) && + wim_has_solid_resources(wim); +} + +/* Update the images' filecount/bytecount stats (in the XML info) to take into + * account any recent modifications. */ +static int +update_image_stats(WIMStruct *wim) +{ + if (!wim_has_metadata(wim)) + return 0; + for (int i = 0; i < wim->hdr.image_count; i++) { + struct wim_image_metadata *imd = wim->image_metadata[i]; + if (imd->stats_outdated) { + int ret = xml_update_image_info(wim, i + 1); + if (ret) + return ret; + imd->stats_outdated = false; + } + } + return 0; +} + /* Write a standalone WIM or split WIM (SWM) part to a new file or to a file * descriptor. */ int @@ -2720,81 +2669,8 @@ write_wim_part(WIMStruct *wim, const u8 *guid) { int ret; - struct wim_header hdr_save; struct list_head blob_table_list; - if (total_parts == 1) - DEBUG("Writing standalone WIM."); - else - DEBUG("Writing split WIM part %u/%u", part_number, total_parts); - if (image == WIMLIB_ALL_IMAGES) - DEBUG("Including all images."); - else - DEBUG("Including image %d only.", image); - if (write_flags & WIMLIB_WRITE_FLAG_FILE_DESCRIPTOR) - DEBUG("File descriptor: %d", *(const int*)path_or_fd); - else - DEBUG("Path: \"%"TS"\"", (const tchar*)path_or_fd); - DEBUG("Write flags: 0x%08x", write_flags); - - if (write_flags & WIMLIB_WRITE_FLAG_CHECK_INTEGRITY) - DEBUG("\tCHECK_INTEGRITY"); - - if (write_flags & WIMLIB_WRITE_FLAG_NO_CHECK_INTEGRITY) - DEBUG("\tNO_CHECK_INTEGRITY"); - - if (write_flags & WIMLIB_WRITE_FLAG_PIPABLE) - DEBUG("\tPIPABLE"); - - if (write_flags & WIMLIB_WRITE_FLAG_NOT_PIPABLE) - DEBUG("\tNOT_PIPABLE"); - - if (write_flags & WIMLIB_WRITE_FLAG_RECOMPRESS) - DEBUG("\tRECOMPRESS"); - - if (write_flags & WIMLIB_WRITE_FLAG_FSYNC) - DEBUG("\tFSYNC"); - - if (write_flags & WIMLIB_WRITE_FLAG_REBUILD) - DEBUG("\tREBUILD"); - - if (write_flags & WIMLIB_WRITE_FLAG_SOFT_DELETE) - DEBUG("\tSOFT_DELETE"); - - if (write_flags & WIMLIB_WRITE_FLAG_IGNORE_READONLY_FLAG) - DEBUG("\tIGNORE_READONLY_FLAG"); - - if (write_flags & WIMLIB_WRITE_FLAG_SKIP_EXTERNAL_WIMS) - DEBUG("\tSKIP_EXTERNAL_WIMS"); - - if (write_flags & WIMLIB_WRITE_FLAG_STREAMS_OK) - DEBUG("\tSTREAMS_OK"); - - if (write_flags & WIMLIB_WRITE_FLAG_RETAIN_GUID) - DEBUG("\tRETAIN_GUID"); - - if (write_flags & WIMLIB_WRITE_FLAG_SOLID) - DEBUG("\tSOLID"); - - if (write_flags & WIMLIB_WRITE_FLAG_FILE_DESCRIPTOR) - DEBUG("\tFILE_DESCRIPTOR"); - - if (write_flags & WIMLIB_WRITE_FLAG_NO_METADATA) - DEBUG("\tNO_METADATA"); - - if (write_flags & WIMLIB_WRITE_FLAG_USE_EXISTING_TOTALBYTES) - DEBUG("\tUSE_EXISTING_TOTALBYTES"); - - if (num_threads == 0) - DEBUG("Number of threads: autodetect"); - else - DEBUG("Number of threads: %u", num_threads); - DEBUG("Progress function: %s", (wim->progfunc ? "yes" : "no")); - DEBUG("Blob list: %s", (blob_list_override ? "specified" : "autodetect")); - DEBUG("GUID: %s", (write_flags & - WIMLIB_WRITE_FLAG_RETAIN_GUID) ? "retain" - : guid ? "explicit" : "generate new"); - /* Internally, this is always called with a valid part number and total * parts. */ wimlib_assert(total_parts >= 1); @@ -2825,164 +2701,172 @@ write_wim_part(WIMStruct *wim, WIMLIB_WRITE_FLAG_NOT_PIPABLE)) return WIMLIB_ERR_INVALID_PARAM; - /* Save previous header, then start initializing the new one. */ - memcpy(&hdr_save, &wim->hdr, sizeof(struct wim_header)); - - /* Set default integrity, pipable, and solid flags. */ - if (!(write_flags & (WIMLIB_WRITE_FLAG_PIPABLE | - WIMLIB_WRITE_FLAG_NOT_PIPABLE))) - if (wim_is_pipable(wim)) { - DEBUG("WIM is pipable; default to PIPABLE."); - write_flags |= WIMLIB_WRITE_FLAG_PIPABLE; - } + /* Only wimlib_overwrite() accepts UNSAFE_COMPACT. */ + if (write_flags & WIMLIB_WRITE_FLAG_UNSAFE_COMPACT) + return WIMLIB_ERR_INVALID_PARAM; + /* Include an integrity table by default if no preference was given and + * the WIM already had an integrity table. */ if (!(write_flags & (WIMLIB_WRITE_FLAG_CHECK_INTEGRITY | - WIMLIB_WRITE_FLAG_NO_CHECK_INTEGRITY))) - if (wim_has_integrity_table(wim)) { - DEBUG("Integrity table present; default to CHECK_INTEGRITY."); + WIMLIB_WRITE_FLAG_NO_CHECK_INTEGRITY))) { + if (wim_has_integrity_table(wim)) write_flags |= WIMLIB_WRITE_FLAG_CHECK_INTEGRITY; - } + } + + /* Write a pipable WIM by default if no preference was given and the WIM + * was already pipable. */ + if (!(write_flags & (WIMLIB_WRITE_FLAG_PIPABLE | + WIMLIB_WRITE_FLAG_NOT_PIPABLE))) { + if (wim_is_pipable(wim)) + write_flags |= WIMLIB_WRITE_FLAG_PIPABLE; + } if ((write_flags & (WIMLIB_WRITE_FLAG_PIPABLE | WIMLIB_WRITE_FLAG_SOLID)) == (WIMLIB_WRITE_FLAG_PIPABLE | WIMLIB_WRITE_FLAG_SOLID)) { - ERROR("Cannot specify both PIPABLE and SOLID!"); + ERROR("Solid compression is unsupported in pipable WIMs"); return WIMLIB_ERR_INVALID_PARAM; } - /* Set appropriate magic number. */ + /* Start initializing the new file header. */ + memset(&wim->out_hdr, 0, sizeof(wim->out_hdr)); + + /* Set the magic number. */ if (write_flags & WIMLIB_WRITE_FLAG_PIPABLE) - wim->hdr.magic = PWM_MAGIC; + wim->out_hdr.magic = PWM_MAGIC; else - wim->hdr.magic = WIM_MAGIC; + wim->out_hdr.magic = WIM_MAGIC; - /* Set appropriate version number. */ + /* Set the version number. */ if ((write_flags & WIMLIB_WRITE_FLAG_SOLID) || wim->out_compression_type == WIMLIB_COMPRESSION_TYPE_LZMS) - wim->hdr.wim_version = WIM_VERSION_SOLID; + wim->out_hdr.wim_version = WIM_VERSION_SOLID; else - wim->hdr.wim_version = WIM_VERSION_DEFAULT; + wim->out_hdr.wim_version = WIM_VERSION_DEFAULT; - /* Clear header flags that will be set automatically. */ - wim->hdr.flags &= ~(WIM_HDR_FLAG_METADATA_ONLY | - WIM_HDR_FLAG_RESOURCE_ONLY | - WIM_HDR_FLAG_SPANNED | - WIM_HDR_FLAG_WRITE_IN_PROGRESS); + /* Default to solid compression if it is valid in the chosen WIM file + * format and the WIMStruct references any solid resources. This is + * useful when exporting an image from a solid WIM. */ + if (should_default_to_solid_compression(wim, write_flags)) + write_flags |= WIMLIB_WRITE_FLAG_SOLID; - /* Set SPANNED header flag if writing part of a split WIM. */ + /* Set the header flags. */ + wim->out_hdr.flags = (wim->hdr.flags & (WIM_HDR_FLAG_RP_FIX | + WIM_HDR_FLAG_READONLY)); if (total_parts != 1) - wim->hdr.flags |= WIM_HDR_FLAG_SPANNED; - - /* Set part number and total parts of split WIM. This will be 1 and 1 - * if the WIM is standalone. */ - wim->hdr.part_number = part_number; - wim->hdr.total_parts = total_parts; - - /* Set compression type if different. */ - if (wim->compression_type != wim->out_compression_type) { - ret = set_wim_hdr_cflags(wim->out_compression_type, &wim->hdr); - wimlib_assert(ret == 0); + wim->out_hdr.flags |= WIM_HDR_FLAG_SPANNED; + if (wim->out_compression_type != WIMLIB_COMPRESSION_TYPE_NONE) { + wim->out_hdr.flags |= WIM_HDR_FLAG_COMPRESSION; + switch (wim->out_compression_type) { + case WIMLIB_COMPRESSION_TYPE_XPRESS: + wim->out_hdr.flags |= WIM_HDR_FLAG_COMPRESS_XPRESS; + break; + case WIMLIB_COMPRESSION_TYPE_LZX: + wim->out_hdr.flags |= WIM_HDR_FLAG_COMPRESS_LZX; + break; + case WIMLIB_COMPRESSION_TYPE_LZMS: + wim->out_hdr.flags |= WIM_HDR_FLAG_COMPRESS_LZMS; + break; + } } - /* Set chunk size if different. */ - wim->hdr.chunk_size = wim->out_chunk_size; + /* Set the chunk size. */ + wim->out_hdr.chunk_size = wim->out_chunk_size; - /* Set GUID. */ - if (!(write_flags & WIMLIB_WRITE_FLAG_RETAIN_GUID)) { - if (guid) - memcpy(wim->hdr.guid, guid, WIMLIB_GUID_LEN); - else - randomize_byte_array(wim->hdr.guid, WIMLIB_GUID_LEN); - } + /* Set the GUID. */ + if (write_flags & WIMLIB_WRITE_FLAG_RETAIN_GUID) + guid = wim->hdr.guid; + if (guid) + copy_guid(wim->out_hdr.guid, guid); + else + generate_guid(wim->out_hdr.guid); - /* Clear references to resources that have not been written yet. */ - zero_reshdr(&wim->hdr.blob_table_reshdr); - zero_reshdr(&wim->hdr.xml_data_reshdr); - zero_reshdr(&wim->hdr.boot_metadata_reshdr); - zero_reshdr(&wim->hdr.integrity_table_reshdr); + /* Set the part number and total parts. */ + wim->out_hdr.part_number = part_number; + wim->out_hdr.total_parts = total_parts; - /* Set image count and boot index correctly for single image writes. */ - if (image != WIMLIB_ALL_IMAGES) { - wim->hdr.image_count = 1; - if (wim->hdr.boot_idx == image) - wim->hdr.boot_idx = 1; - else - wim->hdr.boot_idx = 0; + /* Set the image count. */ + if (image == WIMLIB_ALL_IMAGES) + wim->out_hdr.image_count = wim->hdr.image_count; + else + wim->out_hdr.image_count = 1; + + /* Set the boot index. */ + wim->out_hdr.boot_idx = 0; + if (total_parts == 1) { + if (image == WIMLIB_ALL_IMAGES) + wim->out_hdr.boot_idx = wim->hdr.boot_idx; + else if (image == wim->hdr.boot_idx) + wim->out_hdr.boot_idx = 1; } - /* Split WIMs can't be bootable. */ - if (total_parts != 1) - wim->hdr.boot_idx = 0; + /* Update image stats if needed. */ + ret = update_image_stats(wim); + if (ret) + return ret; - /* Initialize output file descriptor. */ + /* Set up the output file descriptor. */ if (write_flags & WIMLIB_WRITE_FLAG_FILE_DESCRIPTOR) { - /* File descriptor was explicitly provided. Return error if - * file descriptor is not seekable, unless writing a pipable WIM - * was requested. */ - wim->out_fd.fd = *(const int*)path_or_fd; - wim->out_fd.offset = 0; + /* File descriptor was explicitly provided. */ + filedes_init(&wim->out_fd, *(const int *)path_or_fd); if (!filedes_is_seekable(&wim->out_fd)) { + /* The file descriptor is a pipe. */ ret = WIMLIB_ERR_INVALID_PARAM; if (!(write_flags & WIMLIB_WRITE_FLAG_PIPABLE)) - goto out_restore_hdr; + goto out_cleanup; if (write_flags & WIMLIB_WRITE_FLAG_CHECK_INTEGRITY) { ERROR("Can't include integrity check when " "writing pipable WIM to pipe!"); - goto out_restore_hdr; + goto out_cleanup; } } - } else { /* Filename of WIM to write was provided; open file descriptor * to it. */ ret = open_wim_writable(wim, (const tchar*)path_or_fd, O_TRUNC | O_CREAT | O_RDWR); if (ret) - goto out_restore_hdr; + goto out_cleanup; } /* Write initial header. This is merely a "dummy" header since it - * doesn't have all the information yet, so it will be overwritten later - * (unless writing a pipable WIM). */ + * doesn't have resource entries filled in yet, so it will be + * overwritten later (unless writing a pipable WIM). */ if (!(write_flags & WIMLIB_WRITE_FLAG_PIPABLE)) - wim->hdr.flags |= WIM_HDR_FLAG_WRITE_IN_PROGRESS; - ret = write_wim_header(&wim->hdr, &wim->out_fd); - wim->hdr.flags &= ~WIM_HDR_FLAG_WRITE_IN_PROGRESS; + wim->out_hdr.flags |= WIM_HDR_FLAG_WRITE_IN_PROGRESS; + ret = write_wim_header(&wim->out_hdr, &wim->out_fd, wim->out_fd.offset); + wim->out_hdr.flags &= ~WIM_HDR_FLAG_WRITE_IN_PROGRESS; if (ret) - goto out_restore_hdr; + goto out_cleanup; - /* Write file blobs and metadata resources. */ + /* Write file data and metadata resources. */ if (!(write_flags & WIMLIB_WRITE_FLAG_PIPABLE)) { /* Default case: create a normal (non-pipable) WIM. */ - ret = write_file_blobs(wim, image, write_flags, - num_threads, - blob_list_override, - &blob_table_list); + ret = write_file_data(wim, image, write_flags, + num_threads, + blob_list_override, + &blob_table_list); if (ret) - goto out_restore_hdr; + goto out_cleanup; ret = write_metadata_resources(wim, image, write_flags); if (ret) - goto out_restore_hdr; + goto out_cleanup; } else { /* Non-default case: create pipable WIM. */ ret = write_pipable_wim(wim, image, write_flags, num_threads, blob_list_override, &blob_table_list); if (ret) - goto out_restore_hdr; - write_flags |= WIMLIB_WRITE_FLAG_HEADER_AT_END; + goto out_cleanup; } - /* Write blob table, XML data, and (optional) integrity table. */ ret = finish_write(wim, image, write_flags, &blob_table_list); -out_restore_hdr: - memcpy(&wim->hdr, &hdr_save, sizeof(struct wim_header)); +out_cleanup: (void)close_wim_writable(wim, write_flags); - DEBUG("ret=%d", ret); return ret; } @@ -3025,11 +2909,16 @@ wimlib_write_to_fd(WIMStruct *wim, int fd, return write_standalone_wim(wim, &fd, image, write_flags, num_threads); } +/* Have there been any changes to images in the specified WIM, including updates + * as well as deletions and additions of entire images, but excluding changes to + * the XML document? */ static bool -any_images_modified(WIMStruct *wim) +any_images_changed(WIMStruct *wim) { + if (wim->image_deletion_occurred) + return true; for (int i = 0; i < wim->hdr.image_count; i++) - if (wim->image_metadata[i]->modified) + if (!is_image_unchanged_from_wim(wim->image_metadata[i], wim)) return true; return false; } @@ -3048,8 +2937,8 @@ check_resource_offset(struct blob_descriptor *blob, void *_wim) } /* Make sure no file or metadata resources are located after the XML data (or - * integrity table if present)--- otherwise we can't safely overwrite the WIM in - * place and we return WIMLIB_ERR_RESOURCE_ORDER. */ + * integrity table if present)--- otherwise we can't safely append to the WIM + * file and we return WIMLIB_ERR_RESOURCE_ORDER. */ static int check_resource_offsets(WIMStruct *wim, off_t end_offset) { @@ -3069,6 +2958,20 @@ check_resource_offsets(WIMStruct *wim, off_t end_offset) return 0; } +static int +free_blob_if_invalidated(struct blob_descriptor *blob, void *_wim) +{ + const WIMStruct *wim = _wim; + + if (!blob->will_be_in_output_wim && + blob->blob_location == BLOB_IN_WIM && blob->rdesc->wim == wim) + { + blob_table_unlink(wim->blob_table, blob); + free_blob_descriptor(blob); + } + return 0; +} + /* * Overwrite a WIM, possibly appending new resources to it. * @@ -3085,11 +2988,10 @@ check_resource_offsets(WIMStruct *wim, off_t end_offset) * header. This operation is potentially unsafe if the program is abruptly * terminated while the XML data or integrity table are being overwritten, but * before the new header has been written. To partially alleviate this problem, - * a special flag (WIMLIB_WRITE_FLAG_CHECKPOINT_AFTER_XML) is passed to - * finish_write() to cause a temporary WIM header to be written after the XML - * data has been written. This may prevent the WIM from becoming corrupted if - * the program is terminated while the integrity table is being calculated (but - * no guarantees, due to write re-ordering...). + * we write a temporary header after the XML data has been written. This may + * prevent the WIM from becoming corrupted if the program is terminated while + * the integrity table is being calculated (but no guarantees, due to write + * re-ordering...). * * If we are adding new blobs, including new file data as well as any metadata * for any new images, then the blob table needs to be changed, and those blobs @@ -3120,101 +3022,165 @@ check_resource_offsets(WIMStruct *wim, off_t end_offset) * XML data (variable size) * Integrity table (optional) (variable size) * - * This method allows an image to be appended to a large WIM very quickly, and + * This function allows an image to be appended to a large WIM very quickly, and * is crash-safe except in the case of write re-ordering, but the disadvantage * is that a small hole is left in the WIM where the old blob table, xml data, * and integrity table were. (These usually only take up a small amount of * space compared to the blobs, however.) + * + * Finally, this function also supports "compaction" overwrites as an + * alternative to the normal "append" overwrites described above. In a + * compaction, data is written starting immediately from the end of the header. + * All existing resources are written first, in order by file offset. New + * resources are written afterwards, and at the end any extra data is truncated + * from the file. The advantage of this approach is that is that the WIM file + * ends up fully optimized, without any holes remaining. The main disadavantage + * is that this operation is fundamentally unsafe and cannot be interrupted + * without data corruption. Consequently, compactions are only ever done when + * explicitly requested by the library user with the flag + * WIMLIB_WRITE_FLAG_UNSAFE_COMPACT. (Another disadvantage is that a compaction + * can be much slower than an append.) */ static int overwrite_wim_inplace(WIMStruct *wim, int write_flags, unsigned num_threads) { int ret; off_t old_wim_end; - u64 old_blob_table_end, old_xml_begin, old_xml_end; - struct wim_header hdr_save; struct list_head blob_list; struct list_head blob_table_list; struct filter_context filter_ctx; - DEBUG("Overwriting `%"TS"' in-place", wim->filename); - - /* Save original header so it can be restored in case of error */ - memcpy(&hdr_save, &wim->hdr, sizeof(struct wim_header)); - - /* Set default integrity flag. */ + /* Include an integrity table by default if no preference was given and + * the WIM already had an integrity table. */ if (!(write_flags & (WIMLIB_WRITE_FLAG_CHECK_INTEGRITY | WIMLIB_WRITE_FLAG_NO_CHECK_INTEGRITY))) if (wim_has_integrity_table(wim)) write_flags |= WIMLIB_WRITE_FLAG_CHECK_INTEGRITY; - /* Set WIM version if writing solid resources. */ + /* Start preparing the updated file header. */ + memcpy(&wim->out_hdr, &wim->hdr, sizeof(wim->out_hdr)); + + /* If using solid compression, the version number must be set to + * WIM_VERSION_SOLID. */ if (write_flags & WIMLIB_WRITE_FLAG_SOLID) - wim->hdr.wim_version = WIM_VERSION_SOLID; - - /* Set additional flags for overwrite. */ - write_flags |= WIMLIB_WRITE_FLAG_OVERWRITE | - WIMLIB_WRITE_FLAG_STREAMS_OK; - - /* Make sure there is no data after the XML data, except possibily an - * integrity table. If this were the case, then this data would be - * overwritten. */ - old_xml_begin = wim->hdr.xml_data_reshdr.offset_in_wim; - old_xml_end = old_xml_begin + wim->hdr.xml_data_reshdr.size_in_wim; - old_blob_table_end = wim->hdr.blob_table_reshdr.offset_in_wim + - wim->hdr.blob_table_reshdr.size_in_wim; - if (wim->hdr.integrity_table_reshdr.offset_in_wim != 0 && - wim->hdr.integrity_table_reshdr.offset_in_wim < old_xml_end) { - WARNING("Didn't expect the integrity table to be before the XML data"); - ret = WIMLIB_ERR_RESOURCE_ORDER; - goto out_restore_memory_hdr; - } - - if (old_blob_table_end > old_xml_begin) { - WARNING("Didn't expect the blob table to be after the XML data"); - ret = WIMLIB_ERR_RESOURCE_ORDER; - goto out_restore_memory_hdr; - } - - /* Set @old_wim_end, which indicates the point beyond which we don't - * allow any file and metadata resources to appear without returning - * WIMLIB_ERR_RESOURCE_ORDER (due to the fact that we would otherwise - * overwrite these resources). */ - if (!wim->image_deletion_occurred && !any_images_modified(wim)) { - /* If no images have been modified and no images have been - * deleted, a new blob table does not need to be written. We - * shall write the new XML data and optional integrity table - * immediately after the blob table. Note that this may - * overwrite an existing integrity table. */ - DEBUG("Skipping writing blob table " - "(no images modified or deleted)"); - old_wim_end = old_blob_table_end; - write_flags |= WIMLIB_WRITE_FLAG_NO_BLOB_TABLE | - WIMLIB_WRITE_FLAG_CHECKPOINT_AFTER_XML; - } else if (wim->hdr.integrity_table_reshdr.offset_in_wim != 0) { - /* Old WIM has an integrity table; begin writing new blobs after - * it. */ - old_wim_end = wim->hdr.integrity_table_reshdr.offset_in_wim + - wim->hdr.integrity_table_reshdr.size_in_wim; + wim->out_hdr.wim_version = WIM_VERSION_SOLID; + + /* Default to solid compression if it is valid in the chosen WIM file + * format and the WIMStruct references any solid resources. This is + * useful when updating a solid WIM. */ + if (should_default_to_solid_compression(wim, write_flags)) + write_flags |= WIMLIB_WRITE_FLAG_SOLID; + + if (unlikely(write_flags & WIMLIB_WRITE_FLAG_UNSAFE_COMPACT)) { + + /* In-place compaction */ + + WARNING("The WIM file \"%"TS"\" is being compacted in place.\n" + " Do *not* interrupt the operation, or else " + "the WIM file will be\n" + " corrupted!", wim->filename); + wim->being_compacted = 1; + old_wim_end = WIM_HEADER_DISK_SIZE; + + ret = prepare_blob_list_for_write(wim, WIMLIB_ALL_IMAGES, + write_flags, &blob_list, + &blob_table_list, &filter_ctx); + if (ret) + goto out; + + /* Prevent new files from being deduplicated with existing blobs + * in the WIM that we haven't decided to write. Such blobs will + * be overwritten during the compaction. */ + for_blob_in_table(wim->blob_table, free_blob_if_invalidated, wim); + + if (wim_has_metadata(wim)) { + /* Add existing metadata resources to be compacted along + * with the file resources. */ + for (int i = 0; i < wim->hdr.image_count; i++) { + struct wim_image_metadata *imd = wim->image_metadata[i]; + if (is_image_unchanged_from_wim(imd, wim)) { + fully_reference_blob_for_write(imd->metadata_blob, + &blob_list); + } + } + } } else { - /* No existing integrity table; begin writing new blobs after - * the old XML data. */ - old_wim_end = old_xml_end; - } + u64 old_blob_table_end, old_xml_begin, old_xml_end; + + /* Set additional flags for append. */ + write_flags |= WIMLIB_WRITE_FLAG_APPEND | + WIMLIB_WRITE_FLAG_STREAMS_OK; + + /* Make sure there is no data after the XML data, except + * possibily an integrity table. If this were the case, then + * this data would be overwritten. */ + old_xml_begin = wim->hdr.xml_data_reshdr.offset_in_wim; + old_xml_end = old_xml_begin + wim->hdr.xml_data_reshdr.size_in_wim; + if (wim->hdr.blob_table_reshdr.offset_in_wim == 0) + old_blob_table_end = WIM_HEADER_DISK_SIZE; + else + old_blob_table_end = wim->hdr.blob_table_reshdr.offset_in_wim + + wim->hdr.blob_table_reshdr.size_in_wim; + if (wim_has_integrity_table(wim) && + wim->hdr.integrity_table_reshdr.offset_in_wim < old_xml_end) { + WARNING("Didn't expect the integrity table to be " + "before the XML data"); + ret = WIMLIB_ERR_RESOURCE_ORDER; + goto out; + } - ret = check_resource_offsets(wim, old_wim_end); - if (ret) - goto out_restore_memory_hdr; + if (old_blob_table_end > old_xml_begin) { + WARNING("Didn't expect the blob table to be after " + "the XML data"); + ret = WIMLIB_ERR_RESOURCE_ORDER; + goto out; + } + /* Set @old_wim_end, which indicates the point beyond which we + * don't allow any file and metadata resources to appear without + * returning WIMLIB_ERR_RESOURCE_ORDER (due to the fact that we + * would otherwise overwrite these resources). */ + if (!any_images_changed(wim)) { + /* If no images have been modified, added, or deleted, + * then a new blob table does not need to be written. + * We shall write the new XML data and optional + * integrity table immediately after the blob table. + * Note that this may overwrite an existing integrity + * table. */ + old_wim_end = old_blob_table_end; + write_flags |= WIMLIB_WRITE_FLAG_NO_NEW_BLOBS; + } else if (wim_has_integrity_table(wim)) { + /* Old WIM has an integrity table; begin writing new + * blobs after it. */ + old_wim_end = wim->hdr.integrity_table_reshdr.offset_in_wim + + wim->hdr.integrity_table_reshdr.size_in_wim; + } else { + /* No existing integrity table; begin writing new blobs + * after the old XML data. */ + old_wim_end = old_xml_end; + } + + ret = check_resource_offsets(wim, old_wim_end); + if (ret) + goto out; + + ret = prepare_blob_list_for_write(wim, WIMLIB_ALL_IMAGES, + write_flags, &blob_list, + &blob_table_list, &filter_ctx); + if (ret) + goto out; + + if (write_flags & WIMLIB_WRITE_FLAG_NO_NEW_BLOBS) + wimlib_assert(list_empty(&blob_list)); + } - ret = prepare_blob_list_for_write(wim, WIMLIB_ALL_IMAGES, write_flags, - &blob_list, &blob_table_list, - &filter_ctx); + /* Update image stats if needed. */ + ret = update_image_stats(wim); if (ret) - goto out_restore_memory_hdr; + goto out; ret = open_wim_writable(wim, wim->filename, O_RDWR); if (ret) - goto out_restore_memory_hdr; + goto out; ret = lock_wim_for_append(wim); if (ret) @@ -3223,6 +3189,7 @@ overwrite_wim_inplace(WIMStruct *wim, int write_flags, unsigned num_threads) /* Set WIM_HDR_FLAG_WRITE_IN_PROGRESS flag in header. */ wim->hdr.flags |= WIM_HDR_FLAG_WRITE_IN_PROGRESS; ret = write_wim_header_flags(wim->hdr.flags, &wim->out_fd); + wim->hdr.flags &= ~WIM_HDR_FLAG_WRITE_IN_PROGRESS; if (ret) { ERROR_WITH_ERRNO("Error updating WIM header flags"); goto out_unlock_wim; @@ -3231,11 +3198,11 @@ overwrite_wim_inplace(WIMStruct *wim, int write_flags, unsigned num_threads) if (filedes_seek(&wim->out_fd, old_wim_end) == -1) { ERROR_WITH_ERRNO("Can't seek to end of WIM"); ret = WIMLIB_ERR_WRITE; - goto out_restore_physical_hdr; + goto out_restore_hdr; } - ret = wim_write_blob_list(wim, &blob_list, write_flags, - num_threads, &filter_ctx); + ret = write_file_data_blobs(wim, &blob_list, write_flags, + num_threads, &filter_ctx); if (ret) goto out_truncate; @@ -3252,21 +3219,21 @@ overwrite_wim_inplace(WIMStruct *wim, int write_flags, unsigned num_threads) return 0; out_truncate: - if (!(write_flags & WIMLIB_WRITE_FLAG_NO_BLOB_TABLE)) { - WARNING("Truncating `%"TS"' to its original size (%"PRIu64" bytes)", - wim->filename, old_wim_end); - /* Return value of ftruncate() is ignored because this is - * already an error path. */ - (void)ftruncate(wim->out_fd.fd, old_wim_end); - } -out_restore_physical_hdr: - (void)write_wim_header_flags(hdr_save.flags, &wim->out_fd); + if (!(write_flags & (WIMLIB_WRITE_FLAG_NO_NEW_BLOBS | + WIMLIB_WRITE_FLAG_UNSAFE_COMPACT))) { + WARNING("Truncating \"%"TS"\" to its original size " + "(%"PRIu64" bytes)", wim->filename, old_wim_end); + if (ftruncate(wim->out_fd.fd, old_wim_end)) + WARNING_WITH_ERRNO("Failed to truncate WIM file!"); + } +out_restore_hdr: + (void)write_wim_header_flags(wim->hdr.flags, &wim->out_fd); out_unlock_wim: unlock_wim_for_append(wim); out_close_wim: (void)close_wim_writable(wim, write_flags); -out_restore_memory_hdr: - memcpy(&wim->hdr, &hdr_save, sizeof(struct wim_header)); +out: + wim->being_compacted = 0; return ret; } @@ -3276,8 +3243,6 @@ overwrite_wim_via_tmpfile(WIMStruct *wim, int write_flags, unsigned num_threads) size_t wim_name_len; int ret; - DEBUG("Overwriting `%"TS"' via a temporary file", wim->filename); - /* Write the WIM to a temporary file in the same directory as the * original WIM. */ wim_name_len = tstrlen(wim->filename); @@ -3304,7 +3269,6 @@ overwrite_wim_via_tmpfile(WIMStruct *wim, int write_flags, unsigned num_threads) /* Rename the new WIM file to the original WIM file. Note: on Windows * this actually calls win32_rename_replacement(), not _wrename(), so * that removing the existing destination file can be handled. */ - DEBUG("Renaming `%"TS"' to `%"TS"'", tmpfile, wim->filename); ret = trename(tmpfile, wim->filename); if (ret) { ERROR_WITH_ERRNO("Failed to rename `%"TS"' to `%"TS"'", @@ -3325,8 +3289,8 @@ overwrite_wim_via_tmpfile(WIMStruct *wim, int write_flags, unsigned num_threads) &progress, wim->progctx); } -/* Determine if the specified WIM file may be updated by appending in-place - * rather than writing and replacing it with an entirely new file. */ +/* Determine if the specified WIM file may be updated in-place rather than by + * writing and replacing it with an entirely new file. */ static bool can_overwrite_wim_inplace(const WIMStruct *wim, int write_flags) { @@ -3368,6 +3332,20 @@ wimlib_overwrite(WIMStruct *wim, int write_flags, unsigned num_threads) if (!wim->filename) return WIMLIB_ERR_NO_FILENAME; + if (unlikely(write_flags & WIMLIB_WRITE_FLAG_UNSAFE_COMPACT)) { + /* + * In UNSAFE_COMPACT mode: + * - RECOMPRESS is forbidden + * - REBUILD is ignored + * - SOFT_DELETE and NO_SOLID_SORT are implied + */ + if (write_flags & WIMLIB_WRITE_FLAG_RECOMPRESS) + return WIMLIB_ERR_COMPACTION_NOT_POSSIBLE; + write_flags &= ~WIMLIB_WRITE_FLAG_REBUILD; + write_flags |= WIMLIB_WRITE_FLAG_SOFT_DELETE; + write_flags |= WIMLIB_WRITE_FLAG_NO_SOLID_SORT; + } + orig_hdr_flags = wim->hdr.flags; if (write_flags & WIMLIB_WRITE_FLAG_IGNORE_READONLY_FLAG) wim->hdr.flags &= ~WIM_HDR_FLAG_READONLY; @@ -3382,5 +3360,7 @@ wimlib_overwrite(WIMStruct *wim, int write_flags, unsigned num_threads) return ret; WARNING("Falling back to re-building entire WIM"); } + if (write_flags & WIMLIB_WRITE_FLAG_UNSAFE_COMPACT) + return WIMLIB_ERR_COMPACTION_NOT_POSSIBLE; return overwrite_wim_via_tmpfile(wim, write_flags, num_threads); }