X-Git-Url: https://wimlib.net/git/?a=blobdiff_plain;f=src%2Fwrite.c;h=7f13630cceec991aea303afd27b74f0b1b67f888;hb=f370a82a53f55b8713266263096b96e07e02f4a0;hp=9b1de1d3958ee2d8895078abf41c3d002339bee3;hpb=cf13fe6161a69db6cbf17637b60c978fd746078a;p=wimlib diff --git a/src/write.c b/src/write.c index 9b1de1d3..7f13630c 100644 --- a/src/write.c +++ b/src/write.c @@ -1340,17 +1340,6 @@ validate_blob_list(struct list_head *blob_list) } } -static inline bool -blob_is_in_file(const struct blob_descriptor *blob) -{ - return blob->blob_location == BLOB_IN_FILE_ON_DISK -#ifdef __WIN32__ - || blob->blob_location == BLOB_IN_WINNT_FILE_ON_DISK - || blob->blob_location == BLOB_WIN32_ENCRYPTED -#endif - ; -} - static void init_done_with_file_info(struct list_head *blob_list) { @@ -1464,9 +1453,9 @@ init_done_with_file_info(struct list_head *blob_list) * identical to another blob already being written or one that would be filtered * out of the output WIM using blob_filtered() with the context @filter_ctx. * Each such duplicate blob will be removed from @blob_list, its reference count - * transfered to the pre-existing duplicate blob, its memory freed, and will not - * be written. Alternatively, if a blob in @blob_list is a duplicate with any - * blob in @blob_table that has not been marked for writing or would not be + * transferred to the pre-existing duplicate blob, its memory freed, and will + * not be written. Alternatively, if a blob in @blob_list is a duplicate with + * any blob in @blob_table that has not been marked for writing or would not be * hard-filtered, it is freed and the pre-existing duplicate is written instead, * taking ownership of the reference count and slot in the @blob_table_list. * @@ -1551,21 +1540,12 @@ write_blob_list(struct list_head *blob_list, out_ctype, out_chunk_size, &raw_copy_blobs); - /* Copy any compressed resources for which the raw data can be reused - * without decompression. */ - ret = write_raw_copy_resources(&raw_copy_blobs, ctx.out_fd, - &ctx.progress_data); - - if (ret || num_nonraw_bytes == 0) - goto out_destroy_context; - - /* Unless uncompressed output was required, allocate a chunk_compressor - * to do compression. There are serial and parallel implementations of - * the chunk_compressor interface. We default to parallel using the + /* Unless no data needs to be compressed, allocate a chunk_compressor to + * do compression. There are serial and parallel implementations of the + * chunk_compressor interface. We default to parallel using the * specified number of threads, unless the upper bound on the number * bytes needing to be compressed is less than a heuristic value. */ - if (out_ctype != WIMLIB_COMPRESSION_TYPE_NONE) { - + if (num_nonraw_bytes != 0 && out_ctype != WIMLIB_COMPRESSION_TYPE_NONE) { #ifdef ENABLE_MULTITHREADED_COMPRESSION if (num_nonraw_bytes > max(2000000, out_chunk_size)) { ret = new_parallel_chunk_compressor(out_ctype, @@ -1593,9 +1573,6 @@ write_blob_list(struct list_head *blob_list, else ctx.progress_data.progress.write_streams.num_threads = 1; - INIT_LIST_HEAD(&ctx.blobs_being_compressed); - INIT_LIST_HEAD(&ctx.blobs_in_solid_resource); - ret = call_progress(ctx.progress_data.progfunc, WIMLIB_PROGRESS_MSG_WRITE_STREAMS, &ctx.progress_data.progress, @@ -1603,7 +1580,20 @@ write_blob_list(struct list_head *blob_list, if (ret) goto out_destroy_context; + /* Copy any compressed resources for which the raw data can be reused + * without decompression. */ + ret = write_raw_copy_resources(&raw_copy_blobs, ctx.out_fd, + &ctx.progress_data); + + if (ret || num_nonraw_bytes == 0) + goto out_destroy_context; + + INIT_LIST_HEAD(&ctx.blobs_being_compressed); + if (write_resource_flags & WRITE_RESOURCE_FLAG_SOLID) { + + INIT_LIST_HEAD(&ctx.blobs_in_solid_resource); + ret = begin_write_resource(&ctx, num_nonraw_bytes); if (ret) goto out_destroy_context; @@ -2179,20 +2169,28 @@ write_metadata_resources(WIMStruct *wim, int image, int write_flags) struct wim_image_metadata *imd; imd = wim->image_metadata[i - 1]; - /* Build a new metadata resource only if image was modified from - * the original (or was newly added). Otherwise just copy the - * existing one. */ - if (imd->modified) { + if (is_image_dirty(imd)) { + /* The image was modified from the original, or was + * newly added, so we have to build and write a new + * metadata resource. */ ret = write_metadata_resource(wim, i, write_resource_flags); - } else if (write_flags & WIMLIB_WRITE_FLAG_UNSAFE_COMPACT) { - /* For compactions, existing metadata resources are - * written along with the existing file resources. */ - ret = 0; - } else if (write_flags & WIMLIB_WRITE_FLAG_APPEND) { - blob_set_out_reshdr_for_reuse(imd->metadata_blob); + } else if (is_image_unchanged_from_wim(imd, wim) && + (write_flags & (WIMLIB_WRITE_FLAG_UNSAFE_COMPACT | + WIMLIB_WRITE_FLAG_APPEND))) + { + /* The metadata resource is already in the WIM file. + * For appends, we don't need to write it at all. For + * compactions, we re-write existing metadata resources + * along with the existing file resources, not here. */ + if (write_flags & WIMLIB_WRITE_FLAG_APPEND) + blob_set_out_reshdr_for_reuse(imd->metadata_blob); ret = 0; } else { + /* The metadata resource is in a WIM file other than the + * one being written to. We need to rewrite it, + * possibly compressed differently; but rebuilding the + * metadata itself isn't necessary. */ ret = write_wim_resource(imd->metadata_blob, &wim->out_fd, wim->out_compression_type, @@ -2371,10 +2369,8 @@ finish_write(WIMStruct *wim, int image, int write_flags, if (!(write_flags & WIMLIB_WRITE_FLAG_NO_NEW_BLOBS)) { ret = write_blob_table(wim, image, write_flags, blob_table_list); - if (ret) { - free_integrity_table(old_integrity_table); - return ret; - } + if (ret) + goto out; } /* Write XML data. */ @@ -2384,13 +2380,13 @@ finish_write(WIMStruct *wim, int image, int write_flags, ret = write_wim_xml_data(wim, image, xml_totalbytes, &wim->out_hdr.xml_data_reshdr, write_resource_flags); - if (ret) { - free_integrity_table(old_integrity_table); - return ret; - } + if (ret) + goto out; /* Write integrity table if needed. */ - if (write_flags & WIMLIB_WRITE_FLAG_CHECK_INTEGRITY) { + if ((write_flags & WIMLIB_WRITE_FLAG_CHECK_INTEGRITY) && + wim->out_hdr.blob_table_reshdr.offset_in_wim != 0) + { if (write_flags & WIMLIB_WRITE_FLAG_NO_NEW_BLOBS) { /* The XML data we wrote may have overwritten part of * the old integrity table, so while calculating the new @@ -2401,10 +2397,8 @@ finish_write(WIMStruct *wim, int image, int write_flags, zero_reshdr(&checkpoint_hdr.integrity_table_reshdr); checkpoint_hdr.flags |= WIM_HDR_FLAG_WRITE_IN_PROGRESS; ret = write_wim_header(&checkpoint_hdr, &wim->out_fd, 0); - if (ret) { - free_integrity_table(old_integrity_table); - return ret; - } + if (ret) + goto out; } new_blob_table_end = wim->out_hdr.blob_table_reshdr.offset_in_wim + @@ -2414,9 +2408,8 @@ finish_write(WIMStruct *wim, int image, int write_flags, new_blob_table_end, old_blob_table_end, old_integrity_table); - free_integrity_table(old_integrity_table); if (ret) - return ret; + goto out; } else { /* No integrity table. */ zero_reshdr(&wim->out_hdr.integrity_table_reshdr); @@ -2432,13 +2425,17 @@ finish_write(WIMStruct *wim, int image, int write_flags, else ret = write_wim_header(&wim->out_hdr, &wim->out_fd, 0); if (ret) - return ret; + goto out; + ret = WIMLIB_ERR_WRITE; if (unlikely(write_flags & WIMLIB_WRITE_FLAG_UNSAFE_COMPACT)) { /* Truncate any data the compaction freed up. */ - if (ftruncate(wim->out_fd.fd, wim->out_fd.offset)) { + if (ftruncate(wim->out_fd.fd, wim->out_fd.offset) && + errno != EINVAL) /* allow compaction on untruncatable files, + e.g. block devices */ + { ERROR_WITH_ERRNO("Failed to truncate the output WIM file"); - return WIMLIB_ERR_WRITE; + goto out; } } @@ -2448,19 +2445,24 @@ finish_write(WIMStruct *wim, int image, int write_flags, * the system is abruptly terminated when the metadata for the rename * operation has been written to disk, but the new file data has not. */ + ret = WIMLIB_ERR_WRITE; if (write_flags & WIMLIB_WRITE_FLAG_FSYNC) { if (fsync(wim->out_fd.fd)) { ERROR_WITH_ERRNO("Error syncing data to WIM file"); - return WIMLIB_ERR_WRITE; + goto out; } } + ret = WIMLIB_ERR_WRITE; if (close_wim_writable(wim, write_flags)) { ERROR_WITH_ERRNO("Failed to close the output WIM file"); - return WIMLIB_ERR_WRITE; + goto out; } - return 0; + ret = 0; +out: + free_integrity_table(old_integrity_table); + return ret; } #if defined(HAVE_SYS_FILE_H) && defined(HAVE_FLOCK) @@ -2631,6 +2633,25 @@ should_default_to_solid_compression(WIMStruct *wim, int write_flags) wim_has_solid_resources(wim); } +/* Update the images' filecount/bytecount stats (in the XML info) to take into + * account any recent modifications. */ +static int +update_image_stats(WIMStruct *wim) +{ + if (!wim_has_metadata(wim)) + return 0; + for (int i = 0; i < wim->hdr.image_count; i++) { + struct wim_image_metadata *imd = wim->image_metadata[i]; + if (imd->stats_outdated) { + int ret = xml_update_image_info(wim, i + 1); + if (ret) + return ret; + imd->stats_outdated = false; + } + } + return 0; +} + /* Write a standalone WIM or split WIM (SWM) part to a new file or to a file * descriptor. */ int @@ -2778,6 +2799,11 @@ write_wim_part(WIMStruct *wim, wim->out_hdr.boot_idx = 1; } + /* Update image stats if needed. */ + ret = update_image_stats(wim); + if (ret) + return ret; + /* Set up the output file descriptor. */ if (write_flags & WIMLIB_WRITE_FLAG_FILE_DESCRIPTOR) { /* File descriptor was explicitly provided. */ @@ -2880,11 +2906,16 @@ wimlib_write_to_fd(WIMStruct *wim, int fd, return write_standalone_wim(wim, &fd, image, write_flags, num_threads); } +/* Have there been any changes to images in the specified WIM, including updates + * as well as deletions and additions of entire images, but excluding changes to + * the XML document? */ static bool -any_images_modified(WIMStruct *wim) +any_images_changed(WIMStruct *wim) { + if (wim->image_deletion_occurred) + return true; for (int i = 0; i < wim->hdr.image_count; i++) - if (wim->image_metadata[i]->modified) + if (!is_image_unchanged_from_wim(wim->image_metadata[i], wim)) return true; return false; } @@ -2924,6 +2955,20 @@ check_resource_offsets(WIMStruct *wim, off_t end_offset) return 0; } +static int +free_blob_if_invalidated(struct blob_descriptor *blob, void *_wim) +{ + const WIMStruct *wim = _wim; + + if (!blob->will_be_in_output_wim && + blob->blob_location == BLOB_IN_WIM && blob->rdesc->wim == wim) + { + blob_table_unlink(wim->blob_table, blob); + free_blob_descriptor(blob); + } + return 0; +} + /* * Overwrite a WIM, possibly appending new resources to it. * @@ -3040,12 +3085,17 @@ overwrite_wim_inplace(WIMStruct *wim, int write_flags, unsigned num_threads) if (ret) goto out; + /* Prevent new files from being deduplicated with existing blobs + * in the WIM that we haven't decided to write. Such blobs will + * be overwritten during the compaction. */ + for_blob_in_table(wim->blob_table, free_blob_if_invalidated, wim); + if (wim_has_metadata(wim)) { /* Add existing metadata resources to be compacted along * with the file resources. */ for (int i = 0; i < wim->hdr.image_count; i++) { struct wim_image_metadata *imd = wim->image_metadata[i]; - if (!imd->modified) { + if (is_image_unchanged_from_wim(imd, wim)) { fully_reference_blob_for_write(imd->metadata_blob, &blob_list); } @@ -3063,8 +3113,11 @@ overwrite_wim_inplace(WIMStruct *wim, int write_flags, unsigned num_threads) * this data would be overwritten. */ old_xml_begin = wim->hdr.xml_data_reshdr.offset_in_wim; old_xml_end = old_xml_begin + wim->hdr.xml_data_reshdr.size_in_wim; - old_blob_table_end = wim->hdr.blob_table_reshdr.offset_in_wim + - wim->hdr.blob_table_reshdr.size_in_wim; + if (wim->hdr.blob_table_reshdr.offset_in_wim == 0) + old_blob_table_end = WIM_HEADER_DISK_SIZE; + else + old_blob_table_end = wim->hdr.blob_table_reshdr.offset_in_wim + + wim->hdr.blob_table_reshdr.size_in_wim; if (wim_has_integrity_table(wim) && wim->hdr.integrity_table_reshdr.offset_in_wim < old_xml_end) { WARNING("Didn't expect the integrity table to be " @@ -3083,13 +3136,13 @@ overwrite_wim_inplace(WIMStruct *wim, int write_flags, unsigned num_threads) * don't allow any file and metadata resources to appear without * returning WIMLIB_ERR_RESOURCE_ORDER (due to the fact that we * would otherwise overwrite these resources). */ - if (!wim->image_deletion_occurred && !any_images_modified(wim)) { - /* If no images have been modified and no images have - * been deleted, a new blob table does not need to be - * written. We shall write the new XML data and - * optional integrity table immediately after the blob - * table. Note that this may overwrite an existing - * integrity table. */ + if (!any_images_changed(wim)) { + /* If no images have been modified, added, or deleted, + * then a new blob table does not need to be written. + * We shall write the new XML data and optional + * integrity table immediately after the blob table. + * Note that this may overwrite an existing integrity + * table. */ old_wim_end = old_blob_table_end; write_flags |= WIMLIB_WRITE_FLAG_NO_NEW_BLOBS; } else if (wim_has_integrity_table(wim)) { @@ -3117,6 +3170,11 @@ overwrite_wim_inplace(WIMStruct *wim, int write_flags, unsigned num_threads) wimlib_assert(list_empty(&blob_list)); } + /* Update image stats if needed. */ + ret = update_image_stats(wim); + if (ret) + goto out; + ret = open_wim_writable(wim, wim->filename, O_RDWR); if (ret) goto out;