*/
/*
- * Copyright (C) 2012, 2013, 2014, 2015 Eric Biggers
+ * Copyright (C) 2012-2016 Eric Biggers
*
* This file is free software; you can redistribute it and/or modify it under
* the terms of the GNU Lesser General Public License as published by the Free
* details.
*
* You should have received a copy of the GNU Lesser General Public License
- * along with this file; if not, see http://www.gnu.org/licenses/.
+ * along with this file; if not, see https://www.gnu.org/licenses/.
*/
#ifdef HAVE_CONFIG_H
blob_hdr.flags = cpu_to_le32(reshdr_flags);
ret = full_write(out_fd, &blob_hdr, sizeof(blob_hdr));
if (ret)
- ERROR_WITH_ERRNO("Write error");
+ ERROR_WITH_ERRNO("Error writing blob header to WIM file");
return ret;
}
static int
do_write_blobs_progress(struct write_blobs_progress_data *progress_data,
- u64 complete_size, u32 complete_count, bool discarded)
+ u64 complete_size, u64 complete_compressed_size,
+ u32 complete_count, bool discarded)
{
union wimlib_progress_info *progress = &progress_data->progress;
int ret;
}
} else {
progress->write_streams.completed_bytes += complete_size;
+ progress->write_streams.completed_compressed_bytes +=
+ complete_compressed_size;
progress->write_streams.completed_streams += complete_count;
}
* @blobs_being_compressed only when writing a solid resource. */
struct list_head blobs_in_solid_resource;
- /* Current uncompressed offset in the blob being read. */
- u64 cur_read_blob_offset;
-
- /* Uncompressed size of the blob currently being read. */
- u64 cur_read_blob_size;
-
/* Current uncompressed offset in the blob being written. */
u64 cur_write_blob_offset;
reserve_size += sizeof(struct alt_chunk_table_header_disk);
memset(ctx->chunk_csizes, 0, reserve_size);
ret = full_write(ctx->out_fd, ctx->chunk_csizes, reserve_size);
- if (ret)
+ if (ret) {
+ ERROR_WITH_ERRNO("Error reserving space for chunk "
+ "table in WIM file");
return ret;
+ }
}
return 0;
}
0 != (ctx->write_resource_flags &
WRITE_RESOURCE_FLAG_SOLID));
- typedef le64 _may_alias_attribute aliased_le64_t;
- typedef le32 _may_alias_attribute aliased_le32_t;
+ typedef le64 __attribute__((may_alias)) aliased_le64_t;
+ typedef le32 __attribute__((may_alias)) aliased_le32_t;
if (chunk_entry_size == 4) {
aliased_le32_t *entries = (aliased_le32_t*)ctx->chunk_csizes;
return 0;
write_error:
- ERROR_WITH_ERRNO("Write error");
+ ERROR_WITH_ERRNO("Error writing chunk table to WIM file");
return ret;
}
{
int ret;
struct wim_inode *inode;
+ const tchar *path;
tchar *cookie1;
tchar *cookie2;
if (--inode->i_num_remaining_streams > 0)
return 0;
- cookie1 = progress_get_streamless_path(blob->file_on_disk);
- cookie2 = progress_get_win32_path(blob->file_on_disk);
+ path = blob_file_path(blob);
+
+ cookie1 = progress_get_streamless_path(path);
+ cookie2 = progress_get_win32_path(path);
- ret = done_with_file(blob->file_on_disk, progfunc, progctx);
+ ret = done_with_file(path, progfunc, progctx);
progress_put_win32_path(cookie2);
progress_put_streamless_path(cookie1);
wimlib_assert(blob->size > 0);
- ctx->cur_read_blob_offset = 0;
- ctx->cur_read_blob_size = blob->size;
-
/* As an optimization, we allow some blobs to be "unhashed", meaning
* their SHA-1 message digests are unknown. This is the case with blobs
* that are added by scanning a directory tree with wimlib_add_image(),
* output reference count to the duplicate blob
* in the former case. */
ret = do_write_blobs_progress(&ctx->progress_data,
- blob->size, 1, true);
+ blob->size,
+ blob->size,
+ 1, true);
list_del(&blob->write_blobs_list);
list_del(&blob->blob_table_list);
if (new_blob->will_be_in_output_wim)
if (filedes_seek(out_fd, begin_offset) == -1)
return 0;
- ret = extract_blob_to_fd(blob, out_fd);
+ ret = extract_blob_to_fd(blob, out_fd, false);
if (ret) {
/* Error reading the uncompressed data. */
if (out_fd->offset == begin_offset &&
* Exception: if the compressed size happens to be *exactly* the same as
* the uncompressed size, then the blob *must* be written uncompressed
* in order to remain compatible with the Windows Overlay Filesystem
- * Filter Driver (WOF).
+ * filter driver (WOF).
*
* TODO: we are currently assuming that the optimization for
* single-chunk resources in maybe_rewrite_blob_uncompressed() prevents
{
int ret;
struct blob_descriptor *blob;
- u32 completed_blob_count;
- u32 completed_size;
+ u32 completed_blob_count = 0;
blob = list_entry(ctx->blobs_being_compressed.next,
struct blob_descriptor, write_blobs_list);
ctx->cur_write_blob_offset += usize;
- completed_size = usize;
- completed_blob_count = 0;
if (ctx->write_resource_flags & WRITE_RESOURCE_FLAG_SOLID) {
/* Wrote chunk in solid mode. It may have finished multiple
* blobs. */
}
}
- return do_write_blobs_progress(&ctx->progress_data, completed_size,
+ return do_write_blobs_progress(&ctx->progress_data, usize, csize,
completed_blob_count, false);
write_error:
- ERROR_WITH_ERRNO("Write error");
+ ERROR_WITH_ERRNO("Error writing chunk data to WIM file");
return ret;
}
/* Process the next chunk of data to be written to a WIM resource. */
static int
-write_blob_process_chunk(const void *chunk, size_t size, void *_ctx)
+write_blob_process_chunk(const struct blob_descriptor *blob, u64 offset,
+ const void *chunk, size_t size, void *_ctx)
{
struct write_blobs_ctx *ctx = _ctx;
int ret;
ret = write_chunk(ctx, chunk, size, size);
if (ret)
return ret;
- ctx->cur_read_blob_offset += size;
return 0;
}
} else {
needed_chunk_size = min(ctx->out_chunk_size,
ctx->cur_chunk_buf_filled +
- (ctx->cur_read_blob_size -
- ctx->cur_read_blob_offset));
+ (blob->size - offset));
}
bytes_consumed = min(chunkend - chunkptr,
chunkptr, bytes_consumed);
chunkptr += bytes_consumed;
- ctx->cur_read_blob_offset += bytes_consumed;
+ offset += bytes_consumed;
ctx->cur_chunk_buf_filled += bytes_consumed;
if (ctx->cur_chunk_buf_filled == needed_chunk_size) {
{
struct write_blobs_ctx *ctx = _ctx;
- wimlib_assert(ctx->cur_read_blob_offset == ctx->cur_read_blob_size || status);
-
if (!blob->will_be_in_output_wim) {
/* The blob was a duplicate. Now that its data has finished
* being read, it is being discarded in favor of the duplicate
ret = full_pread(in_fd, buf, bytes_to_read,
cur_read_offset);
- if (ret)
+ if (ret) {
+ ERROR_WITH_ERRNO("Error reading raw data "
+ "from WIM file");
return ret;
+ }
ret = full_write(out_fd, buf, bytes_to_read);
- if (ret)
+ if (ret) {
+ ERROR_WITH_ERRNO("Error writing raw data "
+ "to WIM file");
return ret;
+ }
cur_read_offset += bytes_to_read;
blob->rdesc->raw_copy_ok = 1;
list_for_each_entry(blob, raw_copy_blobs, write_blobs_list) {
+ u64 compressed_size = 0;
+
if (blob->rdesc->raw_copy_ok) {
/* Write each solid resource only one time. */
ret = write_raw_copy_resource(blob->rdesc, out_fd);
if (ret)
return ret;
blob->rdesc->raw_copy_ok = 0;
+ compressed_size = blob->rdesc->size_in_wim;
}
ret = do_write_blobs_progress(progress_data, blob->size,
- 1, false);
+ compressed_size, 1, false);
if (ret)
return ret;
}
out_ctype, out_chunk_size,
&raw_copy_blobs);
- /* Copy any compressed resources for which the raw data can be reused
- * without decompression. */
- ret = write_raw_copy_resources(&raw_copy_blobs, ctx.out_fd,
- &ctx.progress_data);
-
- if (ret || num_nonraw_bytes == 0)
- goto out_destroy_context;
-
- /* Unless uncompressed output was required, allocate a chunk_compressor
- * to do compression. There are serial and parallel implementations of
- * the chunk_compressor interface. We default to parallel using the
+ /* Unless no data needs to be compressed, allocate a chunk_compressor to
+ * do compression. There are serial and parallel implementations of the
+ * chunk_compressor interface. We default to parallel using the
* specified number of threads, unless the upper bound on the number
* bytes needing to be compressed is less than a heuristic value. */
- if (out_ctype != WIMLIB_COMPRESSION_TYPE_NONE) {
-
- #ifdef ENABLE_MULTITHREADED_COMPRESSION
+ if (num_nonraw_bytes != 0 && out_ctype != WIMLIB_COMPRESSION_TYPE_NONE) {
if (num_nonraw_bytes > max(2000000, out_chunk_size)) {
ret = new_parallel_chunk_compressor(out_ctype,
out_chunk_size,
wimlib_get_error_string(ret));
}
}
- #endif
if (ctx.compressor == NULL) {
ret = new_serial_chunk_compressor(out_ctype, out_chunk_size,
else
ctx.progress_data.progress.write_streams.num_threads = 1;
- INIT_LIST_HEAD(&ctx.blobs_being_compressed);
- INIT_LIST_HEAD(&ctx.blobs_in_solid_resource);
-
ret = call_progress(ctx.progress_data.progfunc,
WIMLIB_PROGRESS_MSG_WRITE_STREAMS,
&ctx.progress_data.progress,
if (ret)
goto out_destroy_context;
+ /* Copy any compressed resources for which the raw data can be reused
+ * without decompression. */
+ ret = write_raw_copy_resources(&raw_copy_blobs, ctx.out_fd,
+ &ctx.progress_data);
+
+ if (ret || num_nonraw_bytes == 0)
+ goto out_destroy_context;
+
+ INIT_LIST_HEAD(&ctx.blobs_being_compressed);
+
if (write_resource_flags & WRITE_RESOURCE_FLAG_SOLID) {
+
+ INIT_LIST_HEAD(&ctx.blobs_in_solid_resource);
+
ret = begin_write_resource(&ctx, num_nonraw_bytes);
if (ret)
goto out_destroy_context;
struct read_blob_callbacks cbs = {
.begin_blob = write_blob_begin_read,
- .consume_chunk = write_blob_process_chunk,
+ .continue_blob = write_blob_process_chunk,
.end_blob = write_blob_end_read,
.ctx = &ctx,
};
}
blob_set_is_located_in_attached_buffer(&blob, (void *)buf, buf_size);
- sha1_buffer(buf, buf_size, blob.hash);
+ sha1(buf, buf_size, blob.hash);
blob.unhashed = 0;
blob.is_metadata = is_metadata;
if (!(write_flags & WIMLIB_WRITE_FLAG_NO_NEW_BLOBS)) {
ret = write_blob_table(wim, image, write_flags,
blob_table_list);
- if (ret) {
- free_integrity_table(old_integrity_table);
- return ret;
- }
+ if (ret)
+ goto out;
}
/* Write XML data. */
ret = write_wim_xml_data(wim, image, xml_totalbytes,
&wim->out_hdr.xml_data_reshdr,
write_resource_flags);
- if (ret) {
- free_integrity_table(old_integrity_table);
- return ret;
- }
+ if (ret)
+ goto out;
/* Write integrity table if needed. */
- if (write_flags & WIMLIB_WRITE_FLAG_CHECK_INTEGRITY) {
+ if ((write_flags & WIMLIB_WRITE_FLAG_CHECK_INTEGRITY) &&
+ wim->out_hdr.blob_table_reshdr.offset_in_wim != 0)
+ {
if (write_flags & WIMLIB_WRITE_FLAG_NO_NEW_BLOBS) {
/* The XML data we wrote may have overwritten part of
* the old integrity table, so while calculating the new
zero_reshdr(&checkpoint_hdr.integrity_table_reshdr);
checkpoint_hdr.flags |= WIM_HDR_FLAG_WRITE_IN_PROGRESS;
ret = write_wim_header(&checkpoint_hdr, &wim->out_fd, 0);
- if (ret) {
- free_integrity_table(old_integrity_table);
- return ret;
- }
+ if (ret)
+ goto out;
}
new_blob_table_end = wim->out_hdr.blob_table_reshdr.offset_in_wim +
new_blob_table_end,
old_blob_table_end,
old_integrity_table);
- free_integrity_table(old_integrity_table);
if (ret)
- return ret;
+ goto out;
} else {
/* No integrity table. */
zero_reshdr(&wim->out_hdr.integrity_table_reshdr);
else
ret = write_wim_header(&wim->out_hdr, &wim->out_fd, 0);
if (ret)
- return ret;
+ goto out;
+ ret = WIMLIB_ERR_WRITE;
if (unlikely(write_flags & WIMLIB_WRITE_FLAG_UNSAFE_COMPACT)) {
/* Truncate any data the compaction freed up. */
- if (ftruncate(wim->out_fd.fd, wim->out_fd.offset)) {
+ if (ftruncate(wim->out_fd.fd, wim->out_fd.offset) &&
+ errno != EINVAL) /* allow compaction on untruncatable files,
+ e.g. block devices */
+ {
ERROR_WITH_ERRNO("Failed to truncate the output WIM file");
- return WIMLIB_ERR_WRITE;
+ goto out;
}
}
* the system is abruptly terminated when the metadata for the rename
* operation has been written to disk, but the new file data has not.
*/
+ ret = WIMLIB_ERR_WRITE;
if (write_flags & WIMLIB_WRITE_FLAG_FSYNC) {
if (fsync(wim->out_fd.fd)) {
ERROR_WITH_ERRNO("Error syncing data to WIM file");
- return WIMLIB_ERR_WRITE;
+ goto out;
}
}
+ ret = WIMLIB_ERR_WRITE;
if (close_wim_writable(wim, write_flags)) {
ERROR_WITH_ERRNO("Failed to close the output WIM file");
- return WIMLIB_ERR_WRITE;
+ goto out;
}
- return 0;
+ ret = 0;
+out:
+ free_integrity_table(old_integrity_table);
+ return ret;
}
#if defined(HAVE_SYS_FILE_H) && defined(HAVE_FLOCK)
/* At this point, the header at the beginning of the file has already
* been written. */
- /* For efficiency, when wimlib adds an image to the WIM with
- * wimlib_add_image(), the SHA-1 message digests of files are not
- * calculated; instead, they are calculated while the files are being
- * written. However, this does not work when writing a pipable WIM,
- * since when writing a blob to a pipable WIM, its SHA-1 message digest
- * needs to be known before the blob data is written. Therefore, before
- * getting much farther, we need to pre-calculate the SHA-1 message
- * digests of all blobs that will be written. */
+ /*
+ * For efficiency, wimlib normally delays calculating each newly added
+ * stream's hash until while that stream being written, or just before
+ * it is written. However, when writing a pipable WIM (potentially to a
+ * pipe), we first have to write the metadata resources, which contain
+ * all the hashes. Moreover each blob is prefixed with its hash (struct
+ * pwm_blob_hdr). Thus, we have to calculate all the hashes before
+ * writing anything.
+ */
ret = wim_checksum_unhashed_blobs(wim);
if (ret)
return ret;
return 0;
}
+static int
+free_blob_if_invalidated(struct blob_descriptor *blob, void *_wim)
+{
+ const WIMStruct *wim = _wim;
+
+ if (!blob->will_be_in_output_wim &&
+ blob->blob_location == BLOB_IN_WIM && blob->rdesc->wim == wim)
+ {
+ blob_table_unlink(wim->blob_table, blob);
+ free_blob_descriptor(blob);
+ }
+ return 0;
+}
+
/*
* Overwrite a WIM, possibly appending new resources to it.
*
if (ret)
goto out;
+ /* Prevent new files from being deduplicated with existing blobs
+ * in the WIM that we haven't decided to write. Such blobs will
+ * be overwritten during the compaction. */
+ for_blob_in_table(wim->blob_table, free_blob_if_invalidated, wim);
+
if (wim_has_metadata(wim)) {
/* Add existing metadata resources to be compacted along
* with the file resources. */
* this data would be overwritten. */
old_xml_begin = wim->hdr.xml_data_reshdr.offset_in_wim;
old_xml_end = old_xml_begin + wim->hdr.xml_data_reshdr.size_in_wim;
- old_blob_table_end = wim->hdr.blob_table_reshdr.offset_in_wim +
- wim->hdr.blob_table_reshdr.size_in_wim;
+ if (wim->hdr.blob_table_reshdr.offset_in_wim == 0)
+ old_blob_table_end = WIM_HEADER_DISK_SIZE;
+ else
+ old_blob_table_end = wim->hdr.blob_table_reshdr.offset_in_wim +
+ wim->hdr.blob_table_reshdr.size_in_wim;
if (wim_has_integrity_table(wim) &&
wim->hdr.integrity_table_reshdr.offset_in_wim < old_xml_end) {
WARNING("Didn't expect the integrity table to be "
WIMLIB_WRITE_FLAG_UNSAFE_COMPACT))) {
WARNING("Truncating \"%"TS"\" to its original size "
"(%"PRIu64" bytes)", wim->filename, old_wim_end);
- /* Return value of ftruncate() is ignored because this is
- * already an error path. */
- (void)ftruncate(wim->out_fd.fd, old_wim_end);
+ if (ftruncate(wim->out_fd.fd, old_wim_end))
+ WARNING_WITH_ERRNO("Failed to truncate WIM file!");
}
out_restore_hdr:
(void)write_wim_header_flags(wim->hdr.flags, &wim->out_fd);
wim_name_len = tstrlen(wim->filename);
tchar tmpfile[wim_name_len + 10];
tmemcpy(tmpfile, wim->filename, wim_name_len);
- randomize_char_array_with_alnum(tmpfile + wim_name_len, 9);
+ get_random_alnum_chars(tmpfile + wim_name_len, 9);
tmpfile[wim_name_len + 9] = T('\0');
ret = wimlib_write(wim, tmpfile, WIMLIB_ALL_IMAGES,
if (ret) {
ERROR_WITH_ERRNO("Failed to rename `%"TS"' to `%"TS"'",
tmpfile, wim->filename);
- #ifdef __WIN32__
+ #ifdef _WIN32
if (ret < 0)
#endif
{
git://wimlib.net / wimlib / blobdiff