# include <sys/file.h>
#endif
-#include "wimlib/compress_chunks.h"
+#include "wimlib/chunk_compressor.h"
#include "wimlib/endianness.h"
#include "wimlib/error.h"
#include "wimlib/file_io.h"
rspec->wim->chunk_size == out_chunk_size);
}
+ /* XXX: For compatibility, we can't allow multiple packed resources per
+ * WIM. */
+#if 0
if ((rspec->flags & WIM_RESHDR_FLAG_PACKED_STREAMS) &&
(write_resource_flags & WRITE_RESOURCE_FLAG_PACK_STREAMS))
{
return (write_size > rspec->uncompressed_size / 2);
}
+#endif
+
return false;
}
return ret;
}
-#if 0
-static int
-seek_and_truncate(struct filedes *out_fd, off_t offset)
-{
- if (filedes_seek(out_fd, offset) == -1 ||
- ftruncate(out_fd->fd, offset))
- {
- ERROR_WITH_ERRNO("Failed to truncate output WIM file");
- return WIMLIB_ERR_WRITE;
- }
- return 0;
-}
-#endif
-
struct write_streams_progress_data {
wimlib_progress_func_t progress_func;
union wimlib_progress_info progress;
return 0;
}
+/* Rewrite a stream that was just written compressed as uncompressed instead.
+ * This function is optional, but if a stream did not compress to less than its
+ * original size, it might as well be written uncompressed. */
+static int
+write_stream_uncompressed(struct wim_lookup_table_entry *lte,
+ struct filedes *out_fd)
+{
+ int ret;
+ u64 begin_offset = lte->out_reshdr.offset_in_wim;
+ u64 end_offset = out_fd->offset;
+
+ if (filedes_seek(out_fd, begin_offset) == -1)
+ return 0;
+
+ ret = extract_full_stream_to_fd(lte, out_fd);
+ if (ret) {
+ /* Error reading the uncompressed data. */
+ if (out_fd->offset == begin_offset &&
+ filedes_seek(out_fd, end_offset) != -1)
+ {
+ /* Nothing was actually written yet, and we successfully
+ * seeked to the end of the compressed resource, so
+ * don't issue a hard error; just keep the compressed
+ * resource instead. */
+ WARNING("Recovered compressed stream of "
+ "size %"PRIu64", continuing on.",
+ lte->size);
+ return 0;
+ }
+ return ret;
+ }
+
+ wimlib_assert(out_fd->offset - begin_offset == lte->size);
+
+ if (out_fd->offset < end_offset &&
+ 0 != ftruncate(out_fd->fd, out_fd->offset))
+ {
+ ERROR_WITH_ERRNO("Can't truncate output file to "
+ "offset %"PRIu64, out_fd->offset);
+ return WIMLIB_ERR_WRITE;
+ }
+
+ lte->out_reshdr.size_in_wim = lte->size;
+ lte->out_reshdr.flags &= ~(WIM_RESHDR_FLAG_COMPRESSED |
+ WIM_RESHDR_FLAG_PACKED_STREAMS);
+ return 0;
+}
+
/* Write the next chunk of (typically compressed) data to the output WIM,
* handling the writing of the chunk table. */
static int
if (ctx->cur_write_res_offset == ctx->cur_write_res_size &&
!(ctx->write_resource_flags & WRITE_RESOURCE_FLAG_PACK_STREAMS))
{
- struct wim_lookup_table_entry *lte;
-
- lte = list_entry(ctx->pending_streams.next,
- struct wim_lookup_table_entry, write_streams_list);
wimlib_assert(ctx->cur_write_res_offset == lte->size);
/* Finished writing a stream in non-packed mode. */
if (ret)
return ret;
- wimlib_assert(lte->out_reshdr.uncompressed_size == lte->size);
-
lte->out_reshdr.flags = filter_resource_flags(lte->flags);
if (ctx->compressor != NULL)
lte->out_reshdr.flags |= WIM_RESHDR_FLAG_COMPRESSED;
+ if (ctx->compressor != NULL &&
+ lte->out_reshdr.size_in_wim >= lte->out_reshdr.uncompressed_size &&
+ !(ctx->write_resource_flags & WRITE_RESOURCE_FLAG_PIPABLE) &&
+ !(lte->flags & WIM_RESHDR_FLAG_PACKED_STREAMS))
+ {
+ /* Stream did not compress to less than its original
+ * size. If we're not writing a pipable WIM (which
+ * could mean the output file descriptor is
+ * non-seekable), and the stream isn't located in a
+ * resource pack (which would make reading it again
+ * costly), truncate the file to the start of the stream
+ * and write it uncompressed instead. */
+ DEBUG("Stream of size %"PRIu64" did not compress to "
+ "less than original size; writing uncompressed.",
+ lte->size);
+ ret = write_stream_uncompressed(lte, ctx->out_fd);
+ if (ret)
+ return ret;
+ }
+
+ wimlib_assert(lte->out_reshdr.uncompressed_size == lte->size);
+
list_del(<e->write_streams_list);
ctx->cur_write_res_offset = 0;
}
memcpy(&ctx->chunk_buf[ctx->chunk_buf_filled],
chunkptr, bytes_consumed);
- resized_chunk = ctx->chunk_buf;
-
chunkptr += bytes_consumed;
ctx->cur_read_res_offset += bytes_consumed;
ctx->chunk_buf_filled += bytes_consumed;
* no streams are hard-filtered or no streams are unhashed, this parameter
* can be NULL.
*
- * @comp_ctx
- * A location in which to allocate the pointer to the default LZX
- * compression context. This will only be used if @out_ctype is
- * WIMLIB_COMPRESSION_TYPE_LZX.
- *
* @progress_func
* If non-NULL, a progress function that will be called periodically with
* WIMLIB_PROGRESS_MSG_WRITE_STREAMS messages. Note that on-the-fly
unsigned num_threads,
struct wim_lookup_table *lookup_table,
struct filter_context *filter_ctx,
- struct wimlib_lzx_context **comp_ctx,
wimlib_progress_func_t progress_func)
{
int ret;
ctx.write_resource_flags = write_resource_flags;
ctx.filter_ctx = filter_ctx;
- if (out_chunk_size != 0) {
+ if (out_ctype != WIMLIB_COMPRESSION_TYPE_NONE) {
+ wimlib_assert(out_chunk_size != 0);
if (out_chunk_size <= STACK_MAX) {
ctx.chunk_buf = alloca(out_chunk_size);
} else {
* bytes needing to be compressed is less 2000000 (heuristic value). */
if (out_ctype != WIMLIB_COMPRESSION_TYPE_NONE) {
- if (out_ctype == WIMLIB_COMPRESSION_TYPE_LZMS &&
- ctx.lookup_table != NULL) {
- WARNING("LZMS compression not implemented; data will "
- "actually be written uncompressed.");
- }
-
+ #ifdef ENABLE_MULTITHREADED_COMPRESSION
if (ctx.num_bytes_to_compress >= 2000000) {
ret = new_parallel_chunk_compressor(out_ctype,
out_chunk_size,
"(status %d)", ret);
}
}
+ #endif
if (ctx.compressor == NULL) {
- if (out_ctype == WIMLIB_COMPRESSION_TYPE_LZX) {
- ret = wimlib_lzx_alloc_context(out_chunk_size,
- NULL,
- comp_ctx);
- if (ret)
- goto out_destroy_context;
- }
ret = new_serial_chunk_compressor(out_ctype, out_chunk_size,
- *comp_ctx, &ctx.compressor);
+ &ctx.compressor);
if (ret)
goto out_destroy_context;
}
&ctx.progress_data);
out_destroy_context:
- if (out_chunk_size > STACK_MAX)
+ if (out_ctype != WIMLIB_COMPRESSION_TYPE_NONE && out_chunk_size > STACK_MAX)
FREE(ctx.chunk_buf);
FREE(ctx.chunk_csizes);
if (ctx.compressor)
return ret;
}
+static int
+wim_write_stream_list(WIMStruct *wim,
+ struct list_head *stream_list,
+ int write_flags,
+ unsigned num_threads,
+ struct filter_context *filter_ctx,
+ wimlib_progress_func_t progress_func)
+{
+ int out_ctype;
+ u32 out_chunk_size;
+ int write_resource_flags;
+
+ write_resource_flags = write_flags_to_resource_flags(write_flags);
+
+ if (write_resource_flags & WRITE_RESOURCE_FLAG_PACK_STREAMS) {
+ out_chunk_size = wim->out_pack_chunk_size;
+ out_ctype = wim->out_pack_compression_type;
+ } else {
+ out_chunk_size = wim->out_chunk_size;
+ out_ctype = wim->out_compression_type;
+ }
+
+ return write_stream_list(stream_list,
+ &wim->out_fd,
+ write_resource_flags,
+ out_ctype,
+ out_chunk_size,
+ num_threads,
+ wim->lookup_table,
+ filter_ctx,
+ progress_func);
+}
+
static int
write_wim_resource(struct wim_lookup_table_entry *lte,
struct filedes *out_fd,
int out_ctype,
u32 out_chunk_size,
- int write_resource_flags,
- struct wimlib_lzx_context **comp_ctx)
+ int write_resource_flags)
{
LIST_HEAD(stream_list);
list_add(<e->write_streams_list, &stream_list);
1,
NULL,
NULL,
- comp_ctx,
NULL);
}
u32 out_chunk_size,
struct wim_reshdr *out_reshdr,
u8 *hash,
- int write_resource_flags,
- struct wimlib_lzx_context **comp_ctx)
+ int write_resource_flags)
{
int ret;
struct wim_lookup_table_entry *lte;
}
ret = write_wim_resource(lte, out_fd, out_ctype, out_chunk_size,
- write_resource_flags, comp_ctx);
+ write_resource_flags);
if (ret)
goto out_free_lte;
}
}
- return write_stream_list(stream_list,
- &wim->out_fd,
- write_flags_to_resource_flags(write_flags),
- wim->out_compression_type,
- wim->out_chunk_size,
- num_threads,
- wim->lookup_table,
- filter_ctx,
- &wim->lzx_context,
- progress_func);
+ return wim_write_stream_list(wim,
+ stream_list,
+ write_flags,
+ num_threads,
+ filter_ctx,
+ progress_func);
}
static int
&wim->out_fd,
wim->out_compression_type,
wim->out_chunk_size,
- write_resource_flags,
- &wim->lzx_context);
+ write_resource_flags);
}
if (ret)
return ret;
&wim->out_fd,
wim->hdr.part_number,
out_reshdr,
- write_flags_to_resource_flags(write_flags),
- &wim->lzx_context);
+ write_flags_to_resource_flags(write_flags));
}
/*
if (write_flags & WIMLIB_WRITE_FLAG_PACK_STREAMS)
DEBUG("\tPACK_STREAMS");
- if (write_flags & WIMLIB_WRITE_FLAG_NO_PACK_STREAMS)
- DEBUG("\tNO_PACK_STREAMS");
-
if (write_flags & WIMLIB_WRITE_FLAG_FILE_DESCRIPTOR)
DEBUG("\tFILE_DESCRIPTOR");
WIMLIB_WRITE_FLAG_NOT_PIPABLE))
return WIMLIB_ERR_INVALID_PARAM;
- if ((write_flags & (WIMLIB_WRITE_FLAG_PACK_STREAMS |
- WIMLIB_WRITE_FLAG_NO_PACK_STREAMS))
- == (WIMLIB_WRITE_FLAG_PACK_STREAMS |
- WIMLIB_WRITE_FLAG_NO_PACK_STREAMS))
- return WIMLIB_ERR_INVALID_PARAM;
-
/* Save previous header, then start initializing the new one. */
memcpy(&hdr_save, &wim->hdr, sizeof(struct wim_header));
write_flags |= WIMLIB_WRITE_FLAG_CHECK_INTEGRITY;
}
- if (!(write_flags & (WIMLIB_WRITE_FLAG_PACK_STREAMS |
- WIMLIB_WRITE_FLAG_NO_PACK_STREAMS)))
- if (wim->hdr.wim_version == WIM_VERSION_PACKED_STREAMS) {
- DEBUG("WIM version 3584; default to PACK_STREAMS.");
- write_flags |= WIMLIB_WRITE_FLAG_PACK_STREAMS;
- }
-
if ((write_flags & (WIMLIB_WRITE_FLAG_PIPABLE |
WIMLIB_WRITE_FLAG_PACK_STREAMS))
== (WIMLIB_WRITE_FLAG_PIPABLE |
WIMLIB_WRITE_FLAG_PACK_STREAMS))
+ {
+ ERROR("Cannot specify both PIPABLE and PACK_STREAMS!");
return WIMLIB_ERR_INVALID_PARAM;
+ }
/* Set appropriate magic number. */
if (write_flags & WIMLIB_WRITE_FLAG_PIPABLE)
wim->hdr.magic = WIM_MAGIC;
/* Set appropriate version number. */
- if (write_flags & WIMLIB_WRITE_FLAG_PACK_STREAMS)
+ if ((write_flags & WIMLIB_WRITE_FLAG_PACK_STREAMS) ||
+ wim->out_compression_type == WIMLIB_COMPRESSION_TYPE_LZMS)
wim->hdr.wim_version = WIM_VERSION_PACKED_STREAMS;
else
wim->hdr.wim_version = WIM_VERSION_DEFAULT;
if (wim_has_integrity_table(wim))
write_flags |= WIMLIB_WRITE_FLAG_CHECK_INTEGRITY;
- /* Set default packed flag. */
- if (!(write_flags & (WIMLIB_WRITE_FLAG_PACK_STREAMS |
- WIMLIB_WRITE_FLAG_NO_PACK_STREAMS)))
- {
- if (wim->hdr.wim_version == WIM_VERSION_PACKED_STREAMS)
- write_flags |= WIMLIB_WRITE_FLAG_PACK_STREAMS;
- } else if (write_flags & WIMLIB_WRITE_FLAG_PACK_STREAMS) {
+ /* Set WIM version if adding packed streams. */
+ if (write_flags & WIMLIB_WRITE_FLAG_PACK_STREAMS)
wim->hdr.wim_version = WIM_VERSION_PACKED_STREAMS;
- }
/* Set additional flags for overwrite. */
write_flags |= WIMLIB_WRITE_FLAG_OVERWRITE |
goto out_restore_physical_hdr;
}
- ret = write_stream_list(&stream_list,
- &wim->out_fd,
- write_flags_to_resource_flags(write_flags),
- wim->compression_type,
- wim->chunk_size,
- num_threads,
- wim->lookup_table,
- &filter_ctx,
- &wim->lzx_context,
- progress_func);
+ ret = wim_write_stream_list(wim,
+ &stream_list,
+ write_flags,
+ num_threads,
+ &filter_ctx,
+ progress_func);
if (ret)
goto out_truncate;
return 0;
}
+/* Determine if the specified WIM file may be updated by appending in-place
+ * rather than writing and replacing it with an entirely new file. */
static bool
can_overwrite_wim_inplace(const WIMStruct *wim, int write_flags)
{
+ /* REBUILD flag forces full rebuild. */
if (write_flags & WIMLIB_WRITE_FLAG_REBUILD)
return false;
+ /* Deletions cause full rebuild by default. */
if (wim->deletion_occurred && !(write_flags & WIMLIB_WRITE_FLAG_SOFT_DELETE))
return false;
+ /* Pipable WIMs cannot be updated in place, nor can a non-pipable WIM be
+ * turned into a pipable WIM in-place. */
if (wim_is_pipable(wim) || (write_flags & WIMLIB_WRITE_FLAG_PIPABLE))
return false;
+ /* wimlib allows multiple packs in a single WIM, but they don't seem to
+ * be compatible with WIMGAPI, so force all streams to be repacked if
+ * the WIM already may have contained a pack and PACK_STREAMS was
+ * requested. */
+ if (write_flags & WIMLIB_WRITE_FLAG_PACK_STREAMS &&
+ wim->hdr.wim_version == WIM_VERSION_PACKED_STREAMS)
+ return false;
+
+ /* The default compression type and compression chunk size selected for
+ * the output WIM must be the same as those currently used for the WIM.
+ */
if (wim->compression_type != wim->out_compression_type)
return false;
if (wim->chunk_size != wim->out_chunk_size)