#include <limits.h>
-#if defined(__WIN32__) && !defined(INVALID_HANDLE_VALUE)
-# define INVALID_HANDLE_VALUE ((HANDLE)(-1))
-#endif
-
/* Chunk table that's located at the beginning of each compressed resource in
* the WIM. (This is not the on-disk format; the on-disk format just has an
* array of offsets.) */
size_t alloc_size = sizeof(struct chunk_table) + num_chunks * sizeof(u64);
struct chunk_table *chunk_tab = CALLOC(1, alloc_size);
+ DEBUG("Begin chunk table for stream with size %"PRIu64, size);
+
if (!chunk_tab) {
ERROR("Failed to allocate chunk table for %"PRIu64" byte "
"resource", size);
* Returns 0 on success; nonzero on failure.
*/
static int
-write_wim_resource_chunk(const void *chunk, unsigned chunk_size,
- FILE *out_fp, compress_func_t compress,
- struct chunk_table *chunk_tab)
+write_wim_resource_chunk(const void * restrict chunk,
+ unsigned chunk_size,
+ FILE * restrict out_fp,
+ compress_func_t compress,
+ struct chunk_table * restrict chunk_tab)
{
- const u8 *out_chunk;
+ const void *out_chunk;
unsigned out_chunk_size;
if (compress) {
- u8 *compressed_chunk = alloca(chunk_size);
+ void *compressed_chunk = alloca(chunk_size);
- out_chunk_size = compress(chunk, chunk_size, compressed_chunk);
+ out_chunk_size = (*compress)(chunk, chunk_size, compressed_chunk);
if (out_chunk_size) {
/* Write compressed */
out_chunk = compressed_chunk;
* @compressed_size_p.
*/
static int
-finish_wim_resource_chunk_tab(struct chunk_table *chunk_tab,
- FILE *out_fp, u64 *compressed_size_p)
+finish_wim_resource_chunk_tab(struct chunk_table * restrict chunk_tab,
+ FILE * restrict out_fp,
+ u64 * restrict compressed_size_p)
{
size_t bytes_written;
if (fseeko(out_fp, chunk_tab->file_offset, SEEK_SET) != 0) {
}
static int
-finalize_and_check_sha1(SHA_CTX *sha_ctx, struct wim_lookup_table_entry *lte)
+fflush_and_ftruncate(FILE *out_fp, off_t offset)
+{
+ if (fseeko(out_fp, offset, SEEK_SET) ||
+ fflush(out_fp) ||
+ ftruncate(fileno(out_fp), offset))
+ {
+ ERROR_WITH_ERRNO("Failed to flush and/or truncate "
+ "output WIM file");
+ return WIMLIB_ERR_WRITE;
+ } else {
+ return 0;
+ }
+}
+
+static int
+finalize_and_check_sha1(SHA_CTX * restrict sha_ctx,
+ struct wim_lookup_table_entry * restrict lte)
{
u8 md[SHA1_HASH_SIZE];
sha1_final(md, sha_ctx);
};
static int
-write_resource_cb(const void *chunk, size_t chunk_size, void *_ctx)
+write_resource_cb(const void *restrict chunk, size_t chunk_size,
+ void *restrict _ctx)
{
struct write_resource_ctx *ctx = _ctx;
if (ctx->doing_sha)
sha1_update(&ctx->sha_ctx, chunk, chunk_size);
-
- if (ctx->compress) {
- return write_wim_resource_chunk(chunk, chunk_size,
- ctx->out_fp, ctx->compress,
- ctx->chunk_tab);
- } else {
- if (fwrite(chunk, 1, chunk_size, ctx->out_fp) != chunk_size) {
- ERROR_WITH_ERRNO("Error writing to output WIM");
- return WIMLIB_ERR_WRITE;
- } else {
- return 0;
- }
- }
+ return write_wim_resource_chunk(chunk, chunk_size,
+ ctx->out_fp, ctx->compress,
+ ctx->chunk_tab);
}
/*
if (new_size >= wim_resource_size(lte)) {
/* Oops! We compressed the resource to larger than the original
* size. Write the resource uncompressed instead. */
- if (fseeko(out_fp, offset, SEEK_SET) ||
- fflush(out_fp) ||
- ftruncate(fileno(out_fp),
- offset + wim_resource_size(lte)))
- {
- ERROR_WITH_ERRNO("Failed to flush and/or truncate "
- "output WIM file");
- ret = WIMLIB_ERR_WRITE;
- goto out_free_chunk_tab;
- }
DEBUG("Compressed %"PRIu64" => %"PRIu64" bytes; "
"writing uncompressed instead",
wim_resource_size(lte), new_size);
+ ret = fflush_and_ftruncate(out_fp, offset);
+ if (ret)
+ goto out_free_chunk_tab;
write_ctx.compress = NULL;
write_ctx.doing_sha = false;
out_ctype = WIMLIB_COMPRESSION_TYPE_NONE;
* just skip to the next stream. */
DEBUG("Discarding duplicate stream of length %"PRIu64,
wim_resource_size(lte));
+ lte->no_progress = 0;
goto skip_to_progress;
}
}
* the SHA1 message digest yet. */
wimlib_assert(lte->out_refcnt != 0);
lte->deferred = 0;
+ lte->no_progress = 0;
ret = (*write_stream_cb)(lte, write_stream_ctx);
if (ret)
break;
lte->unhashed = 0;
}
skip_to_progress:
- if (progress_func) {
+ if (!lte->no_progress) {
do_write_streams_progress(progress,
progress_func,
wim_resource_size(lte));
static inline int
write_flags_to_resource_flags(int write_flags)
{
- return (write_flags & WIMLIB_WRITE_FLAG_RECOMPRESS) ?
- WIMLIB_RESOURCE_FLAG_RECOMPRESS : 0;
+ int resource_flags = 0;
+
+ if (write_flags & WIMLIB_WRITE_FLAG_RECOMPRESS)
+ resource_flags |= WIMLIB_RESOURCE_FLAG_RECOMPRESS;
+ return resource_flags;
}
static int
wimlib_progress_func_t progress_func,
union wimlib_progress_info *progress)
{
+ DEBUG("Writing stream list (serial version)");
progress->write_streams.num_threads = 1;
if (progress_func)
progress_func(WIMLIB_PROGRESS_MSG_WRITE_STREAMS, progress);
msg->begin_chunk + msg->num_chunks == ctx->cur_chunk_tab->num_chunks)
{
u64 res_csize;
+ off_t offset;
+
ret = finish_wim_resource_chunk_tab(ctx->cur_chunk_tab,
ctx->out_fp,
&res_csize);
return ret;
list_del(&cur_lte->being_compressed_list);
-#if 0
+
+ /* Grab the offset of this stream in the output file
+ * from the chunk table before we free it. */
+ offset = ctx->cur_chunk_tab->file_offset;
+
+ FREE(ctx->cur_chunk_tab);
+ ctx->cur_chunk_tab = NULL;
+
if (res_csize >= wim_resource_size(cur_lte)) {
/* Oops! We compressed the resource to
* larger than the original size. Write
* the resource uncompressed instead. */
- ret = write_uncompressed_resource_and_truncate(
- cur_lte,
- ctx->out_fp,
- ctx->cur_chunk_tab->file_offset,
- &cur_lte->output_resource_entry);
+ DEBUG("Compressed %"PRIu64" => %"PRIu64" bytes; "
+ "writing uncompressed instead",
+ wim_resource_size(cur_lte), res_csize);
+ ret = fflush_and_ftruncate(ctx->out_fp, offset);
if (ret)
- goto out;
- } else
-#endif
- {
+ return ret;
+ ret = write_wim_resource(cur_lte,
+ ctx->out_fp,
+ WIMLIB_COMPRESSION_TYPE_NONE,
+ &cur_lte->output_resource_entry,
+ ctx->write_resource_flags);
+ if (ret)
+ return ret;
+ } else {
cur_lte->output_resource_entry.size =
res_csize;
cur_lte->resource_entry.original_size;
cur_lte->output_resource_entry.offset =
- ctx->cur_chunk_tab->file_offset;
+ offset;
cur_lte->output_resource_entry.flags =
cur_lte->resource_entry.flags |
WIM_RESHDR_FLAG_COMPRESSED;
}
- do_write_streams_progress(ctx->progress, ctx->progress_func,
+
+ do_write_streams_progress(ctx->progress,
+ ctx->progress_func,
wim_resource_size(cur_lte));
- FREE(ctx->cur_chunk_tab);
- ctx->cur_chunk_tab = NULL;
/* Since we just finished writing a stream, write any
* streams that have been added to the serial_streams
* resources that don't need to be compressed because
* the desired compression type is the same as the
* previous compression type). */
- ret = do_write_stream_list_serial(&ctx->serial_streams,
- ctx->lookup_table,
- ctx->out_fp,
- ctx->out_ctype,
- ctx->write_resource_flags,
- ctx->progress_func,
- ctx->progress);
- if (ret)
- return ret;
+ if (!list_empty(&ctx->serial_streams)) {
+ ret = do_write_stream_list_serial(&ctx->serial_streams,
+ ctx->lookup_table,
+ ctx->out_fp,
+ ctx->out_ctype,
+ ctx->write_resource_flags,
+ ctx->progress_func,
+ ctx->progress);
+ if (ret)
+ return ret;
+ }
/* Advance to the next stream to write. */
if (list_empty(&ctx->outstanding_streams)) {
* a fixed number of them, proportional to the number of
* threads) have been sent off to the compressor threads, so we
* receive messages from the compressor threads containing
- * compressed chunks of data.
+ * compressed chunks of data.
*
* We may need to receive multiple messages before one is
* actually available to use because messages received that are
} else {
ret = submit_stream_for_compression(lte, ctx);
}
+ lte->no_progress = 1;
return ret;
}
if (nthreads < 1 || nthreads > UINT_MAX) {
WARNING("Could not determine number of processors! Assuming 1");
goto out_serial;
+ } else if (nthreads == 1) {
+ goto out_serial_quiet;
} else {
num_threads = nthreads;
}
}
+ DEBUG("Writing stream list (parallel version, num_threads=%u)",
+ num_threads);
+
progress->write_streams.num_threads = num_threads;
static const size_t MESSAGES_PER_THREAD = 2;
goto out_join;
ret = do_write_stream_list(stream_list, lookup_table,
main_thread_process_next_stream,
- &ctx, NULL, NULL);
+ &ctx, progress_func, progress);
if (ret)
goto out_destroy_ctx;
return ret;
out_serial:
WARNING("Falling back to single-threaded compression");
+out_serial_quiet:
return write_stream_list_serial(stream_list,
lookup_table,
out_fp,
git://wimlib.net / wimlib / blobdiff