From: Eric Biggers Date: Sun, 11 May 2014 05:23:34 +0000 (-0500) Subject: Don't create unnecessary temporary files X-Git-Tag: v1.7.0~216 X-Git-Url: https://wimlib.net/git/?p=wimlib;a=commitdiff_plain;h=50407ddd59aa2d0906225a4526eaefbf6da8450f Don't create unnecessary temporary files A temporary file is unnecessary (without too much extra trouble) if the whole stream data is already in memory as a result of reading a packed resource. --- diff --git a/include/wimlib/apply.h b/include/wimlib/apply.h index 24648c9b..3b124d31 100644 --- a/include/wimlib/apply.h +++ b/include/wimlib/apply.h @@ -223,6 +223,7 @@ struct apply_ctx { unsigned long invalid_sequence; unsigned long partial_security_descriptors; unsigned long no_security_descriptors; + struct wim_lookup_table_entry *cur_stream; struct filedes tmpfile_fd; tchar *tmpfile_name; u64 num_streams_remaining; diff --git a/include/wimlib/resource.h b/include/wimlib/resource.h index b2ba851b..ae077553 100644 --- a/include/wimlib/resource.h +++ b/include/wimlib/resource.h @@ -216,10 +216,9 @@ skip_wim_stream(struct wim_lookup_table_entry *lte); * @lte: * Stream that is about to be read. * - * @is_partial_res: - * Set to true if the stream is just one of several being read from a - * single pack and therefore would be extra expensive to read - * independently. + * @flags: + * Bitwise OR of BEGIN_STREAM_FLAG_PARTIAL_RESOURCE and/or + * BEGIN_STREAM_FLAG_WHOLE_STREAM. * * @ctx: * User-provided context. @@ -230,9 +229,17 @@ skip_wim_stream(struct wim_lookup_table_entry *lte); * (without calling @consume_chunk or @end_stream). */ typedef int (*read_stream_list_begin_stream_t)(struct wim_lookup_table_entry *lte, - bool is_partial_res, + u32 flags, void *ctx); +/* Set to true if the stream is just one of several being read from a single + * pack and therefore would be extra expensive to read independently. */ +#define BEGIN_STREAM_FLAG_PARTIAL_RESOURCE 0x00000001 + +/* This is purely advisory and indicates that the entire stream data will be + * provided in one call to consume_chunk(). */ +#define BEGIN_STREAM_FLAG_WHOLE_STREAM 0x00000002 + #define BEGIN_STREAM_STATUS_SKIP_STREAM -1 /* diff --git a/src/extract.c b/src/extract.c index f1fcd0f8..bc769eec 100644 --- a/src/extract.c +++ b/src/extract.c @@ -1273,14 +1273,23 @@ need_tmpfile_to_extract(struct wim_lookup_table_entry *lte, } static int -begin_extract_stream_to_tmpfile(struct wim_lookup_table_entry *lte, - bool is_partial_res, - void *_ctx) +begin_extract_stream(struct wim_lookup_table_entry *lte, + u32 flags, void *_ctx) { struct apply_ctx *ctx = _ctx; int ret; - if (!need_tmpfile_to_extract(lte, is_partial_res)) { + if (flags & BEGIN_STREAM_FLAG_WHOLE_STREAM) { + DEBUG("Whole stream (size=%"PRIu64") will be read into memory", + lte->size); + ctx->cur_stream = lte; + filedes_invalidate(&ctx->tmpfile_fd); + return 0; + } + + if (!need_tmpfile_to_extract(lte, + (flags & BEGIN_STREAM_FLAG_PARTIAL_RESOURCE))) + { DEBUG("Temporary file not needed " "for stream (size=%"PRIu64")", lte->size); ret = extract_stream_instances(lte, lte, ctx); @@ -1295,14 +1304,42 @@ begin_extract_stream_to_tmpfile(struct wim_lookup_table_entry *lte, } static int -end_extract_stream_to_tmpfile(struct wim_lookup_table_entry *lte, - int status, void *_ctx) +extract_chunk(const void *chunk, size_t size, void *_ctx) +{ + struct apply_ctx *ctx = _ctx; + int ret; + + if (filedes_valid(&ctx->tmpfile_fd)) { + ret = full_write(&ctx->tmpfile_fd, chunk, size); + if (ret) + ERROR_WITH_ERRNO("Error writing to file descriptor"); + } else { + struct wim_lookup_table_entry lte_override; + + memcpy(<e_override, ctx->cur_stream, + sizeof(struct wim_lookup_table_entry)); + + lte_override.resource_location = RESOURCE_IN_ATTACHED_BUFFER; + lte_override.size = size; + lte_override.attached_buffer = (void *)chunk; + + ret = extract_stream_instances(ctx->cur_stream, <e_override, ctx); + } + return ret; +} + +static int +end_extract_stream(struct wim_lookup_table_entry *lte, + int status, void *_ctx) { struct apply_ctx *ctx = _ctx; struct wim_lookup_table_entry lte_override; int ret; int errno_save = errno; + if (!filedes_valid(&ctx->tmpfile_fd)) + return status; + ret = filedes_close(&ctx->tmpfile_fd); if (status) { @@ -1342,11 +1379,11 @@ static int extract_stream_list(struct apply_ctx *ctx) { struct read_stream_list_callbacks cbs = { - .begin_stream = begin_extract_stream_to_tmpfile, + .begin_stream = begin_extract_stream, .begin_stream_ctx = ctx, - .consume_chunk = extract_chunk_to_fd, - .consume_chunk_ctx = &ctx->tmpfile_fd, - .end_stream = end_extract_stream_to_tmpfile, + .consume_chunk = extract_chunk, + .consume_chunk_ctx = ctx, + .end_stream = end_extract_stream, .end_stream_ctx = ctx, }; return read_stream_list(&ctx->stream_list, diff --git a/src/resource.c b/src/resource.c index b5db1452..9d8fe921 100644 --- a/src/resource.c +++ b/src/resource.c @@ -988,10 +988,17 @@ streamifier_cb(const void *chunk, size_t size, void *_ctx) wimlib_assert(size <= ctx->cur_stream->size - ctx->cur_stream_offset); if (ctx->cur_stream_offset == 0) { + u32 flags; + /* Starting a new stream. */ DEBUG("Begin new stream (size=%"PRIu64").", ctx->cur_stream->size); - ret = (*ctx->cbs.begin_stream)(ctx->cur_stream, true, + + flags = BEGIN_STREAM_FLAG_PARTIAL_RESOURCE; + if (size == ctx->cur_stream->size) + flags |= BEGIN_STREAM_FLAG_WHOLE_STREAM; + ret = (*ctx->cbs.begin_stream)(ctx->cur_stream, + flags, ctx->cbs.begin_stream_ctx); if (ret) return ret; @@ -1037,7 +1044,7 @@ struct hasher_context { /* Callback for starting to read a stream while calculating its SHA1 message * digest. */ static int -hasher_begin_stream(struct wim_lookup_table_entry *lte, bool is_partial_res, +hasher_begin_stream(struct wim_lookup_table_entry *lte, u32 flags, void *_ctx) { struct hasher_context *ctx = _ctx; @@ -1047,7 +1054,7 @@ hasher_begin_stream(struct wim_lookup_table_entry *lte, bool is_partial_res, if (ctx->cbs.begin_stream == NULL) return 0; else - return (*ctx->cbs.begin_stream)(lte, is_partial_res, + return (*ctx->cbs.begin_stream)(lte, flags, ctx->cbs.begin_stream_ctx); } @@ -1137,7 +1144,7 @@ read_full_stream_with_cbs(struct wim_lookup_table_entry *lte, { int ret; - ret = (*cbs->begin_stream)(lte, false, cbs->begin_stream_ctx); + ret = (*cbs->begin_stream)(lte, 0, cbs->begin_stream_ctx); if (ret) return ret; @@ -1165,7 +1172,6 @@ read_full_stream_with_sha1(struct wim_lookup_table_entry *lte, .consume_chunk_ctx = &hasher_ctx, .end_stream = hasher_end_stream, .end_stream_ctx = &hasher_ctx, - }; return read_full_stream_with_cbs(lte, &hasher_cbs); } diff --git a/src/write.c b/src/write.c index 620f8c45..92b30da2 100644 --- a/src/write.c +++ b/src/write.c @@ -613,7 +613,7 @@ end_write_resource(struct write_streams_ctx *ctx, struct wim_reshdr *out_reshdr) /* Begin processing a stream for writing. */ static int write_stream_begin_read(struct wim_lookup_table_entry *lte, - bool is_partial_res, void *_ctx) + u32 flags, void *_ctx) { struct write_streams_ctx *ctx = _ctx; int ret; @@ -637,7 +637,7 @@ write_stream_begin_read(struct wim_lookup_table_entry *lte, ctx->stream_was_duplicate = false; if (ctx->lookup_table != NULL && lte->unhashed && !lte->unique_size) { - wimlib_assert(!is_partial_res); + wimlib_assert(!(flags & BEGIN_STREAM_FLAG_PARTIAL_RESOURCE)); struct wim_lookup_table_entry *lte_new;