Don't create unnecessary temporary files

A temporary file is unnecessary (without too much extra trouble) if the
whole stream data is already in memory as a result of reading a packed
resource.

diff --git a/include/wimlib/apply.h b/include/wimlib/apply.h
index 24648c9b3a378c19581881be787ffeb5f0d45574..3b124d31c4936fb1e55e35247d9e1ef757bac498 100644 (file)
--- a/include/wimlib/apply.h
+++ b/include/wimlib/apply.h
@@ -223,6 +223,7 @@ struct apply_ctx {
        unsigned long invalid_sequence;
        unsigned long partial_security_descriptors;
        unsigned long no_security_descriptors;
+       struct wim_lookup_table_entry *cur_stream;
        struct filedes tmpfile_fd;
        tchar *tmpfile_name;
        u64 num_streams_remaining;

diff --git a/include/wimlib/resource.h b/include/wimlib/resource.h
index b2ba851b3a7819e1b95ac6ce2d642f9462a5b279..ae0775536b726e138000cfc4fd77dcbbd8feba48 100644 (file)
--- a/include/wimlib/resource.h
+++ b/include/wimlib/resource.h
@@ -216,10 +216,9 @@ skip_wim_stream(struct wim_lookup_table_entry *lte);
  * @lte:
  *     Stream that is about to be read.
  *
- * @is_partial_res:
- *     Set to true if the stream is just one of several being read from a
- *     single pack and therefore would be extra expensive to read
- *     independently.
+ * @flags:
+ *     Bitwise OR of BEGIN_STREAM_FLAG_PARTIAL_RESOURCE and/or
+ *     BEGIN_STREAM_FLAG_WHOLE_STREAM.
  *
  * @ctx:
  *     User-provided context.
@@ -230,9 +229,17 @@ skip_wim_stream(struct wim_lookup_table_entry *lte);
  * (without calling @consume_chunk or @end_stream).
  */
 typedef int (*read_stream_list_begin_stream_t)(struct wim_lookup_table_entry *lte,
-                                              bool is_partial_res,
+                                              u32 flags,
                                               void *ctx);
 
+/* Set to true if the stream is just one of several being read from a single
+ * pack and therefore would be extra expensive to read independently.  */
+#define BEGIN_STREAM_FLAG_PARTIAL_RESOURCE     0x00000001
+
+/* This is purely advisory and indicates that the entire stream data will be
+ * provided in one call to consume_chunk().  */
+#define BEGIN_STREAM_FLAG_WHOLE_STREAM         0x00000002
+
 #define BEGIN_STREAM_STATUS_SKIP_STREAM        -1
 
 /*

diff --git a/src/extract.c b/src/extract.c
index f1fcd0f86b5079b9ef020a03e35287c2edfbf831..bc769eec798d284f61ddee9f28fd006304b616bd 100644 (file)
--- a/src/extract.c
+++ b/src/extract.c
@@ -1273,14 +1273,23 @@ need_tmpfile_to_extract(struct wim_lookup_table_entry *lte,
 }
 
 static int
-begin_extract_stream_to_tmpfile(struct wim_lookup_table_entry *lte,
-                               bool is_partial_res,
-                               void *_ctx)
+begin_extract_stream(struct wim_lookup_table_entry *lte,
+                    u32 flags, void *_ctx)
 {
        struct apply_ctx *ctx = _ctx;
        int ret;
 
-       if (!need_tmpfile_to_extract(lte, is_partial_res)) {
+       if (flags & BEGIN_STREAM_FLAG_WHOLE_STREAM) {
+               DEBUG("Whole stream (size=%"PRIu64") will be read into memory",
+                     lte->size);
+               ctx->cur_stream = lte;
+               filedes_invalidate(&ctx->tmpfile_fd);
+               return 0;
+       }
+
+       if (!need_tmpfile_to_extract(lte,
+                                    (flags & BEGIN_STREAM_FLAG_PARTIAL_RESOURCE)))
+       {
                DEBUG("Temporary file not needed "
                      "for stream (size=%"PRIu64")", lte->size);
                ret = extract_stream_instances(lte, lte, ctx);
@@ -1295,14 +1304,42 @@ begin_extract_stream_to_tmpfile(struct wim_lookup_table_entry *lte,
 }
 
 static int
-end_extract_stream_to_tmpfile(struct wim_lookup_table_entry *lte,
-                             int status, void *_ctx)
+extract_chunk(const void *chunk, size_t size, void *_ctx)
+{
+       struct apply_ctx *ctx = _ctx;
+       int ret;
+
+       if (filedes_valid(&ctx->tmpfile_fd)) {
+               ret = full_write(&ctx->tmpfile_fd, chunk, size);
+               if (ret)
+                       ERROR_WITH_ERRNO("Error writing to file descriptor");
+       } else {
+               struct wim_lookup_table_entry lte_override;
+
+               memcpy(&lte_override, ctx->cur_stream,
+                      sizeof(struct wim_lookup_table_entry));
+
+               lte_override.resource_location = RESOURCE_IN_ATTACHED_BUFFER;
+               lte_override.size = size;
+               lte_override.attached_buffer = (void *)chunk;
+
+               ret = extract_stream_instances(ctx->cur_stream, &lte_override, ctx);
+       }
+       return ret;
+}
+
+static int
+end_extract_stream(struct wim_lookup_table_entry *lte,
+                  int status, void *_ctx)
 {
        struct apply_ctx *ctx = _ctx;
        struct wim_lookup_table_entry lte_override;
        int ret;
        int errno_save = errno;
 
+       if (!filedes_valid(&ctx->tmpfile_fd))
+               return status;
+
        ret = filedes_close(&ctx->tmpfile_fd);
 
        if (status) {
@@ -1342,11 +1379,11 @@ static int
 extract_stream_list(struct apply_ctx *ctx)
 {
        struct read_stream_list_callbacks cbs = {
-               .begin_stream           = begin_extract_stream_to_tmpfile,
+               .begin_stream           = begin_extract_stream,
                .begin_stream_ctx       = ctx,
-               .consume_chunk          = extract_chunk_to_fd,
-               .consume_chunk_ctx      = &ctx->tmpfile_fd,
-               .end_stream             = end_extract_stream_to_tmpfile,
+               .consume_chunk          = extract_chunk,
+               .consume_chunk_ctx      = ctx,
+               .end_stream             = end_extract_stream,
                .end_stream_ctx         = ctx,
        };
        return read_stream_list(&ctx->stream_list,

diff --git a/src/resource.c b/src/resource.c
index b5db14527bdcb8cbd9bce60a728596ca1baba692..9d8fe921490a38775975f438f1cb2a5be46c0f81 100644 (file)
--- a/src/resource.c
+++ b/src/resource.c
@@ -988,10 +988,17 @@ streamifier_cb(const void *chunk, size_t size, void *_ctx)
        wimlib_assert(size <= ctx->cur_stream->size - ctx->cur_stream_offset);
 
        if (ctx->cur_stream_offset == 0) {
+               u32 flags;
+
                /* Starting a new stream.  */
                DEBUG("Begin new stream (size=%"PRIu64").",
                      ctx->cur_stream->size);
-               ret = (*ctx->cbs.begin_stream)(ctx->cur_stream, true,
+
+               flags = BEGIN_STREAM_FLAG_PARTIAL_RESOURCE;
+               if (size == ctx->cur_stream->size)
+                       flags |= BEGIN_STREAM_FLAG_WHOLE_STREAM;
+               ret = (*ctx->cbs.begin_stream)(ctx->cur_stream,
+                                              flags,
                                               ctx->cbs.begin_stream_ctx);
                if (ret)
                        return ret;
@@ -1037,7 +1044,7 @@ struct hasher_context {
 /* Callback for starting to read a stream while calculating its SHA1 message
  * digest.  */
 static int
-hasher_begin_stream(struct wim_lookup_table_entry *lte, bool is_partial_res,
+hasher_begin_stream(struct wim_lookup_table_entry *lte, u32 flags,
                    void *_ctx)
 {
        struct hasher_context *ctx = _ctx;
@@ -1047,7 +1054,7 @@ hasher_begin_stream(struct wim_lookup_table_entry *lte, bool is_partial_res,
        if (ctx->cbs.begin_stream == NULL)
                return 0;
        else
-               return (*ctx->cbs.begin_stream)(lte, is_partial_res,
+               return (*ctx->cbs.begin_stream)(lte, flags,
                                                ctx->cbs.begin_stream_ctx);
 }
 
@@ -1137,7 +1144,7 @@ read_full_stream_with_cbs(struct wim_lookup_table_entry *lte,
 {
        int ret;
 
-       ret = (*cbs->begin_stream)(lte, false, cbs->begin_stream_ctx);
+       ret = (*cbs->begin_stream)(lte, 0, cbs->begin_stream_ctx);
        if (ret)
                return ret;
 
@@ -1165,7 +1172,6 @@ read_full_stream_with_sha1(struct wim_lookup_table_entry *lte,
                .consume_chunk_ctx      = &hasher_ctx,
                .end_stream             = hasher_end_stream,
                .end_stream_ctx         = &hasher_ctx,
-
        };
        return read_full_stream_with_cbs(lte, &hasher_cbs);
 }

diff --git a/src/write.c b/src/write.c
index 620f8c455afe4b605cff8b7789a8d4e73f8a745f..92b30da20818e370e5e485cdaee0e47932b24e11 100644 (file)
--- a/src/write.c
+++ b/src/write.c
@@ -613,7 +613,7 @@ end_write_resource(struct write_streams_ctx *ctx, struct wim_reshdr *out_reshdr)
 /* Begin processing a stream for writing.  */
 static int
 write_stream_begin_read(struct wim_lookup_table_entry *lte,
-                       bool is_partial_res, void *_ctx)
+                       u32 flags, void *_ctx)
 {
        struct write_streams_ctx *ctx = _ctx;
        int ret;
@@ -637,7 +637,7 @@ write_stream_begin_read(struct wim_lookup_table_entry *lte,
        ctx->stream_was_duplicate = false;
        if (ctx->lookup_table != NULL && lte->unhashed && !lte->unique_size) {
 
-               wimlib_assert(!is_partial_res);
+               wimlib_assert(!(flags & BEGIN_STREAM_FLAG_PARTIAL_RESOURCE));
 
                struct wim_lookup_table_entry *lte_new;