Fix for rewriting uncompressed resources when exporting from solid WIM

The changes from commit 1ba2a3422b ("Workaround for WOF incompatibility")
are resulting in excessive, expensive decompressions when exporting small
incompressible files from solid blocks.  For now, add an optimization for
single chunk resources which prevents this common case from triggering
the degenerate behavior.

diff --git a/NEWS b/NEWS
index 7d7ab340615e7efdb25cbce3bd83b34ca5795e1c..7ade11fda37c1adb330ead2748acd7a292ff55c5 100644 (file)
--- a/NEWS
+++ b/NEWS
@@ -1,4 +1,9 @@
 Version 1.7.3-BETA:
+       Fix for very slow export from solid WIM / ESD files.
+
+       New progress message: WIMLIB_PROGRESS_MSG_HANDLE_ERROR.  Applications
+       may use this to treat some types of errors as non-fatal.
+
        The library now permits making in-memory changes to a WIMStruct backed
        by a read-only WIM file.
 
@@ -16,13 +21,6 @@ Version 1.7.3-BETA:
                Added a hack to try to work around an intermittent bug in
                Microsoft's WOF (Windows Overlay Filesystem) driver.
 
-       Added a workaround that tries to prevent short filename conflicts.  It
-       can have an effect primarily on versions of Windows that do not support
-       removing short names from files (earlier than Windows 7).
-
-       New progress message: WIMLIB_PROGRESS_MSG_HANDLE_ERROR.  Applications
-       may use this to treat some types of errors as non-fatal.
-
 Version 1.7.2:
        Made more improvements to the XPRESS, LZX, and LZMS compressors.
 

diff --git a/src/write.c b/src/write.c
index 7da18891132f466374a1cd72766e27f7758ade82..908de9adbb17b18ca645aabe33fbfc316bdd685f 100644 (file)
--- a/src/write.c
+++ b/src/write.c
@@ -882,7 +882,15 @@ should_rewrite_stream_uncompressed(const struct write_streams_ctx *ctx,
         * Exception: if the compressed size happens to be *exactly* the same as
         * the uncompressed size, then the stream *must* be written uncompressed
         * in order to remain compatible with the Windows Overlay Filesystem
-        * Filter Driver (WOF).  */
+        * Filter Driver (WOF).
+        *
+        * TODO: we are currently assuming that the optimization for
+        * single-chunk resources in maybe_rewrite_stream_uncompressed()
+        * prevents this case from being triggered too often.  To fully prevent
+        * excessive decompressions in degenerate cases, we really should
+        * obtain the uncompressed data by decompressing the compressed data we
+        * wrote to the output file.
+        */
        if ((lte->flags & WIM_RESHDR_FLAG_PACKED_STREAMS) &&
            (lte->out_reshdr.size_in_wim != lte->out_reshdr.uncompressed_size))
                return false;
@@ -890,6 +898,28 @@ should_rewrite_stream_uncompressed(const struct write_streams_ctx *ctx,
        return true;
 }
 
+static int
+maybe_rewrite_stream_uncompressed(struct write_streams_ctx *ctx,
+                                 struct wim_lookup_table_entry *lte)
+{
+       if (!should_rewrite_stream_uncompressed(ctx, lte))
+               return 0;
+
+       /* Regular (non-solid) WIM resources with exactly one chunk and
+        * compressed size equal to uncompressed size are exactly the same as
+        * the corresponding compressed data --- since there must be 0 entries
+        * in the chunk table and the only chunk must be stored uncompressed.
+        * In this case, there's no need to rewrite anything.  */
+       if (ctx->chunk_index == 1 &&
+           lte->out_reshdr.size_in_wim == lte->out_reshdr.uncompressed_size)
+       {
+               lte->out_reshdr.flags &= ~WIM_RESHDR_FLAG_COMPRESSED;
+               return 0;
+       }
+
+       return write_stream_uncompressed(lte, ctx->out_fd);
+}
+
 /* Write the next chunk of (typically compressed) data to the output WIM,
  * handling the writing of the chunk table.  */
 static int
@@ -996,14 +1026,10 @@ write_chunk(struct write_streams_ctx *ctx, const void *cchunk,
                        if (ctx->compressor != NULL)
                                lte->out_reshdr.flags |= WIM_RESHDR_FLAG_COMPRESSED;
 
-                       if (should_rewrite_stream_uncompressed(ctx, lte)) {
-                               DEBUG("Stream of size %"PRIu64" did not compress to "
-                                     "less than original size; writing uncompressed.",
-                                     lte->size);
-                               ret = write_stream_uncompressed(lte, ctx->out_fd);
-                               if (ret)
-                                       return ret;
-                       }
+                       ret = maybe_rewrite_stream_uncompressed(ctx, lte);
+                       if (ret)
+                               return ret;
+
                        wimlib_assert(lte->out_reshdr.uncompressed_size == lte->size);
 
                        ctx->cur_write_stream_offset = 0;