]> wimlib.net Git - wimlib/blobdiff - src/write.c
header cleanups
[wimlib] / src / write.c
index 5217838192cdb886cf98cdea9474a718e7120ed1..f3c12ffe4cbc8ffacced2e635ff3a352c773a21f 100644 (file)
@@ -6,22 +6,20 @@
  */
 
 /*
- * Copyright (C) 2012, 2013 Eric Biggers
+ * Copyright (C) 2012, 2013, 2014 Eric Biggers
  *
- * This file is part of wimlib, a library for working with WIM files.
+ * This file is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU Lesser General Public License as published by the Free
+ * Software Foundation; either version 3 of the License, or (at your option) any
+ * later version.
  *
- * wimlib is free software; you can redistribute it and/or modify it under the
- * terms of the GNU General Public License as published by the Free
- * Software Foundation; either version 3 of the License, or (at your option)
- * any later version.
- *
- * wimlib is distributed in the hope that it will be useful, but WITHOUT ANY
- * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
- * A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * This file is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
  * details.
  *
- * You should have received a copy of the GNU General Public License
- * along with wimlib; if not, see http://www.gnu.org/licenses/.
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this file; if not, see http://www.gnu.org/licenses/.
  */
 
 #ifdef HAVE_CONFIG_H
 #  include <sys/file.h>
 #endif
 
+#include <errno.h>
+#include <fcntl.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+#include "wimlib/alloca.h"
+#include "wimlib/assert.h"
 #include "wimlib/chunk_compressor.h"
 #include "wimlib/endianness.h"
 #include "wimlib/error.h"
@@ -43,6 +48,7 @@
 #include "wimlib/integrity.h"
 #include "wimlib/lookup_table.h"
 #include "wimlib/metadata.h"
+#include "wimlib/paths.h"
 #include "wimlib/progress.h"
 #include "wimlib/resource.h"
 #ifdef __WIN32__
 #include "wimlib/write.h"
 #include "wimlib/xml.h"
 
-#include <errno.h>
-#include <fcntl.h>
-#include <stdlib.h>
-#include <unistd.h>
-
-#ifdef HAVE_ALLOCA_H
-#  include <alloca.h>
-#endif
 
 /* wimlib internal flags used when writing resources.  */
 #define WRITE_RESOURCE_FLAG_RECOMPRESS         0x00000001
 #define WRITE_RESOURCE_FLAG_PIPABLE            0x00000002
 #define WRITE_RESOURCE_FLAG_PACK_STREAMS       0x00000004
+#define WRITE_RESOURCE_FLAG_SEND_DONE_WITH_FILE        0x00000008
 
 static inline int
 write_flags_to_resource_flags(int write_flags)
@@ -76,6 +75,8 @@ write_flags_to_resource_flags(int write_flags)
                write_resource_flags |= WRITE_RESOURCE_FLAG_PIPABLE;
        if (write_flags & WIMLIB_WRITE_FLAG_PACK_STREAMS)
                write_resource_flags |= WRITE_RESOURCE_FLAG_PACK_STREAMS;
+       if (write_flags & WIMLIB_WRITE_FLAG_SEND_DONE_WITH_FILE_MESSAGES)
+               write_resource_flags |= WRITE_RESOURCE_FLAG_SEND_DONE_WITH_FILE;
        return write_resource_flags;
 }
 
@@ -98,12 +99,15 @@ static int
 stream_filtered(const struct wim_lookup_table_entry *lte,
                const struct filter_context *ctx)
 {
-       int write_flags = ctx->write_flags;
-       WIMStruct *wim = ctx->wim;
+       int write_flags;
+       WIMStruct *wim;
 
        if (ctx == NULL)
                return 0;
 
+       write_flags = ctx->write_flags;
+       wim = ctx->wim;
+
        if (write_flags & WIMLIB_WRITE_FLAG_OVERWRITE &&
            lte->resource_location == RESOURCE_IN_WIM &&
            lte->rspec->wim == wim)
@@ -279,7 +283,6 @@ struct write_streams_progress_data {
 
 static int
 do_write_streams_progress(struct write_streams_progress_data *progress_data,
-                         struct wim_lookup_table_entry *cur_stream,
                          u64 complete_size,
                          u32 complete_count,
                          bool discarded)
@@ -312,10 +315,28 @@ do_write_streams_progress(struct write_streams_progress_data *progress_data,
                if (progress_data->next_progress == progress->write_streams.total_bytes) {
                        progress_data->next_progress = ~(uint64_t)0;
                } else {
+                       /* Handle rate-limiting of messages  */
+
+                       /* Send new message as soon as another 1/128 of the
+                        * total has been written.  (Arbitrary number.)  */
                        progress_data->next_progress =
-                               min(progress->write_streams.total_bytes,
-                                   progress->write_streams.completed_bytes +
-                                       progress->write_streams.total_bytes / 100);
+                               progress->write_streams.completed_bytes +
+                                       progress->write_streams.total_bytes / 128;
+
+                       /* ... Unless that would be more than 5000000 bytes, in
+                        * which case send the next after the next 5000000
+                        * bytes.  (Another arbitrary number.)  */
+                       if (progress->write_streams.completed_bytes + 5000000 <
+                           progress_data->next_progress)
+                               progress_data->next_progress =
+                                       progress->write_streams.completed_bytes + 5000000;
+
+                       /* ... But always send a message as soon as we're
+                        * completely done.  */
+                       if (progress->write_streams.total_bytes <
+                           progress_data->next_progress)
+                               progress_data->next_progress =
+                                       progress->write_streams.total_bytes;
                }
        }
        return 0;
@@ -366,12 +387,6 @@ struct write_streams_ctx {
         * @pending_streams only when writing a packed resource.  */
        struct list_head pack_streams;
 
-       /* Set to true if the stream currently being read was a duplicate, and
-        * therefore the corresponding stream entry needs to be freed once the
-        * read finishes.  (In this case we add the duplicate entry to
-        * pending_streams rather than the entry being read.)  */
-       bool stream_was_duplicate;
-
        /* Current uncompressed offset in the stream being read.  */
        u64 cur_read_stream_offset;
 
@@ -451,7 +466,7 @@ begin_chunk_table(struct write_streams_ctx *ctx, u64 res_expected_size)
                reserve_size = expected_num_chunk_entries *
                               get_chunk_entry_size(res_expected_size,
                                                    0 != (ctx->write_resource_flags &
-                                                         WIM_RESHDR_FLAG_PACKED_STREAMS));
+                                                         WRITE_RESOURCE_FLAG_PACK_STREAMS));
                if (ctx->write_resource_flags & WRITE_RESOURCE_FLAG_PACK_STREAMS)
                        reserve_size += sizeof(struct alt_chunk_table_header_disk);
                memset(ctx->chunk_csizes, 0, reserve_size);
@@ -499,10 +514,10 @@ end_chunk_table(struct write_streams_ctx *ctx, u64 res_actual_size,
 
        chunk_entry_size = get_chunk_entry_size(res_actual_size,
                                                0 != (ctx->write_resource_flags &
-                                                     WIM_RESHDR_FLAG_PACKED_STREAMS));
+                                                     WRITE_RESOURCE_FLAG_PACK_STREAMS));
 
-       typedef le64 __attribute__((may_alias)) aliased_le64_t;
-       typedef le32 __attribute__((may_alias)) aliased_le32_t;
+       typedef le64 _may_alias_attribute aliased_le64_t;
+       typedef le32 _may_alias_attribute aliased_le32_t;
 
        if (chunk_entry_size == 4) {
                aliased_le32_t *entries = (aliased_le32_t*)ctx->chunk_csizes;
@@ -541,7 +556,7 @@ end_chunk_table(struct write_streams_ctx *ctx, u64 res_actual_size,
        if (ctx->write_resource_flags & WRITE_RESOURCE_FLAG_PIPABLE) {
                ret = full_write(ctx->out_fd, ctx->chunk_csizes, chunk_table_size);
                if (ret)
-                       goto error;
+                       goto write_error;
                res_end_offset = ctx->out_fd->offset;
                res_start_offset = ctx->chunks_start_offset;
        } else {
@@ -565,7 +580,7 @@ end_chunk_table(struct write_streams_ctx *ctx, u64 res_actual_size,
                        ret = full_pwrite(ctx->out_fd, &hdr, sizeof(hdr),
                                          chunk_table_offset - sizeof(hdr));
                        if (ret)
-                               goto error;
+                               goto write_error;
                        res_start_offset = chunk_table_offset - sizeof(hdr);
                } else {
                        res_start_offset = chunk_table_offset;
@@ -574,7 +589,7 @@ end_chunk_table(struct write_streams_ctx *ctx, u64 res_actual_size,
                ret = full_pwrite(ctx->out_fd, ctx->chunk_csizes,
                                  chunk_table_size, chunk_table_offset);
                if (ret)
-                       goto error;
+                       goto write_error;
        }
 
        *res_start_offset_ret = res_start_offset;
@@ -582,7 +597,7 @@ end_chunk_table(struct write_streams_ctx *ctx, u64 res_actual_size,
 
        return 0;
 
-error:
+write_error:
        ERROR_WITH_ERRNO("Write error");
        return ret;
 }
@@ -618,10 +633,96 @@ end_write_resource(struct write_streams_ctx *ctx, struct wim_reshdr *out_reshdr)
        return 0;
 }
 
+/* No more data streams of the file at @path are needed.  */
+static int
+done_with_file(const tchar *path, wimlib_progress_func_t progfunc, void *progctx)
+{
+       union wimlib_progress_info info;
+
+       info.done_with_file.path_to_file = path;
+
+       return call_progress(progfunc, WIMLIB_PROGRESS_MSG_DONE_WITH_FILE,
+                            &info, progctx);
+}
+
+static inline bool
+is_file_stream(const struct wim_lookup_table_entry *lte)
+{
+       return lte->resource_location == RESOURCE_IN_FILE_ON_DISK
+#ifdef __WIN32__
+           || lte->resource_location == RESOURCE_IN_WINNT_FILE_ON_DISK
+           || lte->resource_location == RESOURCE_WIN32_ENCRYPTED
+#endif
+          ;
+}
+
+static int
+do_done_with_stream(struct wim_lookup_table_entry *lte,
+                   wimlib_progress_func_t progfunc, void *progctx)
+{
+       int ret;
+       struct wim_inode *inode;
+
+       if (!lte->may_send_done_with_file)
+               return 0;
+
+       inode = lte->file_inode;
+
+       wimlib_assert(inode != NULL);
+       wimlib_assert(inode->num_remaining_streams > 0);
+       if (--inode->num_remaining_streams > 0)
+               return 0;
+
+#ifdef __WIN32__
+       /* XXX: This logic really should be somewhere else.  */
+
+       /* We want the path to the file, but lte->file_on_disk might actually
+        * refer to a named data stream.  Temporarily strip the named data
+        * stream from the path.  */
+       wchar_t *p_colon = NULL;
+       wchar_t *p_question_mark = NULL;
+       const wchar_t *p_stream_name;
+
+       p_stream_name = path_stream_name(lte->file_on_disk);
+       if (unlikely(p_stream_name)) {
+               p_colon = (wchar_t *)(p_stream_name - 1);
+               wimlib_assert(*p_colon == L':');
+               *p_colon = L'\0';
+       }
+
+       /* We also should use a fake Win32 path instead of a NT path  */
+       if (!wcsncmp(lte->file_on_disk, L"\\??\\", 4)) {
+               p_question_mark = &lte->file_on_disk[1];
+               *p_question_mark = L'\\';
+       }
+#endif
+
+       ret = done_with_file(lte->file_on_disk, progfunc, progctx);
+
+#ifdef __WIN32__
+       if (p_colon)
+               *p_colon = L':';
+       if (p_question_mark)
+               *p_question_mark = L'?';
+#endif
+       return ret;
+}
+
+/* Handle WIMLIB_WRITE_FLAG_SEND_DONE_WITH_FILE_MESSAGES mode.  */
+static inline int
+done_with_stream(struct wim_lookup_table_entry *lte,
+                struct write_streams_ctx *ctx)
+{
+       if (likely(!(ctx->write_resource_flags &
+                    WRITE_RESOURCE_FLAG_SEND_DONE_WITH_FILE)))
+               return 0;
+       return do_done_with_stream(lte, ctx->progress_data.progfunc,
+                                  ctx->progress_data.progctx);
+}
+
 /* Begin processing a stream for writing.  */
 static int
-write_stream_begin_read(struct wim_lookup_table_entry *lte,
-                       u32 flags, void *_ctx)
+write_stream_begin_read(struct wim_lookup_table_entry *lte, void *_ctx)
 {
        struct write_streams_ctx *ctx = _ctx;
        int ret;
@@ -642,11 +743,8 @@ write_stream_begin_read(struct wim_lookup_table_entry *lte,
         * still provide the data again to write_stream_process_chunk().  This
         * is okay because an unhashed stream cannot be in a WIM resource, which
         * might be costly to decompress.  */
-       ctx->stream_was_duplicate = false;
        if (ctx->lookup_table != NULL && lte->unhashed && !lte->unique_size) {
 
-               wimlib_assert(!(flags & BEGIN_STREAM_FLAG_PARTIAL_RESOURCE));
-
                struct wim_lookup_table_entry *lte_new;
 
                ret = hash_unhashed_stream(lte, ctx->lookup_table, &lte_new);
@@ -667,7 +765,7 @@ write_stream_begin_read(struct wim_lookup_table_entry *lte,
                                DEBUG("Discarding duplicate stream of "
                                      "length %"PRIu64, lte->size);
                                ret = do_write_streams_progress(&ctx->progress_data,
-                                                               lte, lte->size,
+                                                               lte->size,
                                                                1, true);
                                list_del(&lte->write_streams_list);
                                list_del(&lte->lookup_table_list);
@@ -675,6 +773,8 @@ write_stream_begin_read(struct wim_lookup_table_entry *lte,
                                        lte_new->out_refcnt += lte->out_refcnt;
                                if (ctx->write_resource_flags & WRITE_RESOURCE_FLAG_PACK_STREAMS)
                                        ctx->cur_write_res_size -= lte->size;
+                               if (!ret)
+                                       ret = done_with_stream(lte, ctx);
                                free_lookup_table_entry(lte);
                                if (ret)
                                        return ret;
@@ -692,9 +792,10 @@ write_stream_begin_read(struct wim_lookup_table_entry *lte,
                                             &lte_new->write_streams_list);
                                list_replace(&lte->lookup_table_list,
                                             &lte_new->lookup_table_list);
+                               lte->will_be_in_output_wim = 0;
                                lte_new->out_refcnt = lte->out_refcnt;
                                lte_new->will_be_in_output_wim = 1;
-                               ctx->stream_was_duplicate = true;
+                               lte_new->may_send_done_with_file = 0;
                                lte = lte_new;
                        }
                }
@@ -751,6 +852,73 @@ write_stream_uncompressed(struct wim_lookup_table_entry *lte,
        return 0;
 }
 
+/* Returns true if the specified stream should be truncated from the WIM file
+ * and re-written as uncompressed.  lte->out_reshdr must be filled in from the
+ * initial write of the stream.  */
+static bool
+should_rewrite_stream_uncompressed(const struct write_streams_ctx *ctx,
+                                  const struct wim_lookup_table_entry *lte)
+{
+       /* If the compressed data is smaller than the uncompressed data, prefer
+        * the compressed data.  */
+       if (lte->out_reshdr.size_in_wim < lte->out_reshdr.uncompressed_size)
+               return false;
+
+       /* If we're not actually writing compressed data, then there's no need
+        * for re-writing.  */
+       if (!ctx->compressor)
+               return false;
+
+       /* If writing a pipable WIM, everything we write to the output is final
+        * (it might actually be a pipe!).  */
+       if (ctx->write_resource_flags & WRITE_RESOURCE_FLAG_PIPABLE)
+               return false;
+
+       /* If the stream that would need to be re-read is located in a solid
+        * block in another WIM file, then re-reading it would be costly.  So
+        * don't do it.
+        *
+        * Exception: if the compressed size happens to be *exactly* the same as
+        * the uncompressed size, then the stream *must* be written uncompressed
+        * in order to remain compatible with the Windows Overlay Filesystem
+        * Filter Driver (WOF).
+        *
+        * TODO: we are currently assuming that the optimization for
+        * single-chunk resources in maybe_rewrite_stream_uncompressed()
+        * prevents this case from being triggered too often.  To fully prevent
+        * excessive decompressions in degenerate cases, we really should
+        * obtain the uncompressed data by decompressing the compressed data we
+        * wrote to the output file.
+        */
+       if ((lte->flags & WIM_RESHDR_FLAG_PACKED_STREAMS) &&
+           (lte->out_reshdr.size_in_wim != lte->out_reshdr.uncompressed_size))
+               return false;
+
+       return true;
+}
+
+static int
+maybe_rewrite_stream_uncompressed(struct write_streams_ctx *ctx,
+                                 struct wim_lookup_table_entry *lte)
+{
+       if (!should_rewrite_stream_uncompressed(ctx, lte))
+               return 0;
+
+       /* Regular (non-solid) WIM resources with exactly one chunk and
+        * compressed size equal to uncompressed size are exactly the same as
+        * the corresponding compressed data --- since there must be 0 entries
+        * in the chunk table and the only chunk must be stored uncompressed.
+        * In this case, there's no need to rewrite anything.  */
+       if (ctx->chunk_index == 1 &&
+           lte->out_reshdr.size_in_wim == lte->out_reshdr.uncompressed_size)
+       {
+               lte->out_reshdr.flags &= ~WIM_RESHDR_FLAG_COMPRESSED;
+               return 0;
+       }
+
+       return write_stream_uncompressed(lte, ctx->out_fd);
+}
+
 /* Write the next chunk of (typically compressed) data to the output WIM,
  * handling the writing of the chunk table.  */
 static int
@@ -804,14 +972,14 @@ write_chunk(struct write_streams_ctx *ctx, const void *cchunk,
                        ret = full_write(ctx->out_fd, &chunk_hdr,
                                         sizeof(chunk_hdr));
                        if (ret)
-                               goto error;
+                               goto write_error;
                }
        }
 
        /* Write the chunk data.  */
        ret = full_write(ctx->out_fd, cchunk, csize);
        if (ret)
-               goto error;
+               goto write_error;
 
        ctx->cur_write_stream_offset += usize;
 
@@ -820,36 +988,32 @@ write_chunk(struct write_streams_ctx *ctx, const void *cchunk,
        if (ctx->write_resource_flags & WRITE_RESOURCE_FLAG_PACK_STREAMS) {
                /* Wrote chunk in packed mode.  It may have finished multiple
                 * streams.  */
-               while (ctx->cur_write_stream_offset > lte->size) {
-                       struct wim_lookup_table_entry *next;
+               struct wim_lookup_table_entry *next_lte;
+
+               while (lte && ctx->cur_write_stream_offset >= lte->size) {
 
                        ctx->cur_write_stream_offset -= lte->size;
 
-                       wimlib_assert(!list_is_singular(&ctx->pending_streams) &&
-                                     !list_empty(&ctx->pending_streams));
-                       next = list_entry(lte->write_streams_list.next,
-                                         struct wim_lookup_table_entry,
-                                         write_streams_list);
-                       list_move_tail(&lte->write_streams_list,
-                                      &ctx->pack_streams);
-                       lte = next;
-                       completed_stream_count++;
-               }
-               if (ctx->cur_write_stream_offset == lte->size) {
-                       ctx->cur_write_stream_offset = 0;
-                       list_move_tail(&lte->write_streams_list,
-                                      &ctx->pack_streams);
+                       if (ctx->cur_write_stream_offset)
+                               next_lte = list_entry(lte->write_streams_list.next,
+                                                     struct wim_lookup_table_entry,
+                                                     write_streams_list);
+                       else
+                               next_lte = NULL;
+
+                       ret = done_with_stream(lte, ctx);
+                       if (ret)
+                               return ret;
+                       list_move_tail(&lte->write_streams_list, &ctx->pack_streams);
                        completed_stream_count++;
+
+                       lte = next_lte;
                }
        } else {
                /* Wrote chunk in non-packed mode.  It may have finished a
                 * stream.  */
                if (ctx->cur_write_stream_offset == lte->size) {
 
-                       completed_stream_count++;
-
-                       list_del(&lte->write_streams_list);
-
                        wimlib_assert(ctx->cur_write_stream_offset ==
                                      ctx->cur_write_res_size);
 
@@ -861,36 +1025,27 @@ write_chunk(struct write_streams_ctx *ctx, const void *cchunk,
                        if (ctx->compressor != NULL)
                                lte->out_reshdr.flags |= WIM_RESHDR_FLAG_COMPRESSED;
 
-                       if (ctx->compressor != NULL &&
-                           lte->out_reshdr.size_in_wim >= lte->out_reshdr.uncompressed_size &&
-                           !(ctx->write_resource_flags & WRITE_RESOURCE_FLAG_PIPABLE) &&
-                           !(lte->flags & WIM_RESHDR_FLAG_PACKED_STREAMS))
-                       {
-                               /* Stream did not compress to less than its original
-                                * size.  If we're not writing a pipable WIM (which
-                                * could mean the output file descriptor is
-                                * non-seekable), and the stream isn't located in a
-                                * resource pack (which would make reading it again
-                                * costly), truncate the file to the start of the stream
-                                * and write it uncompressed instead.  */
-                               DEBUG("Stream of size %"PRIu64" did not compress to "
-                                     "less than original size; writing uncompressed.",
-                                     lte->size);
-                               ret = write_stream_uncompressed(lte, ctx->out_fd);
-                               if (ret)
-                                       return ret;
-                       }
+                       ret = maybe_rewrite_stream_uncompressed(ctx, lte);
+                       if (ret)
+                               return ret;
+
                        wimlib_assert(lte->out_reshdr.uncompressed_size == lte->size);
 
                        ctx->cur_write_stream_offset = 0;
+
+                       ret = done_with_stream(lte, ctx);
+                       if (ret)
+                               return ret;
+                       list_del(&lte->write_streams_list);
+                       completed_stream_count++;
                }
        }
 
-       return do_write_streams_progress(&ctx->progress_data, lte,
+       return do_write_streams_progress(&ctx->progress_data,
                                         completed_size, completed_stream_count,
                                         false);
 
-error:
+write_error:
        ERROR_WITH_ERRNO("Write error");
        return ret;
 }
@@ -1006,11 +1161,27 @@ static int
 write_stream_end_read(struct wim_lookup_table_entry *lte, int status, void *_ctx)
 {
        struct write_streams_ctx *ctx = _ctx;
-       if (status == 0)
-               wimlib_assert(ctx->cur_read_stream_offset == ctx->cur_read_stream_size);
-       if (ctx->stream_was_duplicate) {
+
+       wimlib_assert(ctx->cur_read_stream_offset == ctx->cur_read_stream_size || status);
+
+       if (!lte->will_be_in_output_wim) {
+               /* The 'lte' stream was a duplicate.  Now that its data has
+                * finished being read, it is being discarded in favor of the
+                * duplicate entry.  It therefore is no longer needed, and we
+                * can fire the DONE_WITH_FILE callback because the file will
+                * not be read again.
+                *
+                * Note: we can't yet fire DONE_WITH_FILE for non-duplicate
+                * streams, since it needs to be possible to re-read the file if
+                * it does not compress to less than its original size.  */
+               if (!status)
+                       status = done_with_stream(lte, ctx);
                free_lookup_table_entry(lte);
-       } else if (lte->unhashed && ctx->lookup_table != NULL) {
+       } else if (!status && lte->unhashed && ctx->lookup_table != NULL) {
+               /* The 'lte' stream was not a duplicate and was previously
+                * unhashed.  Since we passed COMPUTE_MISSING_STREAM_HASHES to
+                * read_stream_list(), lte->hash is now computed and valid.  So
+                * turn this stream into a "hashed" stream.  */
                list_del(&lte->unhashed_list);
                lookup_table_insert(ctx->lookup_table, lte);
                lte->unhashed = 0;
@@ -1177,7 +1348,7 @@ write_raw_copy_resources(struct list_head *raw_copy_streams,
                                return ret;
                        lte->rspec->raw_copy_ok = 0;
                }
-               ret = do_write_streams_progress(progress_data, lte, lte->size,
+               ret = do_write_streams_progress(progress_data, lte->size,
                                                1, false);
                if (ret)
                        return ret;
@@ -1229,6 +1400,25 @@ remove_zero_length_streams(struct list_head *stream_list)
        }
 }
 
+static void
+init_done_with_file_info(struct list_head *stream_list)
+{
+       struct wim_lookup_table_entry *lte;
+
+       list_for_each_entry(lte, stream_list, write_streams_list) {
+               if (is_file_stream(lte)) {
+                       lte->file_inode->num_remaining_streams = 0;
+                       lte->may_send_done_with_file = 1;
+               } else {
+                       lte->may_send_done_with_file = 0;
+               }
+       }
+
+       list_for_each_entry(lte, stream_list, write_streams_list)
+               if (lte->may_send_done_with_file)
+                       lte->file_inode->num_remaining_streams++;
+}
+
 /*
  * Write a list of streams to the output WIM file.
  *
@@ -1261,9 +1451,8 @@ remove_zero_length_streams(struct list_head *stream_list)
  *
  * @out_ctype
  *     Compression format to use to write the output streams, specified as one
- *     of the WIMLIB_COMPRESSION_TYPE_* constants, excepting
- *     WIMLIB_COMPRESSION_TYPE_INVALID but including
- *     WIMLIB_COMPRESSION_TYPE_NONE.
+ *     of the WIMLIB_COMPRESSION_TYPE_* constants.
+ *     WIMLIB_COMPRESSION_TYPE_NONE is allowed.
  *
  * @out_chunk_size
  *     Chunk size to use to write the streams.  It must be a valid chunk size
@@ -1365,6 +1554,11 @@ write_stream_list(struct list_head *stream_list,
                return 0;
        }
 
+       /* If needed, set auxiliary information so that we can detect when the
+        * library has finished using each external file.  */
+       if (unlikely(write_resource_flags & WRITE_RESOURCE_FLAG_SEND_DONE_WITH_FILE))
+               init_done_with_file_info(stream_list);
+
        memset(&ctx, 0, sizeof(ctx));
 
        /* Pre-sorting the streams is required for compute_stream_list_stats().
@@ -1434,9 +1628,10 @@ write_stream_list(struct list_head *stream_list,
                                                            out_chunk_size,
                                                            num_threads, 0,
                                                            &ctx.compressor);
-                       if (ret) {
-                               DEBUG("Couldn't create parallel chunk compressor "
-                                     "(status %d)", ret);
+                       if (ret > 0) {
+                               WARNING("Couldn't create parallel chunk compressor: %"TS".\n"
+                                       "          Falling back to single-threaded compression.",
+                                       wimlib_get_error_string(ret));
                        }
                }
        #endif
@@ -2265,10 +2460,6 @@ write_wim_lookup_table(WIMStruct *wim, int image, int write_flags,
  *     (private) WIMLIB_WRITE_FLAG_NO_LOOKUP_TABLE:
  *             Don't write the lookup table.
  *
- *     (private) WIMLIB_WRITE_FLAG_REUSE_INTEGRITY_TABLE:
- *             When (if) writing the integrity table, re-use entries from the
- *             existing integrity table, if possible.
- *
  *     (private) WIMLIB_WRITE_FLAG_CHECKPOINT_AFTER_XML:
  *             After writing the XML data but before writing the integrity
  *             table, write a temporary WIM header and flush the stream so that
@@ -2285,6 +2476,9 @@ write_wim_lookup_table(WIMStruct *wim, int image, int write_flags,
  *             Use the existing <TOTALBYTES> stored in the in-memory XML
  *             information, rather than setting it to the offset of the XML
  *             data being written.
+ *     (private) WIMLIB_WRITE_FLAG_OVERWRITE
+ *             The existing WIM file is being updated in-place.  The entries
+ *             from its integrity table may be re-used.
  */
 static int
 finish_write(WIMStruct *wim, int image, int write_flags,
@@ -2293,9 +2487,10 @@ finish_write(WIMStruct *wim, int image, int write_flags,
        int ret;
        off_t hdr_offset;
        int write_resource_flags;
-       off_t old_lookup_table_end;
+       off_t old_lookup_table_end = 0;
        off_t new_lookup_table_end;
        u64 xml_totalbytes;
+       struct integrity_table *old_integrity_table = NULL;
 
        DEBUG("image=%d, write_flags=%08x", image, write_flags);
 
@@ -2314,15 +2509,36 @@ finish_write(WIMStruct *wim, int image, int write_flags,
                                wim->hdr.boot_idx - 1]->metadata_lte->out_reshdr);
        }
 
-       /* Write lookup table.  (Save old position first.)  */
-       old_lookup_table_end = wim->hdr.lookup_table_reshdr.offset_in_wim +
-                              wim->hdr.lookup_table_reshdr.size_in_wim;
+       /* If overwriting the WIM file containing an integrity table in-place,
+        * we'd like to re-use the information in the old integrity table
+        * instead of recalculating it.  But we might overwrite the old
+        * integrity table when we expand the XML data.  Read it into memory
+        * just in case.  */
+       if ((write_flags & (WIMLIB_WRITE_FLAG_OVERWRITE |
+                           WIMLIB_WRITE_FLAG_CHECK_INTEGRITY)) ==
+               (WIMLIB_WRITE_FLAG_OVERWRITE |
+                WIMLIB_WRITE_FLAG_CHECK_INTEGRITY)
+           && wim_has_integrity_table(wim))
+       {
+               old_lookup_table_end = wim->hdr.lookup_table_reshdr.offset_in_wim +
+                                      wim->hdr.lookup_table_reshdr.size_in_wim;
+               (void)read_integrity_table(wim,
+                                          old_lookup_table_end - WIM_HEADER_DISK_SIZE,
+                                          &old_integrity_table);
+               /* If we couldn't read the old integrity table, we can still
+                * re-calculate the full integrity table ourselves.  Hence the
+                * ignoring of the return value.  */
+       }
+
+       /* Write lookup table.  */
        if (!(write_flags & WIMLIB_WRITE_FLAG_NO_LOOKUP_TABLE)) {
                ret = write_wim_lookup_table(wim, image, write_flags,
                                             &wim->hdr.lookup_table_reshdr,
                                             lookup_table_list);
-               if (ret)
+               if (ret) {
+                       free_integrity_table(old_integrity_table);
                        return ret;
+               }
        }
 
        /* Write XML data.  */
@@ -2332,8 +2548,10 @@ finish_write(WIMStruct *wim, int image, int write_flags,
        ret = write_wim_xml_data(wim, image, xml_totalbytes,
                                 &wim->hdr.xml_data_reshdr,
                                 write_resource_flags);
-       if (ret)
+       if (ret) {
+               free_integrity_table(old_integrity_table);
                return ret;
+       }
 
        /* Write integrity table (optional).  */
        if (write_flags & WIMLIB_WRITE_FLAG_CHECK_INTEGRITY) {
@@ -2344,19 +2562,20 @@ finish_write(WIMStruct *wim, int image, int write_flags,
                        checkpoint_hdr.flags |= WIM_HDR_FLAG_WRITE_IN_PROGRESS;
                        ret = write_wim_header_at_offset(&checkpoint_hdr,
                                                         &wim->out_fd, 0);
-                       if (ret)
+                       if (ret) {
+                               free_integrity_table(old_integrity_table);
                                return ret;
+                       }
                }
 
-               if (!(write_flags & WIMLIB_WRITE_FLAG_REUSE_INTEGRITY_TABLE))
-                       old_lookup_table_end = 0;
-
                new_lookup_table_end = wim->hdr.lookup_table_reshdr.offset_in_wim +
                                       wim->hdr.lookup_table_reshdr.size_in_wim;
 
                ret = write_integrity_table(wim,
                                            new_lookup_table_end,
-                                           old_lookup_table_end);
+                                           old_lookup_table_end,
+                                           old_integrity_table);
+               free_integrity_table(old_integrity_table);
                if (ret)
                        return ret;
        } else {
@@ -3028,7 +3247,7 @@ overwrite_wim_inplace(WIMStruct *wim, int write_flags, unsigned num_threads)
         * allow any file and metadata resources to appear without returning
         * WIMLIB_ERR_RESOURCE_ORDER (due to the fact that we would otherwise
         * overwrite these resources). */
-       if (!wim->deletion_occurred && !any_images_modified(wim)) {
+       if (!wim->image_deletion_occurred && !any_images_modified(wim)) {
                /* If no images have been modified and no images have been
                 * deleted, a new lookup table does not need to be written.  We
                 * shall write the new XML data and optional integrity table
@@ -3094,7 +3313,6 @@ overwrite_wim_inplace(WIMStruct *wim, int write_flags, unsigned num_threads)
        if (ret)
                goto out_truncate;
 
-       write_flags |= WIMLIB_WRITE_FLAG_REUSE_INTEGRITY_TABLE;
        ret = finish_write(wim, WIMLIB_ALL_IMAGES, write_flags,
                           &lookup_table_list);
        if (ret)
@@ -3186,8 +3404,9 @@ can_overwrite_wim_inplace(const WIMStruct *wim, int write_flags)
        if (write_flags & WIMLIB_WRITE_FLAG_REBUILD)
                return false;
 
-       /* Deletions cause full rebuild by default.  */
-       if (wim->deletion_occurred && !(write_flags & WIMLIB_WRITE_FLAG_SOFT_DELETE))
+       /* Image deletions cause full rebuild by default.  */
+       if (wim->image_deletion_occurred &&
+           !(write_flags & WIMLIB_WRITE_FLAG_SOFT_DELETE))
                return false;
 
        /* Pipable WIMs cannot be updated in place, nor can a non-pipable WIM be