]> wimlib.net Git - wimlib/blobdiff - src/write.c
write_stream_list(): Fix comments
[wimlib] / src / write.c
index d4393728953bf572be4cd05d8ef6c9767331a89e..3fd1d045f0e5d7fbd582f33c5351cbd658521b78 100644 (file)
@@ -176,6 +176,9 @@ can_raw_copy(const struct wim_lookup_table_entry *lte,
                        rspec->wim->chunk_size == out_chunk_size);
        }
 
+       /* XXX: For compatibility, we can't allow multiple packed resources per
+        * WIM.  */
+#if 0
        if ((rspec->flags & WIM_RESHDR_FLAG_PACKED_STREAMS) &&
            (write_resource_flags & WRITE_RESOURCE_FLAG_PACK_STREAMS))
        {
@@ -198,6 +201,8 @@ can_raw_copy(const struct wim_lookup_table_entry *lte,
 
                return (write_size > rspec->uncompressed_size / 2);
        }
+#endif
+
        return false;
 }
 
@@ -277,23 +282,27 @@ struct write_streams_progress_data {
 
 static void
 do_write_streams_progress(struct write_streams_progress_data *progress_data,
-                         u64 size,
-                         bool discarded,
-                         struct wim_lookup_table_entry *cur_stream)
+                         struct wim_lookup_table_entry *cur_stream,
+                         u64 complete_size,
+                         u32 complete_count,
+                         bool discarded)
 {
        union wimlib_progress_info *progress = &progress_data->progress;
        bool new_wim_part;
 
        if (discarded) {
-               progress->write_streams.total_bytes -= size;
+               progress->write_streams.total_bytes -= complete_size;
+               progress->write_streams.total_streams -= complete_count;
                if (progress_data->next_progress != ~(uint64_t)0 &&
                    progress_data->next_progress > progress->write_streams.total_bytes)
                {
                        progress_data->next_progress = progress->write_streams.total_bytes;
                }
        } else {
-               progress->write_streams.completed_bytes += size;
+               progress->write_streams.completed_bytes += complete_size;
+               progress->write_streams.completed_streams += complete_count;
        }
+
        new_wim_part = false;
        if (cur_stream->resource_location == RESOURCE_IN_WIM &&
            cur_stream->rspec->wim != progress_data->prev_wim_part)
@@ -304,7 +313,7 @@ do_write_streams_progress(struct write_streams_progress_data *progress_data,
                }
                progress_data->prev_wim_part = cur_stream->rspec->wim;
        }
-       progress->write_streams.completed_streams++;
+
        if (progress_data->progress_func
            && (progress->write_streams.completed_bytes >= progress_data->next_progress
                || new_wim_part))
@@ -363,20 +372,24 @@ struct write_streams_ctx {
        /* List of streams that currently have chunks being compressed.  */
        struct list_head pending_streams;
 
+       /* List of streams in the resource pack.  Streams are moved here after
+        * @pending_streams only when writing a packed resource.  */
+       struct list_head pack_streams;
+
        /* Set to true if the stream currently being read was a duplicate, and
         * therefore the corresponding stream entry needs to be freed once the
         * read finishes.  (In this case we add the duplicate entry to
         * pending_streams rather than the entry being read.)  */
        bool stream_was_duplicate;
 
-       /* Current uncompressed offset in the resource being read.  */
-       u64 cur_read_res_offset;
+       /* Current uncompressed offset in the stream being read.  */
+       u64 cur_read_stream_offset;
 
-       /* Uncompressed size of the resource currently being read.  */
-       u64 cur_read_res_size;
+       /* Uncompressed size of the stream currently being read.  */
+       u64 cur_read_stream_size;
 
-       /* Current uncompressed offset in the resource being written.  */
-       u64 cur_write_res_offset;
+       /* Current uncompressed offset in the stream being written.  */
+       u64 cur_write_stream_offset;
 
        /* Uncompressed size of resource currently being written.  */
        u64 cur_write_res_size;
@@ -396,16 +409,6 @@ struct write_streams_ctx {
        u64 chunks_start_offset;
 };
 
-static u64
-get_chunk_entry_size(u64 res_size, int write_resource_flags)
-{
-       if (res_size <= UINT32_MAX ||
-           (write_resource_flags & WIM_RESHDR_FLAG_PACKED_STREAMS))
-               return 4;
-       else
-               return 8;
-}
-
 /* Reserve space for the chunk table and prepare to accumulate the chunk table
  * in memory.  */
 static int
@@ -457,7 +460,8 @@ begin_chunk_table(struct write_streams_ctx *ctx, u64 res_expected_size)
                 * are unknown.  */
                reserve_size = expected_num_chunk_entries *
                               get_chunk_entry_size(res_expected_size,
-                                                   ctx->write_resource_flags);
+                                                   0 != (ctx->write_resource_flags &
+                                                         WIM_RESHDR_FLAG_PACKED_STREAMS));
                if (ctx->write_resource_flags & WRITE_RESOURCE_FLAG_PACK_STREAMS)
                        reserve_size += sizeof(struct alt_chunk_table_header_disk);
                memset(ctx->chunk_csizes, 0, reserve_size);
@@ -484,7 +488,7 @@ begin_write_resource(struct write_streams_ctx *ctx, u64 res_expected_size)
        /* Output file descriptor is now positioned at the offset at which to
         * write the first chunk of the resource.  */
        ctx->chunks_start_offset = ctx->out_fd->offset;
-       ctx->cur_write_res_offset = 0;
+       ctx->cur_write_stream_offset = 0;
        ctx->cur_write_res_size = res_expected_size;
        return 0;
 }
@@ -504,7 +508,8 @@ end_chunk_table(struct write_streams_ctx *ctx, u64 res_actual_size,
                actual_num_chunk_entries--;
 
        chunk_entry_size = get_chunk_entry_size(res_actual_size,
-                                               ctx->write_resource_flags);
+                                               0 != (ctx->write_resource_flags &
+                                                     WIM_RESHDR_FLAG_PACKED_STREAMS));
 
        typedef le64 __attribute__((may_alias)) aliased_le64_t;
        typedef le32 __attribute__((may_alias)) aliased_le32_t;
@@ -602,7 +607,8 @@ end_write_resource(struct write_streams_ctx *ctx, struct wim_reshdr *out_reshdr)
        u64 res_uncompressed_size;
        u64 res_offset_in_wim;
 
-       wimlib_assert(ctx->cur_write_res_size == ctx->cur_write_res_offset);
+       wimlib_assert(ctx->cur_write_stream_offset == ctx->cur_write_res_size ||
+                     (ctx->write_resource_flags & WRITE_RESOURCE_FLAG_PACK_STREAMS));
        res_uncompressed_size = ctx->cur_write_res_size;
 
        if (ctx->compressor) {
@@ -632,8 +638,8 @@ write_stream_begin_read(struct wim_lookup_table_entry *lte,
 
        wimlib_assert(lte->size > 0);
 
-       ctx->cur_read_res_offset = 0;
-       ctx->cur_read_res_size = lte->size;
+       ctx->cur_read_stream_offset = 0;
+       ctx->cur_read_stream_size = lte->size;
 
        /* As an optimization, we allow some streams to be "unhashed", meaning
         * their SHA1 message digests are unknown.  This is the case with
@@ -671,7 +677,8 @@ write_stream_begin_read(struct wim_lookup_table_entry *lte,
                                DEBUG("Discarding duplicate stream of "
                                      "length %"PRIu64, lte->size);
                                do_write_streams_progress(&ctx->progress_data,
-                                                         lte->size, true, lte);
+                                                         lte, lte->size,
+                                                         1, true);
                                list_del(&lte->write_streams_list);
                                list_del(&lte->lookup_table_list);
                                if (lte_new->will_be_in_output_wim)
@@ -704,27 +711,48 @@ write_stream_begin_read(struct wim_lookup_table_entry *lte,
        return 0;
 }
 
+/* Rewrite a stream that was just written compressed as uncompressed instead.
+ * This function is optional, but if a stream did not compress to less than its
+ * original size, it might as well be written uncompressed.  */
 static int
 write_stream_uncompressed(struct wim_lookup_table_entry *lte,
                          struct filedes *out_fd)
 {
        int ret;
+       u64 begin_offset = lte->out_reshdr.offset_in_wim;
+       u64 end_offset = out_fd->offset;
+
+       if (filedes_seek(out_fd, begin_offset) == -1)
+               return 0;
 
-       if (-1 == lseek(out_fd->fd, lte->out_reshdr.offset_in_wim, SEEK_SET) ||
-           0 != ftruncate(out_fd->fd, lte->out_reshdr.offset_in_wim))
+       ret = extract_full_stream_to_fd(lte, out_fd);
+       if (ret) {
+               /* Error reading the uncompressed data.  */
+               if (out_fd->offset == begin_offset &&
+                   filedes_seek(out_fd, end_offset) != -1)
+               {
+                       /* Nothing was actually written yet, and we successfully
+                        * seeked to the end of the compressed resource, so
+                        * don't issue a hard error; just keep the compressed
+                        * resource instead.  */
+                       WARNING("Recovered compressed stream of "
+                               "size %"PRIu64", continuing on.",
+                               lte->size);
+                       return 0;
+               }
+               return ret;
+       }
+
+       wimlib_assert(out_fd->offset - begin_offset == lte->size);
+
+       if (out_fd->offset < end_offset &&
+           0 != ftruncate(out_fd->fd, out_fd->offset))
        {
                ERROR_WITH_ERRNO("Can't truncate output file to "
-                                "offset %"PRIu64, lte->out_reshdr.offset_in_wim);
+                                "offset %"PRIu64, out_fd->offset);
                return WIMLIB_ERR_WRITE;
        }
 
-       out_fd->offset = lte->out_reshdr.offset_in_wim;
-
-       ret = extract_stream_to_fd(lte, out_fd, lte->size);
-       if (ret)
-               return ret;
-
-       wimlib_assert(out_fd->offset - lte->out_reshdr.offset_in_wim == lte->size);
        lte->out_reshdr.size_in_wim = lte->size;
        lte->out_reshdr.flags &= ~(WIM_RESHDR_FLAG_COMPRESSED |
                                   WIM_RESHDR_FLAG_PACKED_STREAMS);
@@ -740,11 +768,13 @@ write_chunk(struct write_streams_ctx *ctx, const void *cchunk,
        int ret;
 
        struct wim_lookup_table_entry *lte;
+       u32 completed_stream_count;
+       u32 completed_size;
 
        lte = list_entry(ctx->pending_streams.next,
                         struct wim_lookup_table_entry, write_streams_list);
 
-       if (ctx->cur_write_res_offset == 0 &&
+       if (ctx->cur_write_stream_offset == 0 &&
            !(ctx->write_resource_flags & WRITE_RESOURCE_FLAG_PACK_STREAMS))
        {
                /* Starting to write a new stream in non-packed mode.  */
@@ -791,56 +821,83 @@ write_chunk(struct write_streams_ctx *ctx, const void *cchunk,
        if (ret)
                goto error;
 
-       ctx->cur_write_res_offset += usize;
+       ctx->cur_write_stream_offset += usize;
 
-       do_write_streams_progress(&ctx->progress_data,
-                                 usize, false, lte);
+       completed_size = usize;
+       completed_stream_count = 0;
+       if (ctx->write_resource_flags & WRITE_RESOURCE_FLAG_PACK_STREAMS) {
+               /* Wrote chunk in packed mode.  It may have finished multiple
+                * streams.  */
+               while (ctx->cur_write_stream_offset > lte->size) {
+                       struct wim_lookup_table_entry *next;
 
-       if (ctx->cur_write_res_offset == ctx->cur_write_res_size &&
-           !(ctx->write_resource_flags & WRITE_RESOURCE_FLAG_PACK_STREAMS))
-       {
-               struct wim_lookup_table_entry *lte;
+                       ctx->cur_write_stream_offset -= lte->size;
 
-               lte = list_entry(ctx->pending_streams.next,
-                                struct wim_lookup_table_entry, write_streams_list);
-               wimlib_assert(ctx->cur_write_res_offset == lte->size);
+                       wimlib_assert(!list_is_singular(&ctx->pending_streams) &&
+                                     !list_empty(&ctx->pending_streams));
+                       next = list_entry(lte->write_streams_list.next,
+                                         struct wim_lookup_table_entry,
+                                         write_streams_list);
+                       list_move_tail(&lte->write_streams_list,
+                                      &ctx->pack_streams);
+                       lte = next;
+                       completed_stream_count++;
+               }
+               if (ctx->cur_write_stream_offset == lte->size) {
+                       ctx->cur_write_stream_offset = 0;
+                       list_move_tail(&lte->write_streams_list,
+                                      &ctx->pack_streams);
+                       completed_stream_count++;
+               }
+       } else {
+               /* Wrote chunk in non-packed mode.  It may have finished a
+                * stream.  */
+               if (ctx->cur_write_stream_offset == lte->size) {
 
-               /* Finished writing a stream in non-packed mode.  */
+                       completed_stream_count++;
 
-               ret = end_write_resource(ctx, &lte->out_reshdr);
-               if (ret)
-                       return ret;
+                       list_del(&lte->write_streams_list);
 
-               lte->out_reshdr.flags = filter_resource_flags(lte->flags);
-               if (ctx->compressor != NULL)
-                       lte->out_reshdr.flags |= WIM_RESHDR_FLAG_COMPRESSED;
+                       wimlib_assert(ctx->cur_write_stream_offset ==
+                                     ctx->cur_write_res_size);
 
-               if (ctx->compressor != NULL &&
-                   lte->out_reshdr.size_in_wim >= lte->out_reshdr.uncompressed_size &&
-                   !(ctx->write_resource_flags & WRITE_RESOURCE_FLAG_PIPABLE) &&
-                   !(lte->flags & WIM_RESHDR_FLAG_PACKED_STREAMS))
-               {
-                       /* Stream did not compress to less than its original
-                        * size.  If we're not writing a pipable WIM (which
-                        * could mean the output file descriptor is
-                        * non-seekable), and the stream isn't located in a
-                        * resource pack (which would make reading it again
-                        * costly), truncate the file to the start of the stream
-                        * and write it uncompressed instead.  */
-                       DEBUG("Stream of size %"PRIu64" did not compress to "
-                             "less than original size; writing uncompressed.",
-                             lte->size);
-                       ret = write_stream_uncompressed(lte, ctx->out_fd);
+                       ret = end_write_resource(ctx, &lte->out_reshdr);
                        if (ret)
                                return ret;
-               }
 
-               wimlib_assert(lte->out_reshdr.uncompressed_size == lte->size);
+                       lte->out_reshdr.flags = filter_resource_flags(lte->flags);
+                       if (ctx->compressor != NULL)
+                               lte->out_reshdr.flags |= WIM_RESHDR_FLAG_COMPRESSED;
+
+                       if (ctx->compressor != NULL &&
+                           lte->out_reshdr.size_in_wim >= lte->out_reshdr.uncompressed_size &&
+                           !(ctx->write_resource_flags & WRITE_RESOURCE_FLAG_PIPABLE) &&
+                           !(lte->flags & WIM_RESHDR_FLAG_PACKED_STREAMS))
+                       {
+                               /* Stream did not compress to less than its original
+                                * size.  If we're not writing a pipable WIM (which
+                                * could mean the output file descriptor is
+                                * non-seekable), and the stream isn't located in a
+                                * resource pack (which would make reading it again
+                                * costly), truncate the file to the start of the stream
+                                * and write it uncompressed instead.  */
+                               DEBUG("Stream of size %"PRIu64" did not compress to "
+                                     "less than original size; writing uncompressed.",
+                                     lte->size);
+                               ret = write_stream_uncompressed(lte, ctx->out_fd);
+                               if (ret)
+                                       return ret;
+                       }
+                       wimlib_assert(lte->out_reshdr.uncompressed_size == lte->size);
 
-               list_del(&lte->write_streams_list);
-               ctx->cur_write_res_offset = 0;
+                       ctx->cur_write_stream_offset = 0;
+               }
        }
 
+       do_write_streams_progress(&ctx->progress_data, lte,
+                                 completed_size, completed_stream_count,
+                                 false);
+
        return 0;
 
 error:
@@ -889,7 +946,7 @@ write_stream_process_chunk(const void *chunk, size_t size, void *_ctx)
                 ret = write_chunk(ctx, chunk, size, size);
                 if (ret)
                         return ret;
-                ctx->cur_read_res_offset += size;
+                ctx->cur_read_stream_offset += size;
                 return 0;
        }
 
@@ -907,8 +964,8 @@ write_stream_process_chunk(const void *chunk, size_t size, void *_ctx)
                } else {
                        u64 res_bytes_remaining;
 
-                       res_bytes_remaining = ctx->cur_read_res_size -
-                                             ctx->cur_read_res_offset;
+                       res_bytes_remaining = ctx->cur_read_stream_size -
+                                             ctx->cur_read_stream_offset;
                        needed_chunk_size = min(ctx->out_chunk_size,
                                                ctx->chunk_buf_filled +
                                                        res_bytes_remaining);
@@ -920,7 +977,7 @@ write_stream_process_chunk(const void *chunk, size_t size, void *_ctx)
                        /* No intermediate buffering needed.  */
                        resized_chunk = chunkptr;
                        chunkptr += needed_chunk_size;
-                       ctx->cur_read_res_offset += needed_chunk_size;
+                       ctx->cur_read_stream_offset += needed_chunk_size;
                } else {
                        /* Intermediate buffering needed.  */
                        size_t bytes_consumed;
@@ -931,10 +988,8 @@ write_stream_process_chunk(const void *chunk, size_t size, void *_ctx)
                        memcpy(&ctx->chunk_buf[ctx->chunk_buf_filled],
                               chunkptr, bytes_consumed);
 
-                       resized_chunk = ctx->chunk_buf;
-
                        chunkptr += bytes_consumed;
-                       ctx->cur_read_res_offset += bytes_consumed;
+                       ctx->cur_read_stream_offset += bytes_consumed;
                        ctx->chunk_buf_filled += bytes_consumed;
                        if (ctx->chunk_buf_filled == needed_chunk_size) {
                                resized_chunk = ctx->chunk_buf;
@@ -962,7 +1017,7 @@ write_stream_end_read(struct wim_lookup_table_entry *lte, int status, void *_ctx
 {
        struct write_streams_ctx *ctx = _ctx;
        if (status == 0)
-               wimlib_assert(ctx->cur_read_res_offset == ctx->cur_read_res_size);
+               wimlib_assert(ctx->cur_read_stream_offset == ctx->cur_read_stream_size);
        if (ctx->stream_was_duplicate) {
                free_lookup_table_entry(lte);
        } else if (lte->unhashed && ctx->lookup_table != NULL) {
@@ -1124,7 +1179,8 @@ write_raw_copy_resources(struct list_head *raw_copy_resources,
                ret = write_raw_copy_resource(lte->rspec, out_fd);
                if (ret)
                        return ret;
-               do_write_streams_progress(progress_data, lte->size, false, lte);
+               do_write_streams_progress(progress_data, lte, lte->size,
+                                         1, false);
        }
        return 0;
 }
@@ -1177,7 +1233,7 @@ remove_zero_length_streams(struct list_head *stream_list)
  * Write a list of streams to the output WIM file.
  *
  * @stream_list
- *     The list of streams to write, specifies a list of `struct
+ *     The list of streams to write, specified by a list of `struct
  *     wim_lookup_table_entry's linked by the 'write_streams_list' member.
  *
  * @out_fd
@@ -1188,20 +1244,20 @@ remove_zero_length_streams(struct list_head *stream_list)
  *
  *     WRITE_RESOURCE_FLAG_RECOMPRESS:
  *             Force compression of all resources, even if they could otherwise
- *             be re-used by caping the raw data, due to being located in a WIM
+ *             be re-used by copying the raw data, due to being located in a WIM
  *             file with compatible compression parameters.
  *
  *     WRITE_RESOURCE_FLAG_PIPABLE:
  *             Write the resources in the wimlib-specific pipable format, and
  *             furthermore do so in such a way that no seeking backwards in
- *             @out_fd will be performed (so it may be a pipe, contrary to the
- *             default behavior).
+ *             @out_fd will be performed (so it may be a pipe).
  *
  *     WRITE_RESOURCE_FLAG_PACK_STREAMS:
  *             Pack all the streams into a single resource rather than writing
- *             them in separate resources.  This format is only valid if the
- *             WIM version number is WIM_VERSION_PACKED_STREAMS.  This flag
- *             currently may not be combined with WRITE_RESOURCE_FLAG_PIPABLE.
+ *             them in separate resources.  This flag is only valid if the WIM
+ *             version number has been, or will be, set to
+ *             WIM_VERSION_PACKED_STREAMS.  This flag may not be combined with
+ *             WRITE_RESOURCE_FLAG_PIPABLE.
  *
  * @out_ctype
  *     Compression format to use to write the output streams, specified as one
@@ -1244,15 +1300,15 @@ remove_zero_length_streams(struct list_head *stream_list)
  * In both cases, the @out_reshdr of the `struct wim_lookup_table_entry' for
  * each stream written will be updated to specify its location, size, and flags
  * in the output WIM.  In the packed resource case,
- * WIM_RESHDR_FLAG_PACKED_STREAMS shall be set in the @flags field of the
- * @out_reshdr, and @out_res_offset_in_wim and @out_res_size_in_wim will also
- * be set to the offset and size, respectively, in the output WIM of the full
- * packed resource containing the corresponding stream.
+ * WIM_RESHDR_FLAG_PACKED_STREAMS will be set in the @flags field of each
+ * @out_reshdr, and furthermore @out_res_offset_in_wim and @out_res_size_in_wim
+ * of each @out_reshdr will be set to the offset and size, respectively, in the
+ * output WIM of the packed resource containing the corresponding stream.
  *
  * Each of the streams to write may be in any location supported by the
  * resource-handling code (specifically, read_stream_list()), such as the
  * contents of external file that has been logically added to the output WIM, or
- * a stream in another WIM file that has been imported, or even stream in the
+ * a stream in another WIM file that has been imported, or even stream in the
  * "same" WIM file of which a modified copy is being written.  In the case that
  * a stream is already in a WIM file and uses compatible compression parameters,
  * by default this function will re-use the raw data instead of decompressing
@@ -1260,32 +1316,31 @@ remove_zero_length_streams(struct list_head *stream_list)
  * specified in @write_resource_flags, this is not done.
  *
  * As a further requirement, this function requires that the
- * @will_be_in_output_wim member be set on all streams in @stream_list as well
- * as any other streams not in @stream_list that will be in the output WIM file,
- * but not on any other streams in the output WIM's lookup table or sharing a
- * packed resource with a stream in @stream_list.  Still furthermore, if
- * on-the-fly deduplication of streams is possible, then all streams in
+ * @will_be_in_output_wim member be set to 1 on all streams in @stream_list as
+ * well as any other streams not in @stream_list that will be in the output WIM
+ * file, but set to 0 on any other streams in the output WIM's lookup table or
+ * sharing a packed resource with a stream in @stream_list.  Still furthermore,
+ * if on-the-fly deduplication of streams is possible, then all streams in
  * @stream_list must also be linked by @lookup_table_list along with any other
  * streams that have @will_be_in_output_wim set.
  *
  * This function handles on-the-fly deduplication of streams for which SHA1
- * message digests have not yet been calculated and it is therefore known
- * whether such streams are already in @stream_list or in the WIM's lookup table
- * at all.  If @lookup_table is non-NULL, then each stream in @stream_list that
- * has @unhashed set but not @unique_size set is checksummed immediately before
- * it would otherwise be read for writing in order to determine if it is
- * identical to another stream already being written or one that would be
- * filtered out of the output WIM using stream_filtered() with the context
- * @filter_ctx.  Each such duplicate stream will be removed from @stream_list, its
- * reference count transfered to the pre-existing duplicate stream, its memory
- * freed, and will not be written.  Alternatively, if a stream in @stream_list
- * is a duplicate with any stream in @lookup_table that has not been marked for
- * writing or would not be hard-filtered, it is freed and the pre-existing
- * duplicate is written instead, taking ownership of the reference count and
- * slot in the @lookup_table_list.
- *
- * Returns 0 if all streams were written successfully (or did not need to be
- * written); otherwise a non-zero error code.
+ * message digests have not yet been calculated.  Such streams may or may not
+ * need to be written.  If @lookup_table is non-NULL, then each stream in
+ * @stream_list that has @unhashed set but not @unique_size set is checksummed
+ * immediately before it would otherwise be read for writing in order to
+ * determine if it is identical to another stream already being written or one
+ * that would be filtered out of the output WIM using stream_filtered() with the
+ * context @filter_ctx.  Each such duplicate stream will be removed from
+ * @stream_list, its reference count transfered to the pre-existing duplicate
+ * stream, its memory freed, and will not be written.  Alternatively, if a
+ * stream in @stream_list is a duplicate with any stream in @lookup_table that
+ * has not been marked for writing or would not be hard-filtered, it is freed
+ * and the pre-existing duplicate is written instead, taking ownership of the
+ * reference count and slot in the @lookup_table_list.
+ *
+ * Returns 0 if every stream was either written successfully or did not need to
+ * be written; otherwise returns a non-zero error code.
  */
 static int
 write_stream_list(struct list_head *stream_list,
@@ -1332,7 +1387,8 @@ write_stream_list(struct list_head *stream_list,
        ctx.write_resource_flags = write_resource_flags;
        ctx.filter_ctx = filter_ctx;
 
-       if (out_chunk_size != 0) {
+       if (out_ctype != WIMLIB_COMPRESSION_TYPE_NONE) {
+               wimlib_assert(out_chunk_size != 0);
                if (out_chunk_size <= STACK_MAX) {
                        ctx.chunk_buf = alloca(out_chunk_size);
                } else {
@@ -1406,6 +1462,7 @@ write_stream_list(struct list_head *stream_list,
              ctx.progress_data.progress.write_streams.num_threads);
 
        INIT_LIST_HEAD(&ctx.pending_streams);
+       INIT_LIST_HEAD(&ctx.pack_streams);
 
        if (ctx.progress_data.progress_func) {
                (*ctx.progress_data.progress_func)(WIMLIB_PROGRESS_MSG_WRITE_STREAMS,
@@ -1459,7 +1516,7 @@ write_stream_list(struct list_head *stream_list,
                      reshdr.uncompressed_size);
 
                offset_in_res = 0;
-               list_for_each_entry(lte, &ctx.pending_streams, write_streams_list) {
+               list_for_each_entry(lte, &ctx.pack_streams, write_streams_list) {
                        lte->out_reshdr.size_in_wim = lte->size;
                        lte->out_reshdr.flags = filter_resource_flags(lte->flags);
                        lte->out_reshdr.flags |= WIM_RESHDR_FLAG_PACKED_STREAMS;
@@ -1480,7 +1537,7 @@ out_write_raw_copy_resources:
                                       &ctx.progress_data);
 
 out_destroy_context:
-       if (out_chunk_size > STACK_MAX)
+       if (out_ctype != WIMLIB_COMPRESSION_TYPE_NONE && out_chunk_size > STACK_MAX)
                FREE(ctx.chunk_buf);
        FREE(ctx.chunk_csizes);
        if (ctx.compressor)
@@ -1489,6 +1546,39 @@ out_destroy_context:
        return ret;
 }
 
+static int
+wim_write_stream_list(WIMStruct *wim,
+                     struct list_head *stream_list,
+                     int write_flags,
+                     unsigned num_threads,
+                     struct filter_context *filter_ctx,
+                     wimlib_progress_func_t progress_func)
+{
+       int out_ctype;
+       u32 out_chunk_size;
+       int write_resource_flags;
+
+       write_resource_flags = write_flags_to_resource_flags(write_flags);
+
+       if (write_resource_flags & WRITE_RESOURCE_FLAG_PACK_STREAMS) {
+               out_chunk_size = wim->out_pack_chunk_size;
+               out_ctype = wim->out_pack_compression_type;
+       } else {
+               out_chunk_size = wim->out_chunk_size;
+               out_ctype = wim->out_compression_type;
+       }
+
+       return write_stream_list(stream_list,
+                                &wim->out_fd,
+                                write_resource_flags,
+                                out_ctype,
+                                out_chunk_size,
+                                num_threads,
+                                wim->lookup_table,
+                                filter_ctx,
+                                progress_func);
+}
+
 static int
 write_wim_resource(struct wim_lookup_table_entry *lte,
                   struct filedes *out_fd,
@@ -1941,15 +2031,12 @@ write_wim_streams(WIMStruct *wim, int image, int write_flags,
                }
        }
 
-       return write_stream_list(stream_list,
-                                &wim->out_fd,
-                                write_flags_to_resource_flags(write_flags),
-                                wim->out_compression_type,
-                                wim->out_chunk_size,
-                                num_threads,
-                                wim->lookup_table,
-                                filter_ctx,
-                                progress_func);
+       return wim_write_stream_list(wim,
+                                    stream_list,
+                                    write_flags,
+                                    num_threads,
+                                    filter_ctx,
+                                    progress_func);
 }
 
 static int
@@ -2507,9 +2594,6 @@ write_wim_part(WIMStruct *wim,
        if (write_flags & WIMLIB_WRITE_FLAG_PACK_STREAMS)
                DEBUG("\tPACK_STREAMS");
 
-       if (write_flags & WIMLIB_WRITE_FLAG_NO_PACK_STREAMS)
-               DEBUG("\tNO_PACK_STREAMS");
-
        if (write_flags & WIMLIB_WRITE_FLAG_FILE_DESCRIPTOR)
                DEBUG("\tFILE_DESCRIPTOR");
 
@@ -2558,12 +2642,6 @@ write_wim_part(WIMStruct *wim,
                                    WIMLIB_WRITE_FLAG_NOT_PIPABLE))
                return WIMLIB_ERR_INVALID_PARAM;
 
-       if ((write_flags & (WIMLIB_WRITE_FLAG_PACK_STREAMS |
-                           WIMLIB_WRITE_FLAG_NO_PACK_STREAMS))
-                               == (WIMLIB_WRITE_FLAG_PACK_STREAMS |
-                                   WIMLIB_WRITE_FLAG_NO_PACK_STREAMS))
-               return WIMLIB_ERR_INVALID_PARAM;
-
        /* Save previous header, then start initializing the new one.  */
        memcpy(&hdr_save, &wim->hdr, sizeof(struct wim_header));
 
@@ -2582,13 +2660,6 @@ write_wim_part(WIMStruct *wim,
                        write_flags |= WIMLIB_WRITE_FLAG_CHECK_INTEGRITY;
                }
 
-       if (!(write_flags & (WIMLIB_WRITE_FLAG_PACK_STREAMS |
-                            WIMLIB_WRITE_FLAG_NO_PACK_STREAMS)))
-               if (wim->hdr.wim_version == WIM_VERSION_PACKED_STREAMS) {
-                       DEBUG("WIM version 3584; default to PACK_STREAMS.");
-                       write_flags |= WIMLIB_WRITE_FLAG_PACK_STREAMS;
-               }
-
        if ((write_flags & (WIMLIB_WRITE_FLAG_PIPABLE |
                            WIMLIB_WRITE_FLAG_PACK_STREAMS))
                                    == (WIMLIB_WRITE_FLAG_PIPABLE |
@@ -2605,7 +2676,8 @@ write_wim_part(WIMStruct *wim,
                wim->hdr.magic = WIM_MAGIC;
 
        /* Set appropriate version number.  */
-       if (write_flags & WIMLIB_WRITE_FLAG_PACK_STREAMS)
+       if ((write_flags & WIMLIB_WRITE_FLAG_PACK_STREAMS) ||
+           wim->out_compression_type == WIMLIB_COMPRESSION_TYPE_LZMS)
                wim->hdr.wim_version = WIM_VERSION_PACKED_STREAMS;
        else
                wim->hdr.wim_version = WIM_VERSION_DEFAULT;
@@ -2895,20 +2967,9 @@ overwrite_wim_inplace(WIMStruct *wim, int write_flags,
                if (wim_has_integrity_table(wim))
                        write_flags |= WIMLIB_WRITE_FLAG_CHECK_INTEGRITY;
 
-       /* Set default packed flag.  */
-       if (!(write_flags & (WIMLIB_WRITE_FLAG_PACK_STREAMS |
-                            WIMLIB_WRITE_FLAG_NO_PACK_STREAMS)))
-       {
-       #if 0
-               if (wim->hdr.wim_version == WIM_VERSION_PACKED_STREAMS)
-                       write_flags |= WIMLIB_WRITE_FLAG_PACK_STREAMS;
-       #endif
-               /* wimlib allows multiple packs in a single WIM, but they don't
-                * seem to be compatible with WIMGAPI.  Write new streams
-                * unpacked.  */
-       } else if (write_flags & WIMLIB_WRITE_FLAG_PACK_STREAMS) {
+       /* Set WIM version if adding packed streams.  */
+       if (write_flags & WIMLIB_WRITE_FLAG_PACK_STREAMS)
                wim->hdr.wim_version = WIM_VERSION_PACKED_STREAMS;
-       }
 
        /* Set additional flags for overwrite.  */
        write_flags |= WIMLIB_WRITE_FLAG_OVERWRITE |
@@ -2993,15 +3054,12 @@ overwrite_wim_inplace(WIMStruct *wim, int write_flags,
                goto out_restore_physical_hdr;
        }
 
-       ret = write_stream_list(&stream_list,
-                               &wim->out_fd,
-                               write_flags_to_resource_flags(write_flags),
-                               wim->compression_type,
-                               wim->chunk_size,
-                               num_threads,
-                               wim->lookup_table,
-                               &filter_ctx,
-                               progress_func);
+       ret = wim_write_stream_list(wim,
+                                   &stream_list,
+                                   write_flags,
+                                   num_threads,
+                                   &filter_ctx,
+                                   progress_func);
        if (ret)
                goto out_truncate;