+ struct wim_lookup_table_entry *lte, *tmp;
+
+ list_for_each_entry_safe(lte, tmp, stream_list, write_streams_list) {
+ wimlib_assert(lte->will_be_in_output_wim);
+ if (lte->size == 0) {
+ list_del(<e->write_streams_list);
+ lte->out_reshdr.offset_in_wim = 0;
+ lte->out_reshdr.size_in_wim = 0;
+ lte->out_reshdr.uncompressed_size = 0;
+ lte->out_reshdr.flags = filter_resource_flags(lte->flags);
+ }
+ }
+}
+
+/*
+ * Write a list of streams to the output WIM file.
+ *
+ * @stream_list
+ * The list of streams to write, specified by a list of `struct
+ * wim_lookup_table_entry's linked by the 'write_streams_list' member.
+ *
+ * @out_fd
+ * The file descriptor, opened for writing, to which to write the streams.
+ *
+ * @write_resource_flags
+ * Flags to modify how the streams are written:
+ *
+ * WRITE_RESOURCE_FLAG_RECOMPRESS:
+ * Force compression of all resources, even if they could otherwise
+ * be re-used by copying the raw data, due to being located in a WIM
+ * file with compatible compression parameters.
+ *
+ * WRITE_RESOURCE_FLAG_PIPABLE:
+ * Write the resources in the wimlib-specific pipable format, and
+ * furthermore do so in such a way that no seeking backwards in
+ * @out_fd will be performed (so it may be a pipe).
+ *
+ * WRITE_RESOURCE_FLAG_PACK_STREAMS:
+ * Pack all the streams into a single resource rather than writing
+ * them in separate resources. This flag is only valid if the WIM
+ * version number has been, or will be, set to
+ * WIM_VERSION_PACKED_STREAMS. This flag may not be combined with
+ * WRITE_RESOURCE_FLAG_PIPABLE.
+ *
+ * @out_ctype
+ * Compression format to use to write the output streams, specified as one
+ * of the WIMLIB_COMPRESSION_TYPE_* constants, excepting
+ * WIMLIB_COMPRESSION_TYPE_INVALID but including
+ * WIMLIB_COMPRESSION_TYPE_NONE.
+ *
+ * @out_chunk_size
+ * Chunk size to use to write the streams. It must be a valid chunk size
+ * for the specified compression format @out_ctype, unless @out_ctype is
+ * WIMLIB_COMPRESSION_TYPE_NONE, in which case this parameter is ignored.
+ *
+ * @num_threads
+ * Number of threads to use to compress data. If 0, a default number of
+ * threads will be chosen. The number of threads still may be decreased
+ * from the specified value if insufficient memory is detected.
+ *
+ * @lookup_table
+ * If on-the-fly deduplication of unhashed streams is desired, this
+ * parameter must be pointer to the lookup table for the WIMStruct on whose
+ * behalf the streams are being written. Otherwise, this parameter can be
+ * NULL.
+ *
+ * @filter_ctx
+ * If on-the-fly deduplication of unhashed streams is desired, this
+ * parameter can be a pointer to a context for stream filtering used to
+ * detect whether the duplicate stream has been hard-filtered or not. If
+ * no streams are hard-filtered or no streams are unhashed, this parameter
+ * can be NULL.
+ *
+ * @progress_func
+ * If non-NULL, a progress function that will be called periodically with
+ * WIMLIB_PROGRESS_MSG_WRITE_STREAMS messages. Note that on-the-fly
+ * deduplication of unhashed streams may result in the total bytes provided
+ * in the progress data to decrease from one message to the next.
+ *
+ * This function will write the streams in @stream_list to resources in
+ * consecutive positions in the output WIM file, or to a single packed resource
+ * if WRITE_RESOURCE_FLAG_PACK_STREAMS was specified in @write_resource_flags.
+ * In both cases, the @out_reshdr of the `struct wim_lookup_table_entry' for
+ * each stream written will be updated to specify its location, size, and flags
+ * in the output WIM. In the packed resource case,
+ * WIM_RESHDR_FLAG_PACKED_STREAMS will be set in the @flags field of each
+ * @out_reshdr, and furthermore @out_res_offset_in_wim and @out_res_size_in_wim
+ * of each @out_reshdr will be set to the offset and size, respectively, in the
+ * output WIM of the packed resource containing the corresponding stream.
+ *
+ * Each of the streams to write may be in any location supported by the
+ * resource-handling code (specifically, read_stream_list()), such as the
+ * contents of external file that has been logically added to the output WIM, or
+ * a stream in another WIM file that has been imported, or even a stream in the
+ * "same" WIM file of which a modified copy is being written. In the case that
+ * a stream is already in a WIM file and uses compatible compression parameters,
+ * by default this function will re-use the raw data instead of decompressing
+ * it, then recompressing it; however, with WRITE_RESOURCE_FLAG_RECOMPRESS
+ * specified in @write_resource_flags, this is not done.
+ *
+ * As a further requirement, this function requires that the
+ * @will_be_in_output_wim member be set to 1 on all streams in @stream_list as
+ * well as any other streams not in @stream_list that will be in the output WIM
+ * file, but set to 0 on any other streams in the output WIM's lookup table or
+ * sharing a packed resource with a stream in @stream_list. Still furthermore,
+ * if on-the-fly deduplication of streams is possible, then all streams in
+ * @stream_list must also be linked by @lookup_table_list along with any other
+ * streams that have @will_be_in_output_wim set.
+ *
+ * This function handles on-the-fly deduplication of streams for which SHA1
+ * message digests have not yet been calculated. Such streams may or may not
+ * need to be written. If @lookup_table is non-NULL, then each stream in
+ * @stream_list that has @unhashed set but not @unique_size set is checksummed
+ * immediately before it would otherwise be read for writing in order to
+ * determine if it is identical to another stream already being written or one
+ * that would be filtered out of the output WIM using stream_filtered() with the
+ * context @filter_ctx. Each such duplicate stream will be removed from
+ * @stream_list, its reference count transfered to the pre-existing duplicate
+ * stream, its memory freed, and will not be written. Alternatively, if a
+ * stream in @stream_list is a duplicate with any stream in @lookup_table that
+ * has not been marked for writing or would not be hard-filtered, it is freed
+ * and the pre-existing duplicate is written instead, taking ownership of the
+ * reference count and slot in the @lookup_table_list.
+ *
+ * Returns 0 if every stream was either written successfully or did not need to
+ * be written; otherwise returns a non-zero error code.
+ */
+static int
+write_stream_list(struct list_head *stream_list,
+ struct filedes *out_fd,
+ int write_resource_flags,
+ int out_ctype,
+ u32 out_chunk_size,
+ unsigned num_threads,
+ struct wim_lookup_table *lookup_table,
+ struct filter_context *filter_ctx,
+ wimlib_progress_func_t progress_func)
+{
+ int ret;
+ struct write_streams_ctx ctx;
+ struct list_head raw_copy_resources;
+
+ wimlib_assert((write_resource_flags &
+ (WRITE_RESOURCE_FLAG_PACK_STREAMS |
+ WRITE_RESOURCE_FLAG_PIPABLE)) !=
+ (WRITE_RESOURCE_FLAG_PACK_STREAMS |
+ WRITE_RESOURCE_FLAG_PIPABLE));
+
+ remove_zero_length_streams(stream_list);
+
+ if (list_empty(stream_list)) {
+ DEBUG("No streams to write.");
+ return 0;
+ }
+
+ memset(&ctx, 0, sizeof(ctx));
+
+ /* Pre-sorting the streams is required for compute_stream_list_stats().
+ * Afterwards, read_stream_list() need not sort them again. */
+ ret = sort_stream_list_by_sequential_order(stream_list,
+ offsetof(struct wim_lookup_table_entry,
+ write_streams_list));
+ if (ret)
+ return ret;
+
+ ctx.out_fd = out_fd;
+ ctx.lookup_table = lookup_table;
+ ctx.out_ctype = out_ctype;
+ ctx.out_chunk_size = out_chunk_size;
+ ctx.write_resource_flags = write_resource_flags;
+ ctx.filter_ctx = filter_ctx;
+
+ if (out_ctype != WIMLIB_COMPRESSION_TYPE_NONE) {
+ wimlib_assert(out_chunk_size != 0);
+ if (out_chunk_size <= STACK_MAX) {
+ ctx.chunk_buf = alloca(out_chunk_size);
+ } else {
+ ctx.chunk_buf = MALLOC(out_chunk_size);
+ if (ctx.chunk_buf == NULL) {
+ ret = WIMLIB_ERR_NOMEM;
+ goto out_destroy_context;
+ }
+ }
+ }
+ ctx.chunk_buf_filled = 0;
+
+ compute_stream_list_stats(stream_list, &ctx);
+
+ ctx.progress_data.progress_func = progress_func;
+
+ ctx.num_bytes_to_compress = find_raw_copy_resources(stream_list,
+ write_resource_flags,
+ out_ctype,
+ out_chunk_size,
+ &raw_copy_resources);
+
+ DEBUG("Writing stream list "
+ "(offset = %"PRIu64", write_resource_flags=0x%08x, "
+ "out_ctype=%d, out_chunk_size=%u, num_threads=%u, "
+ "total_bytes=%"PRIu64", num_bytes_to_compress=%"PRIu64")",
+ out_fd->offset, write_resource_flags,
+ out_ctype, out_chunk_size, num_threads,
+ ctx.progress_data.progress.write_streams.total_bytes,
+ ctx.num_bytes_to_compress);
+
+ if (ctx.num_bytes_to_compress == 0) {
+ DEBUG("No compression needed; skipping to raw copy!");
+ goto out_write_raw_copy_resources;
+ }
+
+ /* Unless uncompressed output was required, allocate a chunk_compressor
+ * to do compression. There are serial and parallel implementations of
+ * the chunk_compressor interface. We default to parallel using the
+ * specified number of threads, unless the upper bound on the number
+ * bytes needing to be compressed is less than a heuristic value. */
+ if (out_ctype != WIMLIB_COMPRESSION_TYPE_NONE) {
+
+ #ifdef ENABLE_MULTITHREADED_COMPRESSION
+ if (ctx.num_bytes_to_compress > max(2000000, out_chunk_size)) {
+ ret = new_parallel_chunk_compressor(out_ctype,
+ out_chunk_size,
+ num_threads, 0,
+ &ctx.compressor);
+ if (ret) {
+ DEBUG("Couldn't create parallel chunk compressor "
+ "(status %d)", ret);
+ }
+ }
+ #endif
+
+ if (ctx.compressor == NULL) {
+ ret = new_serial_chunk_compressor(out_ctype, out_chunk_size,
+ &ctx.compressor);
+ if (ret)
+ goto out_destroy_context;
+ }
+ }
+
+ if (ctx.compressor)
+ ctx.progress_data.progress.write_streams.num_threads = ctx.compressor->num_threads;
+ else
+ ctx.progress_data.progress.write_streams.num_threads = 1;
+
+ DEBUG("Actually using %u threads",
+ ctx.progress_data.progress.write_streams.num_threads);
+
+ INIT_LIST_HEAD(&ctx.pending_streams);
+ INIT_LIST_HEAD(&ctx.pack_streams);
+
+ if (ctx.progress_data.progress_func) {
+ (*ctx.progress_data.progress_func)(WIMLIB_PROGRESS_MSG_WRITE_STREAMS,
+ &ctx.progress_data.progress);
+ }
+
+ if (write_resource_flags & WRITE_RESOURCE_FLAG_PACK_STREAMS) {
+ ret = begin_write_resource(&ctx, ctx.num_bytes_to_compress);
+ if (ret)
+ goto out_destroy_context;
+ }
+
+ /* Read the list of streams needing to be compressed, using the
+ * specified callbacks to execute processing of the data. */
+
+ struct read_stream_list_callbacks cbs = {
+ .begin_stream = write_stream_begin_read,
+ .begin_stream_ctx = &ctx,
+ .consume_chunk = write_stream_process_chunk,
+ .consume_chunk_ctx = &ctx,
+ .end_stream = write_stream_end_read,
+ .end_stream_ctx = &ctx,
+ };
+
+ ret = read_stream_list(stream_list,
+ offsetof(struct wim_lookup_table_entry, write_streams_list),
+ &cbs,
+ STREAM_LIST_ALREADY_SORTED |
+ VERIFY_STREAM_HASHES |
+ COMPUTE_MISSING_STREAM_HASHES);
+
+ if (ret)
+ goto out_destroy_context;
+
+ ret = finish_remaining_chunks(&ctx);
+ if (ret)
+ goto out_destroy_context;
+
+ if (write_resource_flags & WRITE_RESOURCE_FLAG_PACK_STREAMS) {
+ struct wim_reshdr reshdr;
+ struct wim_lookup_table_entry *lte;
+ u64 offset_in_res;
+
+ ret = end_write_resource(&ctx, &reshdr);
+ if (ret)
+ goto out_destroy_context;
+
+ DEBUG("Ending packed resource: %lu %lu %lu.",
+ reshdr.offset_in_wim,
+ reshdr.size_in_wim,
+ reshdr.uncompressed_size);
+
+ offset_in_res = 0;
+ list_for_each_entry(lte, &ctx.pack_streams, write_streams_list) {
+ lte->out_reshdr.size_in_wim = lte->size;
+ lte->out_reshdr.flags = filter_resource_flags(lte->flags);
+ lte->out_reshdr.flags |= WIM_RESHDR_FLAG_PACKED_STREAMS;
+ lte->out_reshdr.uncompressed_size = 0;
+ lte->out_reshdr.offset_in_wim = offset_in_res;
+ lte->out_res_offset_in_wim = reshdr.offset_in_wim;
+ lte->out_res_size_in_wim = reshdr.size_in_wim;
+ /*lte->out_res_uncompressed_size = reshdr.uncompressed_size;*/
+ offset_in_res += lte->size;
+ }
+ wimlib_assert(offset_in_res == reshdr.uncompressed_size);
+ }
+
+out_write_raw_copy_resources:
+ /* Copy any compressed resources for which the raw data can be reused
+ * without decompression. */
+ ret = write_raw_copy_resources(&raw_copy_resources, ctx.out_fd,
+ &ctx.progress_data);
+
+out_destroy_context:
+ if (out_ctype != WIMLIB_COMPRESSION_TYPE_NONE && out_chunk_size > STACK_MAX)
+ FREE(ctx.chunk_buf);
+ FREE(ctx.chunk_csizes);
+ if (ctx.compressor)
+ ctx.compressor->destroy(ctx.compressor);
+ DEBUG("Done (ret=%d)", ret);
+ return ret;
+}
+
+static int
+wim_write_stream_list(WIMStruct *wim,
+ struct list_head *stream_list,
+ int write_flags,
+ unsigned num_threads,
+ struct filter_context *filter_ctx,
+ wimlib_progress_func_t progress_func)
+{
+ int out_ctype;
+ u32 out_chunk_size;
+ int write_resource_flags;
+
+ write_resource_flags = write_flags_to_resource_flags(write_flags);
+
+ if (write_resource_flags & WRITE_RESOURCE_FLAG_PACK_STREAMS) {
+ out_chunk_size = wim->out_pack_chunk_size;
+ out_ctype = wim->out_pack_compression_type;
+ } else {
+ out_chunk_size = wim->out_chunk_size;
+ out_ctype = wim->out_compression_type;
+ }
+
+ return write_stream_list(stream_list,
+ &wim->out_fd,
+ write_resource_flags,
+ out_ctype,
+ out_chunk_size,
+ num_threads,
+ wim->lookup_table,
+ filter_ctx,
+ progress_func);
+}
+
+static int
+write_wim_resource(struct wim_lookup_table_entry *lte,
+ struct filedes *out_fd,
+ int out_ctype,
+ u32 out_chunk_size,
+ int write_resource_flags)
+{
+ LIST_HEAD(stream_list);
+ list_add(<e->write_streams_list, &stream_list);
+ lte->will_be_in_output_wim = 1;
+ return write_stream_list(&stream_list,
+ out_fd,
+ write_resource_flags & ~WRITE_RESOURCE_FLAG_PACK_STREAMS,
+ out_ctype,
+ out_chunk_size,
+ 1,
+ NULL,
+ NULL,
+ NULL);
+}
+
+int
+write_wim_resource_from_buffer(const void *buf, size_t buf_size,
+ int reshdr_flags, struct filedes *out_fd,
+ int out_ctype,
+ u32 out_chunk_size,
+ struct wim_reshdr *out_reshdr,
+ u8 *hash,
+ int write_resource_flags)
+{
+ int ret;
+ struct wim_lookup_table_entry *lte;
+
+ /* Set up a temporary lookup table entry to provide to
+ * write_wim_resource(). */
+
+ lte = new_lookup_table_entry();
+ if (lte == NULL)
+ return WIMLIB_ERR_NOMEM;
+
+ lte->resource_location = RESOURCE_IN_ATTACHED_BUFFER;
+ lte->attached_buffer = (void*)buf;
+ lte->size = buf_size;
+ lte->flags = reshdr_flags;
+
+ if (write_resource_flags & WRITE_RESOURCE_FLAG_PIPABLE) {
+ sha1_buffer(buf, buf_size, lte->hash);
+ lte->unhashed = 0;
+ } else {
+ lte->unhashed = 1;
+ }
+
+ ret = write_wim_resource(lte, out_fd, out_ctype, out_chunk_size,
+ write_resource_flags);
+ if (ret)
+ goto out_free_lte;
+
+ copy_reshdr(out_reshdr, <e->out_reshdr);
+
+ if (hash)
+ copy_hash(hash, lte->hash);
+ ret = 0;
+out_free_lte:
+ lte->resource_location = RESOURCE_NONEXISTENT;
+ free_lookup_table_entry(lte);
+ return ret;
+}
+
+struct stream_size_table {
+ struct hlist_head *array;
+ size_t num_entries;
+ size_t capacity;
+};
+
+static int
+init_stream_size_table(struct stream_size_table *tab, size_t capacity)
+{
+ tab->array = CALLOC(capacity, sizeof(tab->array[0]));
+ if (tab->array == NULL)
+ return WIMLIB_ERR_NOMEM;
+ tab->num_entries = 0;
+ tab->capacity = capacity;
+ return 0;
+}
+
+static void
+destroy_stream_size_table(struct stream_size_table *tab)
+{
+ FREE(tab->array);
+}
+
+static int
+stream_size_table_insert(struct wim_lookup_table_entry *lte, void *_tab)
+{
+ struct stream_size_table *tab = _tab;
+ size_t pos;
+ struct wim_lookup_table_entry *same_size_lte;
+ struct hlist_node *tmp;
+
+ pos = hash_u64(lte->size) % tab->capacity;
+ lte->unique_size = 1;
+ hlist_for_each_entry(same_size_lte, tmp, &tab->array[pos], hash_list_2) {
+ if (same_size_lte->size == lte->size) {
+ lte->unique_size = 0;
+ same_size_lte->unique_size = 0;
+ break;
+ }
+ }
+
+ hlist_add_head(<e->hash_list_2, &tab->array[pos]);
+ tab->num_entries++;
+ return 0;
+}
+
+struct find_streams_ctx {
+ WIMStruct *wim;
+ int write_flags;