+ if (ctx.compressor)
+ ctx.progress_data.progress.write_streams.num_threads = ctx.compressor->num_threads;
+ else
+ ctx.progress_data.progress.write_streams.num_threads = 1;
+
+ INIT_LIST_HEAD(&ctx.blobs_being_compressed);
+ INIT_LIST_HEAD(&ctx.blobs_in_solid_resource);
+
+ ret = call_progress(ctx.progress_data.progfunc,
+ WIMLIB_PROGRESS_MSG_WRITE_STREAMS,
+ &ctx.progress_data.progress,
+ ctx.progress_data.progctx);
+ if (ret)
+ goto out_destroy_context;
+
+ if (write_resource_flags & WRITE_RESOURCE_FLAG_SOLID) {
+ ret = begin_write_resource(&ctx, num_nonraw_bytes);
+ if (ret)
+ goto out_destroy_context;
+ }
+
+ /* Read the list of blobs needing to be compressed, using the specified
+ * callbacks to execute processing of the data. */
+
+ struct read_blob_callbacks cbs = {
+ .begin_blob = write_blob_begin_read,
+ .consume_chunk = write_blob_process_chunk,
+ .end_blob = write_blob_end_read,
+ .ctx = &ctx,
+ };
+
+ ret = read_blob_list(blob_list,
+ offsetof(struct blob_descriptor, write_blobs_list),
+ &cbs,
+ BLOB_LIST_ALREADY_SORTED |
+ VERIFY_BLOB_HASHES |
+ COMPUTE_MISSING_BLOB_HASHES);
+
+ if (ret)
+ goto out_destroy_context;
+
+ ret = finish_remaining_chunks(&ctx);
+ if (ret)
+ goto out_destroy_context;
+
+ if (write_resource_flags & WRITE_RESOURCE_FLAG_SOLID) {
+ struct wim_reshdr reshdr;
+ struct blob_descriptor *blob;
+ u64 offset_in_res;
+
+ ret = end_write_resource(&ctx, &reshdr);
+ if (ret)
+ goto out_destroy_context;
+
+ offset_in_res = 0;
+ list_for_each_entry(blob, &ctx.blobs_in_solid_resource, write_blobs_list) {
+ blob->out_reshdr.size_in_wim = blob->size;
+ blob->out_reshdr.flags = reshdr_flags_for_blob(blob) |
+ WIM_RESHDR_FLAG_SOLID;
+ blob->out_reshdr.uncompressed_size = 0;
+ blob->out_reshdr.offset_in_wim = offset_in_res;
+ blob->out_res_offset_in_wim = reshdr.offset_in_wim;
+ blob->out_res_size_in_wim = reshdr.size_in_wim;
+ blob->out_res_uncompressed_size = reshdr.uncompressed_size;
+ offset_in_res += blob->size;
+ }
+ wimlib_assert(offset_in_res == reshdr.uncompressed_size);
+ }
+
+out_destroy_context:
+ FREE(ctx.chunk_csizes);
+ if (ctx.compressor)
+ ctx.compressor->destroy(ctx.compressor);
+ return ret;
+}
+
+
+static int
+write_file_data_blobs(WIMStruct *wim,
+ struct list_head *blob_list,
+ int write_flags,
+ unsigned num_threads,
+ struct filter_context *filter_ctx)
+{
+ int out_ctype;
+ u32 out_chunk_size;
+ int write_resource_flags;
+
+ write_resource_flags = write_flags_to_resource_flags(write_flags);
+
+ if (write_resource_flags & WRITE_RESOURCE_FLAG_SOLID) {
+ out_chunk_size = wim->out_solid_chunk_size;
+ out_ctype = wim->out_solid_compression_type;
+ } else {
+ out_chunk_size = wim->out_chunk_size;
+ out_ctype = wim->out_compression_type;
+ }
+
+ return write_blob_list(blob_list,
+ &wim->out_fd,
+ write_resource_flags,
+ out_ctype,
+ out_chunk_size,
+ num_threads,
+ wim->blob_table,
+ filter_ctx,
+ wim->progfunc,
+ wim->progctx);
+}
+
+/* Write the contents of the specified blob as a WIM resource. */
+static int
+write_wim_resource(struct blob_descriptor *blob,
+ struct filedes *out_fd,
+ int out_ctype,
+ u32 out_chunk_size,
+ int write_resource_flags)
+{
+ LIST_HEAD(blob_list);
+ list_add(&blob->write_blobs_list, &blob_list);
+ blob->will_be_in_output_wim = 1;
+ return write_blob_list(&blob_list,
+ out_fd,
+ write_resource_flags & ~WRITE_RESOURCE_FLAG_SOLID,
+ out_ctype,
+ out_chunk_size,
+ 1,
+ NULL,
+ NULL,
+ NULL,
+ NULL);
+}
+
+/* Write the contents of the specified buffer as a WIM resource. */
+int
+write_wim_resource_from_buffer(const void *buf,
+ size_t buf_size,
+ bool is_metadata,
+ struct filedes *out_fd,
+ int out_ctype,
+ u32 out_chunk_size,
+ struct wim_reshdr *out_reshdr,
+ u8 *hash_ret,
+ int write_resource_flags)
+{
+ int ret;
+ struct blob_descriptor blob;
+
+ if (unlikely(buf_size == 0)) {
+ zero_reshdr(out_reshdr);
+ if (hash_ret)
+ copy_hash(hash_ret, zero_hash);
+ return 0;
+ }
+
+ blob_set_is_located_in_attached_buffer(&blob, (void *)buf, buf_size);
+ sha1_buffer(buf, buf_size, blob.hash);
+ blob.unhashed = 0;
+ blob.is_metadata = is_metadata;
+
+ ret = write_wim_resource(&blob, out_fd, out_ctype, out_chunk_size,
+ write_resource_flags);
+ if (ret)
+ return ret;
+
+ copy_reshdr(out_reshdr, &blob.out_reshdr);
+
+ if (hash_ret)
+ copy_hash(hash_ret, blob.hash);
+ return 0;
+}
+
+struct blob_size_table {
+ struct hlist_head *array;
+ size_t num_entries;
+ size_t capacity;
+};
+
+static int
+init_blob_size_table(struct blob_size_table *tab, size_t capacity)
+{
+ tab->array = CALLOC(capacity, sizeof(tab->array[0]));
+ if (tab->array == NULL)
+ return WIMLIB_ERR_NOMEM;
+ tab->num_entries = 0;
+ tab->capacity = capacity;
+ return 0;
+}
+
+static void
+destroy_blob_size_table(struct blob_size_table *tab)
+{
+ FREE(tab->array);
+}
+
+static int
+blob_size_table_insert(struct blob_descriptor *blob, void *_tab)
+{
+ struct blob_size_table *tab = _tab;
+ size_t pos;
+ struct blob_descriptor *same_size_blob;
+
+ pos = hash_u64(blob->size) % tab->capacity;
+ blob->unique_size = 1;
+ hlist_for_each_entry(same_size_blob, &tab->array[pos], hash_list_2) {
+ if (same_size_blob->size == blob->size) {
+ blob->unique_size = 0;
+ same_size_blob->unique_size = 0;
+ break;
+ }
+ }
+
+ hlist_add_head(&blob->hash_list_2, &tab->array[pos]);
+ tab->num_entries++;
+ return 0;
+}
+
+struct find_blobs_ctx {
+ WIMStruct *wim;
+ int write_flags;
+ struct list_head blob_list;
+ struct blob_size_table blob_size_tab;
+};
+
+static void
+reference_blob_for_write(struct blob_descriptor *blob,
+ struct list_head *blob_list, u32 nref)
+{
+ if (!blob->will_be_in_output_wim) {
+ blob->out_refcnt = 0;
+ list_add_tail(&blob->write_blobs_list, blob_list);
+ blob->will_be_in_output_wim = 1;
+ }
+ blob->out_refcnt += nref;
+}
+
+static int
+fully_reference_blob_for_write(struct blob_descriptor *blob, void *_blob_list)
+{
+ struct list_head *blob_list = _blob_list;
+ blob->will_be_in_output_wim = 0;
+ reference_blob_for_write(blob, blob_list, blob->refcnt);
+ return 0;
+}
+
+static int
+inode_find_blobs_to_reference(const struct wim_inode *inode,
+ const struct blob_table *table,
+ struct list_head *blob_list)
+{
+ wimlib_assert(inode->i_nlink > 0);
+
+ for (unsigned i = 0; i < inode->i_num_streams; i++) {
+ struct blob_descriptor *blob;
+ const u8 *hash;
+
+ blob = stream_blob(&inode->i_streams[i], table);
+ if (blob) {
+ reference_blob_for_write(blob, blob_list, inode->i_nlink);
+ } else {
+ hash = stream_hash(&inode->i_streams[i]);
+ if (!is_zero_hash(hash))
+ return blob_not_found_error(inode, hash);
+ }
+ }
+ return 0;
+}
+
+static int
+do_blob_set_not_in_output_wim(struct blob_descriptor *blob, void *_ignore)
+{
+ blob->will_be_in_output_wim = 0;
+ return 0;
+}
+
+static int
+image_find_blobs_to_reference(WIMStruct *wim)
+{
+ struct wim_image_metadata *imd;
+ struct wim_inode *inode;
+ struct blob_descriptor *blob;
+ struct list_head *blob_list;
+ int ret;
+
+ imd = wim_get_current_image_metadata(wim);
+
+ image_for_each_unhashed_blob(blob, imd)
+ blob->will_be_in_output_wim = 0;
+
+ blob_list = wim->private;
+ image_for_each_inode(inode, imd) {
+ ret = inode_find_blobs_to_reference(inode,
+ wim->blob_table,
+ blob_list);
+ if (ret)
+ return ret;
+ }
+ return 0;
+}
+
+static int
+prepare_unfiltered_list_of_blobs_in_output_wim(WIMStruct *wim,
+ int image,
+ int blobs_ok,
+ struct list_head *blob_list_ret)
+{
+ int ret;
+
+ INIT_LIST_HEAD(blob_list_ret);
+
+ if (blobs_ok && (image == WIMLIB_ALL_IMAGES ||
+ (image == 1 && wim->hdr.image_count == 1)))
+ {
+ /* Fast case: Assume that all blobs are being written and that
+ * the reference counts are correct. */
+ struct blob_descriptor *blob;
+ struct wim_image_metadata *imd;
+ unsigned i;
+
+ for_blob_in_table(wim->blob_table,
+ fully_reference_blob_for_write,
+ blob_list_ret);
+
+ for (i = 0; i < wim->hdr.image_count; i++) {
+ imd = wim->image_metadata[i];
+ image_for_each_unhashed_blob(blob, imd)
+ fully_reference_blob_for_write(blob, blob_list_ret);
+ }
+ } else {
+ /* Slow case: Walk through the images being written and
+ * determine the blobs referenced. */
+ for_blob_in_table(wim->blob_table,
+ do_blob_set_not_in_output_wim, NULL);
+ wim->private = blob_list_ret;
+ ret = for_image(wim, image, image_find_blobs_to_reference);
+ if (ret)
+ return ret;
+ }
+
+ return 0;
+}
+
+struct insert_other_if_hard_filtered_ctx {
+ struct blob_size_table *tab;
+ struct filter_context *filter_ctx;
+};
+
+static int
+insert_other_if_hard_filtered(struct blob_descriptor *blob, void *_ctx)
+{
+ struct insert_other_if_hard_filtered_ctx *ctx = _ctx;
+
+ if (!blob->will_be_in_output_wim &&
+ blob_hard_filtered(blob, ctx->filter_ctx))
+ blob_size_table_insert(blob, ctx->tab);
+ return 0;
+}
+
+static int
+determine_blob_size_uniquity(struct list_head *blob_list,
+ struct blob_table *lt,
+ struct filter_context *filter_ctx)
+{
+ int ret;
+ struct blob_size_table tab;
+ struct blob_descriptor *blob;
+
+ ret = init_blob_size_table(&tab, 9001);
+ if (ret)
+ return ret;
+
+ if (may_hard_filter_blobs(filter_ctx)) {
+ struct insert_other_if_hard_filtered_ctx ctx = {
+ .tab = &tab,
+ .filter_ctx = filter_ctx,
+ };
+ for_blob_in_table(lt, insert_other_if_hard_filtered, &ctx);
+ }
+
+ list_for_each_entry(blob, blob_list, write_blobs_list)
+ blob_size_table_insert(blob, &tab);
+
+ destroy_blob_size_table(&tab);
+ return 0;
+}
+
+static void
+filter_blob_list_for_write(struct list_head *blob_list,
+ struct filter_context *filter_ctx)
+{
+ struct blob_descriptor *blob, *tmp;
+
+ list_for_each_entry_safe(blob, tmp, blob_list, write_blobs_list) {
+ int status = blob_filtered(blob, filter_ctx);
+
+ if (status == 0) {
+ /* Not filtered. */
+ continue;
+ } else {
+ if (status > 0) {
+ /* Soft filtered. */
+ } else {
+ /* Hard filtered. */
+ blob->will_be_in_output_wim = 0;
+ list_del(&blob->blob_table_list);