+ for (size_t i = 0; i < num_messages; i++)
+ list_add(&msgs[i].list, &available_msgs);
+
+ // outstanding_resources is the list of resources that currently have
+ // had chunks sent off for compression.
+ //
+ // The first stream in outstanding_resources is the stream that is
+ // currently being written (cur_lte).
+ //
+ // The last stream in outstanding_resources is the stream that is
+ // currently being read and chunks fed to the compressor threads
+ // (next_lte).
+ //
+ // Depending on the number of threads and the sizes of the resource,
+ // the outstanding streams list may contain streams between cur_lte and
+ // next_lte that have all their chunks compressed or being compressed,
+ // but haven't been written yet.
+ //
+ LIST_HEAD(outstanding_resources);
+ struct list_head *next_resource = stream_list->next;
+ u64 next_chunk = 0;
+ u64 next_num_chunks = 0;
+
+ // As in write_wim_resource(), each resource we read is checksummed.
+ SHA_CTX next_sha_ctx;
+ u8 next_hash[SHA1_HASH_SIZE];
+
+ // Resources that don't need any chunks compressed are added to this
+ // list and written directly by the main thread.
+ LIST_HEAD(my_resources);
+
+ struct wim_lookup_table_entry *cur_lte = NULL;
+ struct message *msg;
+
+#ifdef WITH_NTFS_3G
+ ntfs_inode *ni = NULL;
+#endif
+
+ DEBUG("Initializing buffers for uncompressed "
+ "and compressed data (%zu bytes needed)",
+ num_messages * MAX_CHUNKS_PER_MSG * WIM_CHUNK_SIZE * 2);
+
+ // Pre-allocate all the buffers that will be needed to do the chunk
+ // compression.
+ for (size_t i = 0; i < num_messages; i++) {
+ for (size_t j = 0; j < MAX_CHUNKS_PER_MSG; j++) {
+ msgs[i].compressed_chunks[j] = MALLOC(WIM_CHUNK_SIZE);
+
+ // The extra 8 bytes is because longest_match() in
+ // lz77.c may read a little bit off the end of the
+ // uncompressed data. It doesn't need to be
+ // initialized--- we really just need to avoid accessing
+ // an unmapped page.
+ msgs[i].uncompressed_chunks[j] = MALLOC(WIM_CHUNK_SIZE + 8);
+ if (msgs[i].compressed_chunks[j] == NULL ||
+ msgs[i].uncompressed_chunks[j] == NULL)
+ {
+ ret = WIMLIB_ERR_NOMEM;
+ goto out;
+ }
+ }
+ }
+
+ // This loop is executed until all resources have been written, except
+ // possibly a few that have been added to the @my_resources list for
+ // writing later.
+ while (1) {
+ // Send chunks to the compressor threads until either (a) there
+ // are no more messages available since they were all sent off,
+ // or (b) there are no more resources that need to be
+ // compressed.
+ while (!list_empty(&available_msgs)) {
+ if (next_chunk == next_num_chunks) {
+ // If next_chunk == next_num_chunks, there are
+ // no more chunks to write in the current
+ // stream. So, check the SHA1 message digest of
+ // the stream that was just finished (unless
+ // next_lte == NULL, which is the case the very
+ // first time this loop is entered, and also
+ // near the very end of the compression when
+ // there are no more streams.) Then, advance to
+ // the next stream (if there is one).
+ if (next_lte != NULL) {
+ #ifdef WITH_NTFS_3G
+ end_wim_resource_read(next_lte, ni);
+ ni = NULL;
+ #else
+ end_wim_resource_read(next_lte);
+ #endif
+ DEBUG2("Finalize SHA1 md (next_num_chunks=%zu)",
+ next_num_chunks);
+ sha1_final(next_hash, &next_sha_ctx);
+ if (!hashes_equal(next_lte->hash, next_hash)) {
+ ERROR("WIM resource has incorrect hash!");
+ if (next_lte->resource_location ==
+ RESOURCE_IN_FILE_ON_DISK)
+ {
+ ERROR("We were reading it from `%"TS"'; "
+ "maybe it changed while we were "
+ "reading it.",
+ next_lte->file_on_disk);
+ }
+ ret = WIMLIB_ERR_INVALID_RESOURCE_HASH;
+ goto out;
+ }
+ }
+
+ // Advance to the next resource.
+ //
+ // If the next resource needs no compression, just write
+ // it with this thread (not now though--- we could be in
+ // the middle of writing another resource.) Keep doing
+ // this until we either get to the end of the resources
+ // list, or we get to a resource that needs compression.
+ while (1) {
+ if (next_resource == stream_list) {
+ // No more resources to send for
+ // compression
+ next_lte = NULL;
+ break;
+ }
+ next_lte = container_of(next_resource,
+ struct wim_lookup_table_entry,
+ staging_list);
+ next_resource = next_resource->next;
+ if ((!(write_flags & WIMLIB_WRITE_FLAG_RECOMPRESS)
+ && wim_resource_compression_type(next_lte) == out_ctype)
+ || wim_resource_size(next_lte) == 0)
+ {
+ list_add_tail(&next_lte->staging_list,
+ &my_resources);
+ } else {
+ list_add_tail(&next_lte->staging_list,
+ &outstanding_resources);
+ next_chunk = 0;
+ next_num_chunks = wim_resource_chunks(next_lte);
+ sha1_init(&next_sha_ctx);
+ INIT_LIST_HEAD(&next_lte->msg_list);
+ #ifdef WITH_NTFS_3G
+ ret = prepare_resource_for_read(next_lte, &ni);
+ #else
+ ret = prepare_resource_for_read(next_lte);
+ #endif
+
+ if (ret != 0)
+ goto out;
+ if (cur_lte == NULL) {
+ // Set cur_lte for the
+ // first time
+ cur_lte = next_lte;
+ }
+ break;
+ }
+ }