4 * Support for writing WIM files; write a WIM file, overwrite a WIM file, write
5 * compressed file resources, etc.
9 * Copyright (C) 2012, 2013 Eric Biggers
11 * This file is part of wimlib, a library for working with WIM files.
13 * wimlib is free software; you can redistribute it and/or modify it under the
14 * terms of the GNU General Public License as published by the Free
15 * Software Foundation; either version 3 of the License, or (at your option)
18 * wimlib is distributed in the hope that it will be useful, but WITHOUT ANY
19 * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
20 * A PARTICULAR PURPOSE. See the GNU General Public License for more
23 * You should have received a copy of the GNU General Public License
24 * along with wimlib; if not, see http://www.gnu.org/licenses/.
29 #if defined(HAVE_SYS_FILE_H) && defined(HAVE_FLOCK)
30 /* On BSD, this should be included before "list.h" so that "list.h" can
31 * overwrite the LIST_HEAD macro. */
32 # include <sys/file.h>
40 #include "wimlib_internal.h"
41 #include "buffer_io.h"
43 #include "lookup_table.h"
48 #ifdef ENABLE_MULTITHREADED_COMPRESSION
57 # include <ntfs-3g/attrib.h>
58 # include <ntfs-3g/inode.h>
59 # include <ntfs-3g/dir.h>
71 fflush_and_ftruncate(FILE *fp, off_t size)
77 ERROR_WITH_ERRNO("Failed to flush data to output WIM file");
78 return WIMLIB_ERR_WRITE;
80 ret = ftruncate(fileno(fp), size);
82 ERROR_WITH_ERRNO("Failed to truncate output WIM file to "
83 "%"PRIu64" bytes", size);
84 return WIMLIB_ERR_WRITE;
89 /* Chunk table that's located at the beginning of each compressed resource in
90 * the WIM. (This is not the on-disk format; the on-disk format just has an
91 * array of offsets.) */
95 u64 original_resource_size;
96 u64 bytes_per_chunk_entry;
104 * Allocates and initializes a chunk table, and reserves space for it in the
108 begin_wim_resource_chunk_tab(const struct wim_lookup_table_entry *lte,
111 struct chunk_table **chunk_tab_ret)
113 u64 size = wim_resource_size(lte);
114 u64 num_chunks = (size + WIM_CHUNK_SIZE - 1) / WIM_CHUNK_SIZE;
115 size_t alloc_size = sizeof(struct chunk_table) + num_chunks * sizeof(u64);
116 struct chunk_table *chunk_tab = CALLOC(1, alloc_size);
120 ERROR("Failed to allocate chunk table for %"PRIu64" byte "
122 ret = WIMLIB_ERR_NOMEM;
125 chunk_tab->file_offset = file_offset;
126 chunk_tab->num_chunks = num_chunks;
127 chunk_tab->original_resource_size = size;
128 chunk_tab->bytes_per_chunk_entry = (size >= (1ULL << 32)) ? 8 : 4;
129 chunk_tab->table_disk_size = chunk_tab->bytes_per_chunk_entry *
131 chunk_tab->cur_offset = 0;
132 chunk_tab->cur_offset_p = chunk_tab->offsets;
134 if (fwrite(chunk_tab, 1, chunk_tab->table_disk_size, out_fp) !=
135 chunk_tab->table_disk_size) {
136 ERROR_WITH_ERRNO("Failed to write chunk table in compressed "
138 ret = WIMLIB_ERR_WRITE;
144 *chunk_tab_ret = chunk_tab;
149 * Pointer to function to compresses a chunk of a WIM resource.
151 * @chunk: Uncompressed data of the chunk.
152 * @chunk_size: Size of the uncompressed chunk in bytes.
153 * @compressed_chunk: Pointer to output buffer of size at least
154 * (@chunk_size - 1) bytes.
155 * @compressed_chunk_len_ret: Pointer to an unsigned int into which the size
156 * of the compressed chunk will be
159 * Returns zero if compressed succeeded, and nonzero if the chunk could not be
160 * compressed to any smaller than @chunk_size. This function cannot fail for
163 typedef int (*compress_func_t)(const void *, unsigned, void *, unsigned *);
166 get_compress_func(int out_ctype)
168 if (out_ctype == WIMLIB_COMPRESSION_TYPE_LZX)
171 return xpress_compress;
175 * Writes a chunk of a WIM resource to an output file.
177 * @chunk: Uncompressed data of the chunk.
178 * @chunk_size: Size of the chunk (<= WIM_CHUNK_SIZE)
179 * @out_fp: FILE * to write tho chunk to.
180 * @out_ctype: Compression type to use when writing the chunk (ignored if no
181 * chunk table provided)
182 * @chunk_tab: Pointer to chunk table being created. It is updated with the
183 * offset of the chunk we write.
185 * Returns 0 on success; nonzero on failure.
188 write_wim_resource_chunk(const u8 chunk[], unsigned chunk_size,
189 FILE *out_fp, compress_func_t compress,
190 struct chunk_table *chunk_tab)
193 unsigned out_chunk_size;
195 u8 *compressed_chunk = alloca(chunk_size);
198 ret = compress(chunk, chunk_size, compressed_chunk,
201 out_chunk = compressed_chunk;
204 out_chunk_size = chunk_size;
206 *chunk_tab->cur_offset_p++ = chunk_tab->cur_offset;
207 chunk_tab->cur_offset += out_chunk_size;
210 out_chunk_size = chunk_size;
212 if (fwrite(out_chunk, 1, out_chunk_size, out_fp) != out_chunk_size) {
213 ERROR_WITH_ERRNO("Failed to write WIM resource chunk");
214 return WIMLIB_ERR_WRITE;
220 * Finishes a WIM chunk table and writes it to the output file at the correct
223 * The final size of the full compressed resource is returned in the
224 * @compressed_size_p.
227 finish_wim_resource_chunk_tab(struct chunk_table *chunk_tab,
228 FILE *out_fp, u64 *compressed_size_p)
230 size_t bytes_written;
231 if (fseeko(out_fp, chunk_tab->file_offset, SEEK_SET) != 0) {
232 ERROR_WITH_ERRNO("Failed to seek to byte %"PRIu64" of output "
233 "WIM file", chunk_tab->file_offset);
234 return WIMLIB_ERR_WRITE;
237 if (chunk_tab->bytes_per_chunk_entry == 8) {
238 array_cpu_to_le64(chunk_tab->offsets, chunk_tab->num_chunks);
240 for (u64 i = 0; i < chunk_tab->num_chunks; i++)
241 ((u32*)chunk_tab->offsets)[i] =
242 cpu_to_le32(chunk_tab->offsets[i]);
244 bytes_written = fwrite((u8*)chunk_tab->offsets +
245 chunk_tab->bytes_per_chunk_entry,
246 1, chunk_tab->table_disk_size, out_fp);
247 if (bytes_written != chunk_tab->table_disk_size) {
248 ERROR_WITH_ERRNO("Failed to write chunk table in compressed "
250 return WIMLIB_ERR_WRITE;
252 if (fseeko(out_fp, 0, SEEK_END) != 0) {
253 ERROR_WITH_ERRNO("Failed to seek to end of output WIM file");
254 return WIMLIB_ERR_WRITE;
256 *compressed_size_p = chunk_tab->cur_offset + chunk_tab->table_disk_size;
260 /* Prepare for multiple reads to a resource by caching a FILE * or NTFS
261 * attribute pointer in the lookup table entry. */
263 prepare_resource_for_read(struct wim_lookup_table_entry *lte
266 , ntfs_inode **ni_ret
270 switch (lte->resource_location) {
271 case RESOURCE_IN_FILE_ON_DISK:
272 if (!lte->file_on_disk_fp) {
273 lte->file_on_disk_fp = fopen(lte->file_on_disk, "rb");
274 if (!lte->file_on_disk_fp) {
275 ERROR_WITH_ERRNO("Failed to open the file "
276 "`%s'", lte->file_on_disk);
277 return WIMLIB_ERR_OPEN;
282 case RESOURCE_IN_NTFS_VOLUME:
284 struct ntfs_location *loc = lte->ntfs_loc;
287 ni = ntfs_pathname_to_inode(*loc->ntfs_vol_p, NULL, loc->path);
289 ERROR_WITH_ERRNO("Failed to open inode `%s' in NTFS "
290 "volume", loc->path);
291 return WIMLIB_ERR_NTFS_3G;
293 lte->attr = ntfs_attr_open(ni,
294 loc->is_reparse_point ? AT_REPARSE_POINT : AT_DATA,
296 loc->stream_name_nchars);
298 ERROR_WITH_ERRNO("Failed to open attribute of `%s' in "
299 "NTFS volume", loc->path);
300 ntfs_inode_close(ni);
301 return WIMLIB_ERR_NTFS_3G;
309 if (!lte->file_on_disk_fp) {
310 lte->file_on_disk_fp = win32_open_file_readonly(lte->file_on_disk);
311 if (!lte->file_on_disk_fp)
312 return WIMLIB_ERR_OPEN;
322 /* Undo prepare_resource_for_read() by closing the cached FILE * or NTFS
325 end_wim_resource_read(struct wim_lookup_table_entry *lte
331 if (lte->resource_location == RESOURCE_IN_FILE_ON_DISK
332 && lte->file_on_disk_fp)
334 fclose(lte->file_on_disk_fp);
335 lte->file_on_disk_fp = NULL;
338 else if (lte->resource_location == RESOURCE_IN_NTFS_VOLUME) {
340 ntfs_attr_close(lte->attr);
344 ntfs_inode_close(ni);
348 else if (lte->resource_location == RESOURCE_WIN32
349 && lte->file_on_disk_fp)
351 win32_close_file(lte->file_on_disk_fp);
352 lte->file_on_disk_fp = NULL;
358 write_uncompressed_resource_and_truncate(struct wim_lookup_table_entry *lte,
361 struct resource_entry *out_res_entry)
364 if (fseeko(out_fp, file_offset, SEEK_SET) != 0) {
365 ERROR_WITH_ERRNO("Failed to seek to byte %"PRIu64" of "
366 "output WIM file", file_offset);
367 return WIMLIB_ERR_WRITE;
369 ret = write_wim_resource(lte, out_fp, WIMLIB_COMPRESSION_TYPE_NONE,
374 return fflush_and_ftruncate(out_fp,
375 file_offset + wim_resource_size(lte));
379 * Writes a WIM resource to a FILE * opened for writing. The resource may be
380 * written uncompressed or compressed depending on the @out_ctype parameter.
382 * If by chance the resource compresses to more than the original size (this may
383 * happen with random data or files than are pre-compressed), the resource is
384 * instead written uncompressed (and this is reflected in the @out_res_entry by
385 * removing the WIM_RESHDR_FLAG_COMPRESSED flag).
387 * @lte: The lookup table entry for the WIM resource.
388 * @out_fp: The FILE * to write the resource to.
389 * @out_ctype: The compression type of the resource to write. Note: if this is
390 * the same as the compression type of the WIM resource we
391 * need to read, we simply copy the data (i.e. we do not
392 * uncompress it, then compress it again).
393 * @out_res_entry: If non-NULL, a resource entry that is filled in with the
394 * offset, original size, compressed size, and compression flag
395 * of the output resource.
397 * Returns 0 on success; nonzero on failure.
400 write_wim_resource(struct wim_lookup_table_entry *lte,
401 FILE *out_fp, int out_ctype,
402 struct resource_entry *out_res_entry,
407 u64 old_compressed_size;
408 u64 new_compressed_size;
411 struct chunk_table *chunk_tab = NULL;
414 compress_func_t compress = NULL;
416 ntfs_inode *ni = NULL;
421 /* Original size of the resource */
422 original_size = wim_resource_size(lte);
424 /* Compressed size of the resource (as it exists now) */
425 old_compressed_size = wim_resource_compressed_size(lte);
427 /* Current offset in output file */
428 file_offset = ftello(out_fp);
429 if (file_offset == -1) {
430 ERROR_WITH_ERRNO("Failed to get offset in output "
432 return WIMLIB_ERR_WRITE;
435 /* Are the compression types the same? If so, do a raw copy (copy
436 * without decompressing and recompressing the data). */
437 raw = (wim_resource_compression_type(lte) == out_ctype
438 && out_ctype != WIMLIB_COMPRESSION_TYPE_NONE
439 && !(flags & WIMLIB_RESOURCE_FLAG_RECOMPRESS));
442 flags |= WIMLIB_RESOURCE_FLAG_RAW;
443 bytes_remaining = old_compressed_size;
445 flags &= ~WIMLIB_RESOURCE_FLAG_RAW;
446 bytes_remaining = original_size;
449 /* Empty resource; nothing needs to be done, so just return success. */
450 if (bytes_remaining == 0)
453 /* Buffer for reading chunks for the resource */
454 u8 buf[min(WIM_CHUNK_SIZE, bytes_remaining)];
456 /* If we are writing a compressed resource and not doing a raw copy, we
457 * need to initialize the chunk table */
458 if (out_ctype != WIMLIB_COMPRESSION_TYPE_NONE && !raw) {
459 ret = begin_wim_resource_chunk_tab(lte, out_fp, file_offset,
465 /* If the WIM resource is in an external file, open a FILE * to it so we
466 * don't have to open a temporary one in read_wim_resource() for each
469 ret = prepare_resource_for_read(lte, &ni);
471 ret = prepare_resource_for_read(lte);
476 /* If we aren't doing a raw copy, we will compute the SHA1 message
477 * digest of the resource as we read it, and verify it's the same as the
478 * hash given in the lookup table entry once we've finished reading the
483 compress = get_compress_func(out_ctype);
487 /* While there are still bytes remaining in the WIM resource, read a
488 * chunk of the resource, update SHA1, then write that chunk using the
489 * desired compression type. */
491 u64 to_read = min(bytes_remaining, WIM_CHUNK_SIZE);
492 ret = read_wim_resource(lte, buf, to_read, offset, flags);
496 sha1_update(&ctx, buf, to_read);
497 ret = write_wim_resource_chunk(buf, to_read, out_fp,
498 compress, chunk_tab);
501 bytes_remaining -= to_read;
503 } while (bytes_remaining);
505 /* Raw copy: The new compressed size is the same as the old compressed
508 * Using WIMLIB_COMPRESSION_TYPE_NONE: The new compressed size is the
511 * Using a different compression type: Call
512 * finish_wim_resource_chunk_tab() and it will provide the new
516 new_compressed_size = old_compressed_size;
518 if (out_ctype == WIMLIB_COMPRESSION_TYPE_NONE)
519 new_compressed_size = original_size;
521 ret = finish_wim_resource_chunk_tab(chunk_tab, out_fp,
522 &new_compressed_size);
528 /* Verify SHA1 message digest of the resource, unless we are doing a raw
529 * write (in which case we never even saw the uncompressed data). Or,
530 * if the hash we had before is all 0's, just re-set it to be the new
533 u8 md[SHA1_HASH_SIZE];
534 sha1_final(md, &ctx);
535 if (is_zero_hash(lte->hash)) {
536 copy_hash(lte->hash, md);
537 } else if (!hashes_equal(md, lte->hash)) {
538 ERROR("WIM resource has incorrect hash!");
539 if (lte->resource_location == RESOURCE_IN_FILE_ON_DISK) {
540 ERROR("We were reading it from `%s'; maybe it changed "
541 "while we were reading it.",
544 ret = WIMLIB_ERR_INVALID_RESOURCE_HASH;
549 if (!raw && new_compressed_size >= original_size &&
550 out_ctype != WIMLIB_COMPRESSION_TYPE_NONE)
552 /* Oops! We compressed the resource to larger than the original
553 * size. Write the resource uncompressed instead. */
554 ret = write_uncompressed_resource_and_truncate(lte,
562 out_res_entry->size = new_compressed_size;
563 out_res_entry->original_size = original_size;
564 out_res_entry->offset = file_offset;
565 out_res_entry->flags = lte->resource_entry.flags
566 & ~WIM_RESHDR_FLAG_COMPRESSED;
567 if (out_ctype != WIMLIB_COMPRESSION_TYPE_NONE)
568 out_res_entry->flags |= WIM_RESHDR_FLAG_COMPRESSED;
574 end_wim_resource_read(lte, ni);
576 end_wim_resource_read(lte);
583 #ifdef ENABLE_MULTITHREADED_COMPRESSION
585 /* Blocking shared queue (solves the producer-consumer problem) */
586 struct shared_queue {
590 unsigned filled_slots;
592 pthread_mutex_t lock;
593 pthread_cond_t msg_avail_cond;
594 pthread_cond_t space_avail_cond;
598 shared_queue_init(struct shared_queue *q, unsigned size)
600 wimlib_assert(size != 0);
601 q->array = CALLOC(sizeof(q->array[0]), size);
603 return WIMLIB_ERR_NOMEM;
608 pthread_mutex_init(&q->lock, NULL);
609 pthread_cond_init(&q->msg_avail_cond, NULL);
610 pthread_cond_init(&q->space_avail_cond, NULL);
615 shared_queue_destroy(struct shared_queue *q)
618 pthread_mutex_destroy(&q->lock);
619 pthread_cond_destroy(&q->msg_avail_cond);
620 pthread_cond_destroy(&q->space_avail_cond);
624 shared_queue_put(struct shared_queue *q, void *obj)
626 pthread_mutex_lock(&q->lock);
627 while (q->filled_slots == q->size)
628 pthread_cond_wait(&q->space_avail_cond, &q->lock);
630 q->back = (q->back + 1) % q->size;
631 q->array[q->back] = obj;
634 pthread_cond_broadcast(&q->msg_avail_cond);
635 pthread_mutex_unlock(&q->lock);
639 shared_queue_get(struct shared_queue *q)
643 pthread_mutex_lock(&q->lock);
644 while (q->filled_slots == 0)
645 pthread_cond_wait(&q->msg_avail_cond, &q->lock);
647 obj = q->array[q->front];
648 q->array[q->front] = NULL;
649 q->front = (q->front + 1) % q->size;
652 pthread_cond_broadcast(&q->space_avail_cond);
653 pthread_mutex_unlock(&q->lock);
657 struct compressor_thread_params {
658 struct shared_queue *res_to_compress_queue;
659 struct shared_queue *compressed_res_queue;
660 compress_func_t compress;
663 #define MAX_CHUNKS_PER_MSG 2
666 struct wim_lookup_table_entry *lte;
667 u8 *uncompressed_chunks[MAX_CHUNKS_PER_MSG];
668 u8 *out_compressed_chunks[MAX_CHUNKS_PER_MSG];
669 u8 *compressed_chunks[MAX_CHUNKS_PER_MSG];
670 unsigned uncompressed_chunk_sizes[MAX_CHUNKS_PER_MSG];
671 unsigned compressed_chunk_sizes[MAX_CHUNKS_PER_MSG];
673 struct list_head list;
679 compress_chunks(struct message *msg, compress_func_t compress)
681 for (unsigned i = 0; i < msg->num_chunks; i++) {
682 DEBUG2("compress chunk %u of %u", i, msg->num_chunks);
683 int ret = compress(msg->uncompressed_chunks[i],
684 msg->uncompressed_chunk_sizes[i],
685 msg->compressed_chunks[i],
686 &msg->compressed_chunk_sizes[i]);
688 msg->out_compressed_chunks[i] = msg->compressed_chunks[i];
690 msg->out_compressed_chunks[i] = msg->uncompressed_chunks[i];
691 msg->compressed_chunk_sizes[i] = msg->uncompressed_chunk_sizes[i];
696 /* Compressor thread routine. This is a lot simpler than the main thread
697 * routine: just repeatedly get a group of chunks from the
698 * res_to_compress_queue, compress them, and put them in the
699 * compressed_res_queue. A NULL pointer indicates that the thread should stop.
702 compressor_thread_proc(void *arg)
704 struct compressor_thread_params *params = arg;
705 struct shared_queue *res_to_compress_queue = params->res_to_compress_queue;
706 struct shared_queue *compressed_res_queue = params->compressed_res_queue;
707 compress_func_t compress = params->compress;
710 DEBUG("Compressor thread ready");
711 while ((msg = shared_queue_get(res_to_compress_queue)) != NULL) {
712 compress_chunks(msg, compress);
713 shared_queue_put(compressed_res_queue, msg);
715 DEBUG("Compressor thread terminating");
718 #endif /* ENABLE_MULTITHREADED_COMPRESSION */
721 do_write_stream_list(struct list_head *my_resources,
724 wimlib_progress_func_t progress_func,
725 union wimlib_progress_info *progress,
726 int write_resource_flags)
729 struct wim_lookup_table_entry *lte, *tmp;
731 list_for_each_entry_safe(lte, tmp, my_resources, staging_list) {
732 ret = write_wim_resource(lte,
735 <e->output_resource_entry,
736 write_resource_flags);
739 list_del(<e->staging_list);
740 progress->write_streams.completed_bytes +=
741 wim_resource_size(lte);
742 progress->write_streams.completed_streams++;
744 progress_func(WIMLIB_PROGRESS_MSG_WRITE_STREAMS,
752 write_stream_list_serial(struct list_head *stream_list,
756 wimlib_progress_func_t progress_func,
757 union wimlib_progress_info *progress)
759 int write_resource_flags;
761 if (write_flags & WIMLIB_WRITE_FLAG_RECOMPRESS)
762 write_resource_flags = WIMLIB_RESOURCE_FLAG_RECOMPRESS;
764 write_resource_flags = 0;
765 progress->write_streams.num_threads = 1;
767 progress_func(WIMLIB_PROGRESS_MSG_WRITE_STREAMS, progress);
768 return do_write_stream_list(stream_list, out_fp,
769 out_ctype, progress_func,
770 progress, write_resource_flags);
773 #ifdef ENABLE_MULTITHREADED_COMPRESSION
775 write_wim_chunks(struct message *msg, FILE *out_fp,
776 struct chunk_table *chunk_tab)
778 for (unsigned i = 0; i < msg->num_chunks; i++) {
779 unsigned chunk_csize = msg->compressed_chunk_sizes[i];
781 DEBUG2("Write wim chunk %u of %u (csize = %u)",
782 i, msg->num_chunks, chunk_csize);
784 if (fwrite(msg->out_compressed_chunks[i], 1, chunk_csize, out_fp)
787 ERROR_WITH_ERRNO("Failed to write WIM chunk");
788 return WIMLIB_ERR_WRITE;
791 *chunk_tab->cur_offset_p++ = chunk_tab->cur_offset;
792 chunk_tab->cur_offset += chunk_csize;
798 * This function is executed by the main thread when the resources are being
799 * compressed in parallel. The main thread is in change of all reading of the
800 * uncompressed data and writing of the compressed data. The compressor threads
801 * *only* do compression from/to in-memory buffers.
803 * Each unit of work given to a compressor thread is up to MAX_CHUNKS_PER_MSG
804 * chunks of compressed data to compress, represented in a `struct message'.
805 * Each message is passed from the main thread to a worker thread through the
806 * res_to_compress_queue, and it is passed back through the
807 * compressed_res_queue.
810 main_writer_thread_proc(struct list_head *stream_list,
813 struct shared_queue *res_to_compress_queue,
814 struct shared_queue *compressed_res_queue,
817 wimlib_progress_func_t progress_func,
818 union wimlib_progress_info *progress)
821 struct chunk_table *cur_chunk_tab = NULL;
822 struct message *msgs = CALLOC(num_messages, sizeof(struct message));
823 struct wim_lookup_table_entry *next_lte = NULL;
825 // Initially, all the messages are available to use.
826 LIST_HEAD(available_msgs);
829 ret = WIMLIB_ERR_NOMEM;
833 for (size_t i = 0; i < num_messages; i++)
834 list_add(&msgs[i].list, &available_msgs);
836 // outstanding_resources is the list of resources that currently have
837 // had chunks sent off for compression.
839 // The first stream in outstanding_resources is the stream that is
840 // currently being written (cur_lte).
842 // The last stream in outstanding_resources is the stream that is
843 // currently being read and chunks fed to the compressor threads
846 // Depending on the number of threads and the sizes of the resource,
847 // the outstanding streams list may contain streams between cur_lte and
848 // next_lte that have all their chunks compressed or being compressed,
849 // but haven't been written yet.
851 LIST_HEAD(outstanding_resources);
852 struct list_head *next_resource = stream_list->next;
854 u64 next_num_chunks = 0;
856 // As in write_wim_resource(), each resource we read is checksummed.
857 SHA_CTX next_sha_ctx;
858 u8 next_hash[SHA1_HASH_SIZE];
860 // Resources that don't need any chunks compressed are added to this
861 // list and written directly by the main thread.
862 LIST_HEAD(my_resources);
864 struct wim_lookup_table_entry *cur_lte = NULL;
868 ntfs_inode *ni = NULL;
871 DEBUG("Initializing buffers for uncompressed "
872 "and compressed data (%zu bytes needed)",
873 num_messages * MAX_CHUNKS_PER_MSG * WIM_CHUNK_SIZE * 2);
875 // Pre-allocate all the buffers that will be needed to do the chunk
877 for (size_t i = 0; i < num_messages; i++) {
878 for (size_t j = 0; j < MAX_CHUNKS_PER_MSG; j++) {
879 msgs[i].compressed_chunks[j] = MALLOC(WIM_CHUNK_SIZE);
881 // The extra 8 bytes is because longest_match() in
882 // lz77.c may read a little bit off the end of the
883 // uncompressed data. It doesn't need to be
884 // initialized--- we really just need to avoid accessing
886 msgs[i].uncompressed_chunks[j] = MALLOC(WIM_CHUNK_SIZE + 8);
887 if (msgs[i].compressed_chunks[j] == NULL ||
888 msgs[i].uncompressed_chunks[j] == NULL)
890 ret = WIMLIB_ERR_NOMEM;
896 // This loop is executed until all resources have been written, except
897 // possibly a few that have been added to the @my_resources list for
900 // Send chunks to the compressor threads until either (a) there
901 // are no more messages available since they were all sent off,
902 // or (b) there are no more resources that need to be
904 while (!list_empty(&available_msgs)) {
905 if (next_chunk == next_num_chunks) {
906 // If next_chunk == next_num_chunks, there are
907 // no more chunks to write in the current
908 // stream. So, check the SHA1 message digest of
909 // the stream that was just finished (unless
910 // next_lte == NULL, which is the case the very
911 // first time this loop is entered, and also
912 // near the very end of the compression when
913 // there are no more streams.) Then, advance to
914 // the next stream (if there is one).
915 if (next_lte != NULL) {
917 end_wim_resource_read(next_lte, ni);
920 end_wim_resource_read(next_lte);
922 DEBUG2("Finalize SHA1 md (next_num_chunks=%zu)",
924 sha1_final(next_hash, &next_sha_ctx);
925 if (!hashes_equal(next_lte->hash, next_hash)) {
926 ERROR("WIM resource has incorrect hash!");
927 if (next_lte->resource_location ==
928 RESOURCE_IN_FILE_ON_DISK)
930 ERROR("We were reading it from `%s'; "
931 "maybe it changed while we were "
933 next_lte->file_on_disk);
935 ret = WIMLIB_ERR_INVALID_RESOURCE_HASH;
940 // Advance to the next resource.
942 // If the next resource needs no compression, just write
943 // it with this thread (not now though--- we could be in
944 // the middle of writing another resource.) Keep doing
945 // this until we either get to the end of the resources
946 // list, or we get to a resource that needs compression.
948 if (next_resource == stream_list) {
949 // No more resources to send for
954 next_lte = container_of(next_resource,
955 struct wim_lookup_table_entry,
957 next_resource = next_resource->next;
958 if ((!(write_flags & WIMLIB_WRITE_FLAG_RECOMPRESS)
959 && wim_resource_compression_type(next_lte) == out_ctype)
960 || wim_resource_size(next_lte) == 0)
962 list_add_tail(&next_lte->staging_list,
965 list_add_tail(&next_lte->staging_list,
966 &outstanding_resources);
968 next_num_chunks = wim_resource_chunks(next_lte);
969 sha1_init(&next_sha_ctx);
970 INIT_LIST_HEAD(&next_lte->msg_list);
972 ret = prepare_resource_for_read(next_lte, &ni);
974 ret = prepare_resource_for_read(next_lte);
979 if (cur_lte == NULL) {
980 // Set cur_lte for the
989 if (next_lte == NULL) {
990 // No more resources to send for compression
994 // Get a message from the available messages
996 msg = container_of(available_msgs.next,
1000 // ... and delete it from the available messages
1002 list_del(&msg->list);
1004 // Initialize the message with the chunks to
1006 msg->num_chunks = min(next_num_chunks - next_chunk,
1007 MAX_CHUNKS_PER_MSG);
1008 msg->lte = next_lte;
1009 msg->complete = false;
1010 msg->begin_chunk = next_chunk;
1012 unsigned size = WIM_CHUNK_SIZE;
1013 for (unsigned i = 0; i < msg->num_chunks; i++) {
1015 // Read chunk @next_chunk of the stream into the
1016 // message so that a compressor thread can
1019 if (next_chunk == next_num_chunks - 1) {
1020 size = MODULO_NONZERO(wim_resource_size(next_lte),
1024 DEBUG2("Read resource (size=%u, offset=%zu)",
1025 size, next_chunk * WIM_CHUNK_SIZE);
1027 msg->uncompressed_chunk_sizes[i] = size;
1029 ret = read_wim_resource(next_lte,
1030 msg->uncompressed_chunks[i],
1032 next_chunk * WIM_CHUNK_SIZE,
1036 sha1_update(&next_sha_ctx,
1037 msg->uncompressed_chunks[i], size);
1041 // Send the compression request
1042 list_add_tail(&msg->list, &next_lte->msg_list);
1043 shared_queue_put(res_to_compress_queue, msg);
1044 DEBUG2("Compression request sent");
1047 // If there are no outstanding resources, there are no more
1048 // resources that need to be written.
1049 if (list_empty(&outstanding_resources)) {
1054 // Get the next message from the queue and process it.
1055 // The message will contain 1 or more data chunks that have been
1057 msg = shared_queue_get(compressed_res_queue);
1058 msg->complete = true;
1060 // Is this the next chunk in the current resource? If it's not
1061 // (i.e., an earlier chunk in a same or different resource
1062 // hasn't been compressed yet), do nothing, and keep this
1063 // message around until all earlier chunks are received.
1065 // Otherwise, write all the chunks we can.
1066 while (cur_lte != NULL &&
1067 !list_empty(&cur_lte->msg_list) &&
1068 (msg = container_of(cur_lte->msg_list.next,
1072 DEBUG2("Complete msg (begin_chunk=%"PRIu64")", msg->begin_chunk);
1073 if (msg->begin_chunk == 0) {
1074 DEBUG2("Begin chunk tab");
1076 // This is the first set of chunks. Leave space
1077 // for the chunk table in the output file.
1078 off_t cur_offset = ftello(out_fp);
1079 if (cur_offset == -1) {
1080 ret = WIMLIB_ERR_WRITE;
1083 ret = begin_wim_resource_chunk_tab(cur_lte,
1091 // Write the compressed chunks from the message.
1092 ret = write_wim_chunks(msg, out_fp, cur_chunk_tab);
1096 list_del(&msg->list);
1098 // This message is available to use for different chunks
1100 list_add(&msg->list, &available_msgs);
1102 // Was this the last chunk of the stream? If so, finish
1104 if (list_empty(&cur_lte->msg_list) &&
1105 msg->begin_chunk + msg->num_chunks == cur_chunk_tab->num_chunks)
1107 DEBUG2("Finish wim chunk tab");
1109 ret = finish_wim_resource_chunk_tab(cur_chunk_tab,
1115 if (res_csize >= wim_resource_size(cur_lte)) {
1116 /* Oops! We compressed the resource to
1117 * larger than the original size. Write
1118 * the resource uncompressed instead. */
1119 ret = write_uncompressed_resource_and_truncate(
1122 cur_chunk_tab->file_offset,
1123 &cur_lte->output_resource_entry);
1127 cur_lte->output_resource_entry.size =
1130 cur_lte->output_resource_entry.original_size =
1131 cur_lte->resource_entry.original_size;
1133 cur_lte->output_resource_entry.offset =
1134 cur_chunk_tab->file_offset;
1136 cur_lte->output_resource_entry.flags =
1137 cur_lte->resource_entry.flags |
1138 WIM_RESHDR_FLAG_COMPRESSED;
1141 progress->write_streams.completed_bytes +=
1142 wim_resource_size(cur_lte);
1143 progress->write_streams.completed_streams++;
1145 if (progress_func) {
1146 progress_func(WIMLIB_PROGRESS_MSG_WRITE_STREAMS,
1150 FREE(cur_chunk_tab);
1151 cur_chunk_tab = NULL;
1153 struct list_head *next = cur_lte->staging_list.next;
1154 list_del(&cur_lte->staging_list);
1156 if (next == &outstanding_resources)
1159 cur_lte = container_of(cur_lte->staging_list.next,
1160 struct wim_lookup_table_entry,
1163 // Since we just finished writing a stream,
1164 // write any streams that have been added to the
1165 // my_resources list for direct writing by the
1166 // main thread (e.g. resources that don't need
1167 // to be compressed because the desired
1168 // compression type is the same as the previous
1169 // compression type).
1170 ret = do_write_stream_list(&my_resources,
1183 if (ret == WIMLIB_ERR_NOMEM) {
1184 ERROR("Could not allocate enough memory for "
1185 "multi-threaded compression");
1190 end_wim_resource_read(next_lte, ni);
1192 end_wim_resource_read(next_lte);
1197 ret = do_write_stream_list(&my_resources, out_fp,
1198 out_ctype, progress_func,
1202 size_t num_available_msgs = 0;
1203 struct list_head *cur;
1205 list_for_each(cur, &available_msgs) {
1206 num_available_msgs++;
1209 while (num_available_msgs < num_messages) {
1210 shared_queue_get(compressed_res_queue);
1211 num_available_msgs++;
1217 for (size_t i = 0; i < num_messages; i++) {
1218 for (size_t j = 0; j < MAX_CHUNKS_PER_MSG; j++) {
1219 FREE(msgs[i].compressed_chunks[j]);
1220 FREE(msgs[i].uncompressed_chunks[j]);
1226 FREE(cur_chunk_tab);
1231 get_default_num_threads()
1234 return win32_get_number_of_processors();
1236 return sysconf(_SC_NPROCESSORS_ONLN);
1241 write_stream_list_parallel(struct list_head *stream_list,
1245 unsigned num_threads,
1246 wimlib_progress_func_t progress_func,
1247 union wimlib_progress_info *progress)
1250 struct shared_queue res_to_compress_queue;
1251 struct shared_queue compressed_res_queue;
1252 pthread_t *compressor_threads = NULL;
1254 if (num_threads == 0) {
1255 long nthreads = get_default_num_threads();
1256 if (nthreads < 1 || nthreads > UINT_MAX) {
1257 WARNING("Could not determine number of processors! Assuming 1");
1260 num_threads = nthreads;
1264 progress->write_streams.num_threads = num_threads;
1265 wimlib_assert(stream_list->next != stream_list);
1267 static const double MESSAGES_PER_THREAD = 2.0;
1268 size_t queue_size = (size_t)(num_threads * MESSAGES_PER_THREAD);
1270 DEBUG("Initializing shared queues (queue_size=%zu)", queue_size);
1272 ret = shared_queue_init(&res_to_compress_queue, queue_size);
1276 ret = shared_queue_init(&compressed_res_queue, queue_size);
1278 goto out_destroy_res_to_compress_queue;
1280 struct compressor_thread_params params;
1281 params.res_to_compress_queue = &res_to_compress_queue;
1282 params.compressed_res_queue = &compressed_res_queue;
1283 params.compress = get_compress_func(out_ctype);
1285 compressor_threads = MALLOC(num_threads * sizeof(pthread_t));
1286 if (!compressor_threads) {
1287 ret = WIMLIB_ERR_NOMEM;
1288 goto out_destroy_compressed_res_queue;
1291 for (unsigned i = 0; i < num_threads; i++) {
1292 DEBUG("pthread_create thread %u", i);
1293 ret = pthread_create(&compressor_threads[i], NULL,
1294 compressor_thread_proc, ¶ms);
1297 ERROR_WITH_ERRNO("Failed to create compressor "
1305 progress_func(WIMLIB_PROGRESS_MSG_WRITE_STREAMS, progress);
1307 ret = main_writer_thread_proc(stream_list,
1310 &res_to_compress_queue,
1311 &compressed_res_queue,
1317 for (unsigned i = 0; i < num_threads; i++)
1318 shared_queue_put(&res_to_compress_queue, NULL);
1320 for (unsigned i = 0; i < num_threads; i++) {
1321 if (pthread_join(compressor_threads[i], NULL)) {
1322 WARNING_WITH_ERRNO("Failed to join compressor "
1326 FREE(compressor_threads);
1327 out_destroy_compressed_res_queue:
1328 shared_queue_destroy(&compressed_res_queue);
1329 out_destroy_res_to_compress_queue:
1330 shared_queue_destroy(&res_to_compress_queue);
1331 if (ret >= 0 && ret != WIMLIB_ERR_NOMEM)
1334 WARNING("Falling back to single-threaded compression");
1335 return write_stream_list_serial(stream_list,
1346 * Write a list of streams to a WIM (@out_fp) using the compression type
1347 * @out_ctype and up to @num_threads compressor threads.
1350 write_stream_list(struct list_head *stream_list, FILE *out_fp,
1351 int out_ctype, int write_flags,
1352 unsigned num_threads,
1353 wimlib_progress_func_t progress_func)
1355 struct wim_lookup_table_entry *lte;
1356 size_t num_streams = 0;
1357 u64 total_bytes = 0;
1358 u64 total_compression_bytes = 0;
1359 union wimlib_progress_info progress;
1361 list_for_each_entry(lte, stream_list, staging_list) {
1363 total_bytes += wim_resource_size(lte);
1364 if (out_ctype != WIMLIB_COMPRESSION_TYPE_NONE
1365 && (wim_resource_compression_type(lte) != out_ctype ||
1366 (write_flags & WIMLIB_WRITE_FLAG_RECOMPRESS)))
1368 total_compression_bytes += wim_resource_size(lte);
1371 progress.write_streams.total_bytes = total_bytes;
1372 progress.write_streams.total_streams = num_streams;
1373 progress.write_streams.completed_bytes = 0;
1374 progress.write_streams.completed_streams = 0;
1375 progress.write_streams.num_threads = num_threads;
1376 progress.write_streams.compression_type = out_ctype;
1378 #ifdef ENABLE_MULTITHREADED_COMPRESSION
1379 if (total_compression_bytes >= 1000000 && num_threads != 1)
1380 return write_stream_list_parallel(stream_list,
1389 return write_stream_list_serial(stream_list,
1397 struct lte_overwrite_prepare_args {
1400 struct list_head *stream_list;
1404 lte_overwrite_prepare(struct wim_lookup_table_entry *lte, void *arg)
1406 struct lte_overwrite_prepare_args *args = arg;
1408 if (lte->resource_location == RESOURCE_IN_WIM &&
1409 lte->wim == args->wim &&
1410 lte->resource_entry.offset + lte->resource_entry.size > args->end_offset)
1412 #ifdef ENABLE_ERROR_MESSAGES
1413 ERROR("The following resource is after the XML data:");
1414 print_lookup_table_entry(lte, stderr);
1416 return WIMLIB_ERR_RESOURCE_ORDER;
1419 lte->out_refcnt = lte->refcnt;
1420 memcpy(<e->output_resource_entry, <e->resource_entry,
1421 sizeof(struct resource_entry));
1422 if (!(lte->resource_entry.flags & WIM_RESHDR_FLAG_METADATA)) {
1423 wimlib_assert(lte->resource_location != RESOURCE_NONEXISTENT);
1424 if (lte->resource_location != RESOURCE_IN_WIM || lte->wim != args->wim)
1425 list_add(<e->staging_list, args->stream_list);
1431 wim_find_new_streams(WIMStruct *wim, off_t end_offset,
1432 struct list_head *stream_list)
1434 struct lte_overwrite_prepare_args args = {
1436 .end_offset = end_offset,
1437 .stream_list = stream_list,
1440 return for_lookup_table_entry(wim->lookup_table,
1441 lte_overwrite_prepare, &args);
1445 inode_find_streams_to_write(struct wim_inode *inode,
1446 struct wim_lookup_table *table,
1447 struct list_head *stream_list)
1449 struct wim_lookup_table_entry *lte;
1450 for (unsigned i = 0; i <= inode->i_num_ads; i++) {
1451 lte = inode_stream_lte(inode, i, table);
1453 if (lte->out_refcnt == 0)
1454 list_add_tail(<e->staging_list, stream_list);
1455 lte->out_refcnt += inode->i_nlink;
1462 image_find_streams_to_write(WIMStruct *w)
1464 struct wim_inode *inode;
1465 struct hlist_node *cur;
1466 struct hlist_head *inode_list;
1468 inode_list = &wim_get_current_image_metadata(w)->inode_list;
1469 hlist_for_each_entry(inode, cur, inode_list, i_hlist) {
1470 inode_find_streams_to_write(inode, w->lookup_table,
1471 (struct list_head*)w->private);
1477 write_wim_streams(WIMStruct *w, int image, int write_flags,
1478 unsigned num_threads,
1479 wimlib_progress_func_t progress_func)
1482 for_lookup_table_entry(w->lookup_table, lte_zero_out_refcnt, NULL);
1483 LIST_HEAD(stream_list);
1484 w->private = &stream_list;
1485 for_image(w, image, image_find_streams_to_write);
1486 return write_stream_list(&stream_list, w->out_fp,
1487 wimlib_get_compression_type(w), write_flags,
1488 num_threads, progress_func);
1492 * Finish writing a WIM file: write the lookup table, xml data, and integrity
1493 * table (optional), then overwrite the WIM header.
1495 * write_flags is a bitwise OR of the following:
1497 * (public) WIMLIB_WRITE_FLAG_CHECK_INTEGRITY:
1498 * Include an integrity table.
1500 * (public) WIMLIB_WRITE_FLAG_SHOW_PROGRESS:
1501 * Show progress information when (if) writing the integrity table.
1503 * (private) WIMLIB_WRITE_FLAG_NO_LOOKUP_TABLE:
1504 * Don't write the lookup table.
1506 * (private) WIMLIB_WRITE_FLAG_REUSE_INTEGRITY_TABLE:
1507 * When (if) writing the integrity table, re-use entries from the
1508 * existing integrity table, if possible.
1510 * (private) WIMLIB_WRITE_FLAG_CHECKPOINT_AFTER_XML:
1511 * After writing the XML data but before writing the integrity
1512 * table, write a temporary WIM header and flush the stream so that
1513 * the WIM is less likely to become corrupted upon abrupt program
1516 * (private) WIMLIB_WRITE_FLAG_FSYNC:
1517 * fsync() the output file before closing it.
1521 finish_write(WIMStruct *w, int image, int write_flags,
1522 wimlib_progress_func_t progress_func)
1525 struct wim_header hdr;
1526 FILE *out = w->out_fp;
1528 /* @hdr will be the header for the new WIM. First copy all the data
1529 * from the header in the WIMStruct; then set all the fields that may
1530 * have changed, including the resource entries, boot index, and image
1532 memcpy(&hdr, &w->hdr, sizeof(struct wim_header));
1534 if (!(write_flags & WIMLIB_WRITE_FLAG_NO_LOOKUP_TABLE)) {
1535 ret = write_lookup_table(w->lookup_table, out, &hdr.lookup_table_res_entry);
1540 ret = write_xml_data(w->wim_info, image, out,
1541 (write_flags & WIMLIB_WRITE_FLAG_NO_LOOKUP_TABLE) ?
1542 wim_info_get_total_bytes(w->wim_info) : 0,
1543 &hdr.xml_res_entry);
1547 if (write_flags & WIMLIB_WRITE_FLAG_CHECK_INTEGRITY) {
1548 if (write_flags & WIMLIB_WRITE_FLAG_CHECKPOINT_AFTER_XML) {
1549 struct wim_header checkpoint_hdr;
1550 memcpy(&checkpoint_hdr, &hdr, sizeof(struct wim_header));
1551 memset(&checkpoint_hdr.integrity, 0, sizeof(struct resource_entry));
1552 if (fseeko(out, 0, SEEK_SET) != 0) {
1553 ERROR_WITH_ERRNO("Failed to seek to beginning "
1554 "of WIM being written");
1555 ret = WIMLIB_ERR_WRITE;
1558 ret = write_header(&checkpoint_hdr, out);
1562 if (fflush(out) != 0) {
1563 ERROR_WITH_ERRNO("Can't write data to WIM");
1564 ret = WIMLIB_ERR_WRITE;
1568 if (fseeko(out, 0, SEEK_END) != 0) {
1569 ERROR_WITH_ERRNO("Failed to seek to end "
1570 "of WIM being written");
1571 ret = WIMLIB_ERR_WRITE;
1576 off_t old_lookup_table_end;
1577 off_t new_lookup_table_end;
1578 if (write_flags & WIMLIB_WRITE_FLAG_REUSE_INTEGRITY_TABLE) {
1579 old_lookup_table_end = w->hdr.lookup_table_res_entry.offset +
1580 w->hdr.lookup_table_res_entry.size;
1582 old_lookup_table_end = 0;
1584 new_lookup_table_end = hdr.lookup_table_res_entry.offset +
1585 hdr.lookup_table_res_entry.size;
1587 ret = write_integrity_table(out,
1589 new_lookup_table_end,
1590 old_lookup_table_end,
1595 memset(&hdr.integrity, 0, sizeof(struct resource_entry));
1599 * In the WIM header, there is room for the resource entry for a
1600 * metadata resource labeled as the "boot metadata". This entry should
1601 * be zeroed out if there is no bootable image (boot_idx 0). Otherwise,
1602 * it should be a copy of the resource entry for the image that is
1603 * marked as bootable. This is not well documented...
1606 /* Set image count and boot index correctly for single image writes */
1607 if (image != WIMLIB_ALL_IMAGES) {
1608 hdr.image_count = 1;
1609 if (hdr.boot_idx == image)
1615 if (hdr.boot_idx == 0) {
1616 memset(&hdr.boot_metadata_res_entry, 0,
1617 sizeof(struct resource_entry));
1619 memcpy(&hdr.boot_metadata_res_entry,
1621 hdr.boot_idx - 1].metadata_lte->output_resource_entry,
1622 sizeof(struct resource_entry));
1625 if (fseeko(out, 0, SEEK_SET) != 0) {
1626 ERROR_WITH_ERRNO("Failed to seek to beginning of WIM "
1628 ret = WIMLIB_ERR_WRITE;
1632 ret = write_header(&hdr, out);
1636 if (write_flags & WIMLIB_WRITE_FLAG_FSYNC) {
1637 if (fflush(out) != 0
1638 || fsync(fileno(out)) != 0)
1640 ERROR_WITH_ERRNO("Error flushing data to WIM file");
1641 ret = WIMLIB_ERR_WRITE;
1645 if (fclose(out) != 0) {
1646 ERROR_WITH_ERRNO("Failed to close the WIM file");
1648 ret = WIMLIB_ERR_WRITE;
1654 #if defined(HAVE_SYS_FILE_H) && defined(HAVE_FLOCK)
1656 lock_wim(WIMStruct *w, FILE *fp)
1659 if (fp && !w->wim_locked) {
1660 ret = flock(fileno(fp), LOCK_EX | LOCK_NB);
1662 if (errno == EWOULDBLOCK) {
1663 ERROR("`%s' is already being modified or has been "
1664 "mounted read-write\n"
1665 " by another process!", w->filename);
1666 ret = WIMLIB_ERR_ALREADY_LOCKED;
1668 WARNING_WITH_ERRNO("Failed to lock `%s'",
1681 open_wim_writable(WIMStruct *w, const mbchar *path,
1682 bool trunc, bool readable)
1693 wimlib_assert(w->out_fp == NULL);
1694 w->out_fp = fopen(path, mode);
1698 ERROR_WITH_ERRNO("Failed to open `%s' for writing", path);
1699 return WIMLIB_ERR_OPEN;
1705 close_wim_writable(WIMStruct *w)
1708 if (fclose(w->out_fp) != 0) {
1709 WARNING_WITH_ERRNO("Failed to close output WIM");
1715 /* Open file stream and write dummy header for WIM. */
1717 begin_write(WIMStruct *w, const mbchar *path, int write_flags)
1720 ret = open_wim_writable(w, path, true,
1721 (write_flags & WIMLIB_WRITE_FLAG_CHECK_INTEGRITY) != 0);
1724 /* Write dummy header. It will be overwritten later. */
1725 return write_header(&w->hdr, w->out_fp);
1728 /* Writes a stand-alone WIM to a file. */
1730 wimlib_write(WIMStruct *w, const mbchar *path,
1731 int image, int write_flags, unsigned num_threads,
1732 wimlib_progress_func_t progress_func)
1737 return WIMLIB_ERR_INVALID_PARAM;
1739 write_flags &= WIMLIB_WRITE_MASK_PUBLIC;
1741 if (image != WIMLIB_ALL_IMAGES &&
1742 (image < 1 || image > w->hdr.image_count))
1743 return WIMLIB_ERR_INVALID_IMAGE;
1745 if (w->hdr.total_parts != 1) {
1746 ERROR("Cannot call wimlib_write() on part of a split WIM");
1747 return WIMLIB_ERR_SPLIT_UNSUPPORTED;
1750 ret = begin_write(w, path, write_flags);
1754 ret = write_wim_streams(w, image, write_flags, num_threads,
1760 progress_func(WIMLIB_PROGRESS_MSG_WRITE_METADATA_BEGIN, NULL);
1762 ret = for_image(w, image, write_metadata_resource);
1767 progress_func(WIMLIB_PROGRESS_MSG_WRITE_METADATA_END, NULL);
1769 ret = finish_write(w, image, write_flags, progress_func);
1771 close_wim_writable(w);
1772 DEBUG("wimlib_write(path=%s) = %d", path, ret);
1777 any_images_modified(WIMStruct *w)
1779 for (int i = 0; i < w->hdr.image_count; i++)
1780 if (w->image_metadata[i].modified)
1786 * Overwrite a WIM, possibly appending streams to it.
1788 * A WIM looks like (or is supposed to look like) the following:
1790 * Header (212 bytes)
1791 * Streams and metadata resources (variable size)
1792 * Lookup table (variable size)
1793 * XML data (variable size)
1794 * Integrity table (optional) (variable size)
1796 * If we are not adding any streams or metadata resources, the lookup table is
1797 * unchanged--- so we only need to overwrite the XML data, integrity table, and
1798 * header. This operation is potentially unsafe if the program is abruptly
1799 * terminated while the XML data or integrity table are being overwritten, but
1800 * before the new header has been written. To partially alleviate this problem,
1801 * a special flag (WIMLIB_WRITE_FLAG_CHECKPOINT_AFTER_XML) is passed to
1802 * finish_write() to cause a temporary WIM header to be written after the XML
1803 * data has been written. This may prevent the WIM from becoming corrupted if
1804 * the program is terminated while the integrity table is being calculated (but
1805 * no guarantees, due to write re-ordering...).
1807 * If we are adding new streams or images (metadata resources), the lookup table
1808 * needs to be changed, and those streams need to be written. In this case, we
1809 * try to perform a safe update of the WIM file by writing the streams *after*
1810 * the end of the previous WIM, then writing the new lookup table, XML data, and
1811 * (optionally) integrity table following the new streams. This will produce a
1812 * layout like the following:
1814 * Header (212 bytes)
1815 * (OLD) Streams and metadata resources (variable size)
1816 * (OLD) Lookup table (variable size)
1817 * (OLD) XML data (variable size)
1818 * (OLD) Integrity table (optional) (variable size)
1819 * (NEW) Streams and metadata resources (variable size)
1820 * (NEW) Lookup table (variable size)
1821 * (NEW) XML data (variable size)
1822 * (NEW) Integrity table (optional) (variable size)
1824 * At all points, the WIM is valid as nothing points to the new data yet. Then,
1825 * the header is overwritten to point to the new lookup table, XML data, and
1826 * integrity table, to produce the following layout:
1828 * Header (212 bytes)
1829 * Streams and metadata resources (variable size)
1830 * Nothing (variable size)
1831 * More Streams and metadata resources (variable size)
1832 * Lookup table (variable size)
1833 * XML data (variable size)
1834 * Integrity table (optional) (variable size)
1836 * This method allows an image to be appended to a large WIM very quickly, and
1837 * is is crash-safe except in the case of write re-ordering, but the
1838 * disadvantage is that a small hole is left in the WIM where the old lookup
1839 * table, xml data, and integrity table were. (These usually only take up a
1840 * small amount of space compared to the streams, however.)
1843 overwrite_wim_inplace(WIMStruct *w, int write_flags,
1844 unsigned num_threads,
1845 wimlib_progress_func_t progress_func)
1848 struct list_head stream_list;
1850 bool found_modified_image;
1852 DEBUG("Overwriting `%s' in-place", w->filename);
1854 /* Make sure that the integrity table (if present) is after the XML
1855 * data, and that there are no stream resources, metadata resources, or
1856 * lookup tables after the XML data. Otherwise, these data would be
1858 if (w->hdr.integrity.offset != 0 &&
1859 w->hdr.integrity.offset < w->hdr.xml_res_entry.offset) {
1860 ERROR("Didn't expect the integrity table to be before the XML data");
1861 return WIMLIB_ERR_RESOURCE_ORDER;
1864 if (w->hdr.lookup_table_res_entry.offset > w->hdr.xml_res_entry.offset) {
1865 ERROR("Didn't expect the lookup table to be after the XML data");
1866 return WIMLIB_ERR_RESOURCE_ORDER;
1870 if (w->hdr.integrity.offset)
1871 old_wim_end = w->hdr.integrity.offset + w->hdr.integrity.size;
1873 old_wim_end = w->hdr.xml_res_entry.offset + w->hdr.xml_res_entry.size;
1875 if (!w->deletion_occurred && !any_images_modified(w)) {
1876 /* If no images have been modified and no images have been
1877 * deleted, a new lookup table does not need to be written. */
1878 old_wim_end = w->hdr.lookup_table_res_entry.offset +
1879 w->hdr.lookup_table_res_entry.size;
1880 write_flags |= WIMLIB_WRITE_FLAG_NO_LOOKUP_TABLE |
1881 WIMLIB_WRITE_FLAG_CHECKPOINT_AFTER_XML;
1883 INIT_LIST_HEAD(&stream_list);
1884 ret = wim_find_new_streams(w, old_wim_end, &stream_list);
1888 ret = open_wim_writable(w, w->filename, false,
1889 (write_flags & WIMLIB_WRITE_FLAG_CHECK_INTEGRITY) != 0);
1893 ret = lock_wim(w, w->out_fp);
1900 if (fseeko(w->out_fp, old_wim_end, SEEK_SET) != 0) {
1901 ERROR_WITH_ERRNO("Can't seek to end of WIM");
1905 return WIMLIB_ERR_WRITE;
1908 if (!list_empty(&stream_list)) {
1909 DEBUG("Writing newly added streams (offset = %"PRIu64")",
1911 ret = write_stream_list(&stream_list, w->out_fp,
1912 wimlib_get_compression_type(w),
1913 write_flags, num_threads,
1918 DEBUG("No new streams were added");
1921 found_modified_image = false;
1922 for (int i = 0; i < w->hdr.image_count; i++) {
1923 if (!found_modified_image)
1924 found_modified_image = w->image_metadata[i].modified;
1925 if (found_modified_image) {
1926 select_wim_image(w, i + 1);
1927 ret = write_metadata_resource(w);
1932 write_flags |= WIMLIB_WRITE_FLAG_REUSE_INTEGRITY_TABLE;
1933 ret = finish_write(w, WIMLIB_ALL_IMAGES, write_flags,
1936 close_wim_writable(w);
1937 if (ret != 0 && !(write_flags & WIMLIB_WRITE_FLAG_NO_LOOKUP_TABLE)) {
1938 WARNING("Truncating `%s' to its original size (%"PRIu64" bytes)",
1939 w->filename, old_wim_end);
1940 /* Return value of truncate() is ignored because this is already
1942 (void)truncate(w->filename, old_wim_end);
1949 overwrite_wim_via_tmpfile(WIMStruct *w, int write_flags,
1950 unsigned num_threads,
1951 wimlib_progress_func_t progress_func)
1953 size_t wim_name_len;
1956 DEBUG("Overwriting `%s' via a temporary file", w->filename);
1958 /* Write the WIM to a temporary file in the same directory as the
1960 wim_name_len = strlen(w->filename);
1961 mbchar tmpfile[wim_name_len + 10];
1962 memcpy(tmpfile, w->filename, wim_name_len);
1963 randomize_char_array_with_alnum(tmpfile + wim_name_len, 9);
1964 tmpfile[wim_name_len + 9] = '\0';
1966 ret = wimlib_write(w, tmpfile, WIMLIB_ALL_IMAGES,
1967 write_flags | WIMLIB_WRITE_FLAG_FSYNC,
1968 num_threads, progress_func);
1970 ERROR("Failed to write the WIM file `%s'", tmpfile);
1974 DEBUG("Renaming `%s' to `%s'", tmpfile, w->filename);
1976 /* Rename the new file to the old file .*/
1977 if (rename(tmpfile, w->filename) != 0) {
1978 ERROR_WITH_ERRNO("Failed to rename `%s' to `%s'",
1979 tmpfile, w->filename);
1980 ret = WIMLIB_ERR_RENAME;
1984 if (progress_func) {
1985 union wimlib_progress_info progress;
1986 progress.rename.from = tmpfile;
1987 progress.rename.to = w->filename;
1988 progress_func(WIMLIB_PROGRESS_MSG_RENAME, &progress);
1991 /* Close the original WIM file that was opened for reading. */
1992 if (w->fp != NULL) {
1997 /* Re-open the WIM read-only. */
1998 w->fp = fopen(w->filename, "rb");
1999 if (w->fp == NULL) {
2000 ret = WIMLIB_ERR_REOPEN;
2001 WARNING_WITH_ERRNO("Failed to re-open `%s' read-only",
2008 /* Remove temporary file. */
2009 if (unlink(tmpfile) != 0)
2010 WARNING_WITH_ERRNO("Failed to remove `%s'", tmpfile);
2015 * Writes a WIM file to the original file that it was read from, overwriting it.
2018 wimlib_overwrite(WIMStruct *w, int write_flags,
2019 unsigned num_threads,
2020 wimlib_progress_func_t progress_func)
2022 write_flags &= WIMLIB_WRITE_MASK_PUBLIC;
2025 return WIMLIB_ERR_NO_FILENAME;
2027 if (w->hdr.total_parts != 1) {
2028 ERROR("Cannot modify a split WIM");
2029 return WIMLIB_ERR_SPLIT_UNSUPPORTED;
2032 if ((!w->deletion_occurred || (write_flags & WIMLIB_WRITE_FLAG_SOFT_DELETE))
2033 && !(write_flags & WIMLIB_WRITE_FLAG_REBUILD))
2036 ret = overwrite_wim_inplace(w, write_flags, num_threads,
2038 if (ret == WIMLIB_ERR_RESOURCE_ORDER)
2039 WARNING("Falling back to re-building entire WIM");
2043 return overwrite_wim_via_tmpfile(w, write_flags, num_threads,