X-Git-Url: https://wimlib.net/git/?a=blobdiff_plain;f=src%2Fwrite.c;h=ba8b697078a2efeb2d23a657b10323d05d1cfc25;hb=fced769402ff6ef0bf75fe9e4dcb2880891469e0;hp=9965f5c06bf2d21f853bb6d1bfedda0cbf3d98a3;hpb=f9695b9f40035f1a20968293255761a8301eaba0;p=wimlib diff --git a/src/write.c b/src/write.c index 9965f5c0..ba8b6970 100644 --- a/src/write.c +++ b/src/write.c @@ -24,30 +24,37 @@ * along with wimlib; if not, see http://www.gnu.org/licenses/. */ -#include "config.h" +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif #if defined(HAVE_SYS_FILE_H) && defined(HAVE_FLOCK) -/* On BSD, this should be included before "list.h" so that "list.h" can +/* On BSD, this should be included before "wimlib/list.h" so that "wimlib/list.h" can * overwrite the LIST_HEAD macro. */ # include #endif +#include "wimlib/endianness.h" +#include "wimlib/error.h" +#include "wimlib/file_io.h" +#include "wimlib/header.h" +#include "wimlib/integrity.h" +#include "wimlib/lookup_table.h" +#include "wimlib/metadata.h" +#include "wimlib/resource.h" +#include "wimlib/write.h" +#include "wimlib/xml.h" + #ifdef __WIN32__ -# include "win32.h" +# include "wimlib/win32.h" /* win32_get_number_of_processors() */ #endif -#include "list.h" -#include "wimlib_internal.h" -#include "buffer_io.h" -#include "dentry.h" -#include "lookup_table.h" -#include "xml.h" - #ifdef ENABLE_MULTITHREADED_COMPRESSION # include #endif #include +#include #include #ifdef WITH_NTFS_3G @@ -65,20 +72,28 @@ #include -#include /* writev() */ +#ifndef __WIN32__ +# include /* for `struct iovec' */ +#endif /* Chunk table that's located at the beginning of each compressed resource in * the WIM. (This is not the on-disk format; the on-disk format just has an * array of offsets.) */ struct chunk_table { off_t file_offset; - u64 num_chunks; u64 original_resource_size; - u64 bytes_per_chunk_entry; + u64 num_chunks; u64 table_disk_size; - u64 cur_offset; - u64 *cur_offset_p; - u64 offsets[0]; + unsigned bytes_per_chunk_entry; + void *cur_offset_p; + union { + u32 cur_offset_u32; + u64 cur_offset_u64; + }; + /* Beginning of chunk offsets, in either 32-bit or 64-bit little endian + * integers, including the first offset of 0, which will not be written. + * */ + u8 offsets[] _aligned_attribute(8); }; /* @@ -92,11 +107,12 @@ begin_wim_resource_chunk_tab(const struct wim_lookup_table_entry *lte, struct chunk_table **chunk_tab_ret) { u64 size = wim_resource_size(lte); - u64 num_chunks = (size + WIM_CHUNK_SIZE - 1) / WIM_CHUNK_SIZE; + u64 num_chunks = wim_resource_chunks(lte); + unsigned bytes_per_chunk_entry = (size > (1ULL << 32)) ? 8 : 4; size_t alloc_size = sizeof(struct chunk_table) + num_chunks * sizeof(u64); struct chunk_table *chunk_tab = CALLOC(1, alloc_size); - DEBUG("Begin chunk table for stream with size %"PRIu64, size); + DEBUG("Beginning chunk table for stream with size %"PRIu64, size); if (!chunk_tab) { ERROR("Failed to allocate chunk table for %"PRIu64" byte " @@ -106,13 +122,15 @@ begin_wim_resource_chunk_tab(const struct wim_lookup_table_entry *lte, chunk_tab->file_offset = file_offset; chunk_tab->num_chunks = num_chunks; chunk_tab->original_resource_size = size; - chunk_tab->bytes_per_chunk_entry = (size >= (1ULL << 32)) ? 8 : 4; + chunk_tab->bytes_per_chunk_entry = bytes_per_chunk_entry; chunk_tab->table_disk_size = chunk_tab->bytes_per_chunk_entry * (num_chunks - 1); - chunk_tab->cur_offset = 0; chunk_tab->cur_offset_p = chunk_tab->offsets; - if (full_write(out_fd, chunk_tab, + /* We don't know the correct offsets yet; this just writes zeroes to + * reserve space for the table, so we can go back to it later after + * we've written the compressed chunks following it. */ + if (full_write(out_fd, chunk_tab->offsets, chunk_tab->table_disk_size) != chunk_tab->table_disk_size) { ERROR_WITH_ERRNO("Failed to write chunk table in compressed " @@ -124,6 +142,22 @@ begin_wim_resource_chunk_tab(const struct wim_lookup_table_entry *lte, return 0; } +/* Add the offset for the next chunk to the chunk table being constructed for a + * compressed stream. */ +static void +chunk_tab_record_chunk(struct chunk_table *chunk_tab, unsigned out_chunk_size) +{ + if (chunk_tab->bytes_per_chunk_entry == 4) { + *(le32*)chunk_tab->cur_offset_p = cpu_to_le32(chunk_tab->cur_offset_u32); + chunk_tab->cur_offset_p = (le32*)chunk_tab->cur_offset_p + 1; + chunk_tab->cur_offset_u32 += out_chunk_size; + } else { + *(le64*)chunk_tab->cur_offset_p = cpu_to_le64(chunk_tab->cur_offset_u64); + chunk_tab->cur_offset_p = (le64*)chunk_tab->cur_offset_p + 1; + chunk_tab->cur_offset_u64 += out_chunk_size; + } +} + /* * compress_func_t- Pointer to a function to compresses a chunk * of a WIM resource. This may be either @@ -164,7 +198,7 @@ get_compress_func(int out_ctype) * * @chunk: Uncompressed data of the chunk. * @chunk_size: Size of the chunk (<= WIM_CHUNK_SIZE) - * @out_fd: FILE descriptor to write the chunk to. + * @out_fd: File descriptor to write the chunk to. * @compress: Compression function to use (NULL if writing uncompressed * data). * @chunk_tab: Pointer to chunk table being created. It is updated with the @@ -175,7 +209,7 @@ get_compress_func(int out_ctype) static int write_wim_resource_chunk(const void * restrict chunk, unsigned chunk_size, - filedes_t out_fd, + int out_fd, compress_func_t compress, struct chunk_table * restrict chunk_tab) { @@ -193,8 +227,7 @@ write_wim_resource_chunk(const void * restrict chunk, out_chunk = chunk; out_chunk_size = chunk_size; } - *chunk_tab->cur_offset_p++ = chunk_tab->cur_offset; - chunk_tab->cur_offset += out_chunk_size; + chunk_tab_record_chunk(chunk_tab, out_chunk_size); } else { /* Write uncompressed */ out_chunk = chunk; @@ -216,19 +249,12 @@ write_wim_resource_chunk(const void * restrict chunk, */ static int finish_wim_resource_chunk_tab(struct chunk_table *chunk_tab, - filedes_t out_fd, u64 *compressed_size_p) + int out_fd, u64 *compressed_size_p) { size_t bytes_written; - if (chunk_tab->bytes_per_chunk_entry == 8) { - array_cpu_to_le64(chunk_tab->offsets, chunk_tab->num_chunks); - } else { - for (u64 i = 0; i < chunk_tab->num_chunks; i++) - ((u32*)chunk_tab->offsets)[i] = - cpu_to_le32(chunk_tab->offsets[i]); - } bytes_written = full_pwrite(out_fd, - (u8*)chunk_tab->offsets + chunk_tab->bytes_per_chunk_entry, + chunk_tab->offsets + chunk_tab->bytes_per_chunk_entry, chunk_tab->table_disk_size, chunk_tab->file_offset); if (bytes_written != chunk_tab->table_disk_size) { @@ -236,12 +262,15 @@ finish_wim_resource_chunk_tab(struct chunk_table *chunk_tab, "file resource"); return WIMLIB_ERR_WRITE; } - *compressed_size_p = chunk_tab->cur_offset + chunk_tab->table_disk_size; + if (chunk_tab->bytes_per_chunk_entry == 4) + *compressed_size_p = chunk_tab->cur_offset_u32 + chunk_tab->table_disk_size; + else + *compressed_size_p = chunk_tab->cur_offset_u64 + chunk_tab->table_disk_size; return 0; } static int -seek_and_truncate(filedes_t out_fd, off_t offset) +seek_and_truncate(int out_fd, off_t offset) { if (lseek(out_fd, offset, SEEK_SET) == -1 || ftruncate(out_fd, offset)) @@ -277,7 +306,7 @@ finalize_and_check_sha1(SHA_CTX * restrict sha_ctx, struct write_resource_ctx { compress_func_t compress; struct chunk_table *chunk_tab; - filedes_t out_fd; + int out_fd; SHA_CTX sha_ctx; bool doing_sha; }; @@ -301,7 +330,7 @@ write_resource_cb(const void *restrict chunk, size_t chunk_size, * @lte: Lookup table entry for the resource, which could be in another WIM, * in an external file, or in another location. * - * @out_fp: File descriptor opened to the output WIM. + * @out_fd: File descriptor opened to the output WIM. * * @out_ctype: One of the WIMLIB_COMPRESSION_TYPE_* constants to indicate * which compression algorithm to use. @@ -321,7 +350,7 @@ write_resource_cb(const void *restrict chunk, size_t chunk_size, */ int write_wim_resource(struct wim_lookup_table_entry *lte, - filedes_t out_fd, int out_ctype, + int out_fd, int out_ctype, struct resource_entry *out_res_entry, int flags) { @@ -596,9 +625,19 @@ compressor_thread_proc(void *arg) static void do_write_streams_progress(union wimlib_progress_info *progress, wimlib_progress_func_t progress_func, - uint64_t size_added) + uint64_t size_added, + bool stream_discarded) { - progress->write_streams.completed_bytes += size_added; + if (stream_discarded) { + progress->write_streams.total_bytes -= size_added; + if (progress->write_streams._private != ~(uint64_t)0 && + progress->write_streams._private > progress->write_streams.total_bytes) + { + progress->write_streams._private = progress->write_streams.total_bytes; + } + } else { + progress->write_streams.completed_bytes += size_added; + } progress->write_streams.completed_streams++; if (progress_func && progress->write_streams.completed_bytes >= progress->write_streams._private) @@ -606,7 +645,7 @@ do_write_streams_progress(union wimlib_progress_info *progress, progress_func(WIMLIB_PROGRESS_MSG_WRITE_STREAMS, progress); if (progress->write_streams._private == progress->write_streams.total_bytes) { - progress->write_streams._private = ~0; + progress->write_streams._private = ~(uint64_t)0; } else { progress->write_streams._private = min(progress->write_streams.total_bytes, @@ -617,7 +656,7 @@ do_write_streams_progress(union wimlib_progress_info *progress, } struct serial_write_stream_ctx { - filedes_t out_fd; + int out_fd; int out_ctype; int write_resource_flags; }; @@ -645,9 +684,11 @@ do_write_stream_list(struct list_head *stream_list, { int ret = 0; struct wim_lookup_table_entry *lte; + bool stream_discarded; /* For each stream in @stream_list ... */ while (!list_empty(stream_list)) { + stream_discarded = false; lte = container_of(stream_list->next, struct wim_lookup_table_entry, write_streams_list); @@ -672,6 +713,7 @@ do_write_stream_list(struct list_head *stream_list, DEBUG("Discarding duplicate stream of length %"PRIu64, wim_resource_size(lte)); lte->no_progress = 0; + stream_discarded = true; goto skip_to_progress; } } @@ -703,7 +745,8 @@ do_write_stream_list(struct list_head *stream_list, if (!lte->no_progress) { do_write_streams_progress(progress, progress_func, - wim_resource_size(lte)); + wim_resource_size(lte), + stream_discarded); } } return ret; @@ -712,7 +755,7 @@ do_write_stream_list(struct list_head *stream_list, static int do_write_stream_list_serial(struct list_head *stream_list, struct wim_lookup_table *lookup_table, - filedes_t out_fd, + int out_fd, int out_ctype, int write_resource_flags, wimlib_progress_func_t progress_func, @@ -744,7 +787,7 @@ write_flags_to_resource_flags(int write_flags) static int write_stream_list_serial(struct list_head *stream_list, struct wim_lookup_table *lookup_table, - filedes_t out_fd, + int out_fd, int out_ctype, int write_resource_flags, wimlib_progress_func_t progress_func, @@ -765,59 +808,24 @@ write_stream_list_serial(struct list_head *stream_list, #ifdef ENABLE_MULTITHREADED_COMPRESSION static int -write_wim_chunks(struct message *msg, filedes_t out_fd, +write_wim_chunks(struct message *msg, int out_fd, struct chunk_table *chunk_tab) { - ssize_t bytes_remaining = msg->total_out_bytes; - struct iovec *vecs = msg->out_chunks; - unsigned nvecs = msg->num_chunks; - int ret; - - wimlib_assert(nvecs != 0); - wimlib_assert(msg->total_out_bytes != 0); - - for (unsigned i = 0; i < msg->num_chunks; i++) { - *chunk_tab->cur_offset_p++ = chunk_tab->cur_offset; - chunk_tab->cur_offset += vecs[i].iov_len; - } - for (;;) { - ssize_t bytes_written; - - bytes_written = writev(out_fd, vecs, nvecs); - if (bytes_written <= 0) { - if (bytes_written < 0 && errno == EINTR) - continue; - else if (bytes_written == 0) - errno = EIO; - ERROR_WITH_ERRNO("Failed to write WIM chunks"); - ret = WIMLIB_ERR_WRITE; - break; - } - bytes_remaining -= bytes_written; - if (bytes_remaining <= 0) { - ret = 0; - break; - } - while (bytes_written >= 0) { - wimlib_assert(nvecs != 0); - if (bytes_written >= vecs[0].iov_len) { - vecs++; - nvecs--; - bytes_written -= vecs[0].iov_len; - } else { - vecs[0].iov_base += bytes_written; - vecs[0].iov_len -= bytes_written; - bytes_written = 0; - } - } + for (unsigned i = 0; i < msg->num_chunks; i++) + chunk_tab_record_chunk(chunk_tab, msg->out_chunks[i].iov_len); + if (full_writev(out_fd, msg->out_chunks, + msg->num_chunks) != msg->total_out_bytes) + { + ERROR_WITH_ERRNO("Failed to write WIM chunks"); + return WIMLIB_ERR_WRITE; } - return ret; + return 0; } struct main_writer_thread_ctx { struct list_head *stream_list; struct wim_lookup_table *lookup_table; - filedes_t out_fd; + int out_fd; int out_ctype; int write_resource_flags; struct shared_queue *res_to_compress_queue; @@ -1046,7 +1054,8 @@ receive_compressed_chunks(struct main_writer_thread_ctx *ctx) do_write_streams_progress(ctx->progress, ctx->progress_func, - wim_resource_size(cur_lte)); + wim_resource_size(cur_lte), + false); /* Since we just finished writing a stream, write any * streams that have been added to the serial_streams @@ -1215,7 +1224,7 @@ main_thread_process_next_stream(struct wim_lookup_table_entry *lte, void *_ctx) } static long -get_default_num_threads() +get_default_num_threads(void) { #ifdef __WIN32__ return win32_get_number_of_processors(); @@ -1253,7 +1262,7 @@ get_default_num_threads() static int write_stream_list_parallel(struct list_head *stream_list, struct wim_lookup_table *lookup_table, - filedes_t out_fd, + int out_fd, int out_ctype, int write_resource_flags, wimlib_progress_func_t progress_func, @@ -1391,7 +1400,7 @@ out_serial_quiet: static int write_stream_list(struct list_head *stream_list, struct wim_lookup_table *lookup_table, - filedes_t out_fd, int out_ctype, int write_flags, + int out_fd, int out_ctype, int write_flags, unsigned num_threads, wimlib_progress_func_t progress_func) { struct wim_lookup_table_entry *lte; @@ -1430,7 +1439,7 @@ write_stream_list(struct list_head *stream_list, progress.write_streams._private = 0; #ifdef ENABLE_MULTITHREADED_COMPRESSION - if (total_compression_bytes >= 1000000 && num_threads != 1) + if (total_compression_bytes >= 2000000 && num_threads != 1) ret = write_stream_list_parallel(stream_list, lookup_table, out_fd, @@ -1538,10 +1547,10 @@ lte_overwrite_prepare_2(struct wim_lookup_table_entry *lte, void *_args) if (lte->resource_entry.offset + lte->resource_entry.size > args->end_offset) { - #ifdef ENABLE_ERROR_MESSAGES - ERROR("The following resource is after the XML data:"); - print_lookup_table_entry(lte, stderr); - #endif + if (wimlib_print_errors) { + ERROR("The following resource is after the XML data:"); + print_lookup_table_entry(lte, stderr); + } return WIMLIB_ERR_RESOURCE_ORDER; } copy_resource_entry(<e->output_resource_entry, @@ -1688,7 +1697,7 @@ prepare_stream_list(WIMStruct *wim, int image, struct list_head *stream_list) return ret; } -/* Writes the streams for the specified @image in @wim to @wim->out_fp. +/* Writes the streams for the specified @image in @wim to @wim->out_fd. */ static int write_wim_streams(WIMStruct *wim, int image, int write_flags, @@ -1719,9 +1728,6 @@ write_wim_streams(WIMStruct *wim, int image, int write_flags, * (public) WIMLIB_WRITE_FLAG_CHECK_INTEGRITY: * Include an integrity table. * - * (public) WIMLIB_WRITE_FLAG_SHOW_PROGRESS: - * Show progress information when (if) writing the integrity table. - * * (private) WIMLIB_WRITE_FLAG_NO_LOOKUP_TABLE: * Don't write the lookup table. * @@ -1835,16 +1841,16 @@ out_close_wim: if (ret == 0) ret = WIMLIB_ERR_WRITE; } - w->out_fd = INVALID_FILEDES; + w->out_fd = -1; return ret; } #if defined(HAVE_SYS_FILE_H) && defined(HAVE_FLOCK) int -lock_wim(WIMStruct *w, filedes_t fd) +lock_wim(WIMStruct *w, int fd) { int ret = 0; - if (fd != INVALID_FILEDES && !w->wim_locked) { + if (fd != -1 && !w->wim_locked) { ret = flock(fd, LOCK_EX | LOCK_NB); if (ret != 0) { if (errno == EWOULDBLOCK) { @@ -1868,9 +1874,8 @@ lock_wim(WIMStruct *w, filedes_t fd) static int open_wim_writable(WIMStruct *w, const tchar *path, int open_flags) { - wimlib_assert(w->out_fd == INVALID_FILEDES); - w->out_fd = open(path, open_flags, 0644); - if (w->out_fd == INVALID_FILEDES) { + w->out_fd = topen(path, open_flags | O_BINARY, 0644); + if (w->out_fd == -1) { ERROR_WITH_ERRNO("Failed to open `%"TS"' for writing", path); return WIMLIB_ERR_OPEN; } @@ -1881,10 +1886,10 @@ open_wim_writable(WIMStruct *w, const tchar *path, int open_flags) void close_wim_writable(WIMStruct *w) { - if (w->out_fd != INVALID_FILEDES) { + if (w->out_fd != -1) { if (close(w->out_fd)) WARNING_WITH_ERRNO("Failed to close output WIM"); - w->out_fd = INVALID_FILEDES; + w->out_fd = -1; } } @@ -1905,8 +1910,8 @@ begin_write(WIMStruct *w, const tchar *path, int write_flags) ret = write_header(&w->hdr, w->out_fd); if (ret) return ret; - if (lseek(w->out_fd, 0, SEEK_END) == -1) { - ERROR_WITH_ERRNO("Failed to seek to end of WIM"); + if (lseek(w->out_fd, WIM_HEADER_DISK_SIZE, SEEK_SET) == -1) { + ERROR_WITH_ERRNO("Failed to seek to end of WIM header"); return WIMLIB_ERR_WRITE; } return 0; @@ -2173,19 +2178,9 @@ overwrite_wim_via_tmpfile(WIMStruct *w, int write_flags, goto out_unlink; } - DEBUG("Renaming `%"TS"' to `%"TS"'", tmpfile, w->filename); - -#ifdef __WIN32__ - /* Windows won't let you delete open files unless FILE_SHARE_DELETE was - * specified to CreateFile(). The WIM was opened with fopen(), which - * didn't provided this flag to CreateFile, so the handle must be closed - * before executing the rename(). */ - if (w->fp != NULL) { - fclose(w->fp); - w->fp = NULL; - } -#endif + close_wim(w); + DEBUG("Renaming `%"TS"' to `%"TS"'", tmpfile, w->filename); /* Rename the new file to the old file .*/ if (trename(tmpfile, w->filename) != 0) { ERROR_WITH_ERRNO("Failed to rename `%"TS"' to `%"TS"'", @@ -2217,15 +2212,16 @@ wimlib_overwrite(WIMStruct *w, int write_flags, unsigned num_threads, wimlib_progress_func_t progress_func) { + int ret; + write_flags &= WIMLIB_WRITE_MASK_PUBLIC; if (!w->filename) return WIMLIB_ERR_NO_FILENAME; - if (w->hdr.total_parts != 1) { - ERROR("Cannot modify a split WIM"); - return WIMLIB_ERR_SPLIT_UNSUPPORTED; - } + ret = can_modify_wim(w); + if (ret) + return ret; if ((!w->deletion_occurred || (write_flags & WIMLIB_WRITE_FLAG_SOFT_DELETE)) && !(write_flags & WIMLIB_WRITE_FLAG_REBUILD))