X-Git-Url: https://wimlib.net/git/?p=wimlib;a=blobdiff_plain;f=src%2Fwrite.c;h=90dcaed4201d0d92ce6cf74dfc9691dd4bf11025;hp=16ce531aacc2850b1d096f243d6e257030c566f9;hb=77b2c74599190197074c82937787f24f83ece0af;hpb=4f953b223bed60d71a7689d414ccb5cc60be537f diff --git a/src/write.c b/src/write.c index 16ce531a..90dcaed4 100644 --- a/src/write.c +++ b/src/write.c @@ -24,30 +24,37 @@ * along with wimlib; if not, see http://www.gnu.org/licenses/. */ -#include "config.h" +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif #if defined(HAVE_SYS_FILE_H) && defined(HAVE_FLOCK) -/* On BSD, this should be included before "list.h" so that "list.h" can +/* On BSD, this should be included before "wimlib/list.h" so that "wimlib/list.h" can * overwrite the LIST_HEAD macro. */ # include #endif +#include "wimlib/endianness.h" +#include "wimlib/error.h" +#include "wimlib/file_io.h" +#include "wimlib/header.h" +#include "wimlib/integrity.h" +#include "wimlib/lookup_table.h" +#include "wimlib/metadata.h" +#include "wimlib/resource.h" +#include "wimlib/write.h" +#include "wimlib/xml.h" + #ifdef __WIN32__ -# include "win32.h" +# include "wimlib/win32.h" /* win32_get_number_of_processors() */ #endif -#include "list.h" -#include "wimlib_internal.h" -#include "buffer_io.h" -#include "dentry.h" -#include "lookup_table.h" -#include "xml.h" - #ifdef ENABLE_MULTITHREADED_COMPRESSION # include #endif #include +#include #include #ifdef WITH_NTFS_3G @@ -65,18 +72,28 @@ #include +#ifndef __WIN32__ +# include /* for `struct iovec' */ +#endif + /* Chunk table that's located at the beginning of each compressed resource in * the WIM. (This is not the on-disk format; the on-disk format just has an * array of offsets.) */ struct chunk_table { off_t file_offset; - u64 num_chunks; u64 original_resource_size; - u64 bytes_per_chunk_entry; + u64 num_chunks; u64 table_disk_size; - u64 cur_offset; - u64 *cur_offset_p; - u64 offsets[0]; + unsigned bytes_per_chunk_entry; + void *cur_offset_p; + union { + u32 cur_offset_u32; + u64 cur_offset_u64; + }; + /* Beginning of chunk offsets, in either 32-bit or 64-bit little endian + * integers, including the first offset of 0, which will not be written. + * */ + u8 offsets[] _aligned_attribute(8); }; /* @@ -90,11 +107,12 @@ begin_wim_resource_chunk_tab(const struct wim_lookup_table_entry *lte, struct chunk_table **chunk_tab_ret) { u64 size = wim_resource_size(lte); - u64 num_chunks = (size + WIM_CHUNK_SIZE - 1) / WIM_CHUNK_SIZE; + u64 num_chunks = wim_resource_chunks(lte); + unsigned bytes_per_chunk_entry = (size > (1ULL << 32)) ? 8 : 4; size_t alloc_size = sizeof(struct chunk_table) + num_chunks * sizeof(u64); struct chunk_table *chunk_tab = CALLOC(1, alloc_size); - DEBUG("Begin chunk table for stream with size %"PRIu64, size); + DEBUG("Beginning chunk table for stream with size %"PRIu64, size); if (!chunk_tab) { ERROR("Failed to allocate chunk table for %"PRIu64" byte " @@ -104,13 +122,15 @@ begin_wim_resource_chunk_tab(const struct wim_lookup_table_entry *lte, chunk_tab->file_offset = file_offset; chunk_tab->num_chunks = num_chunks; chunk_tab->original_resource_size = size; - chunk_tab->bytes_per_chunk_entry = (size >= (1ULL << 32)) ? 8 : 4; + chunk_tab->bytes_per_chunk_entry = bytes_per_chunk_entry; chunk_tab->table_disk_size = chunk_tab->bytes_per_chunk_entry * (num_chunks - 1); - chunk_tab->cur_offset = 0; chunk_tab->cur_offset_p = chunk_tab->offsets; - if (full_write(out_fd, chunk_tab, + /* We don't know the correct offsets yet; this just writes zeroes to + * reserve space for the table, so we can go back to it later after + * we've written the compressed chunks following it. */ + if (full_write(out_fd, chunk_tab->offsets, chunk_tab->table_disk_size) != chunk_tab->table_disk_size) { ERROR_WITH_ERRNO("Failed to write chunk table in compressed " @@ -122,6 +142,22 @@ begin_wim_resource_chunk_tab(const struct wim_lookup_table_entry *lte, return 0; } +/* Add the offset for the next chunk to the chunk table being constructed for a + * compressed stream. */ +static void +chunk_tab_record_chunk(struct chunk_table *chunk_tab, unsigned out_chunk_size) +{ + if (chunk_tab->bytes_per_chunk_entry == 4) { + *(le32*)chunk_tab->cur_offset_p = cpu_to_le32(chunk_tab->cur_offset_u32); + chunk_tab->cur_offset_p = (le32*)chunk_tab->cur_offset_p + 1; + chunk_tab->cur_offset_u32 += out_chunk_size; + } else { + *(le64*)chunk_tab->cur_offset_p = cpu_to_le64(chunk_tab->cur_offset_u64); + chunk_tab->cur_offset_p = (le64*)chunk_tab->cur_offset_p + 1; + chunk_tab->cur_offset_u64 += out_chunk_size; + } +} + /* * compress_func_t- Pointer to a function to compresses a chunk * of a WIM resource. This may be either @@ -173,7 +209,7 @@ get_compress_func(int out_ctype) static int write_wim_resource_chunk(const void * restrict chunk, unsigned chunk_size, - filedes_t out_fd, + int out_fd, compress_func_t compress, struct chunk_table * restrict chunk_tab) { @@ -191,8 +227,7 @@ write_wim_resource_chunk(const void * restrict chunk, out_chunk = chunk; out_chunk_size = chunk_size; } - *chunk_tab->cur_offset_p++ = chunk_tab->cur_offset; - chunk_tab->cur_offset += out_chunk_size; + chunk_tab_record_chunk(chunk_tab, out_chunk_size); } else { /* Write uncompressed */ out_chunk = chunk; @@ -214,19 +249,12 @@ write_wim_resource_chunk(const void * restrict chunk, */ static int finish_wim_resource_chunk_tab(struct chunk_table *chunk_tab, - filedes_t out_fd, u64 *compressed_size_p) + int out_fd, u64 *compressed_size_p) { size_t bytes_written; - if (chunk_tab->bytes_per_chunk_entry == 8) { - array_cpu_to_le64(chunk_tab->offsets, chunk_tab->num_chunks); - } else { - for (u64 i = 0; i < chunk_tab->num_chunks; i++) - ((u32*)chunk_tab->offsets)[i] = - cpu_to_le32(chunk_tab->offsets[i]); - } bytes_written = full_pwrite(out_fd, - (u8*)chunk_tab->offsets + chunk_tab->bytes_per_chunk_entry, + chunk_tab->offsets + chunk_tab->bytes_per_chunk_entry, chunk_tab->table_disk_size, chunk_tab->file_offset); if (bytes_written != chunk_tab->table_disk_size) { @@ -234,12 +262,15 @@ finish_wim_resource_chunk_tab(struct chunk_table *chunk_tab, "file resource"); return WIMLIB_ERR_WRITE; } - *compressed_size_p = chunk_tab->cur_offset + chunk_tab->table_disk_size; + if (chunk_tab->bytes_per_chunk_entry == 4) + *compressed_size_p = chunk_tab->cur_offset_u32 + chunk_tab->table_disk_size; + else + *compressed_size_p = chunk_tab->cur_offset_u64 + chunk_tab->table_disk_size; return 0; } static int -seek_and_truncate(filedes_t out_fd, off_t offset) +seek_and_truncate(int out_fd, off_t offset) { if (lseek(out_fd, offset, SEEK_SET) == -1 || ftruncate(out_fd, offset)) @@ -275,7 +306,7 @@ finalize_and_check_sha1(SHA_CTX * restrict sha_ctx, struct write_resource_ctx { compress_func_t compress; struct chunk_table *chunk_tab; - filedes_t out_fd; + int out_fd; SHA_CTX sha_ctx; bool doing_sha; }; @@ -319,7 +350,7 @@ write_resource_cb(const void *restrict chunk, size_t chunk_size, */ int write_wim_resource(struct wim_lookup_table_entry *lte, - filedes_t out_fd, int out_ctype, + int out_fd, int out_ctype, struct resource_entry *out_res_entry, int flags) { @@ -594,9 +625,19 @@ compressor_thread_proc(void *arg) static void do_write_streams_progress(union wimlib_progress_info *progress, wimlib_progress_func_t progress_func, - uint64_t size_added) + uint64_t size_added, + bool stream_discarded) { - progress->write_streams.completed_bytes += size_added; + if (stream_discarded) { + progress->write_streams.total_bytes -= size_added; + if (progress->write_streams._private != ~(uint64_t)0 && + progress->write_streams._private > progress->write_streams.total_bytes) + { + progress->write_streams._private = progress->write_streams.total_bytes; + } + } else { + progress->write_streams.completed_bytes += size_added; + } progress->write_streams.completed_streams++; if (progress_func && progress->write_streams.completed_bytes >= progress->write_streams._private) @@ -604,7 +645,7 @@ do_write_streams_progress(union wimlib_progress_info *progress, progress_func(WIMLIB_PROGRESS_MSG_WRITE_STREAMS, progress); if (progress->write_streams._private == progress->write_streams.total_bytes) { - progress->write_streams._private = ~0; + progress->write_streams._private = ~(uint64_t)0; } else { progress->write_streams._private = min(progress->write_streams.total_bytes, @@ -615,7 +656,7 @@ do_write_streams_progress(union wimlib_progress_info *progress, } struct serial_write_stream_ctx { - filedes_t out_fd; + int out_fd; int out_ctype; int write_resource_flags; }; @@ -643,9 +684,11 @@ do_write_stream_list(struct list_head *stream_list, { int ret = 0; struct wim_lookup_table_entry *lte; + bool stream_discarded; /* For each stream in @stream_list ... */ while (!list_empty(stream_list)) { + stream_discarded = false; lte = container_of(stream_list->next, struct wim_lookup_table_entry, write_streams_list); @@ -670,6 +713,7 @@ do_write_stream_list(struct list_head *stream_list, DEBUG("Discarding duplicate stream of length %"PRIu64, wim_resource_size(lte)); lte->no_progress = 0; + stream_discarded = true; goto skip_to_progress; } } @@ -701,7 +745,8 @@ do_write_stream_list(struct list_head *stream_list, if (!lte->no_progress) { do_write_streams_progress(progress, progress_func, - wim_resource_size(lte)); + wim_resource_size(lte), + stream_discarded); } } return ret; @@ -710,7 +755,7 @@ do_write_stream_list(struct list_head *stream_list, static int do_write_stream_list_serial(struct list_head *stream_list, struct wim_lookup_table *lookup_table, - filedes_t out_fd, + int out_fd, int out_ctype, int write_resource_flags, wimlib_progress_func_t progress_func, @@ -742,7 +787,7 @@ write_flags_to_resource_flags(int write_flags) static int write_stream_list_serial(struct list_head *stream_list, struct wim_lookup_table *lookup_table, - filedes_t out_fd, + int out_fd, int out_ctype, int write_resource_flags, wimlib_progress_func_t progress_func, @@ -763,13 +808,11 @@ write_stream_list_serial(struct list_head *stream_list, #ifdef ENABLE_MULTITHREADED_COMPRESSION static int -write_wim_chunks(struct message *msg, filedes_t out_fd, +write_wim_chunks(struct message *msg, int out_fd, struct chunk_table *chunk_tab) { - for (unsigned i = 0; i < msg->num_chunks; i++) { - *chunk_tab->cur_offset_p++ = chunk_tab->cur_offset; - chunk_tab->cur_offset += msg->out_chunks[i].iov_len; - } + for (unsigned i = 0; i < msg->num_chunks; i++) + chunk_tab_record_chunk(chunk_tab, msg->out_chunks[i].iov_len); if (full_writev(out_fd, msg->out_chunks, msg->num_chunks) != msg->total_out_bytes) { @@ -782,7 +825,7 @@ write_wim_chunks(struct message *msg, filedes_t out_fd, struct main_writer_thread_ctx { struct list_head *stream_list; struct wim_lookup_table *lookup_table; - filedes_t out_fd; + int out_fd; int out_ctype; int write_resource_flags; struct shared_queue *res_to_compress_queue; @@ -1011,7 +1054,8 @@ receive_compressed_chunks(struct main_writer_thread_ctx *ctx) do_write_streams_progress(ctx->progress, ctx->progress_func, - wim_resource_size(cur_lte)); + wim_resource_size(cur_lte), + false); /* Since we just finished writing a stream, write any * streams that have been added to the serial_streams @@ -1180,7 +1224,7 @@ main_thread_process_next_stream(struct wim_lookup_table_entry *lte, void *_ctx) } static long -get_default_num_threads() +get_default_num_threads(void) { #ifdef __WIN32__ return win32_get_number_of_processors(); @@ -1218,7 +1262,7 @@ get_default_num_threads() static int write_stream_list_parallel(struct list_head *stream_list, struct wim_lookup_table *lookup_table, - filedes_t out_fd, + int out_fd, int out_ctype, int write_resource_flags, wimlib_progress_func_t progress_func, @@ -1356,7 +1400,7 @@ out_serial_quiet: static int write_stream_list(struct list_head *stream_list, struct wim_lookup_table *lookup_table, - filedes_t out_fd, int out_ctype, int write_flags, + int out_fd, int out_ctype, int write_flags, unsigned num_threads, wimlib_progress_func_t progress_func) { struct wim_lookup_table_entry *lte; @@ -1395,7 +1439,7 @@ write_stream_list(struct list_head *stream_list, progress.write_streams._private = 0; #ifdef ENABLE_MULTITHREADED_COMPRESSION - if (total_compression_bytes >= 1000000 && num_threads != 1) + if (total_compression_bytes >= 2000000 && num_threads != 1) ret = write_stream_list_parallel(stream_list, lookup_table, out_fd, @@ -1684,9 +1728,6 @@ write_wim_streams(WIMStruct *wim, int image, int write_flags, * (public) WIMLIB_WRITE_FLAG_CHECK_INTEGRITY: * Include an integrity table. * - * (public) WIMLIB_WRITE_FLAG_SHOW_PROGRESS: - * Show progress information when (if) writing the integrity table. - * * (private) WIMLIB_WRITE_FLAG_NO_LOOKUP_TABLE: * Don't write the lookup table. * @@ -1800,16 +1841,16 @@ out_close_wim: if (ret == 0) ret = WIMLIB_ERR_WRITE; } - w->out_fd = INVALID_FILEDES; + w->out_fd = -1; return ret; } #if defined(HAVE_SYS_FILE_H) && defined(HAVE_FLOCK) int -lock_wim(WIMStruct *w, filedes_t fd) +lock_wim(WIMStruct *w, int fd) { int ret = 0; - if (fd != INVALID_FILEDES && !w->wim_locked) { + if (fd != -1 && !w->wim_locked) { ret = flock(fd, LOCK_EX | LOCK_NB); if (ret != 0) { if (errno == EWOULDBLOCK) { @@ -1833,8 +1874,8 @@ lock_wim(WIMStruct *w, filedes_t fd) static int open_wim_writable(WIMStruct *w, const tchar *path, int open_flags) { - w->out_fd = open(path, open_flags, 0644); - if (w->out_fd == INVALID_FILEDES) { + w->out_fd = topen(path, open_flags | O_BINARY, 0644); + if (w->out_fd == -1) { ERROR_WITH_ERRNO("Failed to open `%"TS"' for writing", path); return WIMLIB_ERR_OPEN; } @@ -1845,10 +1886,10 @@ open_wim_writable(WIMStruct *w, const tchar *path, int open_flags) void close_wim_writable(WIMStruct *w) { - if (w->out_fd != INVALID_FILEDES) { + if (w->out_fd != -1) { if (close(w->out_fd)) WARNING_WITH_ERRNO("Failed to close output WIM"); - w->out_fd = INVALID_FILEDES; + w->out_fd = -1; } }