X-Git-Url: https://wimlib.net/git/?p=wimlib;a=blobdiff_plain;f=src%2Fwrite.c;h=39e8bd0af5a8c75002cb47c17fbd524e0529e694;hp=2b8347da973a1f20bd05dc15eed472f0dff374c3;hb=105b46dcfa04b92674bf0c18f93f9438d8d9ef9a;hpb=b5b9681794d1f5f13350e3567f6f6e74f5c779cf diff --git a/src/write.c b/src/write.c index 2b8347da..39e8bd0a 100644 --- a/src/write.c +++ b/src/write.c @@ -6,7 +6,7 @@ */ /* - * Copyright (C) 2012, 2013 Eric Biggers + * Copyright (C) 2012, 2013, 2014 Eric Biggers * * This file is part of wimlib, a library for working with WIM files. * @@ -43,6 +43,7 @@ #include "wimlib/integrity.h" #include "wimlib/lookup_table.h" #include "wimlib/metadata.h" +#include "wimlib/paths.h" #include "wimlib/progress.h" #include "wimlib/resource.h" #ifdef __WIN32__ @@ -64,6 +65,7 @@ #define WRITE_RESOURCE_FLAG_RECOMPRESS 0x00000001 #define WRITE_RESOURCE_FLAG_PIPABLE 0x00000002 #define WRITE_RESOURCE_FLAG_PACK_STREAMS 0x00000004 +#define WRITE_RESOURCE_FLAG_SEND_DONE_WITH_FILE 0x00000008 static inline int write_flags_to_resource_flags(int write_flags) @@ -76,6 +78,8 @@ write_flags_to_resource_flags(int write_flags) write_resource_flags |= WRITE_RESOURCE_FLAG_PIPABLE; if (write_flags & WIMLIB_WRITE_FLAG_PACK_STREAMS) write_resource_flags |= WRITE_RESOURCE_FLAG_PACK_STREAMS; + if (write_flags & WIMLIB_WRITE_FLAG_SEND_DONE_WITH_FILE_MESSAGES) + write_resource_flags |= WRITE_RESOURCE_FLAG_SEND_DONE_WITH_FILE; return write_resource_flags; } @@ -98,12 +102,15 @@ static int stream_filtered(const struct wim_lookup_table_entry *lte, const struct filter_context *ctx) { - int write_flags = ctx->write_flags; - WIMStruct *wim = ctx->wim; + int write_flags; + WIMStruct *wim; if (ctx == NULL) return 0; + write_flags = ctx->write_flags; + wim = ctx->wim; + if (write_flags & WIMLIB_WRITE_FLAG_OVERWRITE && lte->resource_location == RESOURCE_IN_WIM && lte->rspec->wim == wim) @@ -312,10 +319,28 @@ do_write_streams_progress(struct write_streams_progress_data *progress_data, if (progress_data->next_progress == progress->write_streams.total_bytes) { progress_data->next_progress = ~(uint64_t)0; } else { + /* Handle rate-limiting of messages */ + + /* Send new message as soon as another 1/128 of the + * total has been written. (Arbitrary number.) */ progress_data->next_progress = - min(progress->write_streams.total_bytes, - progress->write_streams.completed_bytes + - progress->write_streams.total_bytes / 100); + progress->write_streams.completed_bytes + + progress->write_streams.total_bytes / 128; + + /* ... Unless that would be more than 5000000 bytes, in + * which case send the next after the next 5000000 + * bytes. (Another arbitrary number.) */ + if (progress->write_streams.completed_bytes + 5000000 < + progress_data->next_progress) + progress_data->next_progress = + progress->write_streams.completed_bytes + 5000000; + + /* ... But always send a message as soon as we're + * completely done. */ + if (progress->write_streams.total_bytes < + progress_data->next_progress) + progress_data->next_progress = + progress->write_streams.total_bytes; } } return 0; @@ -372,6 +397,8 @@ struct write_streams_ctx { * pending_streams rather than the entry being read.) */ bool stream_was_duplicate; + struct wim_inode *stream_inode; + /* Current uncompressed offset in the stream being read. */ u64 cur_read_stream_offset; @@ -451,7 +478,7 @@ begin_chunk_table(struct write_streams_ctx *ctx, u64 res_expected_size) reserve_size = expected_num_chunk_entries * get_chunk_entry_size(res_expected_size, 0 != (ctx->write_resource_flags & - WIM_RESHDR_FLAG_PACKED_STREAMS)); + WRITE_RESOURCE_FLAG_PACK_STREAMS)); if (ctx->write_resource_flags & WRITE_RESOURCE_FLAG_PACK_STREAMS) reserve_size += sizeof(struct alt_chunk_table_header_disk); memset(ctx->chunk_csizes, 0, reserve_size); @@ -499,7 +526,7 @@ end_chunk_table(struct write_streams_ctx *ctx, u64 res_actual_size, chunk_entry_size = get_chunk_entry_size(res_actual_size, 0 != (ctx->write_resource_flags & - WIM_RESHDR_FLAG_PACKED_STREAMS)); + WRITE_RESOURCE_FLAG_PACK_STREAMS)); typedef le64 __attribute__((may_alias)) aliased_le64_t; typedef le32 __attribute__((may_alias)) aliased_le32_t; @@ -618,10 +645,77 @@ end_write_resource(struct write_streams_ctx *ctx, struct wim_reshdr *out_reshdr) return 0; } +/* No more data streams of the file at @path are needed. */ +static int +done_with_file(const tchar *path, wimlib_progress_func_t progfunc, void *progctx) +{ + union wimlib_progress_info info; + + info.done_with_file.path_to_file = path; + + return call_progress(progfunc, WIMLIB_PROGRESS_MSG_DONE_WITH_FILE, + &info, progctx); +} + +/* Handle WIMLIB_WRITE_FLAG_SEND_DONE_WITH_FILE_MESSAGES mode. */ +static int +done_with_stream(struct wim_lookup_table_entry *stream, + wimlib_progress_func_t progfunc, void *progctx, + struct wim_inode *inode) +{ + if (stream->resource_location == RESOURCE_IN_FILE_ON_DISK +#ifdef __WIN32__ + || stream->resource_location == RESOURCE_IN_WINNT_FILE_ON_DISK + || stream->resource_location == RESOURCE_WIN32_ENCRYPTED +#endif + ) + { + int ret; + + wimlib_assert(inode->num_unread_streams > 0); + if (--inode->num_unread_streams > 0) + return 0; + + #ifdef __WIN32__ + /* XXX: This logic really should be somewhere else. */ + + /* We want the path to the file, but stream->file_on_disk might + * actually refer to a named data stream. Temporarily strip the + * named data stream from the path. */ + wchar_t *p_colon = NULL; + wchar_t *p_question_mark = NULL; + const wchar_t *p_stream_name; + + p_stream_name = path_stream_name(stream->file_on_disk); + if (unlikely(p_stream_name)) { + p_colon = (wchar_t *)(p_stream_name - 1); + wimlib_assert(*p_colon == L':'); + *p_colon = L'\0'; + } + + /* We also should use a fake Win32 path instead of a NT path */ + if (!wcsncmp(stream->file_on_disk, L"\\??\\", 4)) { + p_question_mark = &stream->file_on_disk[1]; + *p_question_mark = L'\\'; + } + #endif + + ret = done_with_file(stream->file_on_disk, progfunc, progctx); + + #ifdef __WIN32__ + if (p_colon) + *p_colon = L':'; + if (p_question_mark) + *p_question_mark = L'?'; + #endif + return ret; + } + return 0; +} + /* Begin processing a stream for writing. */ static int -write_stream_begin_read(struct wim_lookup_table_entry *lte, - u32 flags, void *_ctx) +write_stream_begin_read(struct wim_lookup_table_entry *lte, void *_ctx) { struct write_streams_ctx *ctx = _ctx; int ret; @@ -631,6 +725,11 @@ write_stream_begin_read(struct wim_lookup_table_entry *lte, ctx->cur_read_stream_offset = 0; ctx->cur_read_stream_size = lte->size; + if (lte->unhashed) + ctx->stream_inode = lte->back_inode; + else + ctx->stream_inode = NULL; + /* As an optimization, we allow some streams to be "unhashed", meaning * their SHA1 message digests are unknown. This is the case with * streams that are added by scanning a directry tree with @@ -645,8 +744,6 @@ write_stream_begin_read(struct wim_lookup_table_entry *lte, ctx->stream_was_duplicate = false; if (ctx->lookup_table != NULL && lte->unhashed && !lte->unique_size) { - wimlib_assert(!(flags & BEGIN_STREAM_FLAG_PARTIAL_RESOURCE)); - struct wim_lookup_table_entry *lte_new; ret = hash_unhashed_stream(lte, ctx->lookup_table, <e_new); @@ -675,6 +772,14 @@ write_stream_begin_read(struct wim_lookup_table_entry *lte, lte_new->out_refcnt += lte->out_refcnt; if (ctx->write_resource_flags & WRITE_RESOURCE_FLAG_PACK_STREAMS) ctx->cur_write_res_size -= lte->size; + if (!ret && unlikely(ctx->write_resource_flags & + WRITE_RESOURCE_FLAG_SEND_DONE_WITH_FILE)) + { + ret = done_with_stream(lte, + ctx->progress_data.progfunc, + ctx->progress_data.progctx, + ctx->stream_inode); + } free_lookup_table_entry(lte); if (ret) return ret; @@ -863,7 +968,8 @@ write_chunk(struct write_streams_ctx *ctx, const void *cchunk, if (ctx->compressor != NULL && lte->out_reshdr.size_in_wim >= lte->out_reshdr.uncompressed_size && - !(ctx->write_resource_flags & WRITE_RESOURCE_FLAG_PIPABLE) && + !(ctx->write_resource_flags & (WRITE_RESOURCE_FLAG_PIPABLE | + WRITE_RESOURCE_FLAG_SEND_DONE_WITH_FILE)) && !(lte->flags & WIM_RESHDR_FLAG_PACKED_STREAMS)) { /* Stream did not compress to less than its original @@ -1006,15 +1112,31 @@ static int write_stream_end_read(struct wim_lookup_table_entry *lte, int status, void *_ctx) { struct write_streams_ctx *ctx = _ctx; - if (status == 0) - wimlib_assert(ctx->cur_read_stream_offset == ctx->cur_read_stream_size); - if (ctx->stream_was_duplicate) { - free_lookup_table_entry(lte); - } else if (lte->unhashed && ctx->lookup_table != NULL) { + if (status) + goto out; + + wimlib_assert(ctx->cur_read_stream_offset == ctx->cur_read_stream_size); + + if (unlikely(ctx->write_resource_flags & + WRITE_RESOURCE_FLAG_SEND_DONE_WITH_FILE) && + ctx->stream_inode != NULL) + { + status = done_with_stream(lte, + ctx->progress_data.progfunc, + ctx->progress_data.progctx, + ctx->stream_inode); + } + + if (!ctx->stream_was_duplicate && lte->unhashed && + ctx->lookup_table != NULL) + { list_del(<e->unhashed_list); lookup_table_insert(ctx->lookup_table, lte); lte->unhashed = 0; } +out: + if (ctx->stream_was_duplicate) + free_lookup_table_entry(lte); return status; } @@ -1261,9 +1383,8 @@ remove_zero_length_streams(struct list_head *stream_list) * * @out_ctype * Compression format to use to write the output streams, specified as one - * of the WIMLIB_COMPRESSION_TYPE_* constants, excepting - * WIMLIB_COMPRESSION_TYPE_INVALID but including - * WIMLIB_COMPRESSION_TYPE_NONE. + * of the WIMLIB_COMPRESSION_TYPE_* constants. + * WIMLIB_COMPRESSION_TYPE_NONE is allowed. * * @out_chunk_size * Chunk size to use to write the streams. It must be a valid chunk size @@ -1569,7 +1690,7 @@ wim_write_stream_list(WIMStruct *wim, write_resource_flags = write_flags_to_resource_flags(write_flags); - /* wimlib v1.6.3: pack streams by default if the WIM version has been + /* wimlib v1.7.0: pack streams by default if the WIM version has been * set to WIM_VERSION_PACKED_STREAMS and at least one stream in the * WIM's lookup table is located in a packed resource (may be the same * WIM, or a different one in the case of export). */ @@ -2051,6 +2172,17 @@ write_wim_streams(WIMStruct *wim, int image, int write_flags, } } + /* If needed, set auxiliary information so that we can detect when + * wimlib has finished using each external file. */ + if (unlikely(write_flags & WIMLIB_WRITE_FLAG_SEND_DONE_WITH_FILE_MESSAGES)) { + list_for_each_entry(lte, stream_list, write_streams_list) + if (lte->unhashed) + lte->back_inode->num_unread_streams = 0; + list_for_each_entry(lte, stream_list, write_streams_list) + if (lte->unhashed) + lte->back_inode->num_unread_streams++; + } + return wim_write_stream_list(wim, stream_list, write_flags, @@ -2265,10 +2397,6 @@ write_wim_lookup_table(WIMStruct *wim, int image, int write_flags, * (private) WIMLIB_WRITE_FLAG_NO_LOOKUP_TABLE: * Don't write the lookup table. * - * (private) WIMLIB_WRITE_FLAG_REUSE_INTEGRITY_TABLE: - * When (if) writing the integrity table, re-use entries from the - * existing integrity table, if possible. - * * (private) WIMLIB_WRITE_FLAG_CHECKPOINT_AFTER_XML: * After writing the XML data but before writing the integrity * table, write a temporary WIM header and flush the stream so that @@ -2285,6 +2413,9 @@ write_wim_lookup_table(WIMStruct *wim, int image, int write_flags, * Use the existing stored in the in-memory XML * information, rather than setting it to the offset of the XML * data being written. + * (private) WIMLIB_WRITE_FLAG_OVERWRITE + * The existing WIM file is being updated in-place. The entries + * from its integrity table may be re-used. */ static int finish_write(WIMStruct *wim, int image, int write_flags, @@ -2293,9 +2424,10 @@ finish_write(WIMStruct *wim, int image, int write_flags, int ret; off_t hdr_offset; int write_resource_flags; - off_t old_lookup_table_end; + off_t old_lookup_table_end = 0; off_t new_lookup_table_end; u64 xml_totalbytes; + struct integrity_table *old_integrity_table = NULL; DEBUG("image=%d, write_flags=%08x", image, write_flags); @@ -2314,15 +2446,36 @@ finish_write(WIMStruct *wim, int image, int write_flags, wim->hdr.boot_idx - 1]->metadata_lte->out_reshdr); } - /* Write lookup table. (Save old position first.) */ - old_lookup_table_end = wim->hdr.lookup_table_reshdr.offset_in_wim + - wim->hdr.lookup_table_reshdr.size_in_wim; + /* If overwriting the WIM file containing an integrity table in-place, + * we'd like to re-use the information in the old integrity table + * instead of recalculating it. But we might overwrite the old + * integrity table when we expand the XML data. Read it into memory + * just in case. */ + if ((write_flags & (WIMLIB_WRITE_FLAG_OVERWRITE | + WIMLIB_WRITE_FLAG_CHECK_INTEGRITY)) == + (WIMLIB_WRITE_FLAG_OVERWRITE | + WIMLIB_WRITE_FLAG_CHECK_INTEGRITY) + && wim_has_integrity_table(wim)) + { + old_lookup_table_end = wim->hdr.lookup_table_reshdr.offset_in_wim + + wim->hdr.lookup_table_reshdr.size_in_wim; + (void)read_integrity_table(wim, + old_lookup_table_end - WIM_HEADER_DISK_SIZE, + &old_integrity_table); + /* If we couldn't read the old integrity table, we can still + * re-calculate the full integrity table ourselves. Hence the + * ignoring of the return value. */ + } + + /* Write lookup table. */ if (!(write_flags & WIMLIB_WRITE_FLAG_NO_LOOKUP_TABLE)) { ret = write_wim_lookup_table(wim, image, write_flags, &wim->hdr.lookup_table_reshdr, lookup_table_list); - if (ret) + if (ret) { + free_integrity_table(old_integrity_table); return ret; + } } /* Write XML data. */ @@ -2332,8 +2485,10 @@ finish_write(WIMStruct *wim, int image, int write_flags, ret = write_wim_xml_data(wim, image, xml_totalbytes, &wim->hdr.xml_data_reshdr, write_resource_flags); - if (ret) + if (ret) { + free_integrity_table(old_integrity_table); return ret; + } /* Write integrity table (optional). */ if (write_flags & WIMLIB_WRITE_FLAG_CHECK_INTEGRITY) { @@ -2344,19 +2499,20 @@ finish_write(WIMStruct *wim, int image, int write_flags, checkpoint_hdr.flags |= WIM_HDR_FLAG_WRITE_IN_PROGRESS; ret = write_wim_header_at_offset(&checkpoint_hdr, &wim->out_fd, 0); - if (ret) + if (ret) { + free_integrity_table(old_integrity_table); return ret; + } } - if (!(write_flags & WIMLIB_WRITE_FLAG_REUSE_INTEGRITY_TABLE)) - old_lookup_table_end = 0; - new_lookup_table_end = wim->hdr.lookup_table_reshdr.offset_in_wim + wim->hdr.lookup_table_reshdr.size_in_wim; ret = write_integrity_table(wim, new_lookup_table_end, - old_lookup_table_end); + old_lookup_table_end, + old_integrity_table); + free_integrity_table(old_integrity_table); if (ret) return ret; } else { @@ -2400,28 +2556,30 @@ finish_write(WIMStruct *wim, int image, int write_flags, } #if defined(HAVE_SYS_FILE_H) && defined(HAVE_FLOCK) + +/* Set advisory lock on WIM file (if not already done so) */ int -lock_wim(WIMStruct *wim, int fd) +lock_wim_for_append(WIMStruct *wim) { - int ret = 0; - if (fd != -1 && !wim->wim_locked) { - ret = flock(fd, LOCK_EX | LOCK_NB); - if (ret != 0) { - if (errno == EWOULDBLOCK) { - ERROR("`%"TS"' is already being modified or has been " - "mounted read-write\n" - " by another process!", wim->filename); - ret = WIMLIB_ERR_ALREADY_LOCKED; - } else { - WARNING_WITH_ERRNO("Failed to lock `%"TS"'", - wim->filename); - ret = 0; - } - } else { - wim->wim_locked = 1; - } + if (wim->locked_for_append) + return 0; + if (!flock(wim->in_fd.fd, LOCK_EX | LOCK_NB)) { + wim->locked_for_append = 1; + return 0; + } + if (errno != EWOULDBLOCK) + return 0; + return WIMLIB_ERR_ALREADY_LOCKED; +} + +/* Remove advisory lock on WIM file (if present) */ +void +unlock_wim_for_append(WIMStruct *wim) +{ + if (wim->locked_for_append) { + flock(wim->in_fd.fd, LOCK_UN); + wim->locked_for_append = 0; } - return ret; } #endif @@ -3062,7 +3220,7 @@ overwrite_wim_inplace(WIMStruct *wim, int write_flags, unsigned num_threads) if (ret) goto out_restore_memory_hdr; - ret = lock_wim(wim, wim->out_fd.fd); + ret = lock_wim_for_append(wim); if (ret) goto out_close_wim; @@ -3092,13 +3250,12 @@ overwrite_wim_inplace(WIMStruct *wim, int write_flags, unsigned num_threads) if (ret) goto out_truncate; - write_flags |= WIMLIB_WRITE_FLAG_REUSE_INTEGRITY_TABLE; ret = finish_write(wim, WIMLIB_ALL_IMAGES, write_flags, &lookup_table_list); if (ret) goto out_truncate; - wim->wim_locked = 0; + unlock_wim_for_append(wim); return 0; out_truncate: @@ -3112,7 +3269,7 @@ out_truncate: out_restore_physical_hdr: (void)write_wim_header_flags(hdr_save.flags, &wim->out_fd); out_unlock_wim: - wim->wim_locked = 0; + unlock_wim_for_append(wim); out_close_wim: (void)close_wim_writable(wim, write_flags); out_restore_memory_hdr: