From: Eric Biggers Date: Sun, 22 Jun 2014 04:41:05 +0000 (-0500) Subject: Add experimental support for WIMLIB_PROGRESS_MSG_DONE_WITH_FILE X-Git-Tag: v1.7.1~87 X-Git-Url: https://wimlib.net/git/?p=wimlib;a=commitdiff_plain;h=34193844b60cf16bb0a04e344ddf47a0614a7643;hp=7839aa1d5f4983cc0c716219548237f7fca7f1a3 Add experimental support for WIMLIB_PROGRESS_MSG_DONE_WITH_FILE --- diff --git a/NEWS b/NEWS index 5b205d3c..a903d01f 100644 --- a/NEWS +++ b/NEWS @@ -6,7 +6,10 @@ Version 1.7.1-BETA: times in one run of an application program. Library users will now receive WIMLIB_PROGRESS_MSG_WRITE_STREAMS - messages more frequency when writing large WIM files. + messages more frequently when writing large WIM files. + + Added experimental new write flag: + WIMLIB_WRITE_FLAG_SEND_DONE_WITH_FILE_MESSAGES. Version 1.7.0: Improved compression, decompression, and extraction performance. diff --git a/include/wimlib.h b/include/wimlib.h index ecb76353..f56e72cf 100644 --- a/include/wimlib.h +++ b/include/wimlib.h @@ -620,6 +620,12 @@ enum wimlib_progress_msg { /** Starting to unmount a WIM image. @p info will point to * ::wimlib_progress_info.unmount. */ WIMLIB_PROGRESS_MSG_UNMOUNT_BEGIN = 25, + + /** wimlib has used a file's data for the last time (including all data + * streams, if it has multiple). @p info will point to + * ::wimlib_progress_info.done_with_file. This message is only received + * if ::WIMLIB_WRITE_FLAG_SEND_DONE_WITH_FILE_MESSAGES was provided. */ + WIMLIB_PROGRESS_MSG_DONE_WITH_FILE = 26, }; /** Valid return values from user-provided progress functions @@ -1024,6 +1030,28 @@ union wimlib_progress_info { /** Flags passed to wimlib_unmount_image(). */ uint32_t unmount_flags; } unmount; + + /** Valid on messages ::WIMLIB_PROGRESS_MSG_DONE_WITH_FILE. */ + struct wimlib_progress_info_done_with_file { + /* Path to the file whose data has been written to the WIM file, + * or is currently being asynchronously compressed in memory, + * and therefore is no longer needed by wimlib. + * + * WARNING: The file data will not actually be accessible in the + * WIM file until the WIM file has been completely written. + * Ordinarily you should not treat this message as a + * green light to go ahead and delete the specified file, since + * that would result in data loss if the WIM file cannot be + * successfully created for any reason. + * + * If a file has multiple names (hard links), + * ::WIMLIB_PROGRESS_MSG_DONE_WITH_FILE will only be received + * for one name. Also, this message will not be received for + * empty files or reparse points (or symbolic links), unless + * they have nonempty named data streams. + */ + const wimlib_tchar *path_to_file; + } done_with_file; }; /** @@ -1936,6 +1964,13 @@ typedef int (*wimlib_iterate_lookup_table_callback_t)(const struct wimlib_resour */ #define WIMLIB_WRITE_FLAG_PACK_STREAMS 0x00001000 +/** + * Send ::WIMLIB_PROGRESS_MSG_DONE_WITH_FILE messages while writing the WIM + * file. This is only needed in the unusual case that the library user needs to + * know exactly when wimlib has read each file for the last time. + */ +#define WIMLIB_WRITE_FLAG_SEND_DONE_WITH_FILE_MESSAGES 0x00002000 + /** @} */ /** @addtogroup G_general * @{ */ diff --git a/include/wimlib/inode.h b/include/wimlib/inode.h index 4d39c1ba..3b022299 100644 --- a/include/wimlib/inode.h +++ b/include/wimlib/inode.h @@ -176,6 +176,12 @@ struct wim_inode { #endif }; + /* Used during WIM writing with + * WIMLIB_WRITE_FLAG_SEND_DONE_WITH_FILE_MESSAGES: the number + * of data streams this inode has that have not yet been fully + * read. */ + u32 num_unread_streams; + #ifdef WITH_FUSE struct { /* Used only during image mount: Table of file diff --git a/include/wimlib/write.h b/include/wimlib/write.h index d6957ace..61f3bc67 100644 --- a/include/wimlib/write.h +++ b/include/wimlib/write.h @@ -27,7 +27,8 @@ WIMLIB_WRITE_FLAG_SKIP_EXTERNAL_WIMS | \ WIMLIB_WRITE_FLAG_STREAMS_OK | \ WIMLIB_WRITE_FLAG_RETAIN_GUID | \ - WIMLIB_WRITE_FLAG_PACK_STREAMS) + WIMLIB_WRITE_FLAG_PACK_STREAMS | \ + WIMLIB_WRITE_FLAG_SEND_DONE_WITH_FILE_MESSAGES) #if defined(HAVE_SYS_FILE_H) && defined(HAVE_FLOCK) extern int diff --git a/src/write.c b/src/write.c index 56eb5dd4..c45d0b7f 100644 --- a/src/write.c +++ b/src/write.c @@ -43,6 +43,7 @@ #include "wimlib/integrity.h" #include "wimlib/lookup_table.h" #include "wimlib/metadata.h" +#include "wimlib/paths.h" #include "wimlib/progress.h" #include "wimlib/resource.h" #ifdef __WIN32__ @@ -64,6 +65,7 @@ #define WRITE_RESOURCE_FLAG_RECOMPRESS 0x00000001 #define WRITE_RESOURCE_FLAG_PIPABLE 0x00000002 #define WRITE_RESOURCE_FLAG_PACK_STREAMS 0x00000004 +#define WRITE_RESOURCE_FLAG_SEND_DONE_WITH_FILE 0x00000008 static inline int write_flags_to_resource_flags(int write_flags) @@ -76,6 +78,8 @@ write_flags_to_resource_flags(int write_flags) write_resource_flags |= WRITE_RESOURCE_FLAG_PIPABLE; if (write_flags & WIMLIB_WRITE_FLAG_PACK_STREAMS) write_resource_flags |= WRITE_RESOURCE_FLAG_PACK_STREAMS; + if (write_flags & WIMLIB_WRITE_FLAG_SEND_DONE_WITH_FILE_MESSAGES) + write_resource_flags |= WRITE_RESOURCE_FLAG_SEND_DONE_WITH_FILE; return write_resource_flags; } @@ -390,6 +394,8 @@ struct write_streams_ctx { * pending_streams rather than the entry being read.) */ bool stream_was_duplicate; + struct wim_inode *stream_inode; + /* Current uncompressed offset in the stream being read. */ u64 cur_read_stream_offset; @@ -636,6 +642,74 @@ end_write_resource(struct write_streams_ctx *ctx, struct wim_reshdr *out_reshdr) return 0; } +/* No more data streams of the file at @path are needed. */ +static int +done_with_file(const tchar *path, wimlib_progress_func_t progfunc, void *progctx) +{ + union wimlib_progress_info info; + + info.done_with_file.path_to_file = path; + + return call_progress(progfunc, WIMLIB_PROGRESS_MSG_DONE_WITH_FILE, + &info, progctx); +} + +/* Handle WIMLIB_WRITE_FLAG_SEND_DONE_WITH_FILE_MESSAGES mode. */ +static int +done_with_stream(struct wim_lookup_table_entry *stream, + wimlib_progress_func_t progfunc, void *progctx, + struct wim_inode *inode) +{ + if (stream->resource_location == RESOURCE_IN_FILE_ON_DISK +#ifdef __WIN32__ + || stream->resource_location == RESOURCE_IN_WINNT_FILE_ON_DISK + || stream->resource_location == RESOURCE_WIN32_ENCRYPTED +#endif + ) + { + int ret; + + wimlib_assert(inode->num_unread_streams > 0); + if (--inode->num_unread_streams > 0) + return 0; + + #ifdef __WIN32__ + /* XXX: This logic really should be somewhere else. */ + + /* We want the path to the file, but stream->file_on_disk might + * actually refer to a named data stream. Temporarily strip the + * named data stream from the path. */ + wchar_t *p_colon = NULL; + wchar_t *p_question_mark = NULL; + const wchar_t *p_stream_name; + + p_stream_name = path_stream_name(stream->file_on_disk); + if (unlikely(p_stream_name)) { + p_colon = (wchar_t *)(p_stream_name - 1); + wimlib_assert(*p_colon == L':'); + *p_colon = L'\0'; + } + + /* We also should use a fake Win32 path instead of a NT path */ + if (!wcsncmp(stream->file_on_disk, L"\\??\\", 4)) { + p_question_mark = &stream->file_on_disk[1]; + *p_question_mark = L'\\'; + } + #endif + + ret = done_with_file(stream->file_on_disk, progfunc, progctx); + + #ifdef __WIN32__ + if (p_colon) + *p_colon = L':'; + if (p_question_mark) + *p_question_mark = L'?'; + #endif + return ret; + } + return 0; +} + /* Begin processing a stream for writing. */ static int write_stream_begin_read(struct wim_lookup_table_entry *lte, @@ -649,6 +723,11 @@ write_stream_begin_read(struct wim_lookup_table_entry *lte, ctx->cur_read_stream_offset = 0; ctx->cur_read_stream_size = lte->size; + if (lte->unhashed) + ctx->stream_inode = lte->back_inode; + else + ctx->stream_inode = NULL; + /* As an optimization, we allow some streams to be "unhashed", meaning * their SHA1 message digests are unknown. This is the case with * streams that are added by scanning a directry tree with @@ -693,6 +772,14 @@ write_stream_begin_read(struct wim_lookup_table_entry *lte, lte_new->out_refcnt += lte->out_refcnt; if (ctx->write_resource_flags & WRITE_RESOURCE_FLAG_PACK_STREAMS) ctx->cur_write_res_size -= lte->size; + if (!ret && unlikely(ctx->write_resource_flags & + WRITE_RESOURCE_FLAG_SEND_DONE_WITH_FILE)) + { + ret = done_with_stream(lte, + ctx->progress_data.progfunc, + ctx->progress_data.progctx, + ctx->stream_inode); + } free_lookup_table_entry(lte); if (ret) return ret; @@ -881,7 +968,8 @@ write_chunk(struct write_streams_ctx *ctx, const void *cchunk, if (ctx->compressor != NULL && lte->out_reshdr.size_in_wim >= lte->out_reshdr.uncompressed_size && - !(ctx->write_resource_flags & WRITE_RESOURCE_FLAG_PIPABLE) && + !(ctx->write_resource_flags & (WRITE_RESOURCE_FLAG_PIPABLE | + WRITE_RESOURCE_FLAG_SEND_DONE_WITH_FILE)) && !(lte->flags & WIM_RESHDR_FLAG_PACKED_STREAMS)) { /* Stream did not compress to less than its original @@ -1024,15 +1112,31 @@ static int write_stream_end_read(struct wim_lookup_table_entry *lte, int status, void *_ctx) { struct write_streams_ctx *ctx = _ctx; - if (status == 0) - wimlib_assert(ctx->cur_read_stream_offset == ctx->cur_read_stream_size); - if (ctx->stream_was_duplicate) { - free_lookup_table_entry(lte); - } else if (lte->unhashed && ctx->lookup_table != NULL) { + if (status) + goto out; + + wimlib_assert(ctx->cur_read_stream_offset == ctx->cur_read_stream_size); + + if (unlikely(ctx->write_resource_flags & + WRITE_RESOURCE_FLAG_SEND_DONE_WITH_FILE) && + ctx->stream_inode != NULL) + { + status = done_with_stream(lte, + ctx->progress_data.progfunc, + ctx->progress_data.progctx, + ctx->stream_inode); + } + + if (!ctx->stream_was_duplicate && lte->unhashed && + ctx->lookup_table != NULL) + { list_del(<e->unhashed_list); lookup_table_insert(ctx->lookup_table, lte); lte->unhashed = 0; } +out: + if (ctx->stream_was_duplicate) + free_lookup_table_entry(lte); return status; } @@ -2068,6 +2172,17 @@ write_wim_streams(WIMStruct *wim, int image, int write_flags, } } + /* If needed, set auxiliary information so that we can detect when + * wimlib has finished using each external file. */ + if (unlikely(write_flags & WIMLIB_WRITE_FLAG_SEND_DONE_WITH_FILE_MESSAGES)) { + list_for_each_entry(lte, stream_list, write_streams_list) + if (lte->unhashed) + lte->back_inode->num_unread_streams = 0; + list_for_each_entry(lte, stream_list, write_streams_list) + if (lte->unhashed) + lte->back_inode->num_unread_streams++; + } + return wim_write_stream_list(wim, stream_list, write_flags,