From: Eric Biggers Date: Thu, 28 Aug 2014 02:53:02 +0000 (-0500) Subject: Merge remote-tracking branch 'origin/master' into wimboot_fast X-Git-Tag: v1.7.2~48^2~1 X-Git-Url: https://wimlib.net/git/?p=wimlib;a=commitdiff_plain;h=28d78c881095a7b50be60ed8d506aa15cceb9cae;hp=-c Merge remote-tracking branch 'origin/master' into wimboot_fast --- 28d78c881095a7b50be60ed8d506aa15cceb9cae diff --combined include/wimlib/apply.h index ed864810,73ddd871..d8e6d936 --- a/include/wimlib/apply.h +++ b/include/wimlib/apply.h @@@ -33,8 -33,6 +33,8 @@@ struct wim_features struct wim_lookup_table_entry; struct read_stream_list_callbacks; +struct apply_operations; +struct wim_dentry; struct apply_ctx { /* The WIMStruct from which files are being extracted from the currently @@@ -64,7 -62,6 +64,7 @@@ struct wim_features supported_features; /* The members below should not be used outside of extract.c */ + const struct apply_operations *apply_ops; u64 next_progress; unsigned long invalid_sequence; unsigned long num_streams_remaining; @@@ -91,21 -88,22 +91,22 @@@ extract_progress(struct apply_ctx *ctx extern int do_file_extract_progress(struct apply_ctx *ctx, enum wimlib_progress_msg msg); + #define COUNT_PER_FILE_PROGRESS 256 + static inline int maybe_do_file_progress(struct apply_ctx *ctx, enum wimlib_progress_msg msg) { + ctx->progress.extract.current_file_count++; if (unlikely(!--ctx->count_until_file_progress)) return do_file_extract_progress(ctx, msg); return 0; } - /* Call this to reset the counter for report_file_created() and - * report_file_metadata_applied(). */ - static inline void - reset_file_progress(struct apply_ctx *ctx) - { - ctx->count_until_file_progress = 1; - } + extern int + start_file_structure_phase(struct apply_ctx *ctx, uint64_t end_file_count); + + extern int + start_file_metadata_phase(struct apply_ctx *ctx, uint64_t end_file_count); /* Report that a file was created, prior to stream extraction. */ static inline int @@@ -121,6 -119,12 +122,12 @@@ report_file_metadata_applied(struct app return maybe_do_file_progress(ctx, WIMLIB_PROGRESS_MSG_EXTRACT_METADATA); } + extern int + end_file_structure_phase(struct apply_ctx *ctx); + + extern int + end_file_metadata_phase(struct apply_ctx *ctx); + /* Returns any of the aliases of an inode that are being extracted. */ #define inode_first_extraction_dentry(inode) \ list_first_entry(&(inode)->i_extraction_aliases, \ @@@ -130,105 -134,14 +137,105 @@@ extern in extract_stream_list(struct apply_ctx *ctx, const struct read_stream_list_callbacks *cbs); +/* + * Represents an extraction backend. + */ struct apply_operations { + + /* Name of the extraction backend. */ const char *name; + + /* + * Query the features supported by the extraction backend. + * + * @target + * The target string that was provided by the user. (Often a + * directory, but extraction backends are free to interpret this + * differently.) + * + * @supported_features + * A structure, each of whose members represents a feature that may + * be supported by the extraction backend. For each feature that + * the extraction backend supports, this routine must set the + * corresponding member to a nonzero value. + * + * Return 0 if successful; otherwise a positive wimlib error code. + */ int (*get_supported_features)(const tchar *target, struct wim_features *supported_features); + /* + * Main extraction routine. + * + * The extraction backend is provided a list of dentries that have been + * prepared for extraction. It is free to extract them in any way that + * it chooses. Ideally, it should choose a method that maximizes + * performance. + * + * The target string will be provided in ctx->common.target. This might + * be a directory, although extraction backends are free to interpret it + * as they wish. TODO: in some cases, the common extraction code also + * interprets the target string. This should be completely isolated to + * extraction backends. + * + * The extraction flags will be provided in ctx->common.extract_flags. + * Extraction backends should examine them and implement the behaviors + * for as many flags as possible. Some flags are already handled by the + * common extraction code. TODO: this needs to be better formalized. + * + * @dentry_list, the list of dentries, will be ordered such that the + * ancestor of any dentry always precedes any descendents. Unless + * @single_tree_only is set, it's possible that the dentries consist of + * multiple disconnected trees. + * + * 'd_extraction_name' and 'd_extraction_name_nchars' of each dentry + * will be set to indicate the actual name with which the dentry should + * be extracted. This may or may not be the same as 'file_name'. + * TODO: really, the extraction backends should be responsible for + * generating 'd_extraction_name'. + * + * Each dentry will refer to a valid inode in 'd_inode'. + * 'd_inode->i_extraction_aliases' will contain a list of just the + * dentries of that inode being extracted. This will be a (possibly + * nonproper) subset of the 'd_inode->i_dentry' list. + * + * The streams required to be extracted will already be prepared in + * 'apply_ctx'. The extraction backend should call + * extract_stream_list() to extract them. + * + * The will_extract_dentry() utility function, given an arbitrary dentry + * in the WIM image (which may not be in the extraction list), can be + * used to determine if that dentry is in the extraction list. + * + * Return 0 if successful; otherwise a positive wimlib error code. + */ int (*extract)(struct list_head *dentry_list, struct apply_ctx *ctx); + /* + * Query whether the unnamed data stream of the specified file will be + * extracted as "externally backed". If so, the extraction backend is + * assumed to handle this separately, and the common extraction code + * will not register a usage of that stream. + * + * This routine is optional. + * + * Return: + * < 0 if the file will *not* be externally backed. + * = 0 if the file will be externally backed. + * > 0 (wimlib error code) if another error occurred. + */ + int (*will_externally_back)(struct wim_dentry *dentry, struct apply_ctx *ctx); + + /* + * Size of the backend-specific extraction context. It must contain + * 'struct apply_ctx' as its first member. + */ size_t context_size; + + /* + * Set this if the extraction backend only supports extracting dentries + * that form a single tree, not multiple trees. + */ bool single_tree_only; }; diff --combined src/extract.c index caf56b46,8e35994a..0029aa88 --- a/src/extract.c +++ b/src/extract.c @@@ -95,10 -95,49 +95,49 @@@ int do_file_extract_progress(struct apply_ctx *ctx, enum wimlib_progress_msg msg) { - ctx->count_until_file_progress = 512; /* Arbitrary value to limit calls */ + ctx->count_until_file_progress = 500; /* Arbitrary value to limit calls */ return extract_progress(ctx, msg); } + static int + start_file_phase(struct apply_ctx *ctx, uint64_t end_file_count, enum wimlib_progress_msg msg) + { + ctx->progress.extract.current_file_count = 0; + ctx->progress.extract.end_file_count = end_file_count; + return do_file_extract_progress(ctx, msg); + } + + int + start_file_structure_phase(struct apply_ctx *ctx, uint64_t end_file_count) + { + return start_file_phase(ctx, end_file_count, WIMLIB_PROGRESS_MSG_EXTRACT_FILE_STRUCTURE); + } + + int + start_file_metadata_phase(struct apply_ctx *ctx, uint64_t end_file_count) + { + return start_file_phase(ctx, end_file_count, WIMLIB_PROGRESS_MSG_EXTRACT_METADATA); + } + + static int + end_file_phase(struct apply_ctx *ctx, enum wimlib_progress_msg msg) + { + ctx->progress.extract.current_file_count = ctx->progress.extract.end_file_count; + return do_file_extract_progress(ctx, msg); + } + + int + end_file_structure_phase(struct apply_ctx *ctx) + { + return end_file_phase(ctx, WIMLIB_PROGRESS_MSG_EXTRACT_FILE_STRUCTURE); + } + + int + end_file_metadata_phase(struct apply_ctx *ctx) + { + return end_file_phase(ctx, WIMLIB_PROGRESS_MSG_EXTRACT_METADATA); + } + /* Check whether the extraction of a dentry should be skipped completely. */ static bool dentry_is_supported(struct wim_dentry *dentry, @@@ -1041,43 -1080,23 +1080,43 @@@ ref_stream(struct wim_lookup_table_entr } static int -dentry_ref_streams(struct wim_dentry *dentry, struct apply_ctx *ctx) +ref_unnamed_stream(struct wim_dentry *dentry, struct apply_ctx *ctx) { struct wim_inode *inode = dentry->d_inode; int ret; + u16 stream_idx; + struct wim_lookup_table_entry *stream; - /* The unnamed data stream will always be extracted, except in an - * unlikely case. */ - if (!inode_is_encrypted_directory(inode)) { - u16 stream_idx; - struct wim_lookup_table_entry *stream; + if (unlikely(inode_is_encrypted_directory(inode))) + return 0; - stream = inode_unnamed_stream_resolved(inode, &stream_idx); - ret = ref_stream(stream, stream_idx, dentry, ctx); - if (ret) - return ret; + if (unlikely(ctx->apply_ops->will_externally_back)) { + ret = (*ctx->apply_ops->will_externally_back)(dentry, ctx); + if (ret >= 0) { + if (ret) /* Error */ + return ret; + /* Will externally back */ + return 0; + } + /* Won't externally back */ } + stream = inode_unnamed_stream_resolved(inode, &stream_idx); + return ref_stream(stream, stream_idx, dentry, ctx); +} + +static int +dentry_ref_streams(struct wim_dentry *dentry, struct apply_ctx *ctx) +{ + struct wim_inode *inode = dentry->d_inode; + int ret; + + /* The unnamed data stream will almost always be extracted, but there + * exist cases in which it won't be. */ + ret = ref_unnamed_stream(dentry, ctx); + if (ret) + return ret; + /* Named data streams will be extracted only if supported in the current * extraction mode and volume, and to avoid complications, if not doing * a linked extraction. */ @@@ -1414,7 -1433,6 +1453,7 @@@ extract_trees(WIMStruct *wim, struct wi } INIT_LIST_HEAD(&ctx->stream_list); filedes_invalidate(&ctx->tmpfile_fd); + ctx->apply_ops = ops; ret = (*ops->get_supported_features)(target, &ctx->supported_features); if (ret) diff --combined src/win32_apply.c index 21ddfd2f,133eeff2..c1fbc491 --- a/src/win32_apply.c +++ b/src/win32_apply.c @@@ -53,7 -53,6 +53,7 @@@ struct win32_apply_ctx void *mem_prepopulate_pats; u8 wim_lookup_table_hash[SHA1_HASH_SIZE]; bool wof_running; + bool tried_to_load_prepopulate_list; } wimboot; /* Open handle to the target directory */ @@@ -257,8 -256,6 +257,8 @@@ load_prepopulate_pats(struct win32_appl void *mem; struct text_file_section sec; + ctx->wimboot.tried_to_load_prepopulate_list = true; + dentry = get_dentry(ctx->common.wim, path, WIMLIB_CASE_INSENSITIVE); if (!dentry || (dentry->d_inode->i_attributes & (FILE_ATTRIBUTE_DIRECTORY | @@@ -315,95 -312,6 +315,95 @@@ in_prepopulate_list(struct wim_dentry * wcslen(dentry->_full_path), pats); } +static const wchar_t * +current_path(struct win32_apply_ctx *ctx); + +static void +build_extraction_path(const struct wim_dentry *dentry, + struct win32_apply_ctx *ctx); + +#define WIM_BACKING_NOT_ENABLED -1 +#define WIM_BACKING_NOT_POSSIBLE -2 +#define WIM_BACKING_EXCLUDED -3 + +/* + * Determines if the unnamed data stream of a file will be created as an + * external backing, as opposed to a standard extraction. + */ +static int +win32_will_externally_back(struct wim_dentry *dentry, struct apply_ctx *_ctx) +{ + struct win32_apply_ctx *ctx = (struct win32_apply_ctx *)_ctx; + struct wim_lookup_table_entry *stream; + int ret; + + if (!(ctx->common.extract_flags & WIMLIB_EXTRACT_FLAG_WIMBOOT)) + return WIM_BACKING_NOT_ENABLED; + + if (!ctx->wimboot.tried_to_load_prepopulate_list) { + ret = load_prepopulate_pats(ctx); + if (ret == WIMLIB_ERR_NOMEM) + return ret; + } + + if (dentry->d_inode->i_attributes & (FILE_ATTRIBUTE_DIRECTORY | + FILE_ATTRIBUTE_REPARSE_POINT | + FILE_ATTRIBUTE_ENCRYPTED)) + return WIM_BACKING_NOT_POSSIBLE; + + stream = inode_unnamed_lte_resolved(dentry->d_inode); + + if (!stream || + stream->resource_location != RESOURCE_IN_WIM || + stream->rspec->wim != ctx->common.wim || + stream->size != stream->rspec->uncompressed_size) + return WIM_BACKING_NOT_POSSIBLE; + + ret = calculate_dentry_full_path(dentry); + if (ret) + return ret; + + if (in_prepopulate_list(dentry, ctx)) + return WIM_BACKING_EXCLUDED; + + return 0; +} + +static int +set_external_backing(HANDLE h, struct wim_dentry *dentry, struct win32_apply_ctx *ctx) +{ + int ret; + + ret = win32_will_externally_back(dentry, &ctx->common); + if (ret > 0) /* Error. */ + return ret; + + if (ret < 0 && ret != WIM_BACKING_EXCLUDED) + return 0; /* Not externally backing, other than due to exclusion. */ + + build_extraction_path(dentry, ctx); + + if (ret == WIM_BACKING_EXCLUDED) { + /* Not externally backing due to exclusion. */ + union wimlib_progress_info info; + + info.wimboot_exclude.path_in_wim = dentry->_full_path; + info.wimboot_exclude.extraction_path = current_path(ctx); + + return call_progress(ctx->common.progfunc, + WIMLIB_PROGRESS_MSG_WIMBOOT_EXCLUDE, + &info, ctx->common.progctx); + } else { + /* Externally backing. */ + return wimboot_set_pointer(h, + current_path(ctx), + inode_unnamed_lte_resolved(dentry->d_inode), + ctx->wimboot.data_source_id, + ctx->wimboot.wim_lookup_table_hash, + ctx->wimboot.wof_running); + } +} + /* Calculates the SHA-1 message digest of the WIM's lookup table. */ static int hash_lookup_table(WIMStruct *wim, u8 hash[SHA1_HASH_SIZE]) @@@ -420,11 -328,9 +420,11 @@@ start_wimboot_extraction(struct win32_a int ret; WIMStruct *wim = ctx->common.wim; - ret = load_prepopulate_pats(ctx); - if (ret == WIMLIB_ERR_NOMEM) - return ret; + if (!ctx->wimboot.tried_to_load_prepopulate_list) { + ret = load_prepopulate_pats(ctx); + if (ret == WIMLIB_ERR_NOMEM) + return ret; + } if (!wim_info_get_wimboot(wim->wim_info, wim->current_image)) WARNING("Image is not marked as WIMBoot compatible!"); @@@ -744,11 -650,11 +744,11 @@@ prepare_target(struct list_head *dentry /* When creating an inode that will have a short (DOS) name, we create it using * the long name associated with the short name. This ensures that the short * name gets associated with the correct long name. */ -static const struct wim_dentry * +static struct wim_dentry * first_extraction_alias(const struct wim_inode *inode) { - const struct list_head *next = inode->i_extraction_aliases.next; - const struct wim_dentry *dentry; + struct list_head *next = inode->i_extraction_aliases.next; + struct wim_dentry *dentry; do { dentry = list_entry(next, struct wim_dentry, @@@ -1198,16 -1104,15 +1198,15 @@@ create_directories(struct list_head *de /* If the root dentry is being extracted, it was already done so * in prepare_target(). */ - if (dentry_is_root(dentry)) - continue; - - ret = create_directory(dentry, ctx); - if (ret) - return ret; + if (!dentry_is_root(dentry)) { + ret = create_directory(dentry, ctx); + if (ret) + return ret; - ret = create_any_empty_ads(dentry, ctx); - if (ret) - return ret; + ret = create_any_empty_ads(dentry, ctx); + if (ret) + return ret; + } ret = report_file_created(&ctx->common); if (ret) @@@ -1417,7 -1322,7 +1416,7 @@@ create_links(HANDLE h, const struct wim static int create_nondirectory(const struct wim_inode *inode, struct win32_apply_ctx *ctx) { - const struct wim_dentry *first_dentry; + struct wim_dentry *first_dentry; HANDLE h; int ret; @@@ -1436,10 -1341,6 +1435,10 @@@ if (!ret) ret = create_links(h, first_dentry, ctx); + /* "WIMBoot" extraction: set external backing by the WIM file if needed. */ + if (!ret && unlikely(ctx->common.extract_flags & WIMLIB_EXTRACT_FLAG_WIMBOOT)) + ret = set_external_backing(h, first_dentry, ctx); + (*func_NtClose)(h); return ret; } @@@ -1458,11 -1359,11 +1457,11 @@@ create_nondirectories(struct list_head if (inode->i_attributes & FILE_ATTRIBUTE_DIRECTORY) continue; /* Call create_nondirectory() only once per inode */ - if (dentry != inode_first_extraction_dentry(inode)) - continue; - ret = create_nondirectory(inode, ctx); - if (ret) - return ret; + if (dentry == inode_first_extraction_dentry(inode)) { + ret = create_nondirectory(inode, ctx); + if (ret) + return ret; + } ret = report_file_created(&ctx->common); if (ret) return ret; @@@ -1559,6 -1460,42 +1558,6 @@@ begin_extract_stream_instance(const str return prepare_data_buffer(ctx, stream->size); } - /* Extracting unnamed data stream in WIMBoot mode? */ - if (unlikely(ctx->common.extract_flags & WIMLIB_EXTRACT_FLAG_WIMBOOT) - && (stream_name_nchars == 0) - && (stream->resource_location == RESOURCE_IN_WIM) - && (stream->rspec->wim == ctx->common.wim) - && (stream->size == stream->rspec->uncompressed_size)) - { - int ret = calculate_dentry_full_path(dentry); - if (ret) - return ret; - if (in_prepopulate_list(dentry, ctx)) { - union wimlib_progress_info info; - - info.wimboot_exclude.path_in_wim = dentry->_full_path; - info.wimboot_exclude.extraction_path = current_path(ctx); - - ret = call_progress(ctx->common.progfunc, - WIMLIB_PROGRESS_MSG_WIMBOOT_EXCLUDE, - &info, ctx->common.progctx); - FREE(dentry->_full_path); - dentry->_full_path = NULL; - if (ret) - return ret; - /* Go on and open the file for normal extraction. */ - } else { - FREE(dentry->_full_path); - dentry->_full_path = NULL; - return wimboot_set_pointer(&ctx->attr, - current_path(ctx), - stream, - ctx->wimboot.data_source_id, - ctx->wimboot.wim_lookup_table_hash, - ctx->wimboot.wof_running); - } - } - if (ctx->num_open_handles == MAX_OPEN_STREAMS) { /* XXX: Fix this. But because of the checks in * extract_stream_list(), this can now only happen on a @@@ -2233,12 -2170,25 +2232,25 @@@ do_warnings(const struct win32_apply_ct } } + static uint64_t + count_dentries(const struct list_head *dentry_list) + { + const struct list_head *cur; + uint64_t count = 0; + + list_for_each(cur, dentry_list) + count++; + + return count; + } + /* Extract files from a WIM image to a directory on Windows */ static int win32_extract(struct list_head *dentry_list, struct apply_ctx *_ctx) { int ret; struct win32_apply_ctx *ctx = (struct win32_apply_ctx *)_ctx; + uint64_t dentry_count; ret = prepare_target(dentry_list, ctx); if (ret) @@@ -2250,7 -2200,11 +2262,11 @@@ goto out; } - reset_file_progress(&ctx->common); + dentry_count = count_dentries(dentry_list); + + ret = start_file_structure_phase(&ctx->common, dentry_count); + if (ret) + goto out; ret = create_directories(dentry_list, ctx); if (ret) @@@ -2260,6 -2214,10 +2276,10 @@@ if (ret) goto out; + ret = end_file_structure_phase(&ctx->common); + if (ret) + goto out; + struct read_stream_list_callbacks cbs = { .begin_stream = begin_extract_stream, .begin_stream_ctx = ctx, @@@ -2272,12 -2230,18 +2292,18 @@@ if (ret) goto out; - reset_file_progress(&ctx->common); + ret = start_file_metadata_phase(&ctx->common, dentry_count); + if (ret) + goto out; ret = apply_metadata(dentry_list, ctx); if (ret) goto out; + ret = end_file_metadata_phase(&ctx->common); + if (ret) + goto out; + if (unlikely(ctx->common.extract_flags & WIMLIB_EXTRACT_FLAG_WIMBOOT)) { ret = end_wimboot_extraction(ctx); if (ret) @@@ -2305,7 -2269,6 +2331,7 @@@ const struct apply_operations win32_app .name = "Windows", .get_supported_features = win32_get_supported_features, .extract = win32_extract, + .will_externally_back = win32_will_externally_back, .context_size = sizeof(struct win32_apply_ctx), };