From be5acf31aa8292dcd4a2829492faefb0b200d28f Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Thu, 30 Apr 2015 21:26:32 -0500 Subject: [PATCH] Various cleanups --- include/wimlib.h | 12 +- include/wimlib/blob_table.h | 4 +- include/wimlib/inode.h | 3 - include/wimlib/wof.h | 11 +- programs/imagex.c | 10 +- src/blob_table.c | 2 - src/dentry.c | 31 ++--- src/error.c | 2 +- src/export_image.c | 7 +- src/extract.c | 4 +- src/inode.c | 15 --- src/mount_image.c | 5 + src/ntfs-3g_capture.c | 231 +++++++++++++++++++----------------- src/reparse.c | 33 +++--- src/resource.c | 8 +- src/unix_capture.c | 30 +++-- src/verify.c | 8 +- src/wim.c | 20 ++-- src/wimboot.c | 4 +- src/win32_capture.c | 22 ++-- src/write.c | 100 +++++++++------- 21 files changed, 286 insertions(+), 276 deletions(-) diff --git a/include/wimlib.h b/include/wimlib.h index e702fc3b..712202d4 100644 --- a/include/wimlib.h +++ b/include/wimlib.h @@ -1002,7 +1002,7 @@ union wimlib_progress_info { * integrity checks. */ uint64_t total_bytes; - /** The number of bytes that have been processed so far. This + /** The number of bytes that have been checksummed so far. This * starts at 0 and ends at @p total_bytes. */ uint64_t completed_bytes; @@ -1010,8 +1010,8 @@ union wimlib_progress_info { * integrity-checked region is divided into. */ uint32_t total_chunks; - /** The number of chunks that have been processed so far. This - * starts at 0 and ends at @p total_chunks. */ + /** The number of chunks that have been checksummed so far. + * This starts at 0 and ends at @p total_chunks. */ uint32_t completed_chunks; /** The size of each individually checksummed "chunk" in the @@ -4073,9 +4073,9 @@ wimlib_set_print_errors(bool show_messages); * @param wim * Pointer to the ::WIMStruct for a WIM. * @param info - * A struct ::wimlib_wim_info that contains the information to set. Only - * the information explicitly specified in the @p which flags need be - * valid. + * Pointer to a ::wimlib_wim_info structure that contains the information + * to set. Only the information explicitly specified in the @p which flags + * need be valid. * @param which * Flags that specify which information to set. This is a bitwise OR of * ::WIMLIB_CHANGE_READONLY_FLAG, ::WIMLIB_CHANGE_GUID, diff --git a/include/wimlib/blob_table.h b/include/wimlib/blob_table.h index 1adec4ed..672dedf5 100644 --- a/include/wimlib/blob_table.h +++ b/include/wimlib/blob_table.h @@ -12,7 +12,7 @@ enum blob_location { /* The blob's data does not exist. This is a temporary state only. */ BLOB_NONEXISTENT = 0, - /* The blob's data is located in a WIM resource identified by the + /* The blob's data is available in the WIM resource identified by the * `struct wim_resource_descriptor' pointed to by @rdesc. * @offset_in_res identifies the offset at which this particular blob * begins in the uncompressed data of the resource. */ @@ -122,7 +122,7 @@ struct blob_descriptor { #ifdef WITH_FUSE /* Number of open file descriptors to this blob during a FUSE mount of - * the containing image. */ + * a WIM image. */ u16 num_opened_fds; #endif diff --git a/include/wimlib/inode.h b/include/wimlib/inode.h index a3450844..08ce3e91 100644 --- a/include/wimlib/inode.h +++ b/include/wimlib/inode.h @@ -410,9 +410,6 @@ extern int inode_resolve_streams(struct wim_inode *inode, struct blob_table *table, bool force); -extern void -inode_unresolve_streams(struct wim_inode *inode); - extern int blob_not_found_error(const struct wim_inode *inode, const u8 *hash); diff --git a/include/wimlib/wof.h b/include/wimlib/wof.h index fae0801d..2c690c48 100644 --- a/include/wimlib/wof.h +++ b/include/wimlib/wof.h @@ -70,12 +70,13 @@ struct wim_provider_rpdata { u8 blob_table_hash[20]; /* Uncompressed size of the file's unnamed data stream, in bytes. */ - le64 unnamed_data_stream_uncompressed_size; + le64 unnamed_data_stream_size; - /* Compressed size of the file's unnamed data stream, in bytes. If the - * stream is stored uncompressed, set this the same as the uncompressed - * size. */ - le64 unnamed_data_stream_compressed_size; + /* Size of the file's unnamed data stream as stored in the WIM file. + * If this is the same as unnamed_data_stream_size, then the stream is + * uncompressed. If this is the *not* the same as + * unnamed_data_stream_size, then the stream is compressed. */ + le64 unnamed_data_stream_size_in_wim; /* Byte offset of the file's unnamed data stream in the WIM. */ le64 unnamed_data_stream_offset_in_wim; diff --git a/programs/imagex.c b/programs/imagex.c index b3080289..49b5c0f1 100644 --- a/programs/imagex.c +++ b/programs/imagex.c @@ -141,6 +141,7 @@ static FILE *imagex_info_file; enum { IMAGEX_ALLOW_OTHER_OPTION, + IMAGEX_BLOBS_OPTION, IMAGEX_BOOT_OPTION, IMAGEX_CHECK_OPTION, IMAGEX_CHUNK_SIZE_OPTION, @@ -160,7 +161,6 @@ enum { IMAGEX_HEADER_OPTION, IMAGEX_INCLUDE_INVALID_NAMES_OPTION, IMAGEX_LAZY_OPTION, - IMAGEX_BLOBS_OPTION, IMAGEX_METADATA_OPTION, IMAGEX_NEW_IMAGE_OPTION, IMAGEX_NOCHECK_OPTION, @@ -168,8 +168,8 @@ enum { IMAGEX_NOT_PIPABLE_OPTION, IMAGEX_NO_ACLS_OPTION, IMAGEX_NO_ATTRIBUTES_OPTION, - IMAGEX_NO_REPLACE_OPTION, IMAGEX_NO_GLOBS_OPTION, + IMAGEX_NO_REPLACE_OPTION, IMAGEX_NO_SOLID_SORT_OPTION, IMAGEX_NULLGLOB_OPTION, IMAGEX_ONE_FILE_ONLY_OPTION, @@ -183,9 +183,9 @@ enum { IMAGEX_RESUME_OPTION, IMAGEX_RPFIX_OPTION, IMAGEX_SOFT_OPTION, - IMAGEX_SOLID_OPTION, IMAGEX_SOLID_CHUNK_SIZE_OPTION, IMAGEX_SOLID_COMPRESS_OPTION, + IMAGEX_SOLID_OPTION, IMAGEX_SOURCE_LIST_OPTION, IMAGEX_STAGING_DIR_OPTION, IMAGEX_STREAMS_INTERFACE_OPTION, @@ -195,8 +195,8 @@ enum { IMAGEX_UNIX_DATA_OPTION, IMAGEX_UPDATE_OF_OPTION, IMAGEX_VERBOSE_OPTION, - IMAGEX_WIMBOOT_OPTION, IMAGEX_WIMBOOT_CONFIG_OPTION, + IMAGEX_WIMBOOT_OPTION, IMAGEX_XML_OPTION, }; @@ -1278,7 +1278,7 @@ imagex_progress_func(enum wimlib_progress_msg msg, percent_done = TO_PERCENT(info->verify_streams.completed_bytes, info->verify_streams.total_bytes); unit_shift = get_unit(info->verify_streams.total_bytes, &unit_name); - imagex_printf(T("\rVerifying streams: " + imagex_printf(T("\rVerifying file data: " "%"PRIu64" %"TS" of %"PRIu64" %"TS" (%u%%) done"), info->verify_streams.completed_bytes >> unit_shift, unit_name, diff --git a/src/blob_table.c b/src/blob_table.c index 4714b91c..891c0a5a 100644 --- a/src/blob_table.c +++ b/src/blob_table.c @@ -156,8 +156,6 @@ clone_blob_descriptor(const struct blob_descriptor *old) } break; #endif - default: - break; } return new; diff --git a/src/dentry.c b/src/dentry.c index 378e16fe..6a759388 100644 --- a/src/dentry.c +++ b/src/dentry.c @@ -1819,8 +1819,7 @@ write_dentry(const struct wim_dentry * restrict dentry, u8 * restrict p) const struct wim_inode_stream *efs_strm; const u8 *efs_hash; - efs_strm = inode_get_stream(inode, STREAM_TYPE_EFSRPC_RAW_DATA, - NO_STREAM_NAME); + efs_strm = inode_get_unnamed_stream(inode, STREAM_TYPE_EFSRPC_RAW_DATA); efs_hash = efs_strm ? stream_hash(efs_strm) : zero_hash; copy_hash(disk_dentry->default_hash, efs_hash); disk_dentry->num_extra_streams = cpu_to_le16(0); @@ -1836,7 +1835,6 @@ write_dentry(const struct wim_dentry * restrict dentry, u8 * restrict p) */ bool have_named_data_stream = false; bool have_reparse_point_stream = false; - u16 num_extra_streams = 0; const u8 *unnamed_data_stream_hash = zero_hash; const u8 *reparse_point_hash; for (unsigned i = 0; i < inode->i_num_streams; i++) { @@ -1852,7 +1850,9 @@ write_dentry(const struct wim_dentry * restrict dentry, u8 * restrict p) } } - if (have_reparse_point_stream || have_named_data_stream) { + if (unlikely(have_reparse_point_stream || have_named_data_stream)) { + + unsigned num_extra_streams = 0; copy_hash(disk_dentry->default_hash, zero_hash); @@ -1865,19 +1865,22 @@ write_dentry(const struct wim_dentry * restrict dentry, u8 * restrict p) p = write_extra_stream_entry(p, NO_STREAM_NAME, unnamed_data_stream_hash); num_extra_streams++; - } else { - copy_hash(disk_dentry->default_hash, unnamed_data_stream_hash); - } - for (unsigned i = 0; i < inode->i_num_streams; i++) { - const struct wim_inode_stream *strm = &inode->i_streams[i]; - if (stream_is_named_data_stream(strm)) { - p = write_extra_stream_entry(p, strm->stream_name, - stream_hash(strm)); - num_extra_streams++; + for (unsigned i = 0; i < inode->i_num_streams; i++) { + const struct wim_inode_stream *strm = &inode->i_streams[i]; + if (stream_is_named_data_stream(strm)) { + p = write_extra_stream_entry(p, strm->stream_name, + stream_hash(strm)); + num_extra_streams++; + } } + wimlib_assert(num_extra_streams <= 0xFFFF); + + disk_dentry->num_extra_streams = cpu_to_le16(num_extra_streams); + } else { + copy_hash(disk_dentry->default_hash, unnamed_data_stream_hash); + disk_dentry->num_extra_streams = cpu_to_le16(0); } - disk_dentry->num_extra_streams = cpu_to_le16(num_extra_streams); } return p; diff --git a/src/error.c b/src/error.c index 30087aef..06f09f5d 100644 --- a/src/error.c +++ b/src/error.c @@ -250,7 +250,7 @@ static const tchar * const error_strings[] = { [WIMLIB_ERR_INVALID_REPARSE_DATA] = T("The reparse data of a reparse point was invalid"), [WIMLIB_ERR_INVALID_RESOURCE_HASH] - = T("The SHA1 message digest of a WIM resource did not match the expected value"), + = T("The SHA-1 message digest of a WIM resource did not match the expected value"), [WIMLIB_ERR_INVALID_UTF8_STRING] = T("A string provided as input by the user was not a valid UTF-8 string"), [WIMLIB_ERR_INVALID_UTF16_STRING] diff --git a/src/export_image.c b/src/export_image.c index 8d733284..e88107a9 100644 --- a/src/export_image.c +++ b/src/export_image.c @@ -59,13 +59,11 @@ inode_export_blobs(struct wim_inode *inode, struct blob_table *src_blob_table, const u8 *hash; struct blob_descriptor *src_blob, *dest_blob; - inode_unresolve_streams(inode); - for (i = 0; i < inode->i_num_streams; i++) { /* Retrieve SHA-1 message digest of blob to export. */ hash = stream_hash(&inode->i_streams[i]); - if (is_zero_hash(hash)) /* Empty blob? */ + if (is_zero_hash(hash)) /* Empty stream? */ continue; /* Search for the blob (via SHA-1 message digest) in the @@ -75,7 +73,8 @@ inode_export_blobs(struct wim_inode *inode, struct blob_table *src_blob_table, /* Blob not yet present in destination WIM. Search for * it in the source WIM, then export it into the * destination WIM. */ - src_blob = lookup_blob(src_blob_table, hash); + src_blob = stream_blob(&inode->i_streams[i], + src_blob_table); if (!src_blob) return blob_not_found_error(inode, hash); diff --git a/src/extract.c b/src/extract.c index 5945a4d8..9359c53f 100644 --- a/src/extract.c +++ b/src/extract.c @@ -506,8 +506,8 @@ end_extract_blob_wrapper(struct blob_descriptor *blob, int status, void *_ctx) * MAX_OPEN_FILES locations, as measured by the 'out_refcnt' of each blob. * Therefore, the apply_operations implementation need not worry about running * out of file descriptors, unless it might open more than one file descriptor - * per nominal destination (e.g. Win32 currently might because the destination - * file system might not support hard links). + * per 'blob_extraction_target' (e.g. Win32 currently might because the + * destination file system might not support hard links). */ int extract_blob_list(struct apply_ctx *ctx, diff --git a/src/inode.c b/src/inode.c index 4b1b1d3e..4c380bfc 100644 --- a/src/inode.c +++ b/src/inode.c @@ -480,21 +480,6 @@ inode_resolve_streams(struct wim_inode *inode, struct blob_table *table, return 0; } -/* Undo the effects of inode_resolve_streams(). */ -void -inode_unresolve_streams(struct wim_inode *inode) -{ - for (unsigned i = 0; i < inode->i_num_streams; i++) { - - if (!inode->i_streams[i].stream_resolved) - continue; - - copy_hash(inode->i_streams[i]._stream_hash, - stream_hash(&inode->i_streams[i])); - inode->i_streams[i].stream_resolved = 0; - } -} - int blob_not_found_error(const struct wim_inode *inode, const u8 *hash) { diff --git a/src/mount_image.c b/src/mount_image.c index 87620def..3e9566b5 100644 --- a/src/mount_image.c +++ b/src/mount_image.c @@ -2143,6 +2143,11 @@ wimlib_mount_image(WIMStruct *wim, int image, const char *dir, return ret; } + if (wim_has_solid_resources(wim)) { + WARNING("Mounting a WIM file containing solid-compressed data; " + "file access may be slow."); + } + /* If the user did not specify an interface for accessing named * data streams, use the default (extended attributes). */ if (!(mount_flags & (WIMLIB_MOUNT_FLAG_STREAM_INTERFACE_NONE | diff --git a/src/ntfs-3g_capture.c b/src/ntfs-3g_capture.c index 06d46913..dd6cae76 100644 --- a/src/ntfs-3g_capture.c +++ b/src/ntfs-3g_capture.c @@ -48,10 +48,11 @@ #include "wimlib/reparse.h" #include "wimlib/security.h" -static inline ntfschar * -attr_record_name(ATTR_RECORD *ar) +static inline const ntfschar * +attr_record_name(const ATTR_RECORD *record) { - return (ntfschar*)((u8*)ar + le16_to_cpu(ar->name_offset)); + return (const ntfschar *) + ((const u8 *)record + le16_to_cpu(record->name_offset)); } static ntfs_attr * @@ -165,9 +166,103 @@ attr_type_to_wimlib_stream_type(ATTR_TYPES type) } } -/* Load attributes of the specified type from a file in the NTFS volume */ +/* Save information about an NTFS attribute (stream) to a WIM inode. */ static int -load_ntfs_attrs_with_type(struct wim_inode *inode, +scan_ntfs_attr(struct wim_inode *inode, + ntfs_inode *ni, + const char *path, + size_t path_len, + struct list_head *unhashed_blobs, + ntfs_volume *vol, + ATTR_TYPES type, + const ATTR_RECORD *record) +{ + const u64 data_size = ntfs_get_attribute_value_length(record); + const size_t name_nchars = record->name_length; + struct blob_descriptor *blob = NULL; + utf16lechar *stream_name = NULL; + struct wim_inode_stream *strm; + int ret; + + if (unlikely(name_nchars)) { + /* Named stream */ + stream_name = utf16le_dupz(attr_record_name(record), + name_nchars * sizeof(ntfschar)); + if (!stream_name) { + ret = WIMLIB_ERR_NOMEM; + goto out_cleanup; + } + } + + /* If the stream is non-empty, set up a blob descriptor for it. */ + if (data_size != 0) { + blob = new_blob_descriptor(); + if (unlikely(!blob)) { + ret = WIMLIB_ERR_NOMEM; + goto out_cleanup; + } + + blob->ntfs_loc = CALLOC(1, sizeof(struct ntfs_location)); + if (unlikely(!blob->ntfs_loc)) { + ret = WIMLIB_ERR_NOMEM; + goto out_cleanup; + } + + blob->blob_location = BLOB_IN_NTFS_VOLUME; + blob->size = data_size; + blob->ntfs_loc->ntfs_vol = vol; + blob->ntfs_loc->attr_type = type; + blob->ntfs_loc->path = memdup(path, path_len + 1); + if (unlikely(!blob->ntfs_loc->path)) { + ret = WIMLIB_ERR_NOMEM; + goto out_cleanup; + } + + if (unlikely(name_nchars)) { + blob->ntfs_loc->attr_name = utf16le_dup(stream_name); + if (!blob->ntfs_loc->attr_name) { + ret = WIMLIB_ERR_NOMEM; + goto out_cleanup; + } + blob->ntfs_loc->attr_name_nchars = name_nchars; + } + + if (unlikely(type == AT_REPARSE_POINT)) { + if (blob->size < REPARSE_DATA_OFFSET) { + ERROR("Reparse data of \"%s\" " + "is invalid (only %"PRIu64" bytes)!", + path, data_size); + ret = WIMLIB_ERR_INVALID_REPARSE_DATA; + goto out_cleanup; + } + blob->size -= REPARSE_DATA_OFFSET; + ret = read_reparse_tag(ni, blob->ntfs_loc, + &inode->i_reparse_tag); + if (ret) + goto out_cleanup; + } + } + + strm = inode_add_stream(inode, + attr_type_to_wimlib_stream_type(type), + stream_name ? stream_name : NO_STREAM_NAME, + blob); + if (unlikely(!strm)) { + ret = WIMLIB_ERR_NOMEM; + goto out_cleanup; + } + prepare_unhashed_blob(blob, inode, strm->stream_id, unhashed_blobs); + blob = NULL; + ret = 0; +out_cleanup: + free_blob_descriptor(blob); + FREE(stream_name); + return ret; +} + +/* Scan attributes of the specified type from a file in the NTFS volume */ +static int +scan_ntfs_attrs_with_type(struct wim_inode *inode, ntfs_inode *ni, char *path, size_t path_len, @@ -176,127 +271,39 @@ load_ntfs_attrs_with_type(struct wim_inode *inode, ATTR_TYPES type) { ntfs_attr_search_ctx *actx; - struct ntfs_location *ntfs_loc; int ret; - struct blob_descriptor *blob; - utf16lechar *stream_name; - DEBUG("Loading NTFS attributes from \"%s\"", path); + DEBUG("Scanning NTFS attributes from \"%s\"", path); - /* Get context to search the attributes of the NTFS file. */ actx = ntfs_attr_get_search_ctx(ni, NULL); if (!actx) { - ERROR_WITH_ERRNO("Cannot get NTFS attribute search " + ERROR_WITH_ERRNO("Failed to get NTFS attribute search " "context for \"%s\"", path); return WIMLIB_ERR_NTFS_3G; } - /* Save each attribute */ while (!ntfs_attr_lookup(type, NULL, 0, CASE_SENSITIVE, 0, NULL, 0, actx)) { - u64 data_size = ntfs_get_attribute_value_length(actx->attr); - size_t name_nchars = actx->attr->name_length; - struct wim_inode_stream *strm; - - if (name_nchars) { - stream_name = utf16le_dupz(attr_record_name(actx->attr), - name_nchars * sizeof(ntfschar)); - if (!stream_name) { - ret = WIMLIB_ERR_NOMEM; - goto out_put_actx; - } - } else { - stream_name = NULL; - } - - if (data_size == 0) { - /* Empty attribute. No blob is needed. */ - blob = NULL; - ntfs_loc = NULL; - } else { - ntfs_loc = CALLOC(1, sizeof(*ntfs_loc)); - if (!ntfs_loc) { - ret = WIMLIB_ERR_NOMEM; - goto out_free_stream_name; - } - ntfs_loc->ntfs_vol = vol; - ntfs_loc->attr_type = type; - ntfs_loc->path = memdup(path, path_len + 1); - if (!ntfs_loc->path) { - ret = WIMLIB_ERR_NOMEM; - goto out_free_ntfs_loc; - } - if (name_nchars) { - ntfs_loc->attr_name = utf16le_dup(stream_name); - if (!ntfs_loc->attr_name) { - ret = WIMLIB_ERR_NOMEM; - goto out_free_ntfs_loc; - } - ntfs_loc->attr_name_nchars = name_nchars; - } - - blob = new_blob_descriptor(); - if (!blob) { - ret = WIMLIB_ERR_NOMEM; - goto out_free_ntfs_loc; - } - blob->blob_location = BLOB_IN_NTFS_VOLUME; - blob->ntfs_loc = ntfs_loc; - blob->size = data_size; - ntfs_loc = NULL; - if (type == AT_REPARSE_POINT) { - if (data_size < REPARSE_DATA_OFFSET) { - ERROR("Reparse data of \"%s\" " - "is invalid (only %u bytes)!", - path, (unsigned)data_size); - ret = WIMLIB_ERR_NTFS_3G; - goto out_free_blob; - } - blob->size -= REPARSE_DATA_OFFSET; - ret = read_reparse_tag(ni, blob->ntfs_loc, - &inode->i_reparse_tag); - if (ret) - goto out_free_blob; - } - } - - strm = inode_add_stream(inode, - attr_type_to_wimlib_stream_type(type), - stream_name ? stream_name : NO_STREAM_NAME, - blob); - if (!strm) { - ret = WIMLIB_ERR_NOMEM; - goto out_free_blob; - } - prepare_unhashed_blob(blob, inode, strm->stream_id, unhashed_blobs); - - FREE(stream_name); - stream_name = NULL; + ret = scan_ntfs_attr(inode, + ni, + path, + path_len, + unhashed_blobs, + vol, + type, + actx->attr); + if (ret) + goto out_put_actx; } - if (errno == ENOENT) { - ret = 0; - } else { + if (errno != ENOENT) { ERROR_WITH_ERRNO("Error listing NTFS attributes of \"%s\"", path); ret = WIMLIB_ERR_NTFS_3G; + goto out_put_actx; } - goto out_put_actx; -out_free_blob: - free_blob_descriptor(blob); -out_free_ntfs_loc: - if (ntfs_loc) { - FREE(ntfs_loc->path); - FREE(ntfs_loc->attr_name); - FREE(ntfs_loc); - } -out_free_stream_name: - FREE(stream_name); + ret = 0; out_put_actx: ntfs_attr_put_search_ctx(actx); - if (ret == 0) - DEBUG("Successfully loaded NTFS attributes from \"%s\"", path); - else - ERROR("Failed to load NTFS attributes from \"%s\"", path); return ret; } @@ -575,21 +582,21 @@ build_dentry_tree_ntfs_recursive(struct wim_dentry **root_ret, inode->i_attributes = attributes; if (attributes & FILE_ATTRIBUTE_REPARSE_POINT) { - /* Load the reparse point stream. */ - ret = load_ntfs_attrs_with_type(inode, ni, path, path_len, + /* Scan the reparse point stream. */ + ret = scan_ntfs_attrs_with_type(inode, ni, path, path_len, params->unhashed_blobs, vol, AT_REPARSE_POINT); if (ret) goto out; } - /* Load the data streams. + /* Scan the data streams. * * Note: directories should not have an unnamed data stream, but they * may have named data streams. Nondirectories (including reparse * points) can have an unnamed data stream as well as named data * streams. */ - ret = load_ntfs_attrs_with_type(inode, ni, path, path_len, + ret = scan_ntfs_attrs_with_type(inode, ni, path, path_len, params->unhashed_blobs, vol, AT_DATA); if (ret) goto out; diff --git a/src/reparse.c b/src/reparse.c index 2f1b4169..c27fef6e 100644 --- a/src/reparse.c +++ b/src/reparse.c @@ -144,7 +144,7 @@ make_reparse_buffer(const struct reparse_data * restrict rpdata, data = mempcpy(data, rpdata->print_name, rpdata->print_name_nbytes); *(utf16lechar*)data = cpu_to_le16(0); data += 2; - rpbuf_disk->rpdatalen = cpu_to_le16(data - rpbuf - 8); + rpbuf_disk->rpdatalen = cpu_to_le16(data - rpbuf - REPARSE_DATA_OFFSET); *rpbuflen_ret = data - rpbuf; return 0; } @@ -179,8 +179,7 @@ wim_inode_get_reparse_data(const struct wim_inode * restrict inode, } else { struct wim_inode_stream *strm; - strm = inode_get_stream(inode, STREAM_TYPE_REPARSE_POINT, - NO_STREAM_NAME); + strm = inode_get_unnamed_stream(inode, STREAM_TYPE_REPARSE_POINT); if (strm) blob = stream_blob_resolved(strm); else @@ -215,7 +214,7 @@ wim_inode_get_reparse_data(const struct wim_inode * restrict inode, * XXX this could be one of the unknown fields in the WIM dentry. */ rpbuf_disk->rpreserved = cpu_to_le16(0); - *rpbuflen_ret = rpdatalen + 8; + *rpbuflen_ret = rpdatalen + REPARSE_DATA_OFFSET; return 0; } @@ -423,7 +422,7 @@ wim_inode_set_symlink(struct wim_inode *inode, const char *target, ret = tstr_to_utf16le(target, strlen(target), &name_utf16le, &name_utf16le_nbytes); if (ret) - return ret; + goto out; for (size_t i = 0; i < name_utf16le_nbytes / 2; i++) if (name_utf16le[i] == cpu_to_le16('/')) @@ -499,16 +498,22 @@ wim_inode_set_symlink(struct wim_inode *inode, const char *target, } ret = make_reparse_buffer(&rpdata, (u8*)&rpbuf_disk, &rpbuflen); - if (ret == 0) { - if (!inode_add_stream_with_data(inode, - STREAM_TYPE_REPARSE_POINT, - NO_STREAM_NAME, - (u8*)&rpbuf_disk + 8, - rpbuflen - 8, - blob_table)) - ret = WIMLIB_ERR_NOMEM; - } + if (ret) + goto out_free_name; + + ret = WIMLIB_ERR_NOMEM; + if (!inode_add_stream_with_data(inode, + STREAM_TYPE_REPARSE_POINT, + NO_STREAM_NAME, + (u8*)&rpbuf_disk + REPARSE_DATA_OFFSET, + rpbuflen - REPARSE_DATA_OFFSET, + blob_table)) + goto out_free_name; + + ret = 0; +out_free_name: FREE(name_utf16le); +out: return ret; } diff --git a/src/resource.c b/src/resource.c index a49507f3..2c04882e 100644 --- a/src/resource.c +++ b/src/resource.c @@ -769,8 +769,6 @@ read_file_on_disk_prefix(const struct blob_descriptor *blob, u64 size, int raw_fd; struct filedes fd; - wimlib_assert(size <= blob->size); - DEBUG("Reading %"PRIu64" bytes from \"%"TS"\"", size, blob->file_on_disk); raw_fd = topen(blob->file_on_disk, O_BINARY | O_RDONLY); @@ -793,8 +791,6 @@ read_staging_file_prefix(const struct blob_descriptor *blob, u64 size, struct filedes fd; int ret; - wimlib_assert(size <= blob->size); - DEBUG("Reading %"PRIu64" bytes from staging file \"%s\"", size, blob->staging_file_name); @@ -818,7 +814,6 @@ static int read_buffer_prefix(const struct blob_descriptor *blob, u64 size, consume_data_callback_t cb, void *cb_ctx) { - wimlib_assert(size <= blob->size); return (*cb)(blob->attached_buffer, size, cb_ctx); } @@ -863,6 +858,7 @@ read_blob_prefix(const struct blob_descriptor *blob, u64 size, }; wimlib_assert(blob->blob_location < ARRAY_LEN(handlers) && handlers[blob->blob_location] != NULL); + wimlib_assert(size <= blob->size); return handlers[blob->blob_location](blob, size, cb, cb_ctx); } @@ -1085,7 +1081,7 @@ hasher_end_blob(struct blob_descriptor *blob, int status, void *_ctx) tchar actual_hashstr[SHA1_HASH_SIZE * 2 + 1]; sprint_hash(blob->hash, expected_hashstr); sprint_hash(hash, actual_hashstr); - ERROR("The blob is corrupted!\n" + ERROR("The data is corrupted!\n" " (Expected SHA-1=%"TS",\n" " got SHA-1=%"TS")", expected_hashstr, actual_hashstr); diff --git a/src/unix_capture.c b/src/unix_capture.c index a61d91f2..08fd31ef 100644 --- a/src/unix_capture.c +++ b/src/unix_capture.c @@ -103,35 +103,33 @@ static int unix_scan_regular_file(const char *path, u64 size, struct wim_inode *inode, struct list_head *unhashed_blobs) { - struct blob_descriptor *blob; + struct blob_descriptor *blob = NULL; struct wim_inode_stream *strm; inode->i_attributes = FILE_ATTRIBUTE_NORMAL; if (size) { - char *file_on_disk = STRDUP(path); - if (!file_on_disk) - return WIMLIB_ERR_NOMEM; blob = new_blob_descriptor(); - if (!blob) { - FREE(file_on_disk); - return WIMLIB_ERR_NOMEM; - } - blob->file_on_disk = file_on_disk; - blob->file_inode = inode; + if (unlikely(!blob)) + goto err_nomem; + blob->file_on_disk = STRDUP(path); + if (unlikely(!blob->file_on_disk)) + goto err_nomem; blob->blob_location = BLOB_IN_FILE_ON_DISK; blob->size = size; - } else { - blob = NULL; + blob->file_inode = inode; } strm = inode_add_stream(inode, STREAM_TYPE_DATA, NO_STREAM_NAME, blob); - if (!strm) { - free_blob_descriptor(blob); - return WIMLIB_ERR_NOMEM; - } + if (unlikely(!strm)) + goto err_nomem; + prepare_unhashed_blob(blob, inode, strm->stream_id, unhashed_blobs); return 0; + +err_nomem: + free_blob_descriptor(blob); + return WIMLIB_ERR_NOMEM; } static int diff --git a/src/verify.c b/src/verify.c index 81aa96d0..11b47600 100644 --- a/src/verify.c +++ b/src/verify.c @@ -93,8 +93,8 @@ end_verify_blob(struct blob_descriptor *blob, int status, void *_ctx) } static int -verify_image_blobs_present(struct wim_image_metadata *imd, - struct blob_table *blob_table) +verify_file_data_present(struct wim_image_metadata *imd, + struct blob_table *blob_table) { struct wim_inode *inode; int ret; @@ -150,8 +150,8 @@ wimlib_verify_wim(WIMStruct *wim, int verify_flags) if (ret) return ret; - ret = verify_image_blobs_present(wim_get_current_image_metadata(wim), - wim->blob_table); + ret = verify_file_data_present(wim_get_current_image_metadata(wim), + wim->blob_table); if (ret) return ret; diff --git a/src/wim.c b/src/wim.c index 98836a58..522380f1 100644 --- a/src/wim.c +++ b/src/wim.c @@ -929,16 +929,15 @@ static pthread_mutex_t lib_initialization_mutex = PTHREAD_MUTEX_INITIALIZER; WIMLIBAPI int wimlib_global_init(int init_flags) { - int ret; + int ret = 0; if (lib_initialized) - return 0; + goto out; pthread_mutex_lock(&lib_initialization_mutex); - ret = 0; if (lib_initialized) - goto out; + goto out_unlock; #ifdef ENABLE_ERROR_MESSAGES if (!wimlib_error_file) @@ -952,14 +951,14 @@ wimlib_global_init(int init_flags) WIMLIB_INIT_FLAG_STRICT_APPLY_PRIVILEGES | WIMLIB_INIT_FLAG_DEFAULT_CASE_SENSITIVE | WIMLIB_INIT_FLAG_DEFAULT_CASE_INSENSITIVE)) - goto out; + goto out_unlock; ret = WIMLIB_ERR_INVALID_PARAM; if ((init_flags & (WIMLIB_INIT_FLAG_DEFAULT_CASE_SENSITIVE | WIMLIB_INIT_FLAG_DEFAULT_CASE_INSENSITIVE)) == (WIMLIB_INIT_FLAG_DEFAULT_CASE_SENSITIVE | WIMLIB_INIT_FLAG_DEFAULT_CASE_INSENSITIVE)) - goto out; + goto out_unlock; libxml_global_init(); if (!(init_flags & WIMLIB_INIT_FLAG_ASSUME_UTF8)) { @@ -972,7 +971,7 @@ wimlib_global_init(int init_flags) #ifdef __WIN32__ ret = win32_global_init(init_flags); if (ret) - goto out; + goto out_unlock; #endif iconv_global_init(); init_upcase(); @@ -982,8 +981,9 @@ wimlib_global_init(int init_flags) default_ignore_case = true; lib_initialized = true; ret = 0; -out: +out_unlock: pthread_mutex_unlock(&lib_initialization_mutex); +out: return ret; } @@ -997,7 +997,7 @@ wimlib_global_cleanup(void) pthread_mutex_lock(&lib_initialization_mutex); if (!lib_initialized) - goto out; + goto out_unlock; libxml_global_cleanup(); iconv_global_cleanup(); @@ -1008,6 +1008,6 @@ wimlib_global_cleanup(void) wimlib_set_error_file(NULL); lib_initialized = false; -out: +out_unlock: pthread_mutex_unlock(&lib_initialization_mutex); } diff --git a/src/wimboot.c b/src/wimboot.c index b06816e2..351c8e67 100644 --- a/src/wimboot.c +++ b/src/wimboot.c @@ -1150,8 +1150,8 @@ wimboot_set_pointer(HANDLE h, in.wim_info.data_source_id = data_source_id; copy_hash(in.wim_info.unnamed_data_stream_hash, blob->hash); copy_hash(in.wim_info.blob_table_hash, blob_table_hash); - in.wim_info.unnamed_data_stream_uncompressed_size = blob->size; - in.wim_info.unnamed_data_stream_compressed_size = blob->rdesc->size_in_wim; + in.wim_info.unnamed_data_stream_size = blob->size; + in.wim_info.unnamed_data_stream_size_in_wim = blob->rdesc->size_in_wim; in.wim_info.unnamed_data_stream_offset_in_wim = blob->rdesc->offset_in_wim; if (!DeviceIoControl(h, FSCTL_SET_REPARSE_POINT, diff --git a/src/win32_capture.c b/src/win32_capture.c index 5fdc5ab9..13f63650 100644 --- a/src/win32_capture.c +++ b/src/win32_capture.c @@ -824,7 +824,7 @@ win32_get_encrypted_file_size(const wchar_t *path, bool is_dir, u64 *size_ret) } static int -winnt_load_efsrpc_raw_data(struct wim_inode *inode, const wchar_t *nt_path, +winnt_scan_efsrpc_raw_data(struct wim_inode *inode, const wchar_t *nt_path, struct list_head *unhashed_blobs) { struct blob_descriptor *blob; @@ -900,12 +900,12 @@ get_data_stream_name(wchar_t *raw_stream_name, size_t raw_stream_name_nchars, return true; } -/* Build the path to the stream. For unnamed streams, this is simply the path - * to the file. For named streams, this is the path to the file, followed by a - * colon, followed by the stream name. */ +/* Build the path to the data stream. For unnamed streams, this is simply the + * path to the file. For named streams, this is the path to the file, followed + * by a colon, followed by the stream name. */ static wchar_t * -build_stream_path(const wchar_t *path, size_t path_nchars, - const wchar_t *stream_name, size_t stream_name_nchars) +build_data_stream_path(const wchar_t *path, size_t path_nchars, + const wchar_t *stream_name, size_t stream_name_nchars) { size_t stream_path_nchars; wchar_t *stream_path; @@ -952,10 +952,10 @@ winnt_scan_data_stream(const wchar_t *path, size_t path_nchars, blob = new_blob_descriptor(); if (!blob) goto err_nomem; - blob->file_on_disk = build_stream_path(path, - path_nchars, - stream_name, - stream_name_nchars); + blob->file_on_disk = build_data_stream_path(path, + path_nchars, + stream_name, + stream_name_nchars); if (!blob->file_on_disk) goto err_nomem; blob->blob_location = BLOB_IN_WINNT_FILE_ON_DISK; @@ -1352,7 +1352,7 @@ retry_open: * needed. */ (*func_NtClose)(h); h = NULL; - ret = winnt_load_efsrpc_raw_data(inode, full_path, + ret = winnt_scan_efsrpc_raw_data(inode, full_path, params->unhashed_blobs); if (ret) goto out; diff --git a/src/write.c b/src/write.c index 30b063e5..c19d6fcb 100644 --- a/src/write.c +++ b/src/write.c @@ -66,23 +66,28 @@ #define WRITE_RESOURCE_FLAG_SEND_DONE_WITH_FILE 0x00000008 #define WRITE_RESOURCE_FLAG_SOLID_SORT 0x00000010 -static inline int +static int write_flags_to_resource_flags(int write_flags) { int write_resource_flags = 0; if (write_flags & WIMLIB_WRITE_FLAG_RECOMPRESS) write_resource_flags |= WRITE_RESOURCE_FLAG_RECOMPRESS; + if (write_flags & WIMLIB_WRITE_FLAG_PIPABLE) write_resource_flags |= WRITE_RESOURCE_FLAG_PIPABLE; + if (write_flags & WIMLIB_WRITE_FLAG_SOLID) write_resource_flags |= WRITE_RESOURCE_FLAG_SOLID; + if (write_flags & WIMLIB_WRITE_FLAG_SEND_DONE_WITH_FILE_MESSAGES) write_resource_flags |= WRITE_RESOURCE_FLAG_SEND_DONE_WITH_FILE; + if ((write_flags & (WIMLIB_WRITE_FLAG_SOLID | WIMLIB_WRITE_FLAG_NO_SOLID_SORT)) == WIMLIB_WRITE_FLAG_SOLID) write_resource_flags |= WRITE_RESOURCE_FLAG_SOLID_SORT; + return write_resource_flags; } @@ -786,8 +791,8 @@ write_blob_begin_read(struct blob_descriptor *blob, void *_ctx) return 0; } -/* Rewrite a blob that was just written compressed as uncompressed instead. - */ +/* Rewrite a blob that was just written compressed (as a non-solid WIM resource) + * as uncompressed instead. */ static int write_blob_uncompressed(struct blob_descriptor *blob, struct filedes *out_fd) { @@ -808,7 +813,7 @@ write_blob_uncompressed(struct blob_descriptor *blob, struct filedes *out_fd) * seeked to the end of the compressed resource, so * don't issue a hard error; just keep the compressed * resource instead. */ - WARNING("Recovered compressed blob of " + WARNING("Recovered compressed resource of " "size %"PRIu64", continuing on.", blob->size); return 0; } @@ -1708,11 +1713,11 @@ out_destroy_context: static int -wim_write_blob_list(WIMStruct *wim, - struct list_head *blob_list, - int write_flags, - unsigned num_threads, - struct filter_context *filter_ctx) +write_file_data_blobs(WIMStruct *wim, + struct list_head *blob_list, + int write_flags, + unsigned num_threads, + struct filter_context *filter_ctx) { int out_ctype; u32 out_chunk_size; @@ -1887,12 +1892,16 @@ inode_find_blobs_to_reference(const struct wim_inode *inode, for (unsigned i = 0; i < inode->i_num_streams; i++) { struct blob_descriptor *blob; + const u8 *hash; blob = stream_blob(&inode->i_streams[i], table); - if (blob) + if (blob) { reference_blob_for_write(blob, blob_list, inode->i_nlink); - else if (!is_zero_hash(stream_hash(&inode->i_streams[i]))) - return WIMLIB_ERR_RESOURCE_NOT_FOUND; + } else { + hash = stream_hash(&inode->i_streams[i]); + if (!is_zero_hash(hash)) + return blob_not_found_error(inode, hash); + } } return 0; } @@ -2142,10 +2151,10 @@ prepare_blob_list_for_write(WIMStruct *wim, int image, } static int -write_file_blobs(WIMStruct *wim, int image, int write_flags, - unsigned num_threads, - struct list_head *blob_list_override, - struct list_head *blob_table_list_ret) +write_file_data(WIMStruct *wim, int image, int write_flags, + unsigned num_threads, + struct list_head *blob_list_override, + struct list_head *blob_table_list_ret) { int ret; struct list_head _blob_list; @@ -2180,11 +2189,11 @@ write_file_blobs(WIMStruct *wim, int image, int write_flags, } } - return wim_write_blob_list(wim, - blob_list, - write_flags, - num_threads, - filter_ctx); + return write_file_data_blobs(wim, + blob_list, + write_flags, + num_threads, + filter_ctx); } static int @@ -2453,7 +2462,7 @@ finish_write(WIMStruct *wim, int image, int write_flags, && wim_has_integrity_table(wim)) { old_blob_table_end = wim->hdr.blob_table_reshdr.offset_in_wim + - wim->hdr.blob_table_reshdr.size_in_wim; + wim->hdr.blob_table_reshdr.size_in_wim; (void)read_integrity_table(wim, old_blob_table_end - WIM_HEADER_DISK_SIZE, &old_integrity_table); @@ -2606,6 +2615,9 @@ unlock_wim_for_append(WIMStruct *wim) * reading the WIM from a pipe. This copy of the XML data is ignored if the * WIM is read from a seekable file (not a pipe). * + * - Solid resources are not allowed. Each blob is always stored in its own + * resource. + * * - The format of resources, or blobs, has been modified to allow them to be * used before the "blob table" has been read. Each blob is prefixed with a * `struct pwm_blob_hdr' that is basically an abbreviated form of `struct @@ -2667,13 +2679,13 @@ write_pipable_wim(WIMStruct *wim, int image, int write_flags, WARNING("Creating a pipable WIM, which will " "be incompatible\n" - " with Microsoft's software (wimgapi/imagex/Dism)."); + " with Microsoft's software (WIMGAPI/ImageX/DISM)."); /* At this point, the header at the beginning of the file has already * been written. */ /* For efficiency, when wimlib adds an image to the WIM with - * wimlib_add_image(), the SHA-1 message digests of files is not + * wimlib_add_image(), the SHA-1 message digests of files are not * calculated; instead, they are calculated while the files are being * written. However, this does not work when writing a pipable WIM, * since when writing a blob to a pipable WIM, its SHA-1 message digest @@ -2696,11 +2708,11 @@ write_pipable_wim(WIMStruct *wim, int image, int write_flags, if (ret) return ret; - /* Write blobs needed for the image(s) being included in the output WIM, - * or blobs needed for the split WIM part. */ - return write_file_blobs(wim, image, write_flags, - num_threads, blob_list_override, - blob_table_list_ret); + /* Write file data needed for the image(s) being included in the output + * WIM, or file data needed for the split WIM part. */ + return write_file_data(wim, image, write_flags, + num_threads, blob_list_override, + blob_table_list_ret); /* The blob table, XML data, and header at end are handled by * finish_write(). */ @@ -2776,6 +2788,12 @@ write_wim_part(WIMStruct *wim, if (write_flags & WIMLIB_WRITE_FLAG_SOLID) DEBUG("\tSOLID"); + if (write_flags & WIMLIB_WRITE_FLAG_SEND_DONE_WITH_FILE_MESSAGES) + DEBUG("\tSEND_DONE_WITH_FILE_MESSAGES"); + + if (write_flags & WIMLIB_WRITE_FLAG_NO_SOLID_SORT) + DEBUG("\tNO_SOLID_SORT"); + if (write_flags & WIMLIB_WRITE_FLAG_FILE_DESCRIPTOR) DEBUG("\tFILE_DESCRIPTOR"); @@ -2911,14 +2929,12 @@ write_wim_part(WIMStruct *wim, if (total_parts != 1) wim->hdr.boot_idx = 0; - /* Initialize output file descriptor. */ + /* Set up output file descriptor. */ if (write_flags & WIMLIB_WRITE_FLAG_FILE_DESCRIPTOR) { - /* File descriptor was explicitly provided. Return error if - * file descriptor is not seekable, unless writing a pipable WIM - * was requested. */ - wim->out_fd.fd = *(const int*)path_or_fd; - wim->out_fd.offset = 0; + /* File descriptor was explicitly provided. */ + filedes_init(&wim->out_fd, *(const int *)path_or_fd); if (!filedes_is_seekable(&wim->out_fd)) { + /* The file descriptor is a pipe. */ ret = WIMLIB_ERR_INVALID_PARAM; if (!(write_flags & WIMLIB_WRITE_FLAG_PIPABLE)) goto out_restore_hdr; @@ -2948,13 +2964,13 @@ write_wim_part(WIMStruct *wim, if (ret) goto out_restore_hdr; - /* Write file blobs and metadata resources. */ + /* Write file data and metadata resources. */ if (!(write_flags & WIMLIB_WRITE_FLAG_PIPABLE)) { /* Default case: create a normal (non-pipable) WIM. */ - ret = write_file_blobs(wim, image, write_flags, - num_threads, - blob_list_override, - &blob_table_list); + ret = write_file_data(wim, image, write_flags, + num_threads, + blob_list_override, + &blob_table_list); if (ret) goto out_restore_hdr; @@ -3229,8 +3245,8 @@ overwrite_wim_inplace(WIMStruct *wim, int write_flags, unsigned num_threads) goto out_restore_physical_hdr; } - ret = wim_write_blob_list(wim, &blob_list, write_flags, - num_threads, &filter_ctx); + ret = write_file_data_blobs(wim, &blob_list, write_flags, + num_threads, &filter_ctx); if (ret) goto out_truncate; -- 2.43.0