From 7b13472d9515dc87a15bd43a9822ac42217d3af9 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Wed, 14 Dec 2016 20:49:55 -0800 Subject: [PATCH] Extract sparse files as sparse When extracting a stream belonging to an inode with FILE_ATTRIBUTE_SPARSE_FILE set, before writing any data, mark the extracted stream as sparse if needed and skip preallocating space. Then, skip writing zero regions. This makes it so that sparse files are still sparse after extraction. --- NEWS | 2 + doc/man1/wimlib-imagex-apply.1 | 12 +-- include/wimlib/apply.h | 15 ++++ src/extract.c | 64 ++++++++++++++ src/ntfs-3g_apply.c | 87 +++++++++++++++---- src/unix_apply.c | 59 +++++++++++-- src/win32_apply.c | 150 +++++++++++++++++++++++++++------ 7 files changed, 332 insertions(+), 57 deletions(-) diff --git a/NEWS b/NEWS index bbecd4d0..f7f5d7b3 100644 --- a/NEWS +++ b/NEWS @@ -2,6 +2,8 @@ Version 1.11.0-BETA1: Updated 'mkwinpeimg' to work correctly on images that have a "windows" (lower case) directory rather than a "Windows" (upper case) directory. + Sparse files are now extracted as sparse. + On Windows, improved the way in which files deduplicated with Windows' Data Deduplication feature are captured. diff --git a/doc/man1/wimlib-imagex-apply.1 b/doc/man1/wimlib-imagex-apply.1 index b0ab1354..4e8bc8d8 100644 --- a/doc/man1/wimlib-imagex-apply.1 +++ b/doc/man1/wimlib-imagex-apply.1 @@ -63,7 +63,7 @@ Named data streams. .IP \[bu] Reparse points other than symbolic links and junction points. .IP \[bu] -Certain file attributes such as compression, encryption, and sparseness. +Certain file attributes such as compression and encryption. .IP \[bu] Short (DOS) names for files. .IP \[bu] @@ -121,12 +121,8 @@ Win32+DOS namespace, and POSIX namespace. This includes hard links. .IP \[bu] Object IDs. .PP -However, there are also several known limitations of the NTFS volume extraction -mode: -.IP \[bu] 4 -Encrypted files will not be extracted. -.IP \[bu] -Sparse file attributes will not be extracted. +However, a limitation of the NTFS volume extraction mode is that encrypted files +will not be extracted. .PP Regardless, since almost all information from the WIM image is restored in this mode, it is possible (and fully supported) to restore an image of an actual @@ -190,8 +186,6 @@ Additional notes about extracting files on Windows: exact metadata and data of the WIM image, for example due to features mentioned above not being supported by the target filesystem. .IP \[bu] -Sparse file attributes will not be extracted. -.IP \[bu] Since encrypted files (with FILE_ATTRIBUTE_ENCRYPTED) are not stored in plaintext in the WIM image, \fBwimlib-imagex\fR cannot restore encrypted files to filesystems not supporting encryption. Therefore, on such filesystems, diff --git a/include/wimlib/apply.h b/include/wimlib/apply.h index da9087a7..9d9c3c97 100644 --- a/include/wimlib/apply.h +++ b/include/wimlib/apply.h @@ -134,6 +134,21 @@ report_apply_error(struct apply_ctx *ctx, int error_code, const tchar *path) return report_error(ctx->progfunc, ctx->progctx, error_code, path); } +extern bool +detect_sparse_region(const void *data, size_t size, size_t *len_ret); + +static inline bool +maybe_detect_sparse_region(const void *data, size_t size, size_t *len_ret, + bool enabled) +{ + if (!enabled) { + /* Force non-sparse without checking */ + *len_ret = size; + return false; + } + return detect_sparse_region(data, size, len_ret); +} + #define inode_first_extraction_dentry(inode) \ ((inode)->i_first_extraction_alias) diff --git a/src/extract.c b/src/extract.c index d3df861f..a39c1c02 100644 --- a/src/extract.c +++ b/src/extract.c @@ -141,6 +141,70 @@ end_file_metadata_phase(struct apply_ctx *ctx) return end_file_phase(ctx, WIMLIB_PROGRESS_MSG_EXTRACT_METADATA); } +/* Are all bytes in the specified buffer zero? */ +static bool +is_all_zeroes(const u8 *p, const size_t size) +{ + const u8 * const end = p + size; + + for (; (uintptr_t)p % WORDBYTES && p != end; p++) + if (*p) + return false; + + for (; end - p >= WORDBYTES; p += WORDBYTES) + if (*(const machine_word_t *)p) + return false; + + for (; p != end; p++) + if (*p) + return false; + + return true; +} + +/* + * Sparse regions should be detected at the granularity of the filesystem block + * size. For now just assume 4096 bytes, which is the default block size on + * NTFS and most Linux filesystems. + */ +#define SPARSE_UNIT 4096 + +/* + * Detect whether the specified buffer begins with a region of all zero bytes. + * Return %true if a zero region was found or %false if a nonzero region was + * found, and sets *len_ret to the length of the region. This operates at a + * granularity of SPARSE_UNIT bytes, meaning that to extend a zero region, there + * must be SPARSE_UNIT zero bytes with no interruption, but to extend a nonzero + * region, just one nonzero byte in the next SPARSE_UNIT bytes is sufficient. + * + * Note: besides compression, the WIM format doesn't yet have a way to + * efficiently represent zero regions, so that's why we need to detect them + * ourselves. Things will still fall apart badly on extremely large sparse + * files, but this is a start... + */ +bool +detect_sparse_region(const void *data, size_t size, size_t *len_ret) +{ + const void *p = data; + const void * const end = data + size; + size_t len = 0; + bool zeroes = false; + + while (p != end) { + size_t n = min(end - p, SPARSE_UNIT); + bool z = is_all_zeroes(p, n); + + if (len != 0 && z != zeroes) + break; + zeroes = z; + len += n; + p += n; + } + + *len_ret = len; + return zeroes; +} + #define PWM_FOUND_WIM_HDR (-1) /* Read the header for a blob in a pipable WIM. If @pwm_hdr_ret is not NULL, diff --git a/src/ntfs-3g_apply.c b/src/ntfs-3g_apply.c index 27b0ab90..27b512f6 100644 --- a/src/ntfs-3g_apply.c +++ b/src/ntfs-3g_apply.c @@ -61,6 +61,7 @@ ntfs_3g_get_supported_features(const char *target, supported_features->archive_files = 1; supported_features->compressed_files = 1; supported_features->not_context_indexed_files = 1; + supported_features->sparse_files = 1; supported_features->named_data_streams = 1; supported_features->hard_links = 1; supported_features->reparse_points = 1; @@ -84,6 +85,13 @@ struct ntfs_3g_apply_ctx { ntfs_inode *open_inodes[MAX_OPEN_FILES]; unsigned num_open_inodes; + /* For each currently open attribute, whether we're writing to it in + * "sparse" mode or not. */ + bool is_sparse_attr[MAX_OPEN_FILES]; + + /* Whether is_sparse_attr[] is true for any currently open attribute */ + bool any_sparse_attrs; + struct reparse_buffer_disk rpbuf; u8 *reparse_ptr; @@ -329,9 +337,6 @@ ntfs_3g_set_metadata(ntfs_inode *ni, const struct wim_inode *inode, if (!(extract_flags & WIMLIB_EXTRACT_FLAG_NO_ATTRIBUTES)) { u32 attrib = inode->i_attributes; - attrib &= ~(FILE_ATTRIBUTE_SPARSE_FILE | - FILE_ATTRIBUTE_ENCRYPTED); - if (ntfs_set_ntfs_attrib(ni, (const char *)&attrib, sizeof(attrib), 0)) { @@ -680,7 +685,7 @@ ntfs_3g_begin_extract_blob_instance(struct blob_descriptor *blob, struct wim_dentry *one_dentry = inode_first_extraction_dentry(inode); ntfschar *stream_name; size_t stream_name_nchars; - ntfs_attr *attr; + ntfs_attr *na; if (unlikely(strm->stream_type == STREAM_TYPE_REPARSE_POINT)) { @@ -723,14 +728,33 @@ ntfs_3g_begin_extract_blob_instance(struct blob_descriptor *blob, /* This should be ensured by extract_blob_list() */ wimlib_assert(ctx->num_open_attrs < MAX_OPEN_FILES); - attr = ntfs_attr_open(ni, AT_DATA, stream_name, stream_name_nchars); - if (!attr) { + na = ntfs_attr_open(ni, AT_DATA, stream_name, stream_name_nchars); + if (!na) { ERROR_WITH_ERRNO("Failed to open data stream of \"%s\"", dentry_full_path(one_dentry)); return WIMLIB_ERR_NTFS_3G; } - ctx->open_attrs[ctx->num_open_attrs++] = attr; - ntfs_attr_truncate_solid(attr, blob->size); + + /* + * Note: there are problems with trying to combine compression with + * sparseness when extracting. For example, doing ntfs_attr_truncate() + * at the end to extend the attribute to its final size actually extends + * to a compression block size boundary rather than to the requested + * size. Until these problems are solved, we always write the full data + * to compressed attributes. We also don't attempt to preallocate space + * for compressed attributes, since we don't know how much space they + * are going to actually need. + */ + ctx->is_sparse_attr[ctx->num_open_attrs] = false; + if (!(na->data_flags & ATTR_COMPRESSION_MASK)) { + if (inode->i_attributes & FILE_ATTRIBUTE_SPARSE_FILE) { + ctx->is_sparse_attr[ctx->num_open_attrs] = true; + ctx->any_sparse_attrs = true; + } else { + ntfs_attr_truncate_solid(na, blob->size); + } + } + ctx->open_attrs[ctx->num_open_attrs++] = na; return 0; } @@ -753,6 +777,7 @@ ntfs_3g_cleanup_blob_extract(struct ntfs_3g_apply_ctx *ctx) } ctx->num_open_inodes = 0; + ctx->any_sparse_attrs = false; ctx->reparse_ptr = NULL; ctx->num_reparse_inodes = 0; return ret; @@ -842,18 +867,35 @@ ntfs_3g_extract_chunk(const struct blob_descriptor *blob, u64 offset, const void *chunk, size_t size, void *_ctx) { struct ntfs_3g_apply_ctx *ctx = _ctx; - - for (unsigned i = 0; i < ctx->num_open_attrs; i++) { - if (!ntfs_3g_full_pwrite(ctx->open_attrs[i], offset, - size, chunk)) - { - ERROR_WITH_ERRNO("Error writing data to NTFS volume"); - return WIMLIB_ERR_NTFS_3G; + const void * const end = chunk + size; + const void *p; + bool zeroes; + size_t len; + unsigned i; + + /* + * For sparse attributes, only write nonzero regions. This lets the + * filesystem use holes to represent zero regions. + */ + for (p = chunk; p != end; p += len, offset += len) { + zeroes = maybe_detect_sparse_region(p, end - p, &len, + ctx->any_sparse_attrs); + for (i = 0; i < ctx->num_open_attrs; i++) { + if (!zeroes || !ctx->is_sparse_attr[i]) { + if (!ntfs_3g_full_pwrite(ctx->open_attrs[i], + offset, len, p)) + goto err; + } } } + if (ctx->reparse_ptr) ctx->reparse_ptr = mempcpy(ctx->reparse_ptr, chunk, size); return 0; + +err: + ERROR_WITH_ERRNO("Error writing data to NTFS volume"); + return WIMLIB_ERR_NTFS_3G; } static int @@ -867,6 +909,21 @@ ntfs_3g_end_extract_blob(struct blob_descriptor *blob, int status, void *_ctx) goto out; } + /* Extend sparse attributes to their final size. */ + if (ctx->any_sparse_attrs) { + for (unsigned i = 0; i < ctx->num_open_attrs; i++) { + if (!ctx->is_sparse_attr[i]) + continue; + if (ntfs_attr_truncate(ctx->open_attrs[i], blob->size)) + { + ERROR_WITH_ERRNO("Error extending attribute to " + "final size"); + ret = WIMLIB_ERR_WRITE; + goto out; + } + } + } + for (u32 i = 0; i < ctx->num_reparse_inodes; i++) { ret = ntfs_3g_restore_reparse_point(ctx->ntfs_reparse_inodes[i], ctx->wim_reparse_inodes[i], diff --git a/src/unix_apply.c b/src/unix_apply.c index e2ba76f9..b17bd351 100644 --- a/src/unix_apply.c +++ b/src/unix_apply.c @@ -54,6 +54,7 @@ static int unix_get_supported_features(const char *target, struct wim_features *supported_features) { + supported_features->sparse_files = 1; supported_features->hard_links = 1; supported_features->symlink_reparse_points = 1; supported_features->unix_data = 1; @@ -81,6 +82,13 @@ struct unix_apply_ctx { * the beginning of the array. */ unsigned num_open_fds; + /* For each currently open file, whether we're writing to it in "sparse" + * mode or not. */ + bool is_sparse_file[MAX_OPEN_FILES]; + + /* Whether is_sparse_file[] is true for any currently open file */ + bool any_sparse_files; + /* Buffer for reading reparse point data into memory */ u8 reparse_data[REPARSE_DATA_MAX_SIZE]; @@ -541,6 +549,7 @@ unix_cleanup_open_fds(struct unix_apply_ctx *ctx, unsigned offset) for (unsigned i = offset; i < ctx->num_open_fds; i++) filedes_close(&ctx->open_fds[i]); ctx->num_open_fds = 0; + ctx->any_sparse_files = false; } static int @@ -584,10 +593,16 @@ retry_create: ERROR_WITH_ERRNO("Can't create regular file \"%s\"", first_path); return WIMLIB_ERR_OPEN; } - filedes_init(&ctx->open_fds[ctx->num_open_fds++], fd); + if (inode->i_attributes & FILE_ATTRIBUTE_SPARSE_FILE) { + ctx->is_sparse_file[ctx->num_open_fds] = true; + ctx->any_sparse_files = true; + } else { + ctx->is_sparse_file[ctx->num_open_fds] = false; #ifdef HAVE_POSIX_FALLOCATE - posix_fallocate(fd, 0, blob->size); + posix_fallocate(fd, 0, blob->size); #endif + } + filedes_init(&ctx->open_fds[ctx->num_open_fds++], fd); return unix_create_hardlinks(inode, first_dentry, first_path, ctx); } @@ -618,18 +633,37 @@ unix_extract_chunk(const struct blob_descriptor *blob, u64 offset, const void *chunk, size_t size, void *_ctx) { struct unix_apply_ctx *ctx = _ctx; + const void * const end = chunk + size; + const void *p; + bool zeroes; + size_t len; + unsigned i; int ret; - for (unsigned i = 0; i < ctx->num_open_fds; i++) { - ret = full_write(&ctx->open_fds[i], chunk, size); - if (ret) { - ERROR_WITH_ERRNO("Error writing data to filesystem"); - return ret; + /* + * For sparse files, only write nonzero regions. This lets the + * filesystem use holes to represent zero regions. + */ + for (p = chunk; p != end; p += len, offset += len) { + zeroes = maybe_detect_sparse_region(p, end - p, &len, + ctx->any_sparse_files); + for (i = 0; i < ctx->num_open_fds; i++) { + if (!zeroes || !ctx->is_sparse_file[i]) { + ret = full_pwrite(&ctx->open_fds[i], + p, len, offset); + if (ret) + goto err; + } } } + if (ctx->reparse_ptr) ctx->reparse_ptr = mempcpy(ctx->reparse_ptr, chunk, size); return 0; + +err: + ERROR_WITH_ERRNO("Error writing data to filesystem"); + return ret; } /* Called when a blob has been fully read for extraction */ @@ -669,10 +703,17 @@ unix_end_extract_blob(struct blob_descriptor *blob, int status, void *_ctx) if (ret) break; } else { - /* Set metadata on regular file just before closing it. - */ struct filedes *fd = &ctx->open_fds[j]; + /* If the file is sparse, extend it to its final size. */ + if (ctx->is_sparse_file[j] && ftruncate(fd->fd, blob->size)) { + ERROR_WITH_ERRNO("Error extending \"%s\" to final size", + unix_build_inode_extraction_path(inode, ctx)); + ret = WIMLIB_ERR_WRITE; + break; + } + + /* Set metadata on regular file just before closing. */ ret = unix_set_metadata(fd->fd, inode, NULL, ctx); if (ret) break; diff --git a/src/win32_apply.c b/src/win32_apply.c index b55ce924..ada5b919 100644 --- a/src/win32_apply.c +++ b/src/win32_apply.c @@ -125,6 +125,13 @@ struct win32_apply_ctx { * beginning of the array) */ unsigned num_open_handles; + /* For each currently open stream, whether we're writing to it in + * "sparse" mode or not. */ + bool is_sparse_stream[MAX_OPEN_FILES]; + + /* Whether is_sparse_stream[] is true for any currently open stream */ + bool any_sparse_streams; + /* List of dentries, joined by @d_tmp_list, that need to have reparse * data extracted as soon as the whole blob has been read into * @data_buffer. */ @@ -286,7 +293,8 @@ win32_get_supported_features(const wchar_t *target, supported_features->not_context_indexed_files = 1; - /* Don't do anything with FILE_SUPPORTS_SPARSE_FILES. */ + if (vol_flags & FILE_SUPPORTS_SPARSE_FILES) + supported_features->sparse_files = 1; if (vol_flags & FILE_NAMED_STREAMS) supported_features->named_data_streams = 1; @@ -1156,6 +1164,28 @@ adjust_compression_attribute(HANDLE h, const struct wim_dentry *dentry, return WIMLIB_ERR_SET_ATTRIBUTES; } +static bool +need_sparse_flag(const struct wim_inode *inode, + const struct win32_apply_ctx *ctx) +{ + return (inode->i_attributes & FILE_ATTRIBUTE_SPARSE_FILE) && + ctx->common.supported_features.sparse_files; +} + +static int +set_sparse_flag(HANDLE h, struct win32_apply_ctx *ctx) +{ + NTSTATUS status; + + status = winnt_fsctl(h, FSCTL_SET_SPARSE, NULL, 0, NULL, 0, NULL); + if (NT_SUCCESS(status)) + return 0; + + winnt_error(status, L"Can't set sparse flag on \"%ls\"", + current_path(ctx)); + return WIMLIB_ERR_SET_ATTRIBUTES; +} + /* Try to enable short name support on the target volume. If successful, return * true. If unsuccessful, issue a warning and return false. */ static bool @@ -1777,6 +1807,12 @@ create_nondirectory_inode(HANDLE *h_ret, const struct wim_dentry *dentry, if (ret) goto out_close; + if (need_sparse_flag(dentry->d_inode, ctx)) { + ret = set_sparse_flag(h, ctx); + if (ret) + goto out_close; + } + ret = create_empty_streams(dentry, ctx); if (ret) goto out_close; @@ -1957,7 +1993,6 @@ begin_extract_blob_instance(const struct blob_descriptor *blob, const struct wim_inode_stream *strm, struct win32_apply_ctx *ctx) { - FILE_ALLOCATION_INFORMATION alloc_info; HANDLE h; NTSTATUS status; @@ -2023,12 +2058,28 @@ begin_extract_blob_instance(const struct blob_descriptor *blob, return WIMLIB_ERR_OPEN; } + ctx->is_sparse_stream[ctx->num_open_handles] = false; + if (need_sparse_flag(dentry->d_inode, ctx)) { + /* If the stream is unnamed, then the sparse flag was already + * set when the file was created. But if the stream is named, + * then we need to set the sparse flag here. */ + if (unlikely(stream_is_named(strm))) { + int ret = set_sparse_flag(h, ctx); + if (ret) { + NtClose(h); + return ret; + } + } + ctx->is_sparse_stream[ctx->num_open_handles] = true; + ctx->any_sparse_streams = true; + } else { + /* Allocate space for the data. */ + FILE_ALLOCATION_INFORMATION info = + { .AllocationSize = { .QuadPart = blob->size }}; + NtSetInformationFile(h, &ctx->iosb, &info, sizeof(info), + FileAllocationInformation); + } ctx->open_handles[ctx->num_open_handles++] = h; - - /* Allocate space for the data. */ - alloc_info.AllocationSize.QuadPart = blob->size; - NtSetInformationFile(h, &ctx->iosb, &alloc_info, sizeof(alloc_info), - FileAllocationInformation); return 0; } @@ -2267,6 +2318,7 @@ win32_begin_extract_blob(struct blob_descriptor *blob, void *_ctx) ctx->num_open_handles = 0; ctx->data_buffer_ptr = NULL; + ctx->any_sparse_streams = false; INIT_LIST_HEAD(&ctx->reparse_dentries); INIT_LIST_HEAD(&ctx->encrypted_dentries); @@ -2302,31 +2354,58 @@ fail: return ret; } +static int +pwrite_to_handle(HANDLE h, const void *data, size_t size, u64 offset) +{ + const void * const end = data + size; + const void *p; + IO_STATUS_BLOCK iosb; + NTSTATUS status; + + for (p = data; p != end; p += iosb.Information, + offset += iosb.Information) + { + LARGE_INTEGER offs = { .QuadPart = offset }; + + status = NtWriteFile(h, NULL, NULL, NULL, &iosb, + (void *)p, min(INT32_MAX, end - p), + &offs, NULL); + if (!NT_SUCCESS(status)) { + winnt_error(status, + L"Error writing data to target volume"); + return WIMLIB_ERR_WRITE; + } + } + return 0; +} + /* Called when the next chunk of a blob has been read for extraction */ static int win32_extract_chunk(const struct blob_descriptor *blob, u64 offset, const void *chunk, size_t size, void *_ctx) { struct win32_apply_ctx *ctx = _ctx; + const void * const end = chunk + size; + const void *p; + bool zeroes; + size_t len; + unsigned i; + int ret; - /* Write the data chunk to each open handle */ - for (unsigned i = 0; i < ctx->num_open_handles; i++) { - u8 *bufptr = (u8 *)chunk; - size_t bytes_remaining = size; - NTSTATUS status; - while (bytes_remaining) { - ULONG count = min(0xFFFFFFFF, bytes_remaining); - - status = NtWriteFile(ctx->open_handles[i], - NULL, NULL, NULL, - &ctx->iosb, bufptr, count, - NULL, NULL); - if (!NT_SUCCESS(status)) { - winnt_error(status, L"Error writing data to target volume"); - return WIMLIB_ERR_WRITE; + /* + * For sparse streams, only write nonzero regions. This lets the + * filesystem use holes to represent zero regions. + */ + for (p = chunk; p != end; p += len, offset += len) { + zeroes = maybe_detect_sparse_region(p, end - p, &len, + ctx->any_sparse_streams); + for (i = 0; i < ctx->num_open_handles; i++) { + if (!zeroes || !ctx->is_sparse_stream[i]) { + ret = pwrite_to_handle(ctx->open_handles[i], + p, len, offset); + if (ret) + return ret; } - bufptr += ctx->iosb.Information; - bytes_remaining -= ctx->iosb.Information; } } @@ -2598,6 +2677,29 @@ win32_end_extract_blob(struct blob_descriptor *blob, int status, void *_ctx) int ret; const struct wim_dentry *dentry; + /* Extend sparse streams to their final size. */ + if (ctx->any_sparse_streams && !status) { + for (unsigned i = 0; i < ctx->num_open_handles; i++) { + FILE_END_OF_FILE_INFORMATION info = + { .EndOfFile = { .QuadPart = blob->size } }; + NTSTATUS ntstatus; + + if (!ctx->is_sparse_stream[i]) + continue; + + ntstatus = NtSetInformationFile(ctx->open_handles[i], + &ctx->iosb, + &info, sizeof(info), + FileEndOfFileInformation); + if (!NT_SUCCESS(ntstatus)) { + winnt_error(ntstatus, L"Error writing data to " + "target volume (while extending)"); + status = WIMLIB_ERR_WRITE; + break; + } + } + } + close_handles(ctx); if (status) -- 2.43.0