X-Git-Url: https://wimlib.net/git/?p=wimlib;a=blobdiff_plain;f=src%2Fntfs-3g_apply.c;h=c012c8e5b600188d76115949098763eb5b22d6b0;hp=ccfd6fde54703b4531c7d3205d19987517f79c95;hb=a9b5ef0483d60ef1d8bf6014f223dfeaa68c091e;hpb=51df3b63c594a7e35446d2e2e19637e54240b0b2 diff --git a/src/ntfs-3g_apply.c b/src/ntfs-3g_apply.c index ccfd6fde..c012c8e5 100644 --- a/src/ntfs-3g_apply.c +++ b/src/ntfs-3g_apply.c @@ -3,14 +3,14 @@ * * Apply a WIM image directly to an NTFS volume using libntfs-3g. Restore as * much information as possible, including security data, file attributes, DOS - * names, and alternate data streams. + * names, alternate data streams, and object IDs. * - * Note: because NTFS-3g offers inode-based interfaces, we actually don't need + * Note: because NTFS-3G offers inode-based interfaces, we actually don't need * to deal with paths at all! (Other than for error messages.) */ /* - * Copyright (C) 2012, 2013, 2014, 2015 Eric Biggers + * Copyright (C) 2012-2017 Eric Biggers * * This file is free software; you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -35,6 +35,7 @@ #include #include +#include #include #include @@ -46,24 +47,27 @@ #include "wimlib/error.h" #include "wimlib/metadata.h" #include "wimlib/ntfs_3g.h" +#include "wimlib/object_id.h" #include "wimlib/reparse.h" #include "wimlib/security.h" -#include "wimlib/security_descriptor.h" static int ntfs_3g_get_supported_features(const char *target, struct wim_features *supported_features) { - supported_features->archive_files = 1; + supported_features->readonly_files = 1; supported_features->hidden_files = 1; supported_features->system_files = 1; + supported_features->archive_files = 1; supported_features->compressed_files = 1; supported_features->not_context_indexed_files = 1; + supported_features->sparse_files = 1; supported_features->named_data_streams = 1; supported_features->hard_links = 1; supported_features->reparse_points = 1; supported_features->security_descriptors = 1; supported_features->short_names = 1; + supported_features->object_ids = 1; supported_features->timestamps = 1; supported_features->case_sensitive_filenames = 1; return 0; @@ -81,144 +85,21 @@ struct ntfs_3g_apply_ctx { ntfs_inode *open_inodes[MAX_OPEN_FILES]; unsigned num_open_inodes; + /* For each currently open attribute, whether we're writing to it in + * "sparse" mode or not. */ + bool is_sparse_attr[MAX_OPEN_FILES]; + + /* Whether is_sparse_attr[] is true for any currently open attribute */ + bool any_sparse_attrs; + struct reparse_buffer_disk rpbuf; u8 *reparse_ptr; - /* Offset in the blob currently being read */ - u64 offset; - unsigned num_reparse_inodes; ntfs_inode *ntfs_reparse_inodes[MAX_OPEN_FILES]; struct wim_inode *wim_reparse_inodes[MAX_OPEN_FILES]; }; -static size_t -sid_size(const wimlib_SID *sid) -{ - return offsetof(wimlib_SID, sub_authority) + - sizeof(le32) * sid->sub_authority_count; -} - -/* - * sd_fixup - Fix up a Windows NT security descriptor for libntfs-3g. - * - * libntfs-3g validates security descriptors before setting them, but old - * versions contain bugs causing it to reject unusual but valid security - * descriptors: - * - * - Versions before 2013.1.13 reject security descriptors ending with an empty - * SACL (System Access Control List). This bug can be worked around either by - * moving the empty SACL earlier in the security descriptor or by removing the - * SACL entirely. The latter work-around is valid because an empty SACL is - * equivalent to a "null", or non-existent, SACL. - * - Versions before 2014.2.15 reject security descriptors ending with an empty - * DACL (Discretionary Access Control List). This is very similar to the SACL - * bug. However, removing the DACL is not a valid workaround because this - * changes the meaning of the security descriptor--- an empty DACL allows no - * access, whereas a "null" DACL allows all access. - * - * If the security descriptor was fixed, this function returns an allocated - * buffer containing the fixed security descriptor, and its size is updated. - * Otherwise (or if no memory is available) NULL is returned. - */ -static void * -sd_fixup(const void *_desc, size_t *size_p) -{ - u32 owner_offset, group_offset, dacl_offset, sacl_offset; - bool owner_valid, group_valid; - size_t size = *size_p; - const wimlib_SECURITY_DESCRIPTOR_RELATIVE *desc = _desc; - wimlib_SECURITY_DESCRIPTOR_RELATIVE *desc_new; - const wimlib_SID *owner, *group, *sid; - - /* Don't attempt to fix clearly invalid security descriptors. */ - if (size < sizeof(wimlib_SECURITY_DESCRIPTOR_RELATIVE)) - return NULL; - - if (le16_to_cpu(desc->control) & wimlib_SE_DACL_PRESENT) - dacl_offset = le32_to_cpu(desc->dacl_offset); - else - dacl_offset = 0; - - if (le16_to_cpu(desc->control) & wimlib_SE_SACL_PRESENT) - sacl_offset = le32_to_cpu(desc->sacl_offset); - else - sacl_offset = 0; - - /* Check if the security descriptor will be affected by one of the bugs. - * If not, do nothing and return. */ - if (!((sacl_offset != 0 && sacl_offset == size - sizeof(wimlib_ACL)) || - (dacl_offset != 0 && dacl_offset == size - sizeof(wimlib_ACL)))) - return NULL; - - owner_offset = le32_to_cpu(desc->owner_offset); - group_offset = le32_to_cpu(desc->group_offset); - owner = (const wimlib_SID*)((const u8*)desc + owner_offset); - group = (const wimlib_SID*)((const u8*)desc + group_offset); - - /* We'll try to move the owner or group SID to the end of the security - * descriptor to avoid the bug. This is only possible if at least one - * is valid. */ - owner_valid = (owner_offset != 0) && - (owner_offset % 4 == 0) && - (owner_offset <= size - sizeof(SID)) && - (owner_offset + sid_size(owner) <= size) && - (owner_offset >= sizeof(wimlib_SECURITY_DESCRIPTOR_RELATIVE)); - group_valid = (group_offset != 0) && - (group_offset % 4 == 0) && - (group_offset <= size - sizeof(SID)) && - (group_offset + sid_size(group) <= size) && - (group_offset >= sizeof(wimlib_SECURITY_DESCRIPTOR_RELATIVE)); - if (owner_valid) { - sid = owner; - } else if (group_valid) { - sid = group; - } else { - return NULL; - } - - desc_new = MALLOC(size + sid_size(sid)); - if (!desc_new) - return NULL; - - memcpy(desc_new, desc, size); - if (owner_valid) - desc_new->owner_offset = cpu_to_le32(size); - else if (group_valid) - desc_new->group_offset = cpu_to_le32(size); - memcpy((u8*)desc_new + size, sid, sid_size(sid)); - *size_p = size + sid_size(sid); - return desc_new; -} - -/* Set the security descriptor @desc of size @desc_size on the NTFS inode @ni. - */ -static int -ntfs_3g_set_security_descriptor(ntfs_inode *ni, const void *desc, size_t desc_size) -{ - struct SECURITY_CONTEXT sec_ctx; - void *desc_fixed = NULL; - int ret = 0; - - memset(&sec_ctx, 0, sizeof(sec_ctx)); - sec_ctx.vol = ni->vol; - -retry: - if (ntfs_set_ntfs_acl(&sec_ctx, ni, desc, desc_size, 0)) { - if (desc_fixed == NULL) { - desc_fixed = sd_fixup(desc, &desc_size); - if (desc_fixed != NULL) { - desc = desc_fixed; - goto retry; - } - } - ret = WIMLIB_ERR_SET_SECURITY; - } - - FREE(desc_fixed); - return ret; -} - static int ntfs_3g_set_timestamps(ntfs_inode *ni, const struct wim_inode *inode) { @@ -291,14 +172,34 @@ ntfs_3g_restore_dos_name(ntfs_inode *ni, ntfs_inode *dir_ni, ret = -1; } utf16le_put_tstr(dos_name); - if (ret) { + if (unlikely(ret)) { + int err = errno; ERROR_WITH_ERRNO("Failed to set DOS name of \"%s\" in NTFS " "volume", dentry_full_path(dentry)); + if (err == EILSEQ) { + ERROR("This error may have been caused by a known " + "bug in libntfs-3g where it is unable to set " + "DOS names on files whose long names contain " + "unpaired surrogate characters. This bug " + "was fixed in NTFS-3G version 2017.3.23."); + } + if (err == EINVAL) { + utf16lechar c = + dentry->d_name[dentry->d_name_nbytes / 2 - 1]; + if (c == cpu_to_le16('.') || c == cpu_to_le16(' ')) { + ERROR("This error was probably caused by a " + "known bug in libntfs-3g where it is " + "unable to set DOS names on files whose " + "long names end with a dot or space " + "character. This bug was fixed in " + "NTFS-3G version 2017.3.23."); + } + } ret = WIMLIB_ERR_SET_SHORT_NAME; goto out_close; } - /* Unlike most other NTFS-3g functions, ntfs_set_ntfs_dos_name() + /* Unlike most other NTFS-3G functions, ntfs_set_ntfs_dos_name() * changes the directory's last modification timestamp... * Change it back. */ return ntfs_3g_restore_timestamps(vol, dentry->d_parent->d_inode); @@ -331,8 +232,8 @@ ntfs_3g_restore_reparse_point(ntfs_inode *ni, const struct wim_inode *inode, "tag. The preceding error may have been caused " "by a known bug in libntfs-3g where it does not " "correctly validate non-Microsoft reparse " - "points. This bug may be fixed in the 2016 " - "release of libntfs-3g."); + "points. This bug was fixed in NTFS-3G version " + "2016.2.22."); } return WIMLIB_ERR_SET_REPARSE_DATA; } @@ -411,13 +312,29 @@ ntfs_3g_set_metadata(ntfs_inode *ni, const struct wim_inode *inode, sd = wim_get_current_security_data(ctx->common.wim); one_dentry = inode_first_extraction_dentry(inode); + /* Object ID */ + { + u32 len; + const void *object_id = inode_get_object_id(inode, &len); + if (unlikely(object_id != NULL) && + ntfs_set_ntfs_object_id(ni, object_id, len, 0)) + { + if (errno == EEXIST) { + WARNING("Duplicate object ID on file \"%s\"", + dentry_full_path(one_dentry)); + } else { + ERROR_WITH_ERRNO("Failed to set object ID on " + "\"%s\" in NTFS volume", + dentry_full_path(one_dentry)); + return WIMLIB_ERR_NTFS_3G; + } + } + } + /* Attributes */ if (!(extract_flags & WIMLIB_EXTRACT_FLAG_NO_ATTRIBUTES)) { u32 attrib = inode->i_attributes; - attrib &= ~(FILE_ATTRIBUTE_SPARSE_FILE | - FILE_ATTRIBUTE_ENCRYPTED); - if (ntfs_set_ntfs_attrib(ni, (const char *)&attrib, sizeof(attrib), 0)) { @@ -432,13 +349,14 @@ ntfs_3g_set_metadata(ntfs_inode *ni, const struct wim_inode *inode, if (inode_has_security_descriptor(inode) && !(extract_flags & WIMLIB_EXTRACT_FLAG_NO_ACLS)) { + struct SECURITY_CONTEXT sec_ctx = { ctx->vol }; const void *desc; size_t desc_size; desc = sd->descriptors[inode->i_security_id]; desc_size = sd->sizes[inode->i_security_id]; - ret = ntfs_3g_set_security_descriptor(ni, desc, desc_size); + ret = ntfs_set_ntfs_acl(&sec_ctx, ni, desc, desc_size, 0); if (unlikely(ret)) { int err = errno; @@ -449,16 +367,15 @@ ntfs_3g_set_metadata(ntfs_inode *ni, const struct wim_inode *inode, fprintf(wimlib_error_file, "The security descriptor is: "); print_byte_field(desc, desc_size, wimlib_error_file); - fprintf(wimlib_error_file, "\n"); fprintf(wimlib_error_file, - "\nThis error occurred because libntfs-3g thinks " - "the security descriptor is invalid. If you " - "are extracting a Windows 10 image, this may be " - "caused by a known bug in libntfs-3g. See: " - "https://wimlib.net/forums/viewtopic.php?f=1&t=4 " - "for more information.\n\n"); + "\n\nThis error occurred because libntfs-3g thinks " + "the security descriptor is invalid. There " + "are several known bugs with libntfs-3g's " + "security descriptor validation logic in older " + "versions. Please upgrade to NTFS-3G version " + "2016.2.22 or later if you haven't already.\n"); } - return ret; + return WIMLIB_ERR_SET_SECURITY; } } @@ -551,9 +468,10 @@ ntfs_3g_create_directories(struct wim_dentry *root, /* Set the DOS name of any directory that has one. In addition, create * empty attributes for directories that have them. Note that creating - * an empty reparse point attribute must happen *after* setting the - * DOS name in order to work around a case where - * ntfs_set_ntfs_dos_name() fails with EOPNOTSUPP. */ + * an empty reparse point attribute must happen *after* setting the DOS + * name in order to work around a case where ntfs_set_ntfs_dos_name() + * fails with EOPNOTSUPP. This bug was fixed in NTFS-3G version + * 2016.2.22. */ list_for_each_entry(dentry, dentry_list, d_extraction_list_node) { const struct wim_inode *inode = dentry->d_inode; @@ -765,7 +683,7 @@ ntfs_3g_begin_extract_blob_instance(struct blob_descriptor *blob, struct wim_dentry *one_dentry = inode_first_extraction_dentry(inode); ntfschar *stream_name; size_t stream_name_nchars; - ntfs_attr *attr; + ntfs_attr *na; if (unlikely(strm->stream_type == STREAM_TYPE_REPARSE_POINT)) { @@ -808,14 +726,33 @@ ntfs_3g_begin_extract_blob_instance(struct blob_descriptor *blob, /* This should be ensured by extract_blob_list() */ wimlib_assert(ctx->num_open_attrs < MAX_OPEN_FILES); - attr = ntfs_attr_open(ni, AT_DATA, stream_name, stream_name_nchars); - if (!attr) { + na = ntfs_attr_open(ni, AT_DATA, stream_name, stream_name_nchars); + if (!na) { ERROR_WITH_ERRNO("Failed to open data stream of \"%s\"", dentry_full_path(one_dentry)); return WIMLIB_ERR_NTFS_3G; } - ctx->open_attrs[ctx->num_open_attrs++] = attr; - ntfs_attr_truncate_solid(attr, blob->size); + + /* + * Note: there are problems with trying to combine compression with + * sparseness when extracting. For example, doing ntfs_attr_truncate() + * at the end to extend the attribute to its final size actually extends + * to a compression block size boundary rather than to the requested + * size. Until these problems are solved, we always write the full data + * to compressed attributes. We also don't attempt to preallocate space + * for compressed attributes, since we don't know how much space they + * are going to actually need. + */ + ctx->is_sparse_attr[ctx->num_open_attrs] = false; + if (!(na->data_flags & ATTR_COMPRESSION_MASK)) { + if (inode->i_attributes & FILE_ATTRIBUTE_SPARSE_FILE) { + ctx->is_sparse_attr[ctx->num_open_attrs] = true; + ctx->any_sparse_attrs = true; + } else { + ntfs_attr_truncate_solid(na, blob->size); + } + } + ctx->open_attrs[ctx->num_open_attrs++] = na; return 0; } @@ -838,7 +775,7 @@ ntfs_3g_cleanup_blob_extract(struct ntfs_3g_apply_ctx *ctx) } ctx->num_open_inodes = 0; - ctx->offset = 0; + ctx->any_sparse_attrs = false; ctx->reparse_ptr = NULL; ctx->num_reparse_inodes = 0; return ret; @@ -900,11 +837,14 @@ out: return ret; } -/* Note: contrary to its documentation, ntfs_attr_pwrite() can return a short - * count in non-error cases --- specifically, when writing to a compressed - * attribute and the requested count exceeds the size of an NTFS "compression - * block". Therefore, we must continue calling ntfs_attr_pwrite() until all - * bytes have been written or a real error has occurred. */ +/* + * Note: prior to NTFS-3G version 2016.2.22, ntfs_attr_pwrite() could return a + * short count in non-error cases, contrary to its documentation. Specifically, + * a short count could be returned when writing to a compressed attribute and + * the requested count exceeded the size of an NTFS "compression block". + * Therefore, we must continue calling ntfs_attr_pwrite() until all bytes have + * been written or a real error has occurred. + */ static bool ntfs_3g_full_pwrite(ntfs_attr *na, u64 offset, size_t size, const u8 *data) { @@ -921,22 +861,39 @@ ntfs_3g_full_pwrite(ntfs_attr *na, u64 offset, size_t size, const u8 *data) } static int -ntfs_3g_extract_chunk(const void *chunk, size_t size, void *_ctx) +ntfs_3g_extract_chunk(const struct blob_descriptor *blob, u64 offset, + const void *chunk, size_t size, void *_ctx) { struct ntfs_3g_apply_ctx *ctx = _ctx; - - for (unsigned i = 0; i < ctx->num_open_attrs; i++) { - if (!ntfs_3g_full_pwrite(ctx->open_attrs[i], - ctx->offset, size, chunk)) - { - ERROR_WITH_ERRNO("Error writing data to NTFS volume"); - return WIMLIB_ERR_NTFS_3G; + const void * const end = chunk + size; + const void *p; + bool zeroes; + size_t len; + unsigned i; + + /* + * For sparse attributes, only write nonzero regions. This lets the + * filesystem use holes to represent zero regions. + */ + for (p = chunk; p != end; p += len, offset += len) { + zeroes = maybe_detect_sparse_region(p, end - p, &len, + ctx->any_sparse_attrs); + for (i = 0; i < ctx->num_open_attrs; i++) { + if (!zeroes || !ctx->is_sparse_attr[i]) { + if (!ntfs_3g_full_pwrite(ctx->open_attrs[i], + offset, len, p)) + goto err; + } } } + if (ctx->reparse_ptr) ctx->reparse_ptr = mempcpy(ctx->reparse_ptr, chunk, size); - ctx->offset += size; return 0; + +err: + ERROR_WITH_ERRNO("Error writing data to NTFS volume"); + return WIMLIB_ERR_NTFS_3G; } static int @@ -950,6 +907,21 @@ ntfs_3g_end_extract_blob(struct blob_descriptor *blob, int status, void *_ctx) goto out; } + /* Extend sparse attributes to their final size. */ + if (ctx->any_sparse_attrs) { + for (unsigned i = 0; i < ctx->num_open_attrs; i++) { + if (!ctx->is_sparse_attr[i]) + continue; + if (ntfs_attr_truncate(ctx->open_attrs[i], blob->size)) + { + ERROR_WITH_ERRNO("Error extending attribute to " + "final size"); + ret = WIMLIB_ERR_WRITE; + goto out; + } + } + } + for (u32 i = 0; i < ctx->num_reparse_inodes; i++) { ret = ntfs_3g_restore_reparse_point(ctx->ntfs_reparse_inodes[i], ctx->wim_reparse_inodes[i], @@ -992,7 +964,7 @@ ntfs_3g_extract(struct list_head *dentry_list, struct apply_ctx *_ctx) struct wim_dentry *root; int ret; - /* For NTFS-3g extraction mode we require that the dentries to extract + /* For NTFS-3G extraction mode we require that the dentries to extract * form a single tree. */ root = list_first_entry(dentry_list, struct wim_dentry, d_extraction_list_node); @@ -1000,12 +972,22 @@ ntfs_3g_extract(struct list_head *dentry_list, struct apply_ctx *_ctx) /* Mount the NTFS volume. */ vol = ntfs_mount(ctx->common.target, 0); if (!vol) { - ERROR_WITH_ERRNO("Failed to mount \"%s\" with NTFS-3g", + ERROR_WITH_ERRNO("Failed to mount \"%s\" with NTFS-3G", ctx->common.target); return WIMLIB_ERR_NTFS_3G; } ctx->vol = vol; + /* Opening $Secure is required to set security descriptors in NTFS v3.0 + * format, where security descriptors are stored in a per-volume index + * rather than being fully specified for each file. */ + if (ntfs_open_secure(vol) && vol->major_ver >= 3) { + ERROR_WITH_ERRNO("Unable to open security descriptor index of " + "NTFS volume \"%s\"", ctx->common.target); + ret = WIMLIB_ERR_NTFS_3G; + goto out_unmount; + } + /* Create all inodes and aliases, including short names, and set * metadata (attributes, security descriptors, and timestamps). */ @@ -1029,7 +1011,7 @@ ntfs_3g_extract(struct list_head *dentry_list, struct apply_ctx *_ctx) /* Extract blobs. */ struct read_blob_callbacks cbs = { .begin_blob = ntfs_3g_begin_extract_blob, - .consume_chunk = ntfs_3g_extract_chunk, + .continue_blob = ntfs_3g_extract_chunk, .end_blob = ntfs_3g_end_extract_blob, .ctx = ctx, }; @@ -1040,8 +1022,19 @@ ntfs_3g_extract(struct list_head *dentry_list, struct apply_ctx *_ctx) * ntfs_set_ntfs_dos_name() does, but we handle this elsewhere). */ out_unmount: + if (vol->secure_ni) { + ntfs_index_ctx_put(vol->secure_xsii); + ntfs_index_ctx_put(vol->secure_xsdh); + if (ntfs_inode_close(vol->secure_ni) && !ret) { + ERROR_WITH_ERRNO("Failed to close security descriptor " + "index of NTFS volume \"%s\"", + ctx->common.target); + ret = WIMLIB_ERR_NTFS_3G; + } + vol->secure_ni = NULL; + } if (ntfs_umount(ctx->vol, FALSE) && !ret) { - ERROR_WITH_ERRNO("Failed to unmount \"%s\" with NTFS-3g", + ERROR_WITH_ERRNO("Failed to unmount \"%s\" with NTFS-3G", ctx->common.target); ret = WIMLIB_ERR_NTFS_3G; } @@ -1049,15 +1042,9 @@ out_unmount: } const struct apply_operations ntfs_3g_apply_ops = { - .name = "NTFS-3g", + .name = "NTFS-3G", .get_supported_features = ntfs_3g_get_supported_features, .extract = ntfs_3g_extract, .context_size = sizeof(struct ntfs_3g_apply_ctx), .single_tree_only = true, }; - -void -libntfs3g_global_init(void) -{ - ntfs_set_char_encoding(setlocale(LC_ALL, "")); -}