*/
/*
- * Copyright (C) 2012-2016 Eric Biggers
+ * Copyright (C) 2012-2017 Eric Biggers
*
* This file is free software; you can redistribute it and/or modify it under
* the terms of the GNU Lesser General Public License as published by the Free
#include "wimlib/object_id.h"
#include "wimlib/reparse.h"
#include "wimlib/security.h"
-#include "wimlib/security_descriptor.h"
static int
ntfs_3g_get_supported_features(const char *target,
supported_features->archive_files = 1;
supported_features->compressed_files = 1;
supported_features->not_context_indexed_files = 1;
+ supported_features->sparse_files = 1;
supported_features->named_data_streams = 1;
supported_features->hard_links = 1;
supported_features->reparse_points = 1;
ntfs_inode *open_inodes[MAX_OPEN_FILES];
unsigned num_open_inodes;
+ /* For each currently open attribute, whether we're writing to it in
+ * "sparse" mode or not. */
+ bool is_sparse_attr[MAX_OPEN_FILES];
+
+ /* Whether is_sparse_attr[] is true for any currently open attribute */
+ bool any_sparse_attrs;
+
struct reparse_buffer_disk rpbuf;
u8 *reparse_ptr;
- /* Offset in the blob currently being read */
- u64 offset;
-
unsigned num_reparse_inodes;
ntfs_inode *ntfs_reparse_inodes[MAX_OPEN_FILES];
struct wim_inode *wim_reparse_inodes[MAX_OPEN_FILES];
};
-static size_t
-sid_size(const wimlib_SID *sid)
-{
- return offsetof(wimlib_SID, sub_authority) +
- sizeof(le32) * sid->sub_authority_count;
-}
-
-/*
- * sd_fixup - Fix up a Windows NT security descriptor for libntfs-3g.
- *
- * libntfs-3g validates security descriptors before setting them, but old
- * versions contain bugs causing it to reject unusual but valid security
- * descriptors:
- *
- * - Versions before 2013.1.13 reject security descriptors ending with an empty
- * SACL (System Access Control List). This bug can be worked around either by
- * moving the empty SACL earlier in the security descriptor or by removing the
- * SACL entirely. The latter work-around is valid because an empty SACL is
- * equivalent to a "null", or non-existent, SACL.
- * - Versions before 2014.2.15 reject security descriptors ending with an empty
- * DACL (Discretionary Access Control List). This is very similar to the SACL
- * bug. However, removing the DACL is not a valid workaround because this
- * changes the meaning of the security descriptor--- an empty DACL allows no
- * access, whereas a "null" DACL allows all access.
- *
- * If the security descriptor was fixed, this function returns an allocated
- * buffer containing the fixed security descriptor, and its size is updated.
- * Otherwise (or if no memory is available) NULL is returned.
- */
-static void *
-sd_fixup(const void *_desc, size_t *size_p)
-{
- u32 owner_offset, group_offset, dacl_offset, sacl_offset;
- bool owner_valid, group_valid;
- size_t size = *size_p;
- const wimlib_SECURITY_DESCRIPTOR_RELATIVE *desc = _desc;
- wimlib_SECURITY_DESCRIPTOR_RELATIVE *desc_new;
- const wimlib_SID *owner, *group, *sid;
-
- /* Don't attempt to fix clearly invalid security descriptors. */
- if (size < sizeof(wimlib_SECURITY_DESCRIPTOR_RELATIVE))
- return NULL;
-
- if (le16_to_cpu(desc->control) & wimlib_SE_DACL_PRESENT)
- dacl_offset = le32_to_cpu(desc->dacl_offset);
- else
- dacl_offset = 0;
-
- if (le16_to_cpu(desc->control) & wimlib_SE_SACL_PRESENT)
- sacl_offset = le32_to_cpu(desc->sacl_offset);
- else
- sacl_offset = 0;
-
- /* Check if the security descriptor will be affected by one of the bugs.
- * If not, do nothing and return. */
- if (!((sacl_offset != 0 && sacl_offset == size - sizeof(wimlib_ACL)) ||
- (dacl_offset != 0 && dacl_offset == size - sizeof(wimlib_ACL))))
- return NULL;
-
- owner_offset = le32_to_cpu(desc->owner_offset);
- group_offset = le32_to_cpu(desc->group_offset);
- owner = (const wimlib_SID*)((const u8*)desc + owner_offset);
- group = (const wimlib_SID*)((const u8*)desc + group_offset);
-
- /* We'll try to move the owner or group SID to the end of the security
- * descriptor to avoid the bug. This is only possible if at least one
- * is valid. */
- owner_valid = (owner_offset != 0) &&
- (owner_offset % 4 == 0) &&
- (owner_offset <= size - sizeof(SID)) &&
- (owner_offset + sid_size(owner) <= size) &&
- (owner_offset >= sizeof(wimlib_SECURITY_DESCRIPTOR_RELATIVE));
- group_valid = (group_offset != 0) &&
- (group_offset % 4 == 0) &&
- (group_offset <= size - sizeof(SID)) &&
- (group_offset + sid_size(group) <= size) &&
- (group_offset >= sizeof(wimlib_SECURITY_DESCRIPTOR_RELATIVE));
- if (owner_valid) {
- sid = owner;
- } else if (group_valid) {
- sid = group;
- } else {
- return NULL;
- }
-
- desc_new = MALLOC(size + sid_size(sid));
- if (!desc_new)
- return NULL;
-
- memcpy(desc_new, desc, size);
- if (owner_valid)
- desc_new->owner_offset = cpu_to_le32(size);
- else if (group_valid)
- desc_new->group_offset = cpu_to_le32(size);
- memcpy((u8*)desc_new + size, sid, sid_size(sid));
- *size_p = size + sid_size(sid);
- return desc_new;
-}
-
-/* Set the security descriptor @desc of size @desc_size on the NTFS inode @ni.
- */
-static int
-ntfs_3g_set_security_descriptor(ntfs_inode *ni, const void *desc, size_t desc_size)
-{
- struct SECURITY_CONTEXT sec_ctx;
- void *desc_fixed = NULL;
- int ret = 0;
-
- memset(&sec_ctx, 0, sizeof(sec_ctx));
- sec_ctx.vol = ni->vol;
-
-retry:
- if (ntfs_set_ntfs_acl(&sec_ctx, ni, desc, desc_size, 0)) {
- if (desc_fixed == NULL) {
- desc_fixed = sd_fixup(desc, &desc_size);
- if (desc_fixed != NULL) {
- desc = desc_fixed;
- goto retry;
- }
- }
- ret = WIMLIB_ERR_SET_SECURITY;
- }
-
- FREE(desc_fixed);
- return ret;
-}
-
static int
ntfs_3g_set_timestamps(ntfs_inode *ni, const struct wim_inode *inode)
{
ret = -1;
}
utf16le_put_tstr(dos_name);
- if (ret) {
+ if (unlikely(ret)) {
+ int err = errno;
ERROR_WITH_ERRNO("Failed to set DOS name of \"%s\" in NTFS "
"volume", dentry_full_path(dentry));
+ if (err == EILSEQ) {
+ ERROR("This error may have been caused by a known "
+ "bug in libntfs-3g where it is unable to set "
+ "DOS names on files whose long names contain "
+ "unpaired surrogate characters. This bug "
+ "was fixed in NTFS-3G version 2017.3.23.");
+ }
+ if (err == EINVAL) {
+ utf16lechar c =
+ dentry->d_name[dentry->d_name_nbytes / 2 - 1];
+ if (c == cpu_to_le16('.') || c == cpu_to_le16(' ')) {
+ ERROR("This error was probably caused by a "
+ "known bug in libntfs-3g where it is "
+ "unable to set DOS names on files whose "
+ "long names end with a dot or space "
+ "character. This bug was fixed in "
+ "NTFS-3G version 2017.3.23.");
+ }
+ }
ret = WIMLIB_ERR_SET_SHORT_NAME;
goto out_close;
}
"tag. The preceding error may have been caused "
"by a known bug in libntfs-3g where it does not "
"correctly validate non-Microsoft reparse "
- "points. This bug may be fixed in the 2016 "
- "release of libntfs-3g.");
+ "points. This bug was fixed in NTFS-3G version "
+ "2016.2.22.");
}
return WIMLIB_ERR_SET_REPARSE_DATA;
}
if (!(extract_flags & WIMLIB_EXTRACT_FLAG_NO_ATTRIBUTES)) {
u32 attrib = inode->i_attributes;
- attrib &= ~(FILE_ATTRIBUTE_SPARSE_FILE |
- FILE_ATTRIBUTE_ENCRYPTED);
-
if (ntfs_set_ntfs_attrib(ni, (const char *)&attrib,
sizeof(attrib), 0))
{
if (inode_has_security_descriptor(inode)
&& !(extract_flags & WIMLIB_EXTRACT_FLAG_NO_ACLS))
{
+ struct SECURITY_CONTEXT sec_ctx = { ctx->vol };
const void *desc;
size_t desc_size;
desc = sd->descriptors[inode->i_security_id];
desc_size = sd->sizes[inode->i_security_id];
- ret = ntfs_3g_set_security_descriptor(ni, desc, desc_size);
+ ret = ntfs_set_ntfs_acl(&sec_ctx, ni, desc, desc_size, 0);
if (unlikely(ret)) {
int err = errno;
fprintf(wimlib_error_file,
"The security descriptor is: ");
print_byte_field(desc, desc_size, wimlib_error_file);
- fprintf(wimlib_error_file, "\n");
fprintf(wimlib_error_file,
- "\nThis error occurred because libntfs-3g thinks "
- "the security descriptor is invalid. If you "
- "are extracting a Windows 10 image, this may be "
- "caused by a known bug in libntfs-3g. See: "
- "https://wimlib.net/forums/viewtopic.php?f=1&t=4 "
- "for more information.\n\n");
+ "\n\nThis error occurred because libntfs-3g thinks "
+ "the security descriptor is invalid. There "
+ "are several known bugs with libntfs-3g's "
+ "security descriptor validation logic in older "
+ "versions. Please upgrade to NTFS-3G version "
+ "2016.2.22 or later if you haven't already.\n");
}
- return ret;
+ return WIMLIB_ERR_SET_SECURITY;
}
}
/* Set the DOS name of any directory that has one. In addition, create
* empty attributes for directories that have them. Note that creating
- * an empty reparse point attribute must happen *after* setting the
- * DOS name in order to work around a case where
- * ntfs_set_ntfs_dos_name() fails with EOPNOTSUPP. */
+ * an empty reparse point attribute must happen *after* setting the DOS
+ * name in order to work around a case where ntfs_set_ntfs_dos_name()
+ * fails with EOPNOTSUPP. This bug was fixed in NTFS-3G version
+ * 2016.2.22. */
list_for_each_entry(dentry, dentry_list, d_extraction_list_node) {
const struct wim_inode *inode = dentry->d_inode;
struct wim_dentry *one_dentry = inode_first_extraction_dentry(inode);
ntfschar *stream_name;
size_t stream_name_nchars;
- ntfs_attr *attr;
+ ntfs_attr *na;
if (unlikely(strm->stream_type == STREAM_TYPE_REPARSE_POINT)) {
/* This should be ensured by extract_blob_list() */
wimlib_assert(ctx->num_open_attrs < MAX_OPEN_FILES);
- attr = ntfs_attr_open(ni, AT_DATA, stream_name, stream_name_nchars);
- if (!attr) {
+ na = ntfs_attr_open(ni, AT_DATA, stream_name, stream_name_nchars);
+ if (!na) {
ERROR_WITH_ERRNO("Failed to open data stream of \"%s\"",
dentry_full_path(one_dentry));
return WIMLIB_ERR_NTFS_3G;
}
- ctx->open_attrs[ctx->num_open_attrs++] = attr;
- ntfs_attr_truncate_solid(attr, blob->size);
+
+ /*
+ * Note: there are problems with trying to combine compression with
+ * sparseness when extracting. For example, doing ntfs_attr_truncate()
+ * at the end to extend the attribute to its final size actually extends
+ * to a compression block size boundary rather than to the requested
+ * size. Until these problems are solved, we always write the full data
+ * to compressed attributes. We also don't attempt to preallocate space
+ * for compressed attributes, since we don't know how much space they
+ * are going to actually need.
+ */
+ ctx->is_sparse_attr[ctx->num_open_attrs] = false;
+ if (!(na->data_flags & ATTR_COMPRESSION_MASK)) {
+ if (inode->i_attributes & FILE_ATTRIBUTE_SPARSE_FILE) {
+ ctx->is_sparse_attr[ctx->num_open_attrs] = true;
+ ctx->any_sparse_attrs = true;
+ } else {
+ ntfs_attr_truncate_solid(na, blob->size);
+ }
+ }
+ ctx->open_attrs[ctx->num_open_attrs++] = na;
return 0;
}
}
ctx->num_open_inodes = 0;
- ctx->offset = 0;
+ ctx->any_sparse_attrs = false;
ctx->reparse_ptr = NULL;
ctx->num_reparse_inodes = 0;
return ret;
return ret;
}
-/* Note: contrary to its documentation, ntfs_attr_pwrite() can return a short
- * count in non-error cases --- specifically, when writing to a compressed
- * attribute and the requested count exceeds the size of an NTFS "compression
- * block". Therefore, we must continue calling ntfs_attr_pwrite() until all
- * bytes have been written or a real error has occurred. */
+/*
+ * Note: prior to NTFS-3G version 2016.2.22, ntfs_attr_pwrite() could return a
+ * short count in non-error cases, contrary to its documentation. Specifically,
+ * a short count could be returned when writing to a compressed attribute and
+ * the requested count exceeded the size of an NTFS "compression block".
+ * Therefore, we must continue calling ntfs_attr_pwrite() until all bytes have
+ * been written or a real error has occurred.
+ */
static bool
ntfs_3g_full_pwrite(ntfs_attr *na, u64 offset, size_t size, const u8 *data)
{
}
static int
-ntfs_3g_extract_chunk(const void *chunk, size_t size, void *_ctx)
+ntfs_3g_extract_chunk(const struct blob_descriptor *blob, u64 offset,
+ const void *chunk, size_t size, void *_ctx)
{
struct ntfs_3g_apply_ctx *ctx = _ctx;
-
- for (unsigned i = 0; i < ctx->num_open_attrs; i++) {
- if (!ntfs_3g_full_pwrite(ctx->open_attrs[i],
- ctx->offset, size, chunk))
- {
- ERROR_WITH_ERRNO("Error writing data to NTFS volume");
- return WIMLIB_ERR_NTFS_3G;
+ const void * const end = chunk + size;
+ const void *p;
+ bool zeroes;
+ size_t len;
+ unsigned i;
+
+ /*
+ * For sparse attributes, only write nonzero regions. This lets the
+ * filesystem use holes to represent zero regions.
+ */
+ for (p = chunk; p != end; p += len, offset += len) {
+ zeroes = maybe_detect_sparse_region(p, end - p, &len,
+ ctx->any_sparse_attrs);
+ for (i = 0; i < ctx->num_open_attrs; i++) {
+ if (!zeroes || !ctx->is_sparse_attr[i]) {
+ if (!ntfs_3g_full_pwrite(ctx->open_attrs[i],
+ offset, len, p))
+ goto err;
+ }
}
}
+
if (ctx->reparse_ptr)
ctx->reparse_ptr = mempcpy(ctx->reparse_ptr, chunk, size);
- ctx->offset += size;
return 0;
+
+err:
+ ERROR_WITH_ERRNO("Error writing data to NTFS volume");
+ return WIMLIB_ERR_NTFS_3G;
}
static int
goto out;
}
+ /* Extend sparse attributes to their final size. */
+ if (ctx->any_sparse_attrs) {
+ for (unsigned i = 0; i < ctx->num_open_attrs; i++) {
+ if (!ctx->is_sparse_attr[i])
+ continue;
+ if (ntfs_attr_truncate(ctx->open_attrs[i], blob->size))
+ {
+ ERROR_WITH_ERRNO("Error extending attribute to "
+ "final size");
+ ret = WIMLIB_ERR_WRITE;
+ goto out;
+ }
+ }
+ }
+
for (u32 i = 0; i < ctx->num_reparse_inodes; i++) {
ret = ntfs_3g_restore_reparse_point(ctx->ntfs_reparse_inodes[i],
ctx->wim_reparse_inodes[i],
}
ctx->vol = vol;
+ /* Opening $Secure is required to set security descriptors in NTFS v3.0
+ * format, where security descriptors are stored in a per-volume index
+ * rather than being fully specified for each file. */
+ if (ntfs_open_secure(vol) && vol->major_ver >= 3) {
+ ERROR_WITH_ERRNO("Unable to open security descriptor index of "
+ "NTFS volume \"%s\"", ctx->common.target);
+ ret = WIMLIB_ERR_NTFS_3G;
+ goto out_unmount;
+ }
+
/* Create all inodes and aliases, including short names, and set
* metadata (attributes, security descriptors, and timestamps). */
/* Extract blobs. */
struct read_blob_callbacks cbs = {
.begin_blob = ntfs_3g_begin_extract_blob,
- .consume_chunk = ntfs_3g_extract_chunk,
+ .continue_blob = ntfs_3g_extract_chunk,
.end_blob = ntfs_3g_end_extract_blob,
.ctx = ctx,
};
* ntfs_set_ntfs_dos_name() does, but we handle this elsewhere). */
out_unmount:
+ if (vol->secure_ni) {
+ ntfs_index_ctx_put(vol->secure_xsii);
+ ntfs_index_ctx_put(vol->secure_xsdh);
+ if (ntfs_inode_close(vol->secure_ni) && !ret) {
+ ERROR_WITH_ERRNO("Failed to close security descriptor "
+ "index of NTFS volume \"%s\"",
+ ctx->common.target);
+ ret = WIMLIB_ERR_NTFS_3G;
+ }
+ vol->secure_ni = NULL;
+ }
if (ntfs_umount(ctx->vol, FALSE) && !ret) {
ERROR_WITH_ERRNO("Failed to unmount \"%s\" with NTFS-3G",
ctx->common.target);
.context_size = sizeof(struct ntfs_3g_apply_ctx),
.single_tree_only = true,
};
-
-void
-libntfs3g_global_init(void)
-{
- ntfs_set_char_encoding(setlocale(LC_ALL, ""));
-}