Report some extraction errors on Windows
[wimlib] / src / win32_apply.c
index 89ecd21..9ebc161 100644 (file)
@@ -5,20 +5,18 @@
 /*
  * Copyright (C) 2013, 2014 Eric Biggers
  *
- * This file is part of wimlib, a library for working with WIM files.
+ * This file is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU Lesser General Public License as published by the Free
+ * Software Foundation; either version 3 of the License, or (at your option) any
+ * later version.
  *
- * wimlib is free software; you can redistribute it and/or modify it under the
- * terms of the GNU General Public License as published by the Free
- * Software Foundation; either version 3 of the License, or (at your option)
- * any later version.
- *
- * wimlib is distributed in the hope that it will be useful, but WITHOUT ANY
- * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
- * A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * This file is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
  * details.
  *
- * You should have received a copy of the GNU General Public License
- * along with wimlib; if not, see http://www.gnu.org/licenses/.
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this file; if not, see http://www.gnu.org/licenses/.
  */
 
 #ifdef __WIN32__
@@ -38,6 +36,7 @@
 #include "wimlib/reparse.h"
 #include "wimlib/textfile.h"
 #include "wimlib/xml.h"
+#include "wimlib/wildcard.h"
 #include "wimlib/wimboot.h"
 
 struct win32_apply_ctx {
@@ -53,6 +52,7 @@ struct win32_apply_ctx {
                void *mem_prepopulate_pats;
                u8 wim_lookup_table_hash[SHA1_HASH_SIZE];
                bool wof_running;
+               bool tried_to_load_prepopulate_list;
        } wimboot;
 
        /* Open handle to the target directory  */
@@ -126,7 +126,10 @@ struct win32_apply_ctx {
        unsigned long no_security_descriptors;
 
        /* Number of files for which we couldn't set the short name.  */
-       unsigned long num_short_name_failures;
+       unsigned long num_set_short_name_failures;
+
+       /* Number of files for which we couldn't remove the short name.  */
+       unsigned long num_remove_short_name_failures;
 
        /* Have we tried to enable short name support on the target volume yet?
         */
@@ -192,6 +195,30 @@ get_vol_flags(const wchar_t *target, DWORD *vol_flags_ret,
        }
 }
 
+static const wchar_t *
+current_path(struct win32_apply_ctx *ctx);
+
+static void
+build_extraction_path(const struct wim_dentry *dentry,
+                     struct win32_apply_ctx *ctx);
+
+static int
+report_dentry_apply_error(const struct wim_dentry *dentry,
+                         struct win32_apply_ctx *ctx, int ret)
+{
+       build_extraction_path(dentry, ctx);
+       return report_apply_error(&ctx->common, ret, current_path(ctx));
+}
+
+static inline int
+check_apply_error(const struct wim_dentry *dentry,
+                 struct win32_apply_ctx *ctx, int ret)
+{
+       if (unlikely(ret))
+               ret = report_dentry_apply_error(dentry, ctx, ret);
+       return ret;
+}
+
 static int
 win32_get_supported_features(const wchar_t *target,
                             struct wim_features *supported_features)
@@ -256,6 +283,8 @@ load_prepopulate_pats(struct win32_apply_ctx *ctx)
        void *mem;
        struct text_file_section sec;
 
+       ctx->wimboot.tried_to_load_prepopulate_list = true;
+
        dentry = get_dentry(ctx->common.wim, path, WIMLIB_CASE_INSENSITIVE);
        if (!dentry ||
            (dentry->d_inode->i_attributes & (FILE_ATTRIBUTE_DIRECTORY |
@@ -295,12 +324,11 @@ load_prepopulate_pats(struct win32_apply_ctx *ctx)
        return 0;
 }
 
-/* Returns %true if the path to @dentry matches a pattern in [PrepopulateList]
- * of WimBootCompress.ini.  Otherwise returns %false.
- *
- * @dentry must have had its full path calculated.  */
+/* Returns %true if the specified absolute path to a file in the WIM image
+ * matches a pattern in [PrepopulateList] of WimBootCompress.ini.  Otherwise
+ * returns %false.  */
 static bool
-in_prepopulate_list(struct wim_dentry *dentry,
+in_prepopulate_list(const wchar_t *path, size_t path_nchars,
                    const struct win32_apply_ctx *ctx)
 {
        const struct string_set *pats = ctx->wimboot.prepopulate_pats;
@@ -308,8 +336,158 @@ in_prepopulate_list(struct wim_dentry *dentry,
        if (!pats || !pats->num_strings)
                return false;
 
-       return match_pattern_list(dentry->_full_path,
-                                 wcslen(dentry->_full_path), pats);
+       return match_pattern_list(path, path_nchars, pats);
+}
+
+/* Returns %true if the specified absolute path to a file in the WIM image can
+ * be subject to external backing when extracted.  Otherwise returns %false.  */
+static bool
+can_externally_back_path(const wchar_t *path, size_t path_nchars,
+                        const struct win32_apply_ctx *ctx)
+{
+       if (in_prepopulate_list(path, path_nchars, ctx))
+               return false;
+
+       /* Since we attempt to modify the SYSTEM registry after it's extracted
+        * (see end_wimboot_extraction()), it can't be extracted as externally
+        * backed.  This extends to associated files such as SYSTEM.LOG that
+        * also must be writable in order to write to the registry.  Normally,
+        * SYSTEM is in [PrepopulateList], and the SYSTEM.* files match patterns
+        * in [ExclusionList] and therefore are not captured in the WIM at all.
+        * However, a WIM that wasn't specifically captured in "WIMBoot mode"
+        * may contain SYSTEM.* files.  So to make things "just work", hard-code
+        * the pattern.  */
+       if (match_path(path, path_nchars, L"\\Windows\\System32\\config\\SYSTEM*",
+                      OS_PREFERRED_PATH_SEPARATOR, false))
+               return false;
+
+       return true;
+}
+
+#define WIM_BACKING_NOT_ENABLED                -1
+#define WIM_BACKING_NOT_POSSIBLE       -2
+#define WIM_BACKING_EXCLUDED           -3
+
+static int
+will_externally_back_inode(struct wim_inode *inode, struct win32_apply_ctx *ctx,
+                          const struct wim_dentry **excluded_dentry_ret)
+{
+       struct list_head *next;
+       struct wim_dentry *dentry;
+       struct wim_lookup_table_entry *stream;
+       int ret;
+
+       if (inode->i_can_externally_back)
+               return 0;
+
+       /* This may do redundant checks because the cached value
+        * i_can_externally_back is 2-state (as opposed to 3-state:
+        * unknown/no/yes).  But most files can be externally backed, so this
+        * way is fine.  */
+
+       if (inode->i_attributes & (FILE_ATTRIBUTE_DIRECTORY |
+                                  FILE_ATTRIBUTE_REPARSE_POINT |
+                                  FILE_ATTRIBUTE_ENCRYPTED))
+               return WIM_BACKING_NOT_POSSIBLE;
+
+       stream = inode_unnamed_lte_resolved(inode);
+
+       if (!stream ||
+           stream->resource_location != RESOURCE_IN_WIM ||
+           stream->rspec->wim != ctx->common.wim ||
+           stream->size != stream->rspec->uncompressed_size)
+               return WIM_BACKING_NOT_POSSIBLE;
+
+       /*
+        * We need to check the patterns in [PrepopulateList] against every name
+        * of the inode, in case any of them match.
+        */
+       next = inode->i_extraction_aliases.next;
+       do {
+               dentry = list_entry(next, struct wim_dentry,
+                                   d_extraction_alias_node);
+
+               ret = calculate_dentry_full_path(dentry);
+               if (ret)
+                       return ret;
+
+               if (!can_externally_back_path(dentry->_full_path,
+                                             wcslen(dentry->_full_path), ctx))
+               {
+                       if (excluded_dentry_ret)
+                               *excluded_dentry_ret = dentry;
+                       return WIM_BACKING_EXCLUDED;
+               }
+               next = next->next;
+       } while (next != &inode->i_extraction_aliases);
+
+       inode->i_can_externally_back = 1;
+       return 0;
+}
+
+/*
+ * Determines if the unnamed data stream of a file will be created as an
+ * external backing, as opposed to a standard extraction.
+ */
+static int
+win32_will_externally_back(struct wim_dentry *dentry, struct apply_ctx *_ctx)
+{
+       struct win32_apply_ctx *ctx = (struct win32_apply_ctx *)_ctx;
+
+       if (!(ctx->common.extract_flags & WIMLIB_EXTRACT_FLAG_WIMBOOT))
+               return WIM_BACKING_NOT_ENABLED;
+
+       if (!ctx->wimboot.tried_to_load_prepopulate_list)
+               if (load_prepopulate_pats(ctx) == WIMLIB_ERR_NOMEM)
+                       return WIMLIB_ERR_NOMEM;
+
+       return will_externally_back_inode(dentry->d_inode, ctx, NULL);
+}
+
+static int
+set_external_backing(HANDLE h, struct wim_inode *inode, struct win32_apply_ctx *ctx)
+{
+       int ret;
+       const struct wim_dentry *excluded_dentry;
+
+       ret = will_externally_back_inode(inode, ctx, &excluded_dentry);
+       if (ret > 0) /* Error.  */
+               return ret;
+
+       if (ret < 0 && ret != WIM_BACKING_EXCLUDED)
+               return 0; /* Not externally backing, other than due to exclusion.  */
+
+       if (unlikely(ret == WIM_BACKING_EXCLUDED)) {
+               /* Not externally backing due to exclusion.  */
+               union wimlib_progress_info info;
+
+               build_extraction_path(excluded_dentry, ctx);
+
+               info.wimboot_exclude.path_in_wim = excluded_dentry->_full_path;
+               info.wimboot_exclude.extraction_path = current_path(ctx);
+
+               return call_progress(ctx->common.progfunc,
+                                    WIMLIB_PROGRESS_MSG_WIMBOOT_EXCLUDE,
+                                    &info, ctx->common.progctx);
+       } else {
+               /* Externally backing.  */
+               if (unlikely(!wimboot_set_pointer(h,
+                                                 inode_unnamed_lte_resolved(inode),
+                                                 ctx->wimboot.data_source_id,
+                                                 ctx->wimboot.wim_lookup_table_hash,
+                                                 ctx->wimboot.wof_running)))
+               {
+                       const DWORD err = GetLastError();
+
+                       build_extraction_path(inode_first_extraction_dentry(inode), ctx);
+                       set_errno_from_win32_error(err);
+                       ERROR_WITH_ERRNO("\"%ls\": Couldn't set WIMBoot "
+                                        "pointer data (err=%"PRIu32")",
+                                        current_path(ctx), (u32)err);
+                       return WIMLIB_ERR_WIMBOOT;
+               }
+               return 0;
+       }
 }
 
 /* Calculates the SHA-1 message digest of the WIM's lookup table.  */
@@ -328,9 +506,9 @@ start_wimboot_extraction(struct win32_apply_ctx *ctx)
        int ret;
        WIMStruct *wim = ctx->common.wim;
 
-       ret = load_prepopulate_pats(ctx);
-       if (ret == WIMLIB_ERR_NOMEM)
-               return ret;
+       if (!ctx->wimboot.tried_to_load_prepopulate_list)
+               if (load_prepopulate_pats(ctx) == WIMLIB_ERR_NOMEM)
+                       return WIMLIB_ERR_NOMEM;
 
        if (!wim_info_get_wimboot(wim->wim_info, wim->current_image))
                WARNING("Image is not marked as WIMBoot compatible!");
@@ -627,9 +805,11 @@ prepare_target(struct list_head *dentry_list, struct win32_apply_ctx *ctx)
 
        path_max = compute_path_max(dentry_list);
 
-       /* Add some extra for building Win32 paths for the file encryption APIs
-        * ...  */
-       path_max += 2 + (ctx->target_ntpath.Length / sizeof(wchar_t));
+       /* Add some extra for building Win32 paths for the file encryption APIs,
+        * and ensure we have at least enough to potentially use a 8.3 name for
+        * the last component.  */
+       path_max += max(2 + (ctx->target_ntpath.Length / sizeof(wchar_t)),
+                       8 + 1 + 3);
 
        ctx->pathbuf.MaximumLength = path_max * sizeof(wchar_t);
        ctx->pathbuf.Buffer = MALLOC(ctx->pathbuf.MaximumLength);
@@ -650,11 +830,11 @@ prepare_target(struct list_head *dentry_list, struct win32_apply_ctx *ctx)
 /* When creating an inode that will have a short (DOS) name, we create it using
  * the long name associated with the short name.  This ensures that the short
  * name gets associated with the correct long name.  */
-static const struct wim_dentry *
+static struct wim_dentry *
 first_extraction_alias(const struct wim_inode *inode)
 {
-       const struct list_head *next = inode->i_extraction_aliases.next;
-       const struct wim_dentry *dentry;
+       struct list_head *next = inode->i_extraction_aliases.next;
+       struct wim_dentry *dentry;
 
        do {
                dentry = list_entry(next, struct wim_dentry,
@@ -799,27 +979,104 @@ maybe_clear_encryption_attribute(HANDLE *h_ptr, const struct wim_dentry *dentry,
 /* Try to enable short name support on the target volume.  If successful, return
  * true.  If unsuccessful, issue a warning and return false.  */
 static bool
-try_to_enable_short_names(struct win32_apply_ctx *ctx)
+try_to_enable_short_names(const wchar_t *volume)
 {
+       HANDLE h;
        FILE_FS_PERSISTENT_VOLUME_INFORMATION info;
-       NTSTATUS status;
+       BOOL bret;
+       DWORD bytesReturned;
+
+       h = CreateFile(volume, GENERIC_WRITE,
+                      FILE_SHARE_VALID_FLAGS, NULL, OPEN_EXISTING,
+                      FILE_FLAG_BACKUP_SEMANTICS, NULL);
+       if (h == INVALID_HANDLE_VALUE)
+               goto fail;
 
        info.VolumeFlags = 0;
        info.FlagMask = PERSISTENT_VOLUME_STATE_SHORT_NAME_CREATION_DISABLED;
        info.Version = 1;
        info.Reserved = 0;
 
-       status = (*func_NtFsControlFile)(ctx->h_target, NULL, NULL, NULL,
-                                        &ctx->iosb,
-                                        FSCTL_SET_PERSISTENT_VOLUME_STATE,
-                                        &info, sizeof(info), NULL, 0);
+       bret = DeviceIoControl(h, FSCTL_SET_PERSISTENT_VOLUME_STATE,
+                              &info, sizeof(info), NULL, 0,
+                              &bytesReturned, NULL);
+
+       CloseHandle(h);
+
+       if (!bret)
+               goto fail;
+       return true;
+
+fail:
+       WARNING("Failed to enable short name support on %ls "
+               "(err=%"PRIu32")", volume + 4, (u32)GetLastError());
+       return false;
+}
+
+static NTSTATUS
+remove_conflicting_short_name(const struct wim_dentry *dentry, struct win32_apply_ctx *ctx)
+{
+       wchar_t *name;
+       wchar_t *end;
+       NTSTATUS status;
+       HANDLE h;
+       size_t bufsize = offsetof(FILE_NAME_INFORMATION, FileName) +
+                        (13 * sizeof(wchar_t));
+       u8 buf[bufsize] _aligned_attribute(8);
+       bool retried = false;
+       FILE_NAME_INFORMATION *info = (FILE_NAME_INFORMATION *)buf;
+
+       memset(buf, 0, bufsize);
+
+       /* Build the path with the short name.  */
+       name = &ctx->pathbuf.Buffer[ctx->pathbuf.Length / sizeof(wchar_t)];
+       while (name != ctx->pathbuf.Buffer && *(name - 1) != L'\\')
+               name--;
+       end = mempcpy(name, dentry->short_name, dentry->short_name_nbytes);
+       ctx->pathbuf.Length = ((u8 *)end - (u8 *)ctx->pathbuf.Buffer);
+
+       /* Open the conflicting file (by short name).  */
+       status = (*func_NtOpenFile)(&h, GENERIC_WRITE | DELETE,
+                                   &ctx->attr, &ctx->iosb,
+                                   FILE_SHARE_VALID_FLAGS,
+                                   FILE_OPEN_REPARSE_POINT | FILE_OPEN_FOR_BACKUP_INTENT);
        if (!NT_SUCCESS(status)) {
-               WARNING("Failed to enable short name support on target volume "
-                       "(status=0x%08"PRIx32")", (u32)status);
-               return false;
+               WARNING("Can't open \"%ls\" (status=0x%08"PRIx32")",
+                       current_path(ctx), (u32)status);
+               goto out;
        }
 
-       return true;
+#if 0
+       WARNING("Overriding conflicting short name; path=\"%ls\"",
+               current_path(ctx));
+#endif
+
+       /* Try to remove the short name on the conflicting file.  */
+
+retry:
+       status = (*func_NtSetInformationFile)(h, &ctx->iosb, info, bufsize,
+                                             FileShortNameInformation);
+
+       if (status == STATUS_INVALID_PARAMETER && !retried) {
+
+               /* Microsoft forgot to make it possible to remove short names
+                * until Windows 7.  Oops.  Use a random short name instead.  */
+
+               info->FileNameLength = 12 * sizeof(wchar_t);
+               for (int i = 0; i < 8; i++)
+                       info->FileName[i] = 'A' + (rand() % 26);
+               info->FileName[8] = L'.';
+               info->FileName[9] = L'W';
+               info->FileName[10] = L'L';
+               info->FileName[11] = L'B';
+               info->FileName[12] = L'\0';
+               retried = true;
+               goto retry;
+       }
+       (*func_NtClose)(h);
+out:
+       build_extraction_path(dentry, ctx);
+       return status;
 }
 
 /* Set the short name on the open file @h which has been created at the location
@@ -840,16 +1097,31 @@ set_short_name(HANDLE h, const struct wim_dentry *dentry,
        if (!ctx->common.supported_features.short_names)
                return 0;
 
+       /*
+        * Note: The size of the FILE_NAME_INFORMATION buffer must be such that
+        * FileName contains at least 2 wide characters (4 bytes).  Otherwise,
+        * NtSetInformationFile() will return STATUS_INFO_LENGTH_MISMATCH.  This
+        * is despite the fact that FileNameLength can validly be 0 or 2 bytes,
+        * with the former case being removing the existing short name if
+        * present, rather than setting one.
+        *
+        * The null terminator is seemingly optional, but to be safe we include
+        * space for it and zero all unused space.
+        */
+
        size_t bufsize = offsetof(FILE_NAME_INFORMATION, FileName) +
-                        dentry->short_name_nbytes;
+                        max(dentry->short_name_nbytes, sizeof(wchar_t)) +
+                        sizeof(wchar_t);
        u8 buf[bufsize] _aligned_attribute(8);
        FILE_NAME_INFORMATION *info = (FILE_NAME_INFORMATION *)buf;
        NTSTATUS status;
+       bool tried_to_remove_existing = false;
+
+       memset(buf, 0, bufsize);
 
        info->FileNameLength = dentry->short_name_nbytes;
        memcpy(info->FileName, dentry->short_name, dentry->short_name_nbytes);
 
-
 retry:
        status = (*func_NtSetInformationFile)(h, &ctx->iosb, info, bufsize,
                                              FileShortNameInformation);
@@ -860,16 +1132,52 @@ retry:
                if (dentry->short_name_nbytes == 0)
                        return 0;
                if (!ctx->tried_to_enable_short_names) {
+                       wchar_t volume[7];
+                       int ret;
+
                        ctx->tried_to_enable_short_names = true;
-                       if (try_to_enable_short_names(ctx))
+
+                       ret = win32_get_drive_path(ctx->common.target,
+                                                  volume);
+                       if (ret)
+                               return ret;
+                       if (try_to_enable_short_names(volume))
                                goto retry;
                }
        }
 
+       /*
+        * Short names can conflict in several cases:
+        *
+        * - a file being extracted has a short name conflicting with an
+        *   existing file
+        *
+        * - a file being extracted has a short name conflicting with another
+        *   file being extracted (possible, but shouldn't happen)
+        *
+        * - a file being extracted has a short name that conflicts with the
+        *   automatically generated short name of a file we previously
+        *   extracted, but failed to set the short name for.  Sounds unlikely,
+        *   but this actually does happen fairly often on versions of Windows
+        *   prior to Windows 7 because they do not support removing short names
+        *   from files.
+        */
+       if (unlikely(status == STATUS_OBJECT_NAME_COLLISION) &&
+           dentry->short_name_nbytes && !tried_to_remove_existing)
+       {
+               tried_to_remove_existing = true;
+               status = remove_conflicting_short_name(dentry, ctx);
+               if (NT_SUCCESS(status))
+                       goto retry;
+       }
+
        /* By default, failure to set short names is not an error (since short
         * names aren't too important anymore...).  */
        if (!(ctx->common.extract_flags & WIMLIB_EXTRACT_FLAG_STRICT_SHORT_NAMES)) {
-               ctx->num_short_name_failures++;
+               if (dentry->short_name_nbytes)
+                       ctx->num_set_short_name_failures++;
+               else
+                       ctx->num_remove_short_name_failures++;
                return 0;
        }
 
@@ -1073,16 +1381,17 @@ create_directories(struct list_head *dentry_list,
 
                /* If the root dentry is being extracted, it was already done so
                 * in prepare_target().  */
-               if (dentry_is_root(dentry))
-                       continue;
-
-               ret = create_directory(dentry, ctx);
-               if (ret)
-                       return ret;
+               if (!dentry_is_root(dentry)) {
+                       ret = create_directory(dentry, ctx);
+                       ret = check_apply_error(dentry, ctx, ret);
+                       if (ret)
+                               return ret;
 
-               ret = create_any_empty_ads(dentry, ctx);
-               if (ret)
-                       return ret;
+                       ret = create_any_empty_ads(dentry, ctx);
+                       ret = check_apply_error(dentry, ctx, ret);
+                       if (ret)
+                               return ret;
+               }
 
                ret = report_file_created(&ctx->common);
                if (ret)
@@ -1290,9 +1599,9 @@ create_links(HANDLE h, const struct wim_dentry *first_dentry,
 
 /* Create a nondirectory file, including all links.  */
 static int
-create_nondirectory(const struct wim_inode *inode, struct win32_apply_ctx *ctx)
+create_nondirectory(struct wim_inode *inode, struct win32_apply_ctx *ctx)
 {
-       const struct wim_dentry *first_dentry;
+       struct wim_dentry *first_dentry;
        HANDLE h;
        int ret;
 
@@ -1311,6 +1620,10 @@ create_nondirectory(const struct wim_inode *inode, struct win32_apply_ctx *ctx)
        if (!ret)
                ret = create_links(h, first_dentry, ctx);
 
+       /* "WIMBoot" extraction: set external backing by the WIM file if needed.  */
+       if (!ret && unlikely(ctx->common.extract_flags & WIMLIB_EXTRACT_FLAG_WIMBOOT))
+               ret = set_external_backing(h, inode, ctx);
+
        (*func_NtClose)(h);
        return ret;
 }
@@ -1320,8 +1633,8 @@ create_nondirectory(const struct wim_inode *inode, struct win32_apply_ctx *ctx)
 static int
 create_nondirectories(struct list_head *dentry_list, struct win32_apply_ctx *ctx)
 {
-       const struct wim_dentry *dentry;
-       const struct wim_inode *inode;
+       struct wim_dentry *dentry;
+       struct wim_inode *inode;
        int ret;
 
        list_for_each_entry(dentry, dentry_list, d_extraction_list_node) {
@@ -1329,11 +1642,12 @@ create_nondirectories(struct list_head *dentry_list, struct win32_apply_ctx *ctx
                if (inode->i_attributes & FILE_ATTRIBUTE_DIRECTORY)
                        continue;
                /* Call create_nondirectory() only once per inode  */
-               if (dentry != inode_first_extraction_dentry(inode))
-                       continue;
-               ret = create_nondirectory(inode, ctx);
-               if (ret)
-                       return ret;
+               if (dentry == inode_first_extraction_dentry(inode)) {
+                       ret = create_nondirectory(inode, ctx);
+                       ret = check_apply_error(dentry, ctx, ret);
+                       if (ret)
+                               return ret;
+               }
                ret = report_file_created(&ctx->common);
                if (ret)
                        return ret;
@@ -1350,17 +1664,17 @@ close_handles(struct win32_apply_ctx *ctx)
 
 /* Prepare to read the next stream, which has size @stream_size, into an
  * in-memory buffer.  */
-static int
+static bool
 prepare_data_buffer(struct win32_apply_ctx *ctx, u64 stream_size)
 {
        if (stream_size > ctx->data_buffer_size) {
                /* Larger buffer needed.  */
                void *new_buffer;
                if ((size_t)stream_size != stream_size)
-                       return WIMLIB_ERR_NOMEM;
+                       return false;
                new_buffer = REALLOC(ctx->data_buffer, stream_size);
                if (!new_buffer)
-                       return WIMLIB_ERR_NOMEM;
+                       return false;
                ctx->data_buffer = new_buffer;
                ctx->data_buffer_size = stream_size;
        }
@@ -1368,7 +1682,7 @@ prepare_data_buffer(struct win32_apply_ctx *ctx, u64 stream_size)
         * extract_chunk() that the data buffer needs to be filled while reading
         * the stream data.  */
        ctx->data_buffer_ptr = ctx->data_buffer;
-       return 0;
+       return true;
 }
 
 static int
@@ -1404,8 +1718,10 @@ begin_extract_stream_instance(const struct wim_lookup_table_entry *stream,
                 * with FSCTL_SET_REPARSE_POINT, which requires that all the
                 * data be available.  So, stage the data in a buffer.  */
 
+               if (!prepare_data_buffer(ctx, stream->size))
+                       return WIMLIB_ERR_NOMEM;
                list_add_tail(&dentry->tmp_list, &ctx->reparse_dentries);
-               return prepare_data_buffer(ctx, stream->size);
+               return 0;
        }
 
        /* Encrypted file?  */
@@ -1426,44 +1742,10 @@ begin_extract_stream_instance(const struct wim_lookup_table_entry *stream,
                 * TODO: This isn't sufficient for extremely large encrypted
                 * files.  Perhaps we should create an extra thread to write
                 * such files...  */
+               if (!prepare_data_buffer(ctx, stream->size))
+                       return WIMLIB_ERR_NOMEM;
                list_add_tail(&dentry->tmp_list, &ctx->encrypted_dentries);
-               return prepare_data_buffer(ctx, stream->size);
-       }
-
-       /* Extracting unnamed data stream in WIMBoot mode?  */
-       if (unlikely(ctx->common.extract_flags & WIMLIB_EXTRACT_FLAG_WIMBOOT)
-           && (stream_name_nchars == 0)
-           && (stream->resource_location == RESOURCE_IN_WIM)
-           && (stream->rspec->wim == ctx->common.wim)
-           && (stream->size == stream->rspec->uncompressed_size))
-       {
-               int ret = calculate_dentry_full_path(dentry);
-               if (ret)
-                       return ret;
-               if (in_prepopulate_list(dentry, ctx)) {
-                       union wimlib_progress_info info;
-
-                       info.wimboot_exclude.path_in_wim = dentry->_full_path;
-                       info.wimboot_exclude.extraction_path = current_path(ctx);
-
-                       ret = call_progress(ctx->common.progfunc,
-                                           WIMLIB_PROGRESS_MSG_WIMBOOT_EXCLUDE,
-                                           &info, ctx->common.progctx);
-                       FREE(dentry->_full_path);
-                       dentry->_full_path = NULL;
-                       if (ret)
-                               return ret;
-                       /* Go on and open the file for normal extraction.  */
-               } else {
-                       FREE(dentry->_full_path);
-                       dentry->_full_path = NULL;
-                       return wimboot_set_pointer(&ctx->attr,
-                                                  current_path(ctx),
-                                                  stream,
-                                                  ctx->wimboot.data_source_id,
-                                                  ctx->wimboot.wim_lookup_table_hash,
-                                                  ctx->wimboot.wof_running);
-               }
+               return 0;
        }
 
        if (ctx->num_open_handles == MAX_OPEN_STREAMS) {
@@ -1783,6 +2065,7 @@ begin_extract_stream(struct wim_lookup_table_entry *stream, void *_ctx)
                        dentry = inode_first_extraction_dentry(inode);
                        ret = begin_extract_stream_instance(stream, dentry,
                                                            stream_name, ctx);
+                       ret = check_apply_error(dentry, ctx, ret);
                        if (ret)
                                goto fail;
                } else {
@@ -1798,6 +2081,7 @@ begin_extract_stream(struct wim_lookup_table_entry *stream, void *_ctx)
                                                                    dentry,
                                                                    stream_name,
                                                                    ctx);
+                               ret = check_apply_error(dentry, ctx, ret);
                                if (ret)
                                        goto fail;
                                next = next->next;
@@ -1875,7 +2159,8 @@ end_extract_stream(struct wim_lookup_table_entry *stream, int status, void *_ctx
                              "%"PRIu64" bytes (exceeds %u bytes)",
                              current_path(ctx), stream->size,
                              REPARSE_DATA_MAX_SIZE);
-                       return WIMLIB_ERR_INVALID_REPARSE_DATA;
+                       ret = WIMLIB_ERR_INVALID_REPARSE_DATA;
+                       return check_apply_error(dentry, ctx, ret);
                }
                /* In the WIM format, reparse streams are just the reparse data
                 * and omit the header.  But we can reconstruct the header.  */
@@ -1887,6 +2172,7 @@ end_extract_stream(struct wim_lookup_table_entry *stream, int status, void *_ctx
                        ret = set_reparse_data(dentry, &ctx->rpbuf,
                                               stream->size + REPARSE_DATA_OFFSET,
                                               ctx);
+                       ret = check_apply_error(dentry, ctx, ret);
                        if (ret)
                                return ret;
                }
@@ -1896,6 +2182,7 @@ end_extract_stream(struct wim_lookup_table_entry *stream, int status, void *_ctx
                ctx->encrypted_size = stream->size;
                list_for_each_entry(dentry, &ctx->encrypted_dentries, tmp_list) {
                        ret = extract_encrypted_file(dentry, ctx);
+                       ret = check_apply_error(dentry, ctx, ret);
                        if (ret)
                                return ret;
                }
@@ -1915,33 +2202,100 @@ end_extract_stream(struct wim_lookup_table_entry *stream, int status, void *_ctx
 /* Set the security descriptor @desc, of @desc_size bytes, on the file with open
  * handle @h.  */
 static NTSTATUS
-set_security_descriptor(HANDLE h, const void *desc,
+set_security_descriptor(HANDLE h, const void *_desc,
                        size_t desc_size, struct win32_apply_ctx *ctx)
 {
        SECURITY_INFORMATION info;
        NTSTATUS status;
+       SECURITY_DESCRIPTOR_RELATIVE *desc;
+
+       /*
+        * Ideally, we would just pass in the security descriptor buffer as-is.
+        * But it turns out that Windows can mess up the security descriptor
+        * even when using the low-level NtSetSecurityObject() function:
+        *
+        * - Windows will clear SE_DACL_AUTO_INHERITED if it is set in the
+        *   passed buffer.  To actually get Windows to set
+        *   SE_DACL_AUTO_INHERITED, the application must set the non-persistent
+        *   flag SE_DACL_AUTO_INHERIT_REQ.  As usual, Microsoft didn't bother
+        *   to properly document either of these flags.  It's unclear how
+        *   important SE_DACL_AUTO_INHERITED actually is, but to be safe we use
+        *   the SE_DACL_AUTO_INHERIT_REQ workaround to set it if needed.
+        *
+        * - The above also applies to the equivalent SACL flags,
+        *   SE_SACL_AUTO_INHERITED and SE_SACL_AUTO_INHERIT_REQ.
+        *
+        * - If the application says that it's setting
+        *   DACL_SECURITY_INFORMATION, then Windows sets SE_DACL_PRESENT in the
+        *   resulting security descriptor, even if the security descriptor the
+        *   application provided did not have a DACL.  This seems to be
+        *   unavoidable, since omitting DACL_SECURITY_INFORMATION would cause a
+        *   default DACL to remain.  Fortunately, this behavior seems harmless,
+        *   since the resulting DACL will still be "null" --- but it will be
+        *   "the other representation of null".
+        *
+        * - The above also applies to SACL_SECURITY_INFORMATION and
+        *   SE_SACL_PRESENT.  Again, it's seemingly unavoidable but "harmless"
+        *   that Windows changes the representation of a "null SACL".
+        */
+       if (likely(desc_size <= STACK_MAX)) {
+               desc = alloca(desc_size);
+       } else {
+               desc = MALLOC(desc_size);
+               if (!desc)
+                       return STATUS_NO_MEMORY;
+       }
+
+       memcpy(desc, _desc, desc_size);
+
+       if (likely(desc_size >= 4)) {
+
+               if (desc->Control & SE_DACL_AUTO_INHERITED)
+                       desc->Control |= SE_DACL_AUTO_INHERIT_REQ;
+
+               if (desc->Control & SE_SACL_AUTO_INHERITED)
+                       desc->Control |= SE_SACL_AUTO_INHERIT_REQ;
+       }
+
+       /*
+        * More API insanity.  We want to set the entire security descriptor
+        * as-is.  But all available APIs require specifying the specific parts
+        * of the security descriptor being set.  Especially annoying is that
+        * mandatory integrity labels are part of the SACL, but they aren't set
+        * with SACL_SECURITY_INFORMATION.  Instead, applications must also
+        * specify LABEL_SECURITY_INFORMATION (Windows Vista, Windows 7) or
+        * BACKUP_SECURITY_INFORMATION (Windows 8).  But at least older versions
+        * of Windows don't error out if you provide these newer flags...
+        *
+        * Also, if the process isn't running as Administrator, then it probably
+        * doesn't have SE_RESTORE_PRIVILEGE.  In this case, it will always get
+        * the STATUS_PRIVILEGE_NOT_HELD error by trying to set the SACL, even
+        * if the security descriptor it provided did not have a SACL.  By
+        * default, in this case we try to recover and set as much of the
+        * security descriptor as possible --- potentially excluding the DACL, and
+        * even the owner, as well as the SACL.
+        */
 
-       /* We really just want to set entire the security descriptor as-is, but
-        * all available APIs require specifying the specific parts of the
-        * descriptor being set.  Start out by requesting all parts be set.  If
-        * permissions problems are encountered, fall back to omitting some
-        * parts (first the SACL, then the DACL, then the owner), unless the
-        * WIMLIB_EXTRACT_FLAG_STRICT_ACLS flag has been enabled.  */
        info = OWNER_SECURITY_INFORMATION | GROUP_SECURITY_INFORMATION |
-              DACL_SECURITY_INFORMATION | SACL_SECURITY_INFORMATION;
-
-       /* Prefer NtSetSecurityObject() to SetFileSecurity().  SetFileSecurity()
-        * itself necessarily uses NtSetSecurityObject() as the latter is the
-        * underlying system call for setting security information, but
-        * SetFileSecurity() opens the handle with NtCreateFile() without
-        * FILE_OPEN_FILE_BACKUP_INTENT.  Hence, access checks are done and due
-        * to the Windows security model, even a process running as the
-        * Administrator can have access denied.  (Of course, this not mentioned
-        * in the MS "documentation".)  */
+              DACL_SECURITY_INFORMATION | SACL_SECURITY_INFORMATION |
+              LABEL_SECURITY_INFORMATION | BACKUP_SECURITY_INFORMATION;
+
+
+       /*
+        * It's also worth noting that SetFileSecurity() is unusable because it
+        * doesn't request "backup semantics" when it opens the file internally.
+        * NtSetSecurityObject() seems to be the best function to use in backup
+        * applications.  (SetSecurityInfo() should also work, but it's harder
+        * to use and must call NtSetSecurityObject() internally anyway.
+        * BackupWrite() is theoretically usable as well, but it's inflexible
+        * and poorly documented.)
+        */
+
 retry:
-       status = (*func_NtSetSecurityObject)(h, info, (PSECURITY_DESCRIPTOR)desc);
+       status = (*func_NtSetSecurityObject)(h, info, desc);
        if (NT_SUCCESS(status))
-               return status;
+               goto out_maybe_free_desc;
+
        /* Failed to set the requested parts of the security descriptor.  If the
         * error was permissions-related, try to set fewer parts of the security
         * descriptor, unless WIMLIB_EXTRACT_FLAG_STRICT_ACLS is enabled.  */
@@ -1950,7 +2304,9 @@ retry:
            !(ctx->common.extract_flags & WIMLIB_EXTRACT_FLAG_STRICT_ACLS))
        {
                if (info & SACL_SECURITY_INFORMATION) {
-                       info &= ~SACL_SECURITY_INFORMATION;
+                       info &= ~(SACL_SECURITY_INFORMATION |
+                                 LABEL_SECURITY_INFORMATION |
+                                 BACKUP_SECURITY_INFORMATION);
                        ctx->partial_security_descriptors++;
                        goto retry;
                }
@@ -1972,6 +2328,10 @@ retry:
        if (!(info & SACL_SECURITY_INFORMATION))
                ctx->partial_security_descriptors--;
        ctx->no_security_descriptors++;
+
+out_maybe_free_desc:
+       if (unlikely(desc_size > STACK_MAX))
+               FREE(desc);
        return status;
 }
 
@@ -2098,6 +2458,7 @@ apply_metadata(struct list_head *dentry_list, struct win32_apply_ctx *ctx)
        list_for_each_entry_reverse(dentry, dentry_list, d_extraction_list_node)
        {
                ret = apply_metadata_to_file(dentry, ctx);
+               ret = check_apply_error(dentry, ctx, ret);
                if (ret)
                        return ret;
                ret = report_file_metadata_applied(&ctx->common);
@@ -2113,17 +2474,28 @@ apply_metadata(struct list_head *dentry_list, struct win32_apply_ctx *ctx)
 static void
 do_warnings(const struct win32_apply_ctx *ctx)
 {
-       if (ctx->partial_security_descriptors == 0 &&
-           ctx->no_security_descriptors == 0 &&
-           ctx->num_short_name_failures == 0)
+       if (ctx->partial_security_descriptors == 0
+           && ctx->no_security_descriptors == 0
+           && ctx->num_set_short_name_failures == 0
+       #if 0
+           && ctx->num_remove_short_name_failures == 0
+       #endif
+           )
                return;
 
        WARNING("Extraction to \"%ls\" complete, but with one or more warnings:",
                ctx->common.target);
-       if (ctx->num_short_name_failures) {
+       if (ctx->num_set_short_name_failures) {
                WARNING("- Could not set short names on %lu files or directories",
-                       ctx->num_short_name_failures);
+                       ctx->num_set_short_name_failures);
+       }
+#if 0
+       if (ctx->num_remove_short_name_failures) {
+               WARNING("- Could not remove short names on %lu files or directories"
+                       "          (This is expected on Vista and earlier)",
+                       ctx->num_remove_short_name_failures);
        }
+#endif
        if (ctx->partial_security_descriptors) {
                WARNING("- Could only partially set the security descriptor\n"
                        "            on %lu files or directories.",
@@ -2140,12 +2512,25 @@ do_warnings(const struct win32_apply_ctx *ctx)
        }
 }
 
+static uint64_t
+count_dentries(const struct list_head *dentry_list)
+{
+       const struct list_head *cur;
+       uint64_t count = 0;
+
+       list_for_each(cur, dentry_list)
+               count++;
+
+       return count;
+}
+
 /* Extract files from a WIM image to a directory on Windows  */
 static int
 win32_extract(struct list_head *dentry_list, struct apply_ctx *_ctx)
 {
        int ret;
        struct win32_apply_ctx *ctx = (struct win32_apply_ctx *)_ctx;
+       uint64_t dentry_count;
 
        ret = prepare_target(dentry_list, ctx);
        if (ret)
@@ -2157,7 +2542,11 @@ win32_extract(struct list_head *dentry_list, struct apply_ctx *_ctx)
                        goto out;
        }
 
-       reset_file_progress(&ctx->common);
+       dentry_count = count_dentries(dentry_list);
+
+       ret = start_file_structure_phase(&ctx->common, dentry_count);
+       if (ret)
+               goto out;
 
        ret = create_directories(dentry_list, ctx);
        if (ret)
@@ -2167,6 +2556,10 @@ win32_extract(struct list_head *dentry_list, struct apply_ctx *_ctx)
        if (ret)
                goto out;
 
+       ret = end_file_structure_phase(&ctx->common);
+       if (ret)
+               goto out;
+
        struct read_stream_list_callbacks cbs = {
                .begin_stream      = begin_extract_stream,
                .begin_stream_ctx  = ctx,
@@ -2179,12 +2572,18 @@ win32_extract(struct list_head *dentry_list, struct apply_ctx *_ctx)
        if (ret)
                goto out;
 
-       reset_file_progress(&ctx->common);
+       ret = start_file_metadata_phase(&ctx->common, dentry_count);
+       if (ret)
+               goto out;
 
        ret = apply_metadata(dentry_list, ctx);
        if (ret)
                goto out;
 
+       ret = end_file_metadata_phase(&ctx->common);
+       if (ret)
+               goto out;
+
        if (unlikely(ctx->common.extract_flags & WIMLIB_EXTRACT_FLAG_WIMBOOT)) {
                ret = end_wimboot_extraction(ctx);
                if (ret)
@@ -2212,6 +2611,7 @@ const struct apply_operations win32_apply_ops = {
        .name                   = "Windows",
        .get_supported_features = win32_get_supported_features,
        .extract                = win32_extract,
+       .will_externally_back   = win32_will_externally_back,
        .context_size           = sizeof(struct win32_apply_ctx),
 };