*/
/*
- * Copyright (C) 2012, 2013, 2014, 2015 Eric Biggers
+ * Copyright (C) 2012-2018 Eric Biggers
*
* This file is free software; you can redistribute it and/or modify it under
* the terms of the GNU Lesser General Public License as published by the Free
* details.
*
* You should have received a copy of the GNU Lesser General Public License
- * along with this file; if not, see http://www.gnu.org/licenses/.
+ * along with this file; if not, see https://www.gnu.org/licenses/.
*/
/*
* wimlib_extract_pathlist(). Internally, all end up calling
* do_wimlib_extract_paths() and extract_trees().
*
- * Although wimlib supports multiple extraction modes/backends (NTFS-3g, UNIX,
+ * Although wimlib supports multiple extraction modes/backends (NTFS-3G, UNIX,
* Win32), this file does not itself have code to extract files or directories
* to any specific target; instead, it handles generic functionality and relies
* on lower-level callback functions declared in `struct apply_operations' to do
#include <errno.h>
#include <fcntl.h>
+#include <stdlib.h>
#include <sys/stat.h>
#include <unistd.h>
#include "wimlib/endianness.h"
#include "wimlib/error.h"
#include "wimlib/metadata.h"
+#include "wimlib/object_id.h"
#include "wimlib/pathlist.h"
#include "wimlib/paths.h"
#include "wimlib/pattern.h"
#include "wimlib/unix_data.h"
#include "wimlib/wim.h"
#include "wimlib/win32.h" /* for realpath() equivalent */
+#include "wimlib/xattr.h"
#include "wimlib/xml.h"
#define WIMLIB_EXTRACT_FLAG_FROM_PIPE 0x80000000
/* Keep in sync with wimlib.h */
#define WIMLIB_EXTRACT_MASK_PUBLIC \
(WIMLIB_EXTRACT_FLAG_NTFS | \
+ WIMLIB_EXTRACT_FLAG_RECOVER_DATA | \
WIMLIB_EXTRACT_FLAG_UNIX_DATA | \
WIMLIB_EXTRACT_FLAG_NO_ACLS | \
WIMLIB_EXTRACT_FLAG_STRICT_ACLS | \
return end_file_phase(ctx, WIMLIB_PROGRESS_MSG_EXTRACT_METADATA);
}
+/* Are all bytes in the specified buffer zero? */
+static bool
+is_all_zeroes(const u8 *p, const size_t size)
+{
+ const u8 * const end = p + size;
+
+ for (; (uintptr_t)p % WORDBYTES && p != end; p++)
+ if (*p)
+ return false;
+
+ for (; end - p >= WORDBYTES; p += WORDBYTES)
+ if (*(const machine_word_t *)p)
+ return false;
+
+ for (; p != end; p++)
+ if (*p)
+ return false;
+
+ return true;
+}
+
+/*
+ * Sparse regions should be detected at the granularity of the filesystem block
+ * size. For now just assume 4096 bytes, which is the default block size on
+ * NTFS and most Linux filesystems.
+ */
+#define SPARSE_UNIT 4096
+
+/*
+ * Detect whether the specified buffer begins with a region of all zero bytes.
+ * Return %true if a zero region was found or %false if a nonzero region was
+ * found, and sets *len_ret to the length of the region. This operates at a
+ * granularity of SPARSE_UNIT bytes, meaning that to extend a zero region, there
+ * must be SPARSE_UNIT zero bytes with no interruption, but to extend a nonzero
+ * region, just one nonzero byte in the next SPARSE_UNIT bytes is sufficient.
+ *
+ * Note: besides compression, the WIM format doesn't yet have a way to
+ * efficiently represent zero regions, so that's why we need to detect them
+ * ourselves. Things will still fall apart badly on extremely large sparse
+ * files, but this is a start...
+ */
+bool
+detect_sparse_region(const void *data, size_t size, size_t *len_ret)
+{
+ const void *p = data;
+ const void * const end = data + size;
+ size_t len = 0;
+ bool zeroes = false;
+
+ while (p != end) {
+ size_t n = min(end - p, SPARSE_UNIT);
+ bool z = is_all_zeroes(p, n);
+
+ if (len != 0 && z != zeroes)
+ break;
+ zeroes = z;
+ len += n;
+ p += n;
+ }
+
+ *len_ret = len;
+ return zeroes;
+}
+
#define PWM_FOUND_WIM_HDR (-1)
/* Read the header for a blob in a pipable WIM. If @pwm_hdr_ret is not NULL,
&& (blob->out_refcnt))
{
wim_reshdr_to_desc_and_blob(&reshdr, ctx->wim, &rdesc, blob);
- ret = read_blob_with_sha1(blob, cbs);
+ ret = read_blob_with_sha1(blob, cbs,
+ ctx->extract_flags &
+ WIMLIB_EXTRACT_FLAG_RECOVER_DATA);
blob_unset_is_located_in_wim_resource(blob);
if (ret)
return ret;
return 0;
}
+static int
+handle_pwm_metadata_resource(WIMStruct *pwm, int image, bool is_needed)
+{
+ struct blob_descriptor *blob;
+ struct wim_reshdr reshdr;
+ struct wim_resource_descriptor *rdesc;
+ int ret;
+
+ ret = WIMLIB_ERR_NOMEM;
+ blob = new_blob_descriptor();
+ if (!blob)
+ goto out;
+
+ ret = read_pwm_blob_header(pwm, blob->hash, &reshdr, NULL);
+ if (ret)
+ goto out;
+
+ ret = WIMLIB_ERR_INVALID_PIPABLE_WIM;
+ if (!(reshdr.flags & WIM_RESHDR_FLAG_METADATA)) {
+ ERROR("Expected metadata resource, but found non-metadata "
+ "resource");
+ goto out;
+ }
+
+ ret = WIMLIB_ERR_NOMEM;
+ rdesc = MALLOC(sizeof(*rdesc));
+ if (!rdesc)
+ goto out;
+
+ wim_reshdr_to_desc_and_blob(&reshdr, pwm, rdesc, blob);
+ pwm->refcnt++;
+
+ ret = WIMLIB_ERR_NOMEM;
+ pwm->image_metadata[image - 1] = new_unloaded_image_metadata(blob);
+ if (!pwm->image_metadata[image - 1])
+ goto out;
+ blob = NULL;
+
+ /* If the metadata resource is for the image being extracted, then parse
+ * it and save the metadata in memory. Otherwise, skip over it. */
+ if (is_needed)
+ ret = select_wim_image(pwm, image);
+ else
+ ret = skip_wim_resource(rdesc);
+out:
+ free_blob_descriptor(blob);
+ return ret;
+}
+
/* Creates a temporary file opened for writing. The open file descriptor is
* returned in @fd_ret and its name is returned in @name_ret (dynamically
* allocated). */
create_temporary_file(struct filedes *fd_ret, tchar **name_ret)
{
tchar *name;
- int open_flags;
int raw_fd;
+#ifdef _WIN32
retry:
- name = ttempnam(NULL, T("wimlib"));
+ name = _wtempnam(NULL, L"wimlib");
if (!name) {
ERROR_WITH_ERRNO("Failed to create temporary filename");
return WIMLIB_ERR_NOMEM;
}
-
- open_flags = O_WRONLY | O_CREAT | O_EXCL | O_BINARY;
-#ifdef __WIN32__
- open_flags |= _O_SHORT_LIVED;
-#endif
- raw_fd = topen(name, open_flags, 0600);
+ raw_fd = _wopen(name, O_WRONLY | O_CREAT | O_EXCL | O_BINARY |
+ _O_SHORT_LIVED, 0600);
+ if (raw_fd < 0 && errno == EEXIST) {
+ FREE(name);
+ goto retry;
+ }
+#else /* _WIN32 */
+ const char *tmpdir = getenv("TMPDIR");
+ if (!tmpdir)
+ tmpdir = P_tmpdir;
+ name = MALLOC(strlen(tmpdir) + 1 + 6 + 6 + 1);
+ if (!name)
+ return WIMLIB_ERR_NOMEM;
+ sprintf(name, "%s/wimlibXXXXXX", tmpdir);
+ raw_fd = mkstemp(name);
+#endif /* !_WIN32 */
if (raw_fd < 0) {
- if (errno == EEXIST) {
- FREE(name);
- goto retry;
- }
ERROR_WITH_ERRNO("Failed to create temporary file "
"\"%"TS"\"", name);
FREE(name);
}
static int
-begin_extract_blob_wrapper(struct blob_descriptor *blob, void *_ctx)
+begin_extract_blob(struct blob_descriptor *blob, void *_ctx)
{
struct apply_ctx *ctx = _ctx;
- ctx->cur_blob = blob;
- ctx->cur_blob_offset = 0;
-
if (unlikely(blob->out_refcnt > MAX_OPEN_FILES))
return create_temporary_file(&ctx->tmpfile_fd, &ctx->tmpfile_name);
}
static int
-extract_chunk_wrapper(const void *chunk, size_t size, void *_ctx)
+extract_chunk(const struct blob_descriptor *blob, u64 offset,
+ const void *chunk, size_t size, void *_ctx)
{
struct apply_ctx *ctx = _ctx;
union wimlib_progress_info *progress = &ctx->progress;
+ bool last = (offset + size == blob->size);
int ret;
- ctx->cur_blob_offset += size;
-
if (likely(ctx->supported_features.hard_links)) {
progress->extract.completed_bytes +=
- (u64)size * ctx->cur_blob->out_refcnt;
- if (ctx->cur_blob_offset == ctx->cur_blob->size)
- progress->extract.completed_streams += ctx->cur_blob->out_refcnt;
+ (u64)size * blob->out_refcnt;
+ if (last)
+ progress->extract.completed_streams += blob->out_refcnt;
} else {
const struct blob_extraction_target *targets =
- blob_extraction_targets(ctx->cur_blob);
- for (u32 i = 0; i < ctx->cur_blob->out_refcnt; i++) {
+ blob_extraction_targets(blob);
+ for (u32 i = 0; i < blob->out_refcnt; i++) {
const struct wim_inode *inode = targets[i].inode;
const struct wim_dentry *dentry;
inode_for_each_extraction_alias(dentry, inode) {
progress->extract.completed_bytes += size;
- if (ctx->cur_blob_offset == ctx->cur_blob->size)
+ if (last)
progress->extract.completed_streams++;
}
}
return ret;
}
- return call_consume_chunk(chunk, size, ctx->saved_cbs);
+ return call_continue_blob(blob, offset, chunk, size, ctx->saved_cbs);
}
/* Copy the blob's data from the temporary file to each of its targets.
for (u32 i = 0; i < orig_blob->out_refcnt; i++) {
tmpfile_blob.inline_blob_extraction_targets[0] = targets[i];
- ret = read_blob_with_cbs(&tmpfile_blob, cbs);
+ ret = read_blob_with_cbs(&tmpfile_blob, cbs, false);
if (ret)
return ret;
}
return 0;
}
+static void
+warn_about_corrupted_file(struct wim_dentry *dentry,
+ const struct wim_inode_stream *stream)
+{
+ WARNING("Corruption in %s\"%"TS"\"! Extracting anyway since data recovery mode is enabled.",
+ stream_is_unnamed_data_stream(stream) ? "" : "alternate stream of ",
+ dentry_full_path(dentry));
+}
+
static int
-end_extract_blob_wrapper(struct blob_descriptor *blob, int status, void *_ctx)
+end_extract_blob(struct blob_descriptor *blob, int status, void *_ctx)
{
struct apply_ctx *ctx = _ctx;
+ if ((ctx->extract_flags & WIMLIB_EXTRACT_FLAG_RECOVER_DATA) &&
+ !status && blob->corrupted) {
+ const struct blob_extraction_target *targets =
+ blob_extraction_targets(blob);
+ for (u32 i = 0; i < blob->out_refcnt; i++) {
+ struct wim_dentry *dentry =
+ inode_first_extraction_dentry(targets[i].inode);
+
+ warn_about_corrupted_file(dentry, targets[i].stream);
+ }
+ }
+
if (unlikely(filedes_valid(&ctx->tmpfile_fd))) {
filedes_close(&ctx->tmpfile_fd);
if (!status)
extract_blob_list(struct apply_ctx *ctx, const struct read_blob_callbacks *cbs)
{
struct read_blob_callbacks wrapper_cbs = {
- .begin_blob = begin_extract_blob_wrapper,
- .consume_chunk = extract_chunk_wrapper,
- .end_blob = end_extract_blob_wrapper,
+ .begin_blob = begin_extract_blob,
+ .continue_blob = extract_chunk,
+ .end_blob = end_extract_blob,
.ctx = ctx,
};
ctx->saved_cbs = cbs;
if (ctx->extract_flags & WIMLIB_EXTRACT_FLAG_FROM_PIPE) {
return read_blobs_from_pipe(ctx, &wrapper_cbs);
} else {
+ int flags = VERIFY_BLOB_HASHES;
+
+ if (ctx->extract_flags & WIMLIB_EXTRACT_FLAG_RECOVER_DATA)
+ flags |= RECOVER_DATA;
+
return read_blob_list(&ctx->blob_list,
offsetof(struct blob_descriptor,
extraction_list),
- &wrapper_cbs, VERIFY_BLOB_HASHES);
+ &wrapper_cbs, flags);
}
}
* unnamed data stream only. */
static int
extract_dentry_to_stdout(struct wim_dentry *dentry,
- const struct blob_table *blob_table)
+ const struct blob_table *blob_table, int extract_flags)
{
struct wim_inode *inode = dentry->d_inode;
struct blob_descriptor *blob;
struct filedes _stdout;
+ bool recover = (extract_flags & WIMLIB_EXTRACT_FLAG_RECOVER_DATA);
+ int ret;
if (inode->i_attributes & (FILE_ATTRIBUTE_REPARSE_POINT |
FILE_ATTRIBUTE_DIRECTORY |
}
filedes_init(&_stdout, STDOUT_FILENO);
- return extract_blob_to_fd(blob, &_stdout);
+ ret = extract_blob_to_fd(blob, &_stdout, recover);
+ if (ret)
+ return ret;
+ if (recover && blob->corrupted)
+ warn_about_corrupted_file(dentry,
+ inode_get_unnamed_data_stream(inode));
+ return 0;
}
static int
extract_dentries_to_stdout(struct wim_dentry **dentries, size_t num_dentries,
- const struct blob_table *blob_table)
+ const struct blob_table *blob_table,
+ int extract_flags)
{
for (size_t i = 0; i < num_dentries; i++) {
- int ret = extract_dentry_to_stdout(dentries[i], blob_table);
+ int ret = extract_dentry_to_stdout(dentries[i], blob_table,
+ extract_flags);
if (ret)
return ret;
}
FREE(blob->blob_extraction_targets);
}
-#ifdef __WIN32__
+#ifdef _WIN32
static const utf16lechar replacement_char = cpu_to_le16(0xfffd);
#else
static const utf16lechar replacement_char = cpu_to_le16('?');
return true;
for (i = 0; i < num_chars; i++) {
switch (le16_to_cpu(name[i])) {
- #ifdef __WIN32__
+ #ifdef _WIN32
case '\x01'...'\x1F':
case '\\':
case ':':
if (!ctx->supported_features.case_sensitive_filenames) {
struct wim_dentry *other;
- list_for_each_entry(other, &dentry->d_ci_conflict_list,
- d_ci_conflict_list)
- {
+ dentry_for_each_ci_match(other, dentry) {
if (will_extract_dentry(other)) {
if (ctx->extract_flags &
WIMLIB_EXTRACT_FLAG_ALL_CASE_CONFLICTS) {
inode_tally_features(const struct wim_inode *inode,
struct wim_features *features)
{
- if (inode->i_attributes & FILE_ATTRIBUTE_ARCHIVE)
- features->archive_files++;
+ if (inode->i_attributes & FILE_ATTRIBUTE_READONLY)
+ features->readonly_files++;
if (inode->i_attributes & FILE_ATTRIBUTE_HIDDEN)
features->hidden_files++;
if (inode->i_attributes & FILE_ATTRIBUTE_SYSTEM)
features->system_files++;
+ if (inode->i_attributes & FILE_ATTRIBUTE_ARCHIVE)
+ features->archive_files++;
if (inode->i_attributes & FILE_ATTRIBUTE_COMPRESSED)
features->compressed_files++;
if (inode->i_attributes & FILE_ATTRIBUTE_ENCRYPTED) {
features->security_descriptors++;
if (inode_has_unix_data(inode))
features->unix_data++;
+ if (inode_has_object_id(inode))
+ features->object_ids++;
+ if (inode_has_xattrs(inode))
+ features->xattrs++;
}
/* Tally features necessary to extract a dentry and the corresponding inode. */
/* File attributes. */
if (!(extract_flags & WIMLIB_EXTRACT_FLAG_NO_ATTRIBUTES)) {
- /* Note: Don't bother the user about FILE_ATTRIBUTE_ARCHIVE.
- * We're an archive program, so theoretically we can do what we
- * want with it. */
+
+ if (required_features->readonly_files &&
+ !supported_features->readonly_files)
+ WARNING("Ignoring FILE_ATTRIBUTE_READONLY of %lu files",
+ required_features->readonly_files);
if (required_features->hidden_files &&
!supported_features->hidden_files)
WARNING("Ignoring FILE_ATTRIBUTE_SYSTEM of %lu files",
required_features->system_files);
+ /* Note: Don't bother the user about FILE_ATTRIBUTE_ARCHIVE.
+ * We're an archive program, so theoretically we can do what we
+ * want with it. */
+
if (required_features->compressed_files &&
!supported_features->compressed_files)
WARNING("Ignoring FILE_ATTRIBUTE_COMPRESSED of %lu files",
WARNING("Ignoring Windows NT security descriptors of %lu files",
required_features->security_descriptors);
- /* UNIX data. */
- if ((extract_flags & WIMLIB_EXTRACT_FLAG_UNIX_DATA) &&
- required_features->unix_data && !supported_features->unix_data)
+ /* Standard UNIX metadata */
+ if (required_features->unix_data &&
+ (!supported_features->unix_data ||
+ !(extract_flags & WIMLIB_EXTRACT_FLAG_UNIX_DATA)))
{
- ERROR("Extraction backend does not support UNIX data!");
- return WIMLIB_ERR_UNSUPPORTED;
+ if (extract_flags & WIMLIB_EXTRACT_FLAG_UNIX_DATA) {
+ ERROR("Requested UNIX metadata extraction, but "
+ "extraction backend does not support it!");
+ return WIMLIB_ERR_UNSUPPORTED;
+ }
+ WARNING("Ignoring UNIX metadata (uid/gid/mode/rdev) of %lu files%"TS,
+ required_features->unix_data,
+ (supported_features->unix_data ?
+ T("\n (use --unix-data mode to extract these)") : T("")));
}
- if (required_features->unix_data &&
- !(extract_flags & WIMLIB_EXTRACT_FLAG_UNIX_DATA))
+ /* Extended attributes */
+ if (required_features->xattrs &&
+ (!supported_features->xattrs ||
+ (supported_features->unix_data &&
+ !(extract_flags & WIMLIB_EXTRACT_FLAG_UNIX_DATA))))
{
- WARNING("Ignoring UNIX metadata of %lu files",
- required_features->unix_data);
+ WARNING("Ignoring extended attributes of %lu files%"TS,
+ required_features->xattrs,
+ (supported_features->xattrs ?
+ T("\n (use --unix-data mode to extract these)") : T("")));
+ }
+
+ /* Object IDs. */
+ if (required_features->object_ids && !supported_features->object_ids) {
+ WARNING("Ignoring object IDs of %lu files",
+ required_features->object_ids);
}
/* DOS Names. */
if (extract_flags & WIMLIB_EXTRACT_FLAG_NTFS)
return &ntfs_3g_apply_ops;
#endif
-#ifdef __WIN32__
+#ifdef _WIN32
return &win32_apply_ops;
#else
return &unix_apply_ops;
if (extract_flags & WIMLIB_EXTRACT_FLAG_TO_STDOUT) {
ret = extract_dentries_to_stdout(trees, num_trees,
- wim->blob_table);
+ wim->blob_table,
+ extract_flags);
goto out;
}
if (errno == EEXIST)
return 0;
-#ifdef __WIN32__
+#ifdef _WIN32
/* _wmkdir() fails with EACCES if called on a drive root directory. */
if (errno == EACCES)
return 0;
#ifndef WITH_NTFS_3G
if (extract_flags & WIMLIB_EXTRACT_FLAG_NTFS) {
- ERROR("wimlib was compiled without support for NTFS-3g, so\n"
+ ERROR("wimlib was compiled without support for NTFS-3G, so\n"
" it cannot apply a WIM image directly to an NTFS volume.");
return WIMLIB_ERR_UNSUPPORTED;
}
#endif
if (extract_flags & WIMLIB_EXTRACT_FLAG_WIMBOOT) {
-#ifdef __WIN32__
+#ifdef _WIN32
if (!wim->filename)
return WIMLIB_ERR_NO_FILENAME;
#else
WIMLIB_EXTRACT_FLAG_COMPACT_XPRESS16K |
WIMLIB_EXTRACT_FLAG_COMPACT_LZX))
{
- #ifdef __WIN32__
+ #ifdef _WIN32
int count = 0;
count += ((extract_flags & WIMLIB_EXTRACT_FLAG_COMPACT_XPRESS4K) != 0);
count += ((extract_flags & WIMLIB_EXTRACT_FLAG_COMPACT_XPRESS8K) != 0);
}
static const tchar * const filename_forbidden_chars =
-T(
-#ifdef __WIN32__
-"<>:\"/\\|?*"
+#ifdef _WIN32
+T("<>:\"/\\|?*");
#else
-"/"
+T("/");
#endif
-);
/* This function checks if it is okay to use a WIM image's name as a directory
* name. */
ret = read_path_list_file(path_list_file, &paths, &num_paths, &mem);
if (ret) {
ERROR("Failed to read path list file \"%"TS"\"",
- path_list_file);
+ path_list_file ? path_list_file : T("<stdin>"));
return ret;
}
/* Load the needed metadata resource. */
for (i = 1; i <= pwm->hdr.image_count; i++) {
- struct wim_image_metadata *imd;
- struct wim_reshdr reshdr;
- struct wim_resource_descriptor *metadata_rdesc;
-
- imd = pwm->image_metadata[i - 1];
-
- ret = WIMLIB_ERR_NOMEM;
- imd->metadata_blob = new_blob_descriptor();
- if (!imd->metadata_blob)
- goto out_wimlib_free;
-
- imd->metadata_blob->is_metadata = 1;
-
- ret = read_pwm_blob_header(pwm, imd->metadata_blob->hash,
- &reshdr, NULL);
+ ret = handle_pwm_metadata_resource(pwm, i, i == image);
if (ret)
goto out_wimlib_free;
-
- if (!(reshdr.flags & WIM_RESHDR_FLAG_METADATA)) {
- ERROR("Expected metadata resource, but found "
- "non-metadata resource");
- ret = WIMLIB_ERR_INVALID_PIPABLE_WIM;
- goto out_wimlib_free;
- }
-
- ret = WIMLIB_ERR_NOMEM;
- metadata_rdesc = MALLOC(sizeof(struct wim_resource_descriptor));
- if (!metadata_rdesc)
- goto out_wimlib_free;
- wim_reshdr_to_desc_and_blob(&reshdr, pwm, metadata_rdesc,
- imd->metadata_blob);
-
- if (i == image) {
- /* Metadata resource is for the image being extracted.
- * Parse it and save the metadata in memory. */
- ret = read_metadata_resource(imd);
- if (ret)
- goto out_wimlib_free;
- imd->modified = 1;
- } else {
- /* Metadata resource is not for the image being
- * extracted. Skip over it. */
- ret = skip_wim_resource(metadata_rdesc);
- if (ret)
- goto out_wimlib_free;
- }
}
/* Extract the image. */
extract_flags |= WIMLIB_EXTRACT_FLAG_FROM_PIPE;