Merge branch with pipable WIM support
authorEric Biggers <ebiggers3@gmail.com>
Tue, 13 Aug 2013 06:19:23 +0000 (01:19 -0500)
committerEric Biggers <ebiggers3@gmail.com>
Tue, 13 Aug 2013 06:19:23 +0000 (01:19 -0500)
Many changes; see changes to NEWS for additional details.

78 files changed:
Makefile.am
NEWS
README
archlinux/PKGBUILD
configure.ac
debian/changelog
doc/imagex-apply.1.in
doc/imagex-capture.1.in
doc/imagex-delete.1.in
doc/imagex-dir.1.in
doc/imagex-export.1.in
doc/imagex-extract.1.in
doc/imagex-info.1.in
doc/imagex-join.1.in
doc/imagex-mount.1.in
doc/imagex-optimize.1.in
doc/imagex-split.1.in
doc/imagex-update.1.in
doc/imagex.1.in
doc/mkwinpeimg.1.in
include/wimlib.h
include/wimlib/apply.h
include/wimlib/dentry.h
include/wimlib/file_io.h
include/wimlib/header.h
include/wimlib/integrity.h
include/wimlib/lookup_table.h
include/wimlib/reparse.h
include/wimlib/resource.h
include/wimlib/wim.h
include/wimlib/win32_common.h
include/wimlib/write.h
include/wimlib/xml.h
include/wimlib_tchar.h
make-release
programs/imagex-win32.c
programs/imagex-win32.h
programs/imagex.c
rpm/wimlib-without-fuse-or-ntfs-3g.spec
rpm/wimlib.spec
src/add_image.c
src/decompress.c
src/delete_image.c
src/dentry.c
src/export_image.c
src/extract.c
src/file_io.c
src/hardlink.c
src/header.c
src/integrity.c
src/join.c
src/lookup_table.c
src/lzx-compress.c
src/lzx-decompress.c
src/metadata_resource.c
src/mount_image.c
src/ntfs-3g_apply.c
src/paths.c
src/reparse.c
src/resource.c
src/security.c
src/split.c
src/unix_apply.c
src/update_image.c
src/util.c
src/verify.c
src/wim.c
src/win32_apply.c
src/win32_capture.c
src/win32_common.c
src/win32_replacements.c
src/write.c
src/xml.c
src/xpress-compress.c
src/xpress-decompress.c
tests/test-imagex
tests/test-imagex-capture_and_apply
tests/tests-common.sh

index 8f6dc96..b6d4da9 100644 (file)
@@ -15,7 +15,7 @@ endif
 
 lib_LTLIBRARIES = libwim.la
 
-libwim_la_LDFLAGS = -version-info 8:0:1 $(WINDOWS_LDFLAGS)
+libwim_la_LDFLAGS = -version-info 9:0:0 $(WINDOWS_LDFLAGS)
 
 libwim_la_SOURCES =            \
        src/add_image.c         \
diff --git a/NEWS b/NEWS
index 1d8e643..d0d2575 100644 (file)
--- a/NEWS
+++ b/NEWS
@@ -1,5 +1,35 @@
 Only the most important changes more recent than version 0.6 are noted here.
 
+Version 1.5.0:
+       Added support for "pipable" WIMs.  This support allows capturing images
+       to standard output or applying images from standard input, but they are
+       not compatible with Microsoft's software and are not created by default.
+       See documentation for --pipable flag of `wimlib-imagex capture' for more
+       information.  Two new functions have been added to the library to fully
+       support this: wimlib_write_to_fd() and wimlib_extract_image_from_pipe().
+
+       wimlib now preserve WIM integrity tables by default, even if
+       WIMLIB_WRITE_FLAG_CHECK_INTEGRITY is not specified.  This affects
+       wimlib-imagex's behavior when --check is not specified.
+
+       Security descriptors are now extracted correctly on Windows.
+
+       `mkwinpeimg' now supports grabbing files from the WAIK supplement rather
+       than the WAIK itself.
+
+       The test suite no longer fails when run in a locale where the decimal
+       separator is not a period.
+
+       WIMLIB_PROGRESS_MSG_EXTRACT_DENTRY has been removed and
+       WIMLIB_EXTRACT_FLAG_VERBOSE re-reserved for future use.
+
+       The extraction code has been rewritten and it will now be easier to
+       support new features on all supported backends (currently Win32, UNIX,
+       and NTFS-3g).  For example, hard-linked extraction mode (--hardlink) is
+       now supported on all backends, not just UNIX.
+
+       A few changes were made to the error codes returned by library routines.
+
 Version 1.4.2:
        Fixed bug in `wimlib-imagex export' that made it impossible to export an
        image from a WIM that is readonly at the filesystem level.
diff --git a/README b/README
index 881833b..f473d72 100644 (file)
--- a/README
+++ b/README
@@ -1,6 +1,6 @@
                                   INTRODUCTION
 
-This is wimlib version 1.4.2 (June 2013).  wimlib is a C library for creating,
+This is wimlib version 1.5.0 (August 2013).  wimlib is a C library for creating,
 modifying, extracting, and mounting files in the Windows Imaging Format (WIM
 files).  These files are normally created by using the `imagex.exe' utility on
 Windows, but wimlib is distributed with a free implementation of ImageX called
index 837caf6..2887ca6 100644 (file)
@@ -1,7 +1,7 @@
 # Maintainer:  Eric Biggers <ebiggers3 at gmail dot com>
 
 pkgname=wimlib
-pkgver=1.4.2
+pkgver=1.5.0
 pkgrel=1
 pkgdesc="A library to extract, create, and modify WIM files"
 arch=("i686" "x86_64")
index 50ca230..8d96cc3 100644 (file)
@@ -1,4 +1,4 @@
-AC_INIT([wimlib], [1.4.2], [ebiggers3@gmail.com])
+AC_INIT([wimlib], [1.5.0], [ebiggers3@gmail.com])
 AC_CONFIG_SRCDIR([src/wim.c])
 AC_CONFIG_MACRO_DIR([m4])
 AC_CONFIG_AUX_DIR([build-aux])
index f7aaf1f..0990ed1 100644 (file)
@@ -1,3 +1,9 @@
+wimlib (1.5.0-1) unstable; urgency=low
+
+  * Update to v1.5.0
+
+ -- Eric Biggers <ebiggers3@gmail.com>  Tue, 13 Aug 2013 00:58:52 -0500
+
 wimlib (1.4.2-1) unstable; urgency=low
 
   * Update to v1.4.2
index 439a3bd..4b1a79a 100644 (file)
@@ -1,11 +1,12 @@
-.TH IMAGEX "1" "June 2013" "@IMAGEX_PROGNAME@ @VERSION@" "User Commands"
+.TH IMAGEX "1" "August 2013" "@IMAGEX_PROGNAME@ @VERSION@" "User Commands"
 .SH NAME
 @IMAGEX_PROGNAME@-apply \- Extract one image, or all images, from a WIM archive
 .SH SYNOPSIS
 \fB@IMAGEX_PROGNAME@ apply\fR \fIWIMFILE\fR [\fIIMAGE\fR] \fITARGET\fR [\fIOPTION\fR...]
 .SH DESCRIPTION
 \fB@IMAGEX_PROGNAME@ apply\fR extracts an image, or all images, from the Windows
-Imaging (WIM) file \fIWIMFILE\fR.
+Imaging (WIM) file \fIWIMFILE\fR.  \fIWIMFILE\fR may be "-" to read the WIM from
+standard input, but see \fBPIPABLE WIMS\fR.
 .PP
 This command is designed to extract, or "apply", one or more full WIM images.
 If you instead want to extract only certain files or directories contained in a
@@ -237,6 +238,29 @@ To apply the first image of this split WIM to the directory "dir", run:
 @IMAGEX_PROGNAME@ apply mywim.swm 1 dir --ref="mywim*.swm"
 .RE
 .PP
+As a special case, if you are applying an image from standard input from a split
+WIM that is also pipable (as described in \fBPIPABLE WIMS\fR), the \fB--ref\fR
+option is unneeded; instead you must ensure that all the split WIM parts are
+concatenated together on standard input.  They can be provided in any other,
+with the exception of the first part, which must be first.
+.SH PIPABLE WIMS
+As of wimlib 1.5.0, \fB@IMAGEX_PROGNAME@ apply\fR supports applying a WIM from a
+nonseekable file, such as a pipe, provided that the WIM was captured with
+\fB--pipable\fR (see \fB@IMAGEX_PROGNAME@ capture\fR(1)).  To use standard input
+as the WIM, specify "-" as \fIWIMFILE\fR.  A useful use of this ability is to
+apply an image from a WIM while streaming it from a webserver; for example, to
+apply the first image from a WIM file to a NTFS volume on /dev/sda1:
+.PP
+.RS
+wget -O - http://myserver/mywim.wim | @IMAGEX_PROGNAME@ apply - 1 /dev/sda1
+.RE
+.PP
+Note: WIM files are \fInot\fR pipable by default; you have to explicitly capture
+them with \fB--pipable\fR, and they are \fInot\fR compatible with Microsoft's
+software.  See \fB@IMAGEX_PROGNAME@ capture\fR(1) for more information.
+.PP
+It is possible to apply an image from a pipable WIM split into multiple parts;
+see \fBSPLIT WIMS\fR.
 .SH OPTIONS
 .TP 6
 \fB--check\fR
@@ -266,12 +290,12 @@ Print the path to of each file or directory within the WIM image as it is
 extracted.
 .TP
 \fB--hardlink\fR
-(UNIX only) When extracting a file from the WIM that is identical to a file that
-has already extracted, create a hard link rather than creating a separate file.
-This option causes all identical files to be hard-linked, overriding the hard
-link groups that are specified in the WIM image(s).  In the case of extracting
-all images from the WIM, files may be hard-linked even if they are in different
-WIM images.  This option is not available in the NTFS extraction mode.
+When extracting a file from the WIM that is identical to a file that has already
+extracted, create a hard link rather than creating a separate file.  This option
+causes all identical files to be hard-linked, overriding the hard link groups
+that are specified in the WIM image(s).  In the case of extracting all images
+from the WIM, files may be hard-linked even if they are in different WIM images.
+This option is not available in the NTFS extraction mode.
 .TP
 \fB--symlink\fR
 (UNIX only) This option is similar to \fB--hardlink\fR, except symbolic links are created
@@ -330,9 +354,6 @@ Microsoft's ImageX does.  Please note that this is separate from the integrity
 table of the WIM, which provides SHA1 message digests over raw chunks of the
 entire WIM file and is checked separately if the \fB--check\fR option is
 specified.
-.PP
-You cannot use \fB@IMAGEX_PROGNAME@ apply\fR to apply a WIM from a pipe (such as standard
-input) because the WIM file format is not designed for this.
 .SH EXAMPLES
 .SS Applying a WIM image to a directory (both UNIX and Windows)
 Extract the first image from the Windows PE image from the Windows Vista/7/8
index 8930fe4..cb65165 100644 (file)
@@ -1,4 +1,4 @@
-.TH IMAGEX "1" "June 2013" "@IMAGEX_PROGNAME@ @VERSION@" "User Commands"
+.TH IMAGEX "1" "August 2013" "@IMAGEX_PROGNAME@ @VERSION@" "User Commands"
 .SH NAME
 @IMAGEX_PROGNAME@-capture, @IMAGEX_PROGNAME@-append \- Create or append a WIM image
 .SH SYNOPSIS
@@ -31,6 +31,10 @@ which a WIM image is to be captured.
 the new image.  If \fIIMAGE_NAME\fR is not given, it is taken to be the same as
 the base name of \fISOURCE\fR.  If \fIIMAGE_DESCRIPTION\fR is not given, no
 description is given to the new image.
+.PP
+As a special case, if \fIWIMFILE\fR is "-", the \fB--pipable\fR option is
+assumed and the WIM file is written to standard output in a special pipable
+format.   See the documentation for \fB--pipable\fR for more details.
 .SH NORMAL MODE (UNIX)
 This section documents how files are captured from a directory on UNIX.  See
 \fBWINDOWS VERSION\fR for the corresponding documentation for the Windows
@@ -131,9 +135,13 @@ anyway).
 Specifies that the new image is to be made the bootable image of the WIM archive.
 .TP
 \fB--check\fR
-Include an integrity table in the new WIM file or the modified WIM file.  If
-this option is not specified, no integrity table is included in \fIWIMFILE\fR,
-even if there was one before in the case of \fB@IMAGEX_PROGNAME@ append\fR.
+For \fB@IMAGEX_PROGNAME@ append\fR, check the integrity of \fIWIMFILE\fR if an
+integrity was present.  Furthermore, include an integrity table in the new WIM
+file (\fB@IMAGEX_PROGNAME@ capture\fR) or the modified WIM file
+(\fB@IMAGEX_PROGNAME@ append\fR).  If this option is not specified, no integrity
+table is included in a WIM file created with \fB@IMAGEX_PROGNAME@ capture\fR,
+while a WIM file updated with \fB@IMAGEX_PROGNAME@ append\fR will be written
+with an integrity table if and only if one was present before.
 .TP
 \fB--compress\fR=\fITYPE\fR
 Specifies the compression type for the new WIM file.  This flag is only valid
@@ -220,7 +228,8 @@ used:
 is done by adding a special alternate data stream to each directory entry that
 contains this information.  Please note that this flag is for convenience only,
 in case you want to use \fB@IMAGEX_PROGNAME@\fR to archive files on UNIX.
-Microsoft's software will not understand this special information.
+Microsoft's software will not understand this special information.  You also may
+run into problems when applying an image with UNIX data from a pipable WIM.
 .TP
 \fB--no-acls\fR
 Do not capture files' security descriptors.  This option is available in the
@@ -300,6 +309,43 @@ input rather than an external file.
 .IP ""
 The NTFS capture mode cannot be used with \fB--source-list\fR, as only capturing
 a full NTFS volume is supported.
+.TP
+\fB--pipable\fR
+Create a "pipable" WIM, which can be applied fully sequentially, including from
+a pipe.  An image in the resulting WIM can be applied with \fB@IMAGEX_PROGNAME@
+apply\fR, either normally by specifying the WIM file name, or with
+\fB@IMAGEX_PROGNAME@ apply -\fR to read the WIM from standard input.  See
+\fB@IMAGEX_PROGNAME@ apply\fR(1) for more details.
+.IP ""
+For append operations, this option will result in a full rebuild of the WIM to
+make it pipable.  For capture operations, the captured WIM is simply created as
+pipable.  Beware that the more images you add to a pipable WIM, the less
+efficient piping it will be, since more unneeded data will be sent through the
+pipe.
+.IP ""
+When wimlib creates a pipable WIM, it carefully re-arranges the components of
+the WIM so that they can be read sequentially, and also makes several other
+modifications.  As a result, these "pipable" WIMs are \fInot compatible with
+Microsoft's software\fR, so keep this in mind if you're going to use them.  If
+desired, you can use \fB@IMAGEX_PROGNAME@ optimize --not-pipable\fR to re-write
+a pipable WIM as a regular WIM.  (\fB@IMAGEX_PROGNAME@ export\fR also provides
+the capability to export images from a pipable WIM into a non-pipable WIM, or
+vice versa.)
+.IP ""
+For the most part, wimlib operates on pipable WIMs transparently.  You can
+modify them, add or delete images, export images, and even create split pipable
+WIMs.  The main disadvantages are that appending is (currently) less efficient,
+and also they aren't compatible with Microsoft's software.
+.IP ""
+\fB@IMAGEX_PROGNAME@ capture\fR and \fB@IMAGEX_PROGNAME@ append\fR can both
+write a pipable WIM directly to standard output; this is done automatically if
+\fIWIMFILE\fR is specified as "-".  (In that case, \fB--pipable\fR is assumed.)
+.TP
+\fB--not-pipable\fR
+Ensure the resulting WIM is in the normal, non-pipable WIM format.  This is the
+default for \fB@IMAGEX_PROGNAME@ capture\fR, except when writing to standard
+output, and for \fB@IMAGEX_PROGNAME@ append\fR, except when appending to a WIM
+that is already pipable.
 .SH NOTES
 \fB@IMAGEX_PROGNAME@ append\fR does not support appending an image to a split WIM.
 .PP
@@ -308,9 +354,9 @@ specify a special WIM format.  A WIM file can contain images captured using
 different modes.  However, all images in a WIM must have the same compression
 type, and \fB@IMAGEX_PROGNAME@\fR always enforces this.
 .PP
-\fB@IMAGEX_PROGNAME@\fR writes WIMs having the version number 0x10d00 and a compressed
-stream chunk size of 32768.  The only WIMs I've seen that are different from
-this are some pre-Vista WIMs that had a different version number.
+\fB@IMAGEX_PROGNAME@\fR writes WIMs having the version number 0x10d00 and a
+compressed stream chunk size of 32768.  The only WIMs I've seen that are
+different from this are some pre-Vista WIMs that had a different version number.
 .PP
 It is safe to abort an \fB@IMAGEX_PROGNAME@ append\fR command partway through;
 however, after doing this, it is recommended to run \fB@IMAGEX_PROGNAME@
@@ -349,5 +395,12 @@ integrity table to be discarded.
 @IMAGEX_PROGNAME@ append /dev/sda2 mywim.wim --check "Windows 7"
 .RE
 .PP
+Capture a WIM from a NTFS volume and write it directly to standard output, using
+the wimlib-specific pipable WIM format:
+.RS
+.PP
+@IMAGEX_PROGNAME@ capture /dev/sda1 -
+.RE
+.PP
 .SH SEE ALSO
 .BR @IMAGEX_PROGNAME@ (1)
index 3dfd674..b6158c4 100644 (file)
@@ -1,4 +1,4 @@
-.TH IMAGEX "1" "June 2013" "@IMAGEX_PROGNAME@ @VERSION@" "User Commands"
+.TH IMAGEX "1" "August 2013" "@IMAGEX_PROGNAME@ @VERSION@" "User Commands"
 .SH NAME
 @IMAGEX_PROGNAME@-delete \- Delete an image from a WIM archive
 .SH SYNOPSIS
@@ -25,9 +25,11 @@ WIM and have a WIM with 0 images, although such a file wouldn't be very useful.
 .TP 6
 \fB--check\fR
 When reading \fIWIMFILE\fR, verify its integrity if the integrity table is
-present; additionally, when rewriting \fIWIMFILE\fR after the specified image was
-deleted, write an integrity table.  If this option is not specified, no integrity
-table is included in the new WIM file, even if there was one before.
+present; additionally, when rewriting \fIWIMFILE\fR after the specified image
+was deleted, always write an integrity table.  If this option is not specified,
+the integrity of \fIWIMFILE\fR will not be checked when it's opened, but an
+integrity table will be written in the updated WIM if and only if one was
+present before.
 .TP 6
 \fB--soft\fR
 Perform a "soft delete".  Specifying this flag overrides the default behavior of
index 92534bd..a000340 100644 (file)
@@ -1,4 +1,4 @@
-.TH IMAGEX "1" "June 2013" "@IMAGEX_PROGNAME@ @VERSION@" "User Commands"
+.TH IMAGEX "1" "August 2013" "@IMAGEX_PROGNAME@ @VERSION@" "User Commands"
 .SH NAME
 @IMAGEX_PROGNAME@-dir \- Show the files contained in a WIM archive
 .SH SYNOPSIS
index ee071ea..daca7ea 100644 (file)
@@ -1,4 +1,4 @@
-.TH IMAGEX "1" "June 2013" "@IMAGEX_PROGNAME@ @VERSION@" "User Commands"
+.TH IMAGEX "1" "August 2013" "@IMAGEX_PROGNAME@ @VERSION@" "User Commands"
 .SH NAME
 @IMAGEX_PROGNAME@-export \- Exports an image from a WIM archive to an existing or new WIM archive
 .SH SYNOPSIS
@@ -29,6 +29,10 @@ image being exported to \fIDEST_WIMFILE\fR.  The default is its description in
 from split WIMs.  However, you cannot export an image to a split WIM.  See
 \fBSPLIT WIMS\fR.
 .PP
+\fB@IMAGEX_PROGNAME@ export\fR also supports exporting images from a non-pipable
+WIM into a (possibly new) pipable WIM, and vice versa.  See \fB--pipable\fR and
+\fB--not-pipable\fR.
+.PP
 .SH OPTIONS
 .TP 6
 \fB--boot\fR
@@ -43,9 +47,17 @@ error.
 \fB--check\fR
 When reading \fISRC_WIMFILE\fR, and \fIDEST_WIMFILE\fR if it exists, verify the
 file's integrity if the integrity table is present; additionally, when writing
-\fIDEST_WIMFILE\fR with the new image added, write an integrity table.  If this
-option is not specified, no integrity table is included in \fIDEST_WIMFILE\fR,
-even if there was one before.
+\fIDEST_WIMFILE\fR with the new image(s) added, write an integrity table.
+If neither \fB--check\fR nor \fB--nocheck\fR is specified, an integrity
+table is included in \fIDEST_WIMFILE\fR if and only if \fIDEST_WIMFILE\fR
+already existed and it had an integrity table before.
+.TP
+\fB--nocheck\fR
+When writing \fIDEST_WIMFILE\fR with the new image(s) added, do not write an
+integrity table.
+If neither \fB--check\fR nor \fB--nocheck\fR is specified, an integrity
+table is included in \fIDEST_WIMFILE\fR if and only if \fIDEST_WIMFILE\fR
+already existed and it had an integrity table before.
 .TP
 \fB--compress\fR=\fITYPE\fR
 Specifies the compression type for \fIDEST_WIMFILE\fR.  This is only valid if
@@ -74,6 +86,18 @@ little bit of space that would otherwise be left as a hole in the WIM.  Also see
 \fB--ref\fR="\fIGLOB\fR"
 File glob of additional split WIM parts that are part of the split WIM being
 exported.  See \fBSPLIT_WIMS\fR.
+.TP
+\fB--pipable\fR
+Build, or rebuild, \fIDEST_WIMFILE\fR as a "pipable WIM" so that it can be
+applied fully sequentially, including from a pipe.  See \fB@IMAGEX_PROGNAME@
+capture\fR(1) for more details about creating pipable WIMs.  The default without
+this option is to make \fIDEST_WIMFILE\fR pipable if and only if it already
+existed and was already pipable.
+.TP
+\fB--not-pipable\fR
+Build, or rebuld, \fIDEST_WIMFILE\fR as a normal, non-pipable WIM.  This is the
+default behavior, unless \fIDEST_WIMFILE\fR already existed and was already
+pipable.
 .SH SPLIT WIMS
 You may use \fB@IMAGEX_PROGNAME@ export\fR to export images from a split WIM.  The
 \fISRC_WIMFILE\fR argument is used to specify the first part of the split WIM, and
index b6f7bdc..dee6eb2 100644 (file)
@@ -1,4 +1,4 @@
-.TH IMAGEX "1" "June 2013" "@IMAGEX_PROGNAME@ @VERSION@" "User Commands"
+.TH IMAGEX "1" "August 2013" "@IMAGEX_PROGNAME@ @VERSION@" "User Commands"
 .SH NAME
 @IMAGEX_PROGNAME@-extract \- Extract files or directories from a WIM image
 .SH SYNOPSIS
index 59c12af..b3b4a5b 100644 (file)
@@ -1,4 +1,4 @@
-.TH IMAGEX "1" "June 2013" "@IMAGEX_PROGNAME@ @VERSION@" "User Commands"
+.TH IMAGEX "1" "August 2013" "@IMAGEX_PROGNAME@ @VERSION@" "User Commands"
 .SH NAME
 @IMAGEX_PROGNAME@-info \- Display information about a WIM file, or change information about
 an image
@@ -33,8 +33,14 @@ archive.
 When reading \fIWIMFILE\fR, verify its integrity if the integrity table is
 present; additionally if an action that requires modifying the WIM archive is
 specified, include an integrity table in the modified WIM.  If this option is
-not specified and \fIWIMFILE\fR is modified, no integrity table is included in
-the modified WIM, even if there was one before.
+not specified and \fIWIMFILE\fR is modified, an integrity table will be included
+in the modified WIM if and only if there was one before.
+.TP
+\fB--nocheck\fR
+If an action that requires modifying the WIM archive is specified, do not
+include an integrity table in the modified WIM.  If this option is not specified
+and \fIWIMFILE\fR is modified, an integrity table will be included in the
+modified WIM if and only if there was one before.
 .TP
 \fB--extract-xml\fR=\fIFILE\fR
 Extracts the raw data from the XML resource in the WIM file to \fIFILE\fR.
index 048adf9..f2d2c36 100644 (file)
@@ -1,4 +1,4 @@
-.TH IMAGEX "1" "June 2013" "@IMAGEX_PROGNAME@ @VERSION@" "User Commands"
+.TH IMAGEX "1" "August 2013" "@IMAGEX_PROGNAME@ @VERSION@" "User Commands"
 .SH NAME
 @IMAGEX_PROGNAME@-join \- Join split WIMs into a standalone one-part WIM
 .SH SYNOPSIS
@@ -8,13 +8,16 @@ Joins the \fISPLIT_WIMs\fR into a standalone one-part WIM \fIOUT_WIMFILE\fR.
 .PP
 All parts of the split WIM  must be specified.  You probably want to do so using
 a shell wildcard.
+.PP
+\fB@IMAGEX_PROGNAME@ join\fR can join both non-pipable and pipable split WIMs.
 .SH OPTIONS
 .TP 6
 \fB--check\fR
-When reading each \fISPLIT_WIM\fR, verify its integrity if the integrity table is
-present; additionally, when writing \fIOUT_WIMFILE\fR,
-write an integrity table.  If this option is not specified, no integrity table
-is included in the new WIM file, even if there was one before.
+When reading each \fISPLIT_WIM\fR, verify its integrity if the integrity table
+is present; additionally, when writing \fIOUT_WIMFILE\fR, write an integrity
+table.  If this option is not specified, an integrity table will be included in
+\fIOUT_WIMFILE\fR if and only if one was present in the first part of the split
+WIM.
 .SH EXAMPLES
 Join a split WIM, with the parts named `windows*.swm' where the * is anything
 (usually the number of the part, except for the first part which may have no
index 2a08732..6c040f7 100644 (file)
@@ -1,4 +1,4 @@
-.TH IMAGEX "1" "June 2013" "@IMAGEX_PROGNAME@ @VERSION@" "User Commands"
+.TH IMAGEX "1" "August 2013" "@IMAGEX_PROGNAME@ @VERSION@" "User Commands"
 .SH NAME
 @IMAGEX_PROGNAME@-mount, @IMAGEX_PROGNAME@-mountrw, @IMAGEX_PROGNAME@-unmount \- Mount and unmount an image from a WIM archive
 .SH SYNOPSIS
@@ -140,7 +140,8 @@ mount is read-only.
 .TP
 \fB--check\fR
 When writing \fIWIMFILE\fR, include an integrity table.  Has no effect if the
-mount is read-only or if \fB--commit\fR was not specified.
+mount is read-only or if \fB--commit\fR was not specified.  The default behavior
+is to include an integrity table if and only if there was one present before.
 .TP
 \fB--rebuild\fR
 Rebuild the entire WIM rather than appending any new data to the end of it.
index 9a575aa..aa3b947 100644 (file)
@@ -1,8 +1,8 @@
-.TH IMAGEX "1" "June 2013" "@IMAGEX_PROGNAME@ @VERSION@" "User Commands"
+.TH IMAGEX "1" "August 2013" "@IMAGEX_PROGNAME@ @VERSION@" "User Commands"
 .SH NAME
 @IMAGEX_PROGNAME@-optimize \- Optimize a WIM archive
 .SH SYNOPSIS
-\fB@IMAGEX_PROGNAME@ optimize\fR \fIWIMFILE\fR [--check] [--recompress]
+\fB@IMAGEX_PROGNAME@ optimize\fR \fIWIMFILE\fR [\fIOPTION\fR...]
 .SH DESCRIPTION
 \fB@IMAGEX_PROGNAME@ optimize\fR will rebuild the stand-alone WIM \fIWIMFILE\fR.  The new
 WIM is written to a temporary file, and it is renamed to the original file when
@@ -13,10 +13,15 @@ In addition, some errors in the original WIM may be fixed by re-writing it
 .SH OPTIONS
 .TP 6
 \fB--check\fR
-When reading \fIWIMFILE\fR, verify its integrity if the integrity table is
-present; in addition, include an integrity table in the optimized WIM.  If this
-option is not specified, no integrity table is included in the optimized WIM,
-even if there was one in the original WIM.
+When reading \fIWIMFILE\fR, verify its integrity if an integrity table is
+present.  In addition, include an integrity table in the optimized WIM.  If this
+option is not specified, by default the integrity table (if present) is not
+checked, but an integrity table is included in the new WIM if there was one in
+the old WIM.
+.TP
+\fB--nocheck\fR
+Neither verify the integrity of \fIWIMFILE\fR using the integrity table, nor
+include an integrity table in the rebuilt WIM file.
 .TP
 \fB--recompress\fR
 Recompress all compressed streams in \fIWIMFILE\fR when rebuilding it.  This
@@ -36,13 +41,24 @@ created on Windows with Microsoft's ImageX.
 Number of threads to use for compressing data.  Default: autodetect (number of
 processors).  This parameter is only meaningful when \fB--recompress\fR is also
 specified.
+.TP
+\fB--pipable\fR
+Rebuild the WIM that it can be applied fully sequentially, including from a
+pipe.  See \fB@IMAGEX_PROGNAME@ capture\fR(1) for more details about creating
+pipable WIMs.  By default, when neither \fB--pipable\fR or \fB--not-pipable\fR
+is specified, the rebuilt WIM will be pipable if and only if it was already
+pipable.
+.TP
+\fB--not-pipable\fR
+Rebuild the WIM in the non-pipable format.  (This is the default if
+\fIWIMFILE\fR is not pipable.)
 .SH NOTES
 \fB@IMAGEX_PROGNAME@ optimize\fR does not support split WIMs.
 .PP
 \fB@IMAGEX_PROGNAME@ optimize\fR is roughly equivalent to:
 .RS
 .PP
-\fB@IMAGEX_PROGNAME@ export\fR \fIWIMFILE\fR all tmp.wim [--check] && mv tmp.wim \fIWIMFILE\fR
+\fB@IMAGEX_PROGNAME@ export\fR \fIWIMFILE\fR all tmp.wim && mv tmp.wim \fIWIMFILE\fR
 .RE
 .PP
 .SH SEE ALSO
index fa53ee2..5c21576 100644 (file)
@@ -1,19 +1,21 @@
-.TH IMAGEX "1" "June 2013" "@IMAGEX_PROGNAME@ @VERSION@" "User Commands"
+.TH IMAGEX "1" "August 2013" "@IMAGEX_PROGNAME@ @VERSION@" "User Commands"
 .SH NAME
 @IMAGEX_PROGNAME@-split \- Split a WIM into multiple parts
 .SH SYNOPSIS
-\fB@IMAGEX_PROGNAME@ split\fR \fIWIMFILE\fR \fISPLIT_WIMFILE\fR \fIPART_SIZE\fR [\fIOPTION...\fR]
+\fB@IMAGEX_PROGNAME@ split\fR \fIWIMFILE\fR \fISPLIT_WIM_PART\fR \fIPART_SIZE\fR [\fIOPTION...\fR]
 .SH DESCRIPTION
 Splits \fIWIMFILE\fR into parts with size at most \fIPART_SIZE\fR mebibytes,
-with the first part having the name \fISPLIT_WIMFILE\fR and the other parts
+with the first part having the name \fISPLIT_WIM_PART\fR and the other parts
 having names numbered in order of the parts.
+.PP
+\fB@IMAGEX_PROGNAME@ split\fR can split both non-pipable and pipable WIMs.
 .SH OPTIONS
 .TP 6
 \fB--check\fR
 When reading \fIWIMFILE\fR, verify its integrity if the integrity table is
-present; additionally, when writing each \fISPLIT_WIMFILE\fR, write an integrity
-table.  If this option is not specified, no integrity tables are included in the
-split WIM files, even if there was one in the original WIM.
+present; additionally, when writing each \fISPLIT_WIM_PART\fR, write an integrity
+table.  If this option is not specified, integrity tables will be included in
+the split WIMs if and only if one was present in \fIWIMFILE\fR.
 .SH EXAMPLES
 Splits the WIM 'windows.wim' into 'windows.swm', 'windows2.swm', 'windows3.swm',
 etc. where each part is at most 100 MiB:
index 77fecc3..fe2ef9b 100644 (file)
@@ -1,4 +1,4 @@
-.TH IMAGEX "1" "June 2013" "@IMAGEX_PROGNAME@ @VERSION@" "User Commands"
+.TH IMAGEX "1" "August 2013" "@IMAGEX_PROGNAME@ @VERSION@" "User Commands"
 .SH NAME
 @IMAGEX_PROGNAME@-update \- Update a WIM image
 .SH SYNOPSIS
@@ -113,8 +113,8 @@ Use \fB--recursive\fR for all \fBdelete\fR commands.
 \fB--check\fR
 When reading \fIWIMFILE\fR, verify its integrity if the integrity table is
 present; in addition, include an integrity table in the updated WIM.  If this
-option is not specified, no integrity table is included in the updated WIM, even
-if there was one in the original WIM.
+option is not specified, an integrity table will be included in the updated WIM
+if and only if one was present before.
 .TP
 \fB--threads\fR=\fINUM_THREADS\fR
 Number of threads to use for compressing newly added files.  Default: autodetect
index 69222a8..b913c45 100644 (file)
@@ -1,4 +1,4 @@
-.TH IMAGEX 1 "June 2013" "@IMAGEX_PROGNAME@ @VERSION@" "User Commands"
+.TH IMAGEX 1 "August 2013" "@IMAGEX_PROGNAME@ @VERSION@" "User Commands"
 .SH NAME
 @IMAGEX_PROGNAME@ \- Create, modify, extract, mount, or unmount a WIM (Windows Imaging Format) archive
 .SH SYNOPSIS
@@ -48,12 +48,18 @@ but not Windows):
 .IP \[bu] 4
 Create a stand-alone WIM from a directory or NTFS volume (\fB@IMAGEX_PROGNAME@ capture\fR)
 .IP \[bu]
+Capture a WIM image directly to standard output in a special pipable format
+(\fB@IMAGEX_PROGNAME@ capture\fR)
+.IP \[bu]
 Append a directory or NTFS volume onto a stand-alone WIM as a new image (\fB@IMAGEX_PROGNAME@
 append\fR)
 .IP \[bu]
 Apply an image from a stand-alone or split WIM to a directory or NTFS volume
 (\fB@IMAGEX_PROGNAME@ apply\fR)
 .IP \[bu]
+Apply an image from a special pipable WIM format sent over standard input
+(\fB@IMAGEX_PROGNAME@ apply\fR)
+.IP \[bu]
 Mount an image from a stand-alone or split WIM read-only (\fB@IMAGEX_PROGNAME@ mount\fR)
 .IP \[bu]
 Mount an image from a stand-alone WIM read-write (\fB@IMAGEX_PROGNAME@ mountrw\fR)
@@ -92,6 +98,16 @@ thus, much functionality was designed around this.
 .IP \[bu]
 The command-line syntax of the two programs is similar but not exactly the same.
 .IP \[bu]
+As of wimlib v1.5.0, for convenience \fB@IMAGEX_PROGNAME@\fR automatically
+preserves the integrity table in WIMs that have one, even when \fB--check\fR is
+not specified.
+.IP \[bu]
+As of wimlib v1.5.0, a special "pipable" WIM format that is not compatible with
+Microsoft's software is supported.  This allows capturing and applying images
+directly to standard output or from standard input, respectively; this can be
+used to pipe images to or from a server over the network to implement fast
+filesystem imaging and restore.
+.IP \[bu]
 On UNIX, because Microsoft designed the WIM file format to accomodate
 Windows-specific and NTFS-specific features, wimlib must have two separate image
 capture and application modes (although the \fB@IMAGEX_PROGNAME@\fR subcommands
index fb47fd2..3095023 100644 (file)
@@ -1,4 +1,4 @@
-.TH MKWINPEIMG "1" "June 2013" "mkwinpeimg (wimlib @VERSION@)" "User Commands"
+.TH MKWINPEIMG "1" "August 2013" "mkwinpeimg (wimlib @VERSION@)" "User Commands"
 .SH NAME
 mkwinpeimg \- Make a customized bootable image of Windows PE
 .SH SYNOPSIS
index d80089c..faececa 100644 (file)
@@ -31,7 +31,7 @@
  *
  * \section intro Introduction
  *
- * This is the documentation for the library interface of wimlib 1.4.2, a C
+ * This is the documentation for the library interface of wimlib 1.5.0, a C
  * library for creating, modifying, extracting, and mounting files in the
  * Windows Imaging Format.  This documentation is intended for developers only.
  * If you have installed wimlib and want to know how to use the @b wimlib-imagex
  * After creating or modifying a WIM file, you can write it to a file using
  * wimlib_write().  Alternatively,  if the WIM was originally read from a file
  * (using wimlib_open_wim() rather than wimlib_create_new_wim()), you can use
- * wimlib_overwrite() to overwrite the original file.
+ * wimlib_overwrite() to overwrite the original file.  Still alternatively, you
+ * can write a WIM directly to a file descriptor by calling wimlib_write_to_fd()
+ * instead.
+ *
+ * wimlib supports a special "pipable" WIM format (which unfortunately is @b not
+ * compatible with Microsoft's software).  To create a pipable WIM, call
+ * wimlib_write(), wimlib_write_to_fd(), or wimlib_overwrite() with
+ * ::WIMLIB_WRITE_FLAG_PIPABLE specified.  Pipable WIMs are pipable in both
+ * directions, so wimlib_write_to_fd() can be used to write a pipable WIM to a
+ * pipe, and wimlib_extract_image_from_pipe() can be used to apply an image from
+ * a pipable WIM.
  *
  * Please note: merely by calling wimlib_add_image() or many of the other
  * functions in this library that operate on ::WIMStruct's, you are @b not
 #define WIMLIB_MAJOR_VERSION 1
 
 /** Minor version of the library (for example, the 2 in 1.2.5). */
-#define WIMLIB_MINOR_VERSION 4
+#define WIMLIB_MINOR_VERSION 5
 
 /** Patch version of the library (for example, the 5 in 1.2.5). */
-#define WIMLIB_PATCH_VERSION 2
+#define WIMLIB_PATCH_VERSION 0
 
 /**
  * Opaque structure that represents a WIM file.  This is an in-memory structure
@@ -250,10 +260,8 @@ enum wimlib_progress_msg {
         * info will point to ::wimlib_progress_info.extract. */
        WIMLIB_PROGRESS_MSG_EXTRACT_STREAMS,
 
-       /** A file or directory is being extracted.  @a info will point to
-        * ::wimlib_progress_info.extract, and the @a cur_path member will be
-        * valid. */
-       WIMLIB_PROGRESS_MSG_EXTRACT_DENTRY,
+       /** Reserved.  */
+       WIMLIB_PROGRESS_MSG_EXTRACT_RESERVED,
 
        /** All the WIM files and directories have been extracted, and
         * timestamps are about to be applied.  @a info will point to
@@ -432,14 +440,12 @@ union wimlib_progress_info {
                 * extracted. */
                const wimlib_tchar *target;
 
-               /** Current dentry being extracted.  (Valid only if message is
-                * ::WIMLIB_PROGRESS_MSG_EXTRACT_DENTRY.) */
+               /** Reserved.  */
                const wimlib_tchar *cur_path;
 
                /** Number of bytes of uncompressed data that will be extracted.
                 * Takes into account hard links (they are not counted for each
-                * link.)
-                * */
+                * link.)  */
                uint64_t total_bytes;
 
                /** Number of bytes that have been written so far.  Will be 0
@@ -548,6 +554,9 @@ union wimlib_progress_info {
                 * finished (::WIMLIB_PROGRESS_MSG_SPLIT_END_PART). */
                unsigned cur_part_number;
 
+               /** Total number of split WIM parts that are being written.  */
+               unsigned total_parts;
+
                /** Name of the split WIM part that is about to be started
                 * (::WIMLIB_PROGRESS_MSG_SPLIT_BEGIN_PART) or has just been
                 * finished (::WIMLIB_PROGRESS_MSG_SPLIT_END_PART). */
@@ -714,7 +723,10 @@ struct wimlib_wim_info {
        uint32_t write_in_progress : 1;
        uint32_t metadata_only : 1;
        uint32_t resource_only : 1;
-       uint32_t reserved_flags : 23;
+
+       /** 1 if the WIM is pipable (see ::WIMLIB_WRITE_FLAG_PIPABLE).  */
+       uint32_t pipable : 1;
+       uint32_t reserved_flags : 22;
        uint32_t reserved[9];
 };
 
@@ -989,10 +1001,7 @@ typedef int (*wimlib_iterate_lookup_table_callback_t)(const struct wimlib_resour
  * together.  Cannot be used with ::WIMLIB_EXTRACT_FLAG_NTFS. */
 #define WIMLIB_EXTRACT_FLAG_SYMLINK                    0x00000004
 
-/** Call the progress function with the argument
- * ::WIMLIB_PROGRESS_MSG_EXTRACT_DENTRY each time a file or directory is
- * extracted.  Note: these calls will be interspersed with calls for the message
- * ::WIMLIB_PROGRESS_MSG_EXTRACT_STREAMS. */
+/** This flag no longer does anything but is reserved for future use.  */
 #define WIMLIB_EXTRACT_FLAG_VERBOSE                    0x00000008
 
 /** Read the WIM file sequentially while extracting the image. */
@@ -1039,6 +1048,19 @@ typedef int (*wimlib_iterate_lookup_table_callback_t)(const struct wimlib_resour
  * one. */
 #define WIMLIB_EXTRACT_FLAG_ALL_CASE_CONFLICTS         0x00001000
 
+/** Do not ignore failure to set timestamps on extracted files.  */
+#define WIMLIB_EXTRACT_FLAG_STRICT_TIMESTAMPS          0x00002000
+
+/** Do not ignore failure to set short names on extracted files.  */
+#define WIMLIB_EXTRACT_FLAG_STRICT_SHORT_NAMES          0x00004000
+
+/** Do not ignore failure to extract symbolic links (and junction points, on
+ * Windows) due to permissions problems.  By default, such failures are ignored
+ * since the default configuration of Windows only allows the Administrator to
+ * create symbolic links.  */
+#define WIMLIB_EXTRACT_FLAG_STRICT_SYMLINKS             0x00008000
+
+
 /******************************
  * WIMLIB_MOUNT_FLAG_*
  ******************************/
@@ -1123,37 +1145,62 @@ typedef int (*wimlib_iterate_lookup_table_callback_t)(const struct wimlib_resour
  * WIMLIB_WRITE_FLAG_*
  ******************************/
 
-/** Include an integrity table in the new WIM file. */
+/** Include an integrity table in the WIM.
+ *
+ * For WIMs created with wimlib_open_wim(), the default behavior is to include
+ * an integrity table if and only if one was present before.  For WIMs created
+ * with wimlib_create_new_wim(), the default behavior is to not include an
+ * integrity table.  */
 #define WIMLIB_WRITE_FLAG_CHECK_INTEGRITY              0x00000001
 
-/** Re-build the entire WIM file rather than appending data to it, if possible.
- * (Applies to wimlib_overwrite(), not wimlib_write()). */
-#define WIMLIB_WRITE_FLAG_REBUILD                      0x00000002
-
-/** Recompress all resources, even if they could otherwise be copied from a
- * different WIM with the same compression type (in the case of
- * wimlib_export_image() being called previously). */
-#define WIMLIB_WRITE_FLAG_RECOMPRESS                   0x00000004
-
-/** Call fsync() when the WIM file is closed */
-#define WIMLIB_WRITE_FLAG_FSYNC                                0x00000008
+/** Do not include an integrity table in the new WIM file.  This is the default
+ * behavior, unless the WIM already included an integrity table.  */
+#define WIMLIB_WRITE_FLAG_NO_CHECK_INTEGRITY           0x00000002
 
-/* Specifying this flag overrides the default behavior of wimlib_overwrite()
- * after one or more calls to wimlib_delete_image(), which is to rebuild the
- * entire WIM.
+/** Write the WIM as "pipable".  After writing a WIM with this flag specified,
+ * images from it can be applied directly from a pipe using
+ * wimlib_extract_image_from_pipe().  See the documentation for the --pipable
+ * flag of `wimlib-imagex capture' for more information.  Beware: WIMs written
+ * with this flag will not be compatible with Microsoft's software.
  *
- * If you specifiy this flag to wimlib_overwrite(), only minimal changes to
- * correctly remove the image from the WIM will be taken.  In particular, all
- * streams will be left alone, even if they are no longer referenced.  This is
- * probably not what you want, because almost no space will be spaced by
- * deleting an image in this way. */
-#define WIMLIB_WRITE_FLAG_SOFT_DELETE                  0x00000010
+ * For WIMs created with wimlib_open_wim(), the default behavior is to write the
+ * WIM as pipable if and only if it was pipable before.  For WIMs created with
+ * wimlib_create_new_wim(), the default behavior is to write the WIM as
+ * non-pipable.  */
+#define WIMLIB_WRITE_FLAG_PIPABLE                      0x00000004
+
+/** Do not write the WIM as "pipable".  This is the default behavior, unless the
+ * WIM was pipable already.  */
+#define WIMLIB_WRITE_FLAG_NOT_PIPABLE                  0x00000008
 
-/** With wimlib_overwrite(), allow overwriting the WIM even if the readonly flag
- * is set in the WIM header; this can be used in combination with
+/** Recompress all resources, even if they could otherwise be copied from a
+ * different WIM with the same compression type (in the case of
+ * wimlib_export_image() being called previously).  This flag is also valid in
+ * the @p wim_write_flags of wimlib_join(), in which case all resources included
+ * in the joined WIM file will be recompressed.  */
+#define WIMLIB_WRITE_FLAG_RECOMPRESS                   0x00000010
+
+/** Call fsync() just before the WIM file is closed.  */
+#define WIMLIB_WRITE_FLAG_FSYNC                                0x00000020
+
+/** wimlib_overwrite() only:  Re-build the entire WIM file rather than appending
+ * data to it if possible.  */
+#define WIMLIB_WRITE_FLAG_REBUILD                      0x00000040
+
+/** wimlib_overwrite() only:  Specifying this flag overrides the default
+ * behavior of wimlib_overwrite() after one or more calls to
+ * wimlib_delete_image(), which is to rebuild the entire WIM.  With this flag,
+ * only minimal changes to correctly remove the image from the WIM will be
+ * taken.  In particular, all streams will be left alone, even if they are no
+ * longer referenced.  This is probably not what you want, because almost no
+ * space will be saved by deleting an image in this way. */
+#define WIMLIB_WRITE_FLAG_SOFT_DELETE                  0x00000080
+
+/** wimlib_overwrite() only:  Allow overwriting the WIM even if the readonly
+ * flag is set in the WIM header.  This can be used in combination with
  * wimlib_set_wim_info() with the ::WIMLIB_CHANGE_READONLY_FLAG flag to actually
- * set the readonly flag on the on-disk WIM file. */
-#define WIMLIB_WRITE_FLAG_IGNORE_READONLY_FLAG         0x00000020
+ * set the readonly flag on the on-disk WIM file.  */
+#define WIMLIB_WRITE_FLAG_IGNORE_READONLY_FLAG         0x00000100
 
 /******************************
  * WIMLIB_INIT_FLAG_*
@@ -1244,7 +1291,6 @@ struct wimlib_extract_command {
 enum wimlib_error_code {
        WIMLIB_ERR_SUCCESS = 0,
        WIMLIB_ERR_ALREADY_LOCKED,
-       WIMLIB_ERR_COMPRESSED_LOOKUP_TABLE,
        WIMLIB_ERR_DECOMPRESSION,
        WIMLIB_ERR_DELETE_STAGING_DIR,
        WIMLIB_ERR_FILESYSTEM_DAEMON_CRASHED,
@@ -1254,23 +1300,22 @@ enum wimlib_error_code {
        WIMLIB_ERR_ICONV_NOT_AVAILABLE,
        WIMLIB_ERR_IMAGE_COUNT,
        WIMLIB_ERR_IMAGE_NAME_COLLISION,
-       WIMLIB_ERR_INSUFFICIENT_PRIVILEGES_TO_EXTRACT,
        WIMLIB_ERR_INTEGRITY,
        WIMLIB_ERR_INVALID_CAPTURE_CONFIG,
        WIMLIB_ERR_INVALID_CHUNK_SIZE,
        WIMLIB_ERR_INVALID_COMPRESSION_TYPE,
-       WIMLIB_ERR_INVALID_DENTRY,
-       WIMLIB_ERR_INVALID_HEADER_SIZE,
+       WIMLIB_ERR_INVALID_HEADER,
        WIMLIB_ERR_INVALID_IMAGE,
        WIMLIB_ERR_INVALID_INTEGRITY_TABLE,
        WIMLIB_ERR_INVALID_LOOKUP_TABLE_ENTRY,
+       WIMLIB_ERR_INVALID_METADATA_RESOURCE,
        WIMLIB_ERR_INVALID_MULTIBYTE_STRING,
        WIMLIB_ERR_INVALID_OVERLAY,
        WIMLIB_ERR_INVALID_PARAM,
        WIMLIB_ERR_INVALID_PART_NUMBER,
+       WIMLIB_ERR_INVALID_PIPABLE_WIM,
        WIMLIB_ERR_INVALID_REPARSE_DATA,
        WIMLIB_ERR_INVALID_RESOURCE_HASH,
-       WIMLIB_ERR_INVALID_RESOURCE_SIZE,
        WIMLIB_ERR_INVALID_SECURITY_DATA,
        WIMLIB_ERR_INVALID_UNMOUNT_MESSAGE,
        WIMLIB_ERR_INVALID_UTF16_STRING,
@@ -1285,6 +1330,7 @@ enum wimlib_error_code {
        WIMLIB_ERR_NOTEMPTY,
        WIMLIB_ERR_NOT_A_REGULAR_FILE,
        WIMLIB_ERR_NOT_A_WIM_FILE,
+       WIMLIB_ERR_NOT_PIPABLE,
        WIMLIB_ERR_NO_FILENAME,
        WIMLIB_ERR_NTFS_3G,
        WIMLIB_ERR_OPEN,
@@ -1295,20 +1341,24 @@ enum wimlib_error_code {
        WIMLIB_ERR_RENAME,
        WIMLIB_ERR_REOPEN,
        WIMLIB_ERR_REPARSE_POINT_FIXUP_FAILED,
+       WIMLIB_ERR_RESOURCE_NOT_FOUND,
        WIMLIB_ERR_RESOURCE_ORDER,
+       WIMLIB_ERR_SET_ATTRIBUTES,
+       WIMLIB_ERR_SET_SECURITY,
+       WIMLIB_ERR_SET_TIMESTAMPS,
        WIMLIB_ERR_SPECIAL_FILE,
        WIMLIB_ERR_SPLIT_INVALID,
        WIMLIB_ERR_SPLIT_UNSUPPORTED,
        WIMLIB_ERR_STAT,
        WIMLIB_ERR_TIMEOUT,
+       WIMLIB_ERR_UNEXPECTED_END_OF_FILE,
        WIMLIB_ERR_UNICODE_STRING_NOT_REPRESENTABLE,
        WIMLIB_ERR_UNKNOWN_VERSION,
        WIMLIB_ERR_UNSUPPORTED,
        WIMLIB_ERR_VOLUME_LACKS_FEATURES,
+       WIMLIB_ERR_WIM_IS_READONLY,
        WIMLIB_ERR_WRITE,
        WIMLIB_ERR_XML,
-       WIMLIB_ERR_WIM_IS_READONLY,
-       WIMLIB_ERR_RESOURCE_NOT_FOUND,
 };
 
 
@@ -1321,7 +1371,11 @@ enum wimlib_error_code {
 /**
  * Appends an empty image to a WIM file.  This empty image will initially
  * contain no files or directories, although if written without further
- * modifications, a root directory will be created automatically for it.
+ * modifications, a root directory will be created automatically for it.  After
+ * calling this function, you can use wimlib_update_image() to add files to the
+ * new WIM image.  This gives you slightly more control over making the new
+ * image compared to calling wimlib_add_image() or
+ * wimlib_add_image_multisource() directly.
  *
  * @param wim
  *     Pointer to the ::WIMStruct for the WIM file to which the image is to be
@@ -1478,9 +1532,6 @@ wimlib_create_new_wim(int ctype, WIMStruct **wim_ret);
  *
  * @retval ::WIMLIB_ERR_DECOMPRESSION
  *     Could not decompress the metadata resource for @a image.
- * @retval ::WIMLIB_ERR_INVALID_DENTRY
- *     A directory entry in the metadata resource for @a image in the WIM is
- *     invalid.
  * @retval ::WIMLIB_ERR_INVALID_IMAGE
  *     @a image does not exist in the WIM and is not ::WIMLIB_ALL_IMAGES.
  * @retval ::WIMLIB_ERR_INVALID_RESOURCE_SIZE
@@ -1571,9 +1622,6 @@ wimlib_delete_image(WIMStruct *wim, int image);
  * @retval ::WIMLIB_ERR_IMAGE_NAME_COLLISION
  *     One or more of the names being given to an exported image was already in
  *     use in the destination WIM.
- * @retval ::WIMLIB_ERR_INVALID_DENTRY
- *     A directory entry in the metadata resource for @a src_image in @a
- *     src_wim is invalid.
  * @retval ::WIMLIB_ERR_INVALID_IMAGE
  *     @a src_image does not exist in @a src_wim.
  * @retval ::WIMLIB_ERR_INVALID_PARAM
@@ -1755,9 +1803,6 @@ wimlib_extract_files(WIMStruct *wim,
  * @retval ::WIMLIB_ERR_DECOMPRESSION
  *     Could not decompress a resource (file or metadata) for @a image in @a
  *     wim.
- * @retval ::WIMLIB_ERR_INVALID_DENTRY
- *     A directory entry in the metadata resource for @a image in @a wim is
- *     invalid.
  * @retval ::WIMLIB_ERR_INVALID_PARAM
  *     @a target was @c NULL, or both ::WIMLIB_EXTRACT_FLAG_HARDLINK and
  *     ::WIMLIB_EXTRACT_FLAG_SYMLINK were specified in @a extract_flags; or
@@ -1796,6 +1841,16 @@ wimlib_extract_files(WIMStruct *wim,
  *     One of the dentries in the image referenced a stream not present in the
  *     WIM's lookup table (or in any of the lookup tables of the split WIM
  *     parts).
+ * @retval ::WIMLIB_ERR_SET_ATTRIBUTES
+ *     Failed to set attributes on a file.
+ * @retval ::WIMLIB_ERR_SET_SECURITY
+ *     Failed to set security descriptor on a file
+ *     (only if ::WIMLIB_EXTRACT_FLAG_STRICT_ACLS) specified in @p
+ *     extract_flags.
+ * @retval ::WIMLIB_ERR_SET_SECURITY
+ *     Failed to set security descriptor on a file
+ *     (only if ::WIMLIB_EXTRACT_FLAG_STRICT_ACLS) specified in @p
+ *     extract_flags.
  * @retval ::WIMLIB_ERR_SPLIT_INVALID
  *     The WIM is a split WIM, but the parts specified do not form a complete
  *     split WIM because they do not include all the parts of the original WIM,
@@ -1817,6 +1872,49 @@ wimlib_extract_image(WIMStruct *wim, int image,
                     wimlib_progress_func_t progress_func);
 
 /**
+ * Extract one or more images from a pipe on which a pipable WIM is being sent.
+ *
+ * See the documentation for ::WIMLIB_WRITE_FLAG_PIPABLE for more information
+ * about pipable WIMs.
+ *
+ * This function operates in a special way to read the WIM fully sequentially.
+ * As a result, there is no ::WIMStruct is made visible to library users, and
+ * you cannot call wimlib_open_wim() on the pipe.  (You can, however, use
+ * wimlib_open_wim() to transparently open a pipable WIM if it's available as a
+ * seekable file, not a pipe.)
+ *
+ * @param pipe_fd
+ *     File descriptor, which may be a pipe, opened for reading and positioned
+ *     at the start of the pipable WIM.
+ * @param image_num_or_name
+ *     String that specifies the 1-based index or name of the image to extract.
+ *     It is translated to an image index using the same rules that
+ *     wimlib_resolve_image() uses.  However, unlike wimlib_extract_image(),
+ *     only a single image (not all images) can be specified.
+ * @param target
+ *     Same as the corresponding parameter to wimlib_extract_image().
+ * @param extract_flags
+ *     Same as the corresponding parameter to wimlib_extract_image(), except
+ *     for the following exceptions:  ::WIMLIB_EXTRACT_FLAG_SEQUENTIAL is
+ *     always implied, since data is always read from @p pipe_fd sequentially
+ *     in this mode; also, ::WIMLIB_EXTRACT_FLAG_TO_STDOUT is invalid and will
+ *     result in ::WIMLIB_ERR_INVALID_PARAM being returned.
+ * @param progress_func
+ *     Same as the corresponding parameter to wimlib_extract_image().
+ *
+ * @return 0 on success; nonzero on error.  The possible error codes include
+ * those returned by wimlib_extract_image() as well as the following:
+ *
+ * @retval ::WIMLIB_ERR_NOT_PIPABLE
+ *     The WIM being piped in a @p pipe_fd is a normal WIM, not a pipable WIM.
+ */
+extern int
+wimlib_extract_image_from_pipe(int pipe_fd,
+                              const wimlib_tchar *image_num_or_name,
+                              const wimlib_tchar *target, int extract_flags,
+                              wimlib_progress_func_t progress_func);
+
+/**
  * Extracts the XML data of a WIM file to a file stream.  Every WIM file
  * includes a string of XML that describes the images contained in the WIM.
  * This function works on standalone WIMs as well as split WIM parts.
@@ -2238,9 +2336,6 @@ wimlib_lzx_decompress(const void *compressed_data, unsigned compressed_len,
  *     Could not decompress the metadata resource for @a image in @a wim.
  * @retval ::WIMLIB_ERR_FUSE
  *     A non-zero status was returned by @c fuse_main().
- * @retval ::WIMLIB_ERR_INVALID_DENTRY
- *     A directory entry in the metadata resource for @a image in @a wim is
- *     invalid.
  * @retval ::WIMLIB_ERR_INVALID_IMAGE
  *     @a image does not specify an existing, single image in @a wim.
  * @retval ::WIMLIB_ERR_INVALID_PARAM
@@ -2327,7 +2422,7 @@ wimlib_mount_image(WIMStruct *wim,
  * @retval ::WIMLIB_ERR_INVALID_COMPRESSION_TYPE
  *     The header of @a wim_file says that resources in the WIM are compressed,
  *     but the header flag indicating LZX or XPRESS compression is not set.
- * @retval ::WIMLIB_ERR_INVALID_HEADER_SIZE
+ * @retval ::WIMLIB_ERR_INVALID_HEADER
  *     The length field of the WIM header is not 208.
  * @retval ::WIMLIB_ERR_INVALID_INTEGRITY_TABLE
  *     ::WIMLIB_OPEN_FLAG_CHECK_INTEGRITY was specified in @a open_flags and @a
@@ -2337,6 +2432,8 @@ wimlib_mount_image(WIMStruct *wim,
  *     The lookup table for the WIM contained duplicate entries that are not
  *     for metadata resources, or it contained an entry with a SHA1 message
  *     digest of all 0's.
+ * @retval ::WIMLIB_ERR_INVALID_PARAM
+ *     @p wim_ret was @c NULL.
  * @retval ::WIMLIB_ERR_NOMEM
  *     Failed to allocated needed memory.
  * @retval ::WIMLIB_ERR_NOT_A_WIM_FILE
@@ -2735,7 +2832,7 @@ wimlib_set_print_errors(bool show_messages);
 extern int
 wimlib_split(WIMStruct *wim,
             const wimlib_tchar *swm_name,
-            size_t part_size,
+            uint64_t part_size,
             int write_flags,
             wimlib_progress_func_t progress_func);
 
@@ -2833,8 +2930,6 @@ wimlib_unmount_image(const wimlib_tchar *dir,
  * @retval ::WIMLIB_ERR_INVALID_CAPTURE_CONFIG
  *     The capture configuration structure specified for an add command was
  *     invalid.
- * @retval ::WIMLIB_ERR_INVALID_DENTRY
- *     A directory entry for @a image in @a wim is invalid.
  * @retval ::WIMLIB_ERR_INVALID_IMAGE
  *     @a image did not specify a single, existing image in @a wim.
  * @retval ::WIMLIB_ERR_INVALID_OVERLAY
@@ -2932,12 +3027,10 @@ wimlib_update_image(WIMStruct *wim,
  * @param path
  *     The path to the file to write the WIM to.
  * @param image
- *     The image inside the WIM to write.  Use ::WIMLIB_ALL_IMAGES to include all
- *     images.
+ *     The 1-based index of the image inside the WIM to write.  Use
+ *     ::WIMLIB_ALL_IMAGES to include all images.
  * @param write_flags
- *     Bitwise OR of the flags ::WIMLIB_WRITE_FLAG_CHECK_INTEGRITY,
- *     ::WIMLIB_WRITE_FLAG_RECOMPRESS, ::WIMLIB_WRITE_FLAG_FSYNC, and/or
- *     ::WIMLIB_WRITE_FLAG_SOFT_DELETE.
+ *     Bitwise OR of any of the flags prefixed with @c WIMLIB_WRITE_FLAG.
  * @param num_threads
  *     Number of threads to use for compressing data.  If 0, the number of
  *     threads is taken to be the number of online processors.  Note: if no
@@ -2948,14 +3041,14 @@ wimlib_update_image(WIMStruct *wim,
  *     write_flags).
  * @param progress_func
  *     If non-NULL, a function that will be called periodically with the
- *     progress of the current operation.
+ *     progress of the current operation.  The possible messages are
+ *     ::WIMLIB_PROGRESS_MSG_WRITE_METADATA_BEGIN,
+ *     ::WIMLIB_PROGRESS_MSG_WRITE_METADATA_END, and
+ *     ::WIMLIB_PROGRESS_MSG_WRITE_STREAMS.
  *
  * @return 0 on success; nonzero on error.
  * @retval ::WIMLIB_ERR_DECOMPRESSION
  *     Failed to decompress a metadata or file resource in @a wim.
- * @retval ::WIMLIB_ERR_INVALID_DENTRY
- *     A directory entry in the metadata resource for @a image in @a wim is
- *     invalid.
  * @retval ::WIMLIB_ERR_INVALID_IMAGE
  *     @a image does not specify a single existing image in @a wim, and is not
  *     ::WIMLIB_ALL_IMAGES.
@@ -2964,7 +3057,7 @@ wimlib_update_image(WIMStruct *wim,
  *     wimlib_add_image() was concurrently modified, so it failed the SHA1
  *     message digest check.
  * @retval ::WIMLIB_ERR_INVALID_PARAM
- *     @a path was @c NULL.
+ *     @p path was @c NULL.
  * @retval ::WIMLIB_ERR_INVALID_RESOURCE_SIZE
  *     The metadata resource for @a image in @a wim is invalid.
  * @retval ::WIMLIB_ERR_INVALID_SECURITY_DATA
@@ -2972,20 +3065,18 @@ wimlib_update_image(WIMStruct *wim,
  * @retval ::WIMLIB_ERR_NOMEM
  *     Failed to allocate needed memory.
  * @retval ::WIMLIB_ERR_OPEN
- *     Failed to open @a path for writing, or some file resources in @a
- *     wim refer to files in the outside filesystem, and one of these files
- *     could not be opened for reading.
+ *     Failed to open @a path for writing, or some file resources in @a wim
+ *     refer to files in the outside filesystem, and one of these files could
+ *     not be opened for reading.
  * @retval ::WIMLIB_ERR_READ
  *     An error occurred when trying to read data from the WIM file associated
  *     with @a wim, or some file resources in @a wim refer to files in the
  *     outside filesystem, and a read error occurred when reading one of these
  *     files.
  * @retval ::WIMLIB_ERR_SPLIT_UNSUPPORTED
- *     @a wim is part of a split WIM.  You may not call this function on a
- *     split WIM.
+ *     @a wim is part of a split WIM, not a standalone WIM.
  * @retval ::WIMLIB_ERR_WRITE
- *     An error occurred when trying to write data to the new WIM file at @a
- *     path.
+ *     An error occurred when trying to write data to the new WIM file.
  */
 extern int
 wimlib_write(WIMStruct *wim,
@@ -2996,6 +3087,28 @@ wimlib_write(WIMStruct *wim,
             wimlib_progress_func_t progress_func);
 
 /**
+ * Same as wimlib_write(), but write the WIM directly to a file descriptor,
+ * which need not be seekable if the write is done in a special pipable WIM
+ * format by providing ::WIMLIB_WRITE_FLAG_PIPABLE in @p write_flags.  This can,
+ * for example, allow capturing a WIM image and streaming it over the network.
+ * See the documentation for ::WIMLIB_WRITE_FLAG_PIPABLE for more information
+ * about pipable WIMs.
+ *
+ * Return values are mostly the same as wimlib_write(), but also:
+ *
+ * @retval ::WIMLIB_ERR_INVALID_PARAM
+ *     @p fd was not seekable, but ::WIMLIB_WRITE_FLAG_PIPABLE was not
+ *     specified in @p write_flags.
+ */
+extern int
+wimlib_write_to_fd(WIMStruct *wim,
+                  int fd,
+                  int image,
+                  int write_flags,
+                  unsigned num_threads,
+                  wimlib_progress_func_t progress_func);
+
+/**
  * This function is equivalent to wimlib_lzx_compress(), but instead compresses
  * the data using "XPRESS" compression.
  */
index 4fd22f8..5063f80 100644 (file)
 #ifndef _WIMLIB_APPLY_H
 #define _WIMLIB_APPLY_H
 
-#include "wimlib.h"
 #include "wimlib/types.h"
+#include "wimlib/list.h"
+#include "wimlib.h"
 
-#ifdef WITH_NTFS_3G
-struct _ntfs_volume;
-#endif
+struct wim_lookup_table_entry;
+struct wimlib_unix_data;
+struct wim_dentry;
+struct apply_ctx;
 
-struct apply_args {
-       WIMStruct *wim;
+/*
+ * struct apply_operations -  Callback functions for a specific extraction
+ * mode/backend.  These are lower-level functions that are called by the generic
+ * code in extract.c.
+ *
+ * Unless otherwise specified, the callbacks in this structure are expected to
+ * return 0 on success or a WIMLIB_ERR_* value on failure as well as set errno.
+ * When possible, error messages should NOT be printed as they are handled by
+ * the generic code.
+ *
+ * Many callbacks are optional, but to extract the most data from the WIM
+ * format, as many as possible should be provided, and the corresponding
+ * features should be marked as supported in start_extract().
+ */
+struct apply_operations {
 
-       /* Directory to which we're extracting the WIM image or directory tree,
-        * in user-specified form (may be slightly altered) */
-       const tchar *target;
-       unsigned target_nchars;
+       /* OPTIONAL:  Name of this extraction mode.  */
+       const tchar *name;
 
-#ifdef __WIN32__
-       /* \\?\-prefixed full path to the above directory; needed to work around
-        * lack of default support for long paths on Windoze. */
-       tchar *target_lowlevel_path;
-       unsigned target_lowlevel_path_nchars;
-#endif
+       /* REQUIRED:  Fill in ctx->supported_features with nonzero values for
+        * features supported by the extraction mode and volume.  This callback
+        * can also be used to do any setup needed to access the volume.  */
+       int (*start_extract)
+               (const tchar *path, struct apply_ctx *ctx);
 
-       /* Absolute path to the above directory; on UNIX this is simply a path
-        * beginning with /, while on Windoze this will be a path beginning with
-        * a drive letter followed by a backslash, but not with \\?\. */
-       tchar *target_realpath;
-       unsigned target_realpath_len;
+       /* OPTIONAL:  If root_directory_is_special is set:  provide this
+        * callback to determine whether the path corresponds to the root of the
+        * target volume (%true) or not (%false).  */
+       bool (*target_is_root)
+               (const tchar *target);
 
-       struct wim_dentry *extract_root;
-       unsigned long invalid_sequence;
-       int extract_flags;
-       union wimlib_progress_info progress;
-       wimlib_progress_func_t progress_func;
-       int (*apply_dentry)(struct wim_dentry *, void *);
-       union {
-       #ifdef WITH_NTFS_3G
-               struct {
-                       /* NTFS apply only */
-                       struct _ntfs_volume *vol;
-               };
-       #endif
-       #ifdef __WIN32__
-               struct {
-                       /* Normal apply only (Win32) */
-                       unsigned long num_set_sacl_priv_notheld;
-                       unsigned long num_set_sd_access_denied;
-                       unsigned vol_flags;
-                       unsigned long num_hard_links_failed;
-                       unsigned long num_soft_links_failed;
-                       unsigned long num_long_paths;
-                       bool have_vol_flags;
-               };
-       #else
-               struct {
-                       /* Normal apply only (UNIX) */
-                       unsigned long num_utime_warnings;
-               };
-       #endif
-       };
+       /* REQUIRED:  Create a file.  */
+       int (*create_file)
+               (const tchar *path, struct apply_ctx *ctx);
+
+       /* REQUIRED:  Create a directory.  */
+       int (*create_directory)
+               (const tchar *path, struct apply_ctx *ctx);
+
+       /* OPTIONAL:  Create a hard link.  In start_extract(), set
+        * ctx->supported_features.hard_links if supported.  */
+       int (*create_hardlink)
+               (const tchar *oldpath, const tchar *newpath,
+                struct apply_ctx *ctx);
+
+       /* OPTIONAL:  Create a symbolic link.  In start_extract(), set
+        * ctx->supported_features.symlink_reparse_points if supported.  */
+       int (*create_symlink)
+               (const tchar *oldpath, const tchar *newpath,
+                struct apply_ctx *ctx);
+
+       /* REQUIRED:  Extract unnamed data stream.  */
+       int (*extract_unnamed_stream)
+               (const tchar *path, struct wim_lookup_table_entry *lte,
+                struct apply_ctx *ctx);
+
+       /* OPTIONAL:  Extracted named data stream.  In start_extract(), set
+        * ctx->supported_features.alternate_data_streams if supported.  */
+       int (*extract_named_stream)
+               (const tchar *path, const utf16lechar *stream_name,
+                size_t stream_name_nchars, struct wim_lookup_table_entry *lte,
+                struct apply_ctx *ctx);
+
+       /* OPTIONAL:  Extracted encrypted stream.  In start_extract(), set
+        * ctx->supported_features.encrypted_files if supported.  */
+       int (*extract_encrypted_stream)
+               (const tchar *path, struct wim_lookup_table_entry *lte,
+                struct apply_ctx *ctx);
+
+       /* OPTIONAL:  Set file attributes.  Calling code calls this if non-NULL.
+        */
+       int (*set_file_attributes)
+               (const tchar *path, u32 attributes, struct apply_ctx *ctx);
+
+       /* OPTIONAL:  Set reparse data.  In start_extract(), set
+        * ctx->supported_features.reparse_data if supported.  */
+       int (*set_reparse_data)
+               (const tchar *path, const u8 *rpbuf, u16 rpbuflen,
+                struct apply_ctx *ctx);
+
+       /* OPTIONAL:  Set short (DOS) filename.  In start_extract(), set
+        * ctx->supported_features.short_name if supported.  */
+       int (*set_short_name)
+               (const tchar *path, const utf16lechar *short_name,
+                size_t short_name_nchars, struct apply_ctx *ctx);
+
+       /* OPTIONAL:  Set Windows NT security descriptor.  In start_extract(),
+        * set ctx->supported_features.security_descriptors if supported.  */
+       int (*set_security_descriptor)
+               (const tchar *path, const u8 *desc, size_t desc_size,
+                struct apply_ctx *ctx, bool strict);
+
+       /* OPTIONAL:  Set wimlib-specific UNIX data.  In start_extract(), set
+        * ctx->supported_features.unix_data if supported.  */
+       int (*set_unix_data)
+               (const tchar *path, const struct wimlib_unix_data *data,
+                struct apply_ctx *ctx);
+
+       /* OPTIONAL:  Set timestamps.  Calling code calls this if non-NULL.  */
+       int (*set_timestamps)
+               (const tchar *path, u64 creation_time, u64 last_write_time,
+                u64 last_access_time, struct apply_ctx *ctx);
+
+       /* OPTIONAL:  Called after the extraction operation has succeeded.  */
+       int (*finish_extract)
+               (struct apply_ctx *ctx);
+
+       /* OPTIONAL:  Called after the extraction operation has failed.  */
+       int (*abort_extract)
+               (struct apply_ctx *ctx);
+
+       /* REQUIRED:  Path separator character to use when building paths.  */
+       tchar path_separator;
+
+       /* REQUIRED:  Maximum path length, in tchars, including the
+        * null-terminator.  */
+       unsigned path_max;
+
+       /* OPTIONAL:  String to prefix every path with.  */
+       const tchar *path_prefix;
+
+       /* OPTIONAL:  Length of path_prefix in tchars.  */
+       unsigned path_prefix_nchars;
+
+       /* OPTIONAL:  Set to 1 if paths must be prefixed by the name of the
+        * extraction target (i.e. if it's interpreted as a directory).  */
+       unsigned requires_target_in_paths : 1;
+
+       /* OPTIONAL:  Like above, but operations require real (absolute) path.
+        * */
+       unsigned requires_realtarget_in_paths : 1;
+
+       /* OPTIONAL:  Set to 1 if realpath() can be used to get the real
+        * (absolute) path of a file on the target volume before it's been
+        * created.  */
+       unsigned realpath_works_on_nonexisting_files : 1;
+
+       /* OPTIONAL:  Set to 1 if this extraction mode supports case sensitive
+        * filenames.  */
+       unsigned supports_case_sensitive_filenames : 1;
+
+       /* OPTIONAL:  Set to 1 if the root directory of the volume (see
+        * target_is_root() callback) should not be explicitly extracted.  */
+       unsigned root_directory_is_special : 1;
 };
 
-#ifdef WITH_NTFS_3G
-extern int
-apply_dentry_ntfs(struct wim_dentry *dentry, void *arg);
+struct wim_features {
+       unsigned long archive_files;
+       unsigned long hidden_files;
+       unsigned long system_files;
+       unsigned long compressed_files;
+       unsigned long encrypted_files;
+       unsigned long not_context_indexed_files;
+       unsigned long sparse_files;
+       unsigned long named_data_streams;
+       unsigned long hard_links;
+       unsigned long reparse_points;
+       unsigned long symlink_reparse_points;
+       unsigned long other_reparse_points;
+       unsigned long security_descriptors;
+       unsigned long short_names;
+       unsigned long unix_data;
+};
 
-extern int
-apply_dentry_timestamps_ntfs(struct wim_dentry *dentry, void *arg);
-#endif
+/* Context for an apply (extract) operation.  */
+struct apply_ctx {
+       WIMStruct *wim;
+       int extract_flags;
+       const tchar *target;
+       size_t target_nchars;
+       wimlib_progress_func_t progress_func;
+       union wimlib_progress_info progress;
+       struct wim_dentry *extract_root;
+       const struct apply_operations *ops;
+       struct wim_features supported_features;
+       struct list_head stream_list;
+       tchar *realtarget;
+       size_t realtarget_nchars;
+       unsigned long invalid_sequence;
+       u64 num_streams_remaining;
+       bool root_dentry_is_special;
+       uint64_t next_progress;
+       intptr_t private[8];
+};
 
 #ifdef __WIN32__
-extern int
-win32_do_apply_dentry(const tchar *output_path,
-                     size_t output_path_nbytes,
-                     struct wim_dentry *dentry,
-                     struct apply_args *args);
-
-extern int
-win32_do_apply_dentry_timestamps(const tchar *output_path,
-                                size_t output_path_nbytes,
-                                struct wim_dentry *dentry,
-                                struct apply_args *args);
-#else /* __WIN32__ */
-extern int
-unix_do_apply_dentry(const tchar *output_path, size_t output_path_nbytes,
-                    struct wim_dentry *dentry, struct apply_args *args);
-extern int
-unix_do_apply_dentry_timestamps(const tchar *output_path,
-                               size_t output_path_nbytes,
-                               struct wim_dentry *dentry,
-                               struct apply_args *args);
-#endif /* !__WIN32__ */
-
-/* Internal use only */
-#define WIMLIB_EXTRACT_FLAG_MULTI_IMAGE                0x80000000
-#define WIMLIB_EXTRACT_FLAG_NO_STREAMS         0x40000000
-#define WIMLIB_EXTRACT_MASK_PUBLIC             0x3fffffff
+  extern const struct apply_operations win32_apply_ops;
+#else
+  extern const struct apply_operations unix_apply_ops;
+#endif
 
+#ifdef WITH_NTFS_3G
+  extern const struct apply_operations ntfs_3g_apply_ops;
+#endif
 
 #endif /* _WIMLIB_APPLY_H */
index 9439366..6d8c557 100644 (file)
@@ -153,16 +153,11 @@ struct wim_dentry {
         * including the terminating null character. */
        u32 full_path_nbytes;
 
-       /* For extraction operations, a subtree of dentries will have this flag
-        * set so we can keep track of which dentries still need to be
-        * extracted.  Otherwise this will always be 0.  */
-       u8 needs_extraction : 1;
-
        /* For extraction operations, this flag will be set when a dentry in the
         * tree being extracted is not being extracted for some reason (file
         * type not supported by target filesystem or contains invalid
         * characters).  Otherwise this will always be 0. */
-       u8 not_extracted : 1;
+       u8 extraction_skipped : 1;
 
        /* When capturing from a NTFS volume using NTFS-3g, this flag is set on
         * dentries that were created from a filename in the WIN32 or WIN32+DOS
@@ -177,6 +172,10 @@ struct wim_dentry {
         * names.  */
        u8 dos_name_invalid : 1;
 
+       u8 tmp_flag : 1;
+
+       u8 was_hardlinked : 1;
+
        /* Temporary list field used to make lists of dentries in a few places.
         * */
        struct list_head tmp_list;
@@ -312,10 +311,6 @@ struct wim_inode {
         * error paths.  */
        u8 i_visited : 1;
 
-       /* For NTFS-3g extraction:  Set after the DOS name for this inode has
-        * been extracted.  */
-       u8 i_dos_name_extracted : 1;
-
        /* Pointer to a malloc()ed array of i_num_ads alternate data stream
         * entries for this inode.  */
        struct wim_ads_entry *i_ads_entries;
@@ -522,6 +517,9 @@ inode_add_ads_with_data(struct wim_inode *inode, const tchar *name,
                        const void *value, size_t size,
                        struct wim_lookup_table *lookup_table);
 
+bool
+inode_has_named_stream(const struct wim_inode *inode);
+
 extern int
 inode_set_unnamed_stream(struct wim_inode *inode, const void *data, size_t len,
                         struct wim_lookup_table *lookup_table);
@@ -537,6 +535,22 @@ inode_remove_ads(struct wim_inode *inode, u16 idx,
 #define WIMLIB_UNIX_DATA_TAG_UTF16LE "$\0$\0_\0_\0w\0i\0m\0l\0i\0b\0_\0U\0N\0I\0X\0_\0d\0a\0t\0a\0"
 #define WIMLIB_UNIX_DATA_TAG_UTF16LE_NBYTES (sizeof(WIMLIB_UNIX_DATA_TAG_UTF16LE) - 1)
 
+static inline bool
+ads_entry_is_unix_data(const struct wim_ads_entry *entry)
+{
+       return (entry->stream_name_nbytes ==
+                       WIMLIB_UNIX_DATA_TAG_UTF16LE_NBYTES) &&
+               !memcmp(entry->stream_name, WIMLIB_UNIX_DATA_TAG_UTF16LE,
+                       WIMLIB_UNIX_DATA_TAG_UTF16LE_NBYTES);
+}
+
+static inline bool
+ads_entry_is_named_stream(const struct wim_ads_entry *entry)
+{
+       return entry->stream_name_nbytes != 0 && !ads_entry_is_unix_data(entry);
+}
+
+#ifndef __WIN32__
 /* Format for special alternate data stream entries to store UNIX data for files
  * and directories (see: WIMLIB_ADD_FLAG_UNIX_DATA) */
 struct wimlib_unix_data {
@@ -546,8 +560,6 @@ struct wimlib_unix_data {
        u16 mode;
 } _packed_attribute;
 
-#ifndef __WIN32__
-
 #define NO_UNIX_DATA (-1)
 #define BAD_UNIX_DATA (-2)
 extern int
@@ -563,7 +575,10 @@ inode_get_unix_data(const struct wim_inode *inode,
 extern int
 inode_set_unix_data(struct wim_inode *inode, uid_t uid, gid_t gid, mode_t mode,
                    struct wim_lookup_table *lookup_table, int which);
-#endif
+#endif /* __WIN32__ */
+
+extern bool
+inode_has_unix_data(const struct wim_inode *inode);
 
 extern int
 read_dentry(const u8 * restrict metadata_resource,
@@ -594,6 +609,14 @@ inode_is_directory(const struct wim_inode *inode)
 }
 
 static inline bool
+inode_is_encrypted_directory(const struct wim_inode *inode)
+{
+       return ((inode->i_attributes & (FILE_ATTRIBUTE_DIRECTORY |
+                                       FILE_ATTRIBUTE_ENCRYPTED))
+               == (FILE_ATTRIBUTE_DIRECTORY | FILE_ATTRIBUTE_ENCRYPTED));
+}
+
+static inline bool
 dentry_is_directory(const struct wim_dentry *dentry)
 {
        return inode_is_directory(dentry->d_inode);
index 1d25898..de59cfa 100644 (file)
@@ -4,18 +4,29 @@
 #include <stddef.h>
 #include <sys/types.h>
 
-extern size_t
-full_read(int fd, void *buf, size_t n);
+struct filedes {
+       int fd;
+       unsigned int is_pipe : 1;
+       off_t offset;
+};
+
+extern int
+full_read(struct filedes *fd, void *buf, size_t n);
 
-extern size_t
-full_write(int fd, const void *buf, size_t n);
+extern int
+full_pread(struct filedes *fd, void *buf, size_t nbyte, off_t offset);
 
-extern size_t
-full_pread(int fd, void *buf, size_t nbyte, off_t offset);
+extern int
+full_write(struct filedes *fd, const void *buf, size_t n);
 
-extern size_t
-full_pwrite(int fd, const void *buf, size_t count, off_t offset);
+extern int
+full_pwrite(struct filedes *fd, const void *buf, size_t count, off_t offset);
 
+extern ssize_t
+raw_pread(struct filedes *fd, void *buf, size_t nbyte, off_t offset);
+
+extern ssize_t
+raw_pwrite(struct filedes *fd, const void *buf, size_t count, off_t offset);
 
 #ifdef __WIN32__
 struct iovec {
@@ -26,14 +37,42 @@ struct iovec {
 struct iovec;
 #endif
 
-extern size_t
-full_writev(int fd, struct iovec *iov, int iovcnt);
-
-extern off_t
-filedes_offset(int fd);
+extern int
+full_writev(struct filedes *fd, struct iovec *iov, int iovcnt);
 
 #ifndef __WIN32__
 #  define O_BINARY 0
 #endif
 
+extern off_t
+filedes_seek(struct filedes *fd, off_t offset);
+
+extern bool
+filedes_is_seekable(struct filedes *fd);
+
+static inline void filedes_init(struct filedes *fd, int raw_fd)
+{
+       fd->fd = raw_fd;
+       fd->offset = 0;
+       fd->is_pipe = 0;
+}
+
+static inline void filedes_invalidate(struct filedes *fd)
+{
+       fd->fd = -1;
+}
+
+static inline void filedes_copy(struct filedes *dst, const struct filedes *src)
+{
+       *dst = *src;
+}
+
+#define filedes_close(f) close((f)->fd)
+
+static inline bool
+filedes_valid(const struct filedes *fd)
+{
+       return fd->fd != -1;
+}
+
 #endif /* _WIMLIB_FILE_IO_H */
index 0702b45..238b6b2 100644 (file)
@@ -3,24 +3,54 @@
 
 #include "wimlib/resource.h"
 #include "wimlib/types.h"
+#include "wimlib/endianness.h"
 
+/* Length of "Globally Unique ID" field in WIM header.  */
 #define WIM_GID_LEN    16
 
-/* Length of the WIM header on disk. */
+/* Length of the WIM header on disk.  */
 #define WIM_HEADER_DISK_SIZE 208
 
 /* Compressed resources in the WIM are divided into separated compressed chunks
- * of this size. */
+ * of this size.  This value is unfortunately not configurable (at least when
+ * compatibility with Microsoft's software is desired).  */
 #define WIM_CHUNK_SIZE 32768
 
-/* Version of the WIM file.  There is an older version, but we don't support it
- * yet.  The differences between the versions are undocumented. */
+/* Version of the WIM file.  There is an older version, but wimlib doesn't
+ * support it.  The differences between the versions are undocumented.  */
 #define WIM_VERSION 0x10d00
 
+/* WIM magic characters, translated to a single 64-bit little endian number.  */
+#define WIM_MAGIC \
+               cpu_to_le64(((u64)'M' << 0) |           \
+                           ((u64)'S' << 8) |           \
+                           ((u64)'W' << 16) |          \
+                           ((u64)'I' << 24) |          \
+                           ((u64)'M' << 32) |          \
+                           ((u64)'\0' << 40) |         \
+                           ((u64)'\0' << 48) |         \
+                           ((u64)'\0' << 54))
+
+/* wimlib pipable WIM magic characters, translated to a single 64-bit little
+ * endian number.  */
+#define PWM_MAGIC \
+               cpu_to_le64(((u64)'W' << 0) |           \
+                           ((u64)'L' << 8) |           \
+                           ((u64)'P' << 16) |          \
+                           ((u64)'W' << 24) |          \
+                           ((u64)'M' << 32) |          \
+                           ((u64)'\0' << 40) |         \
+                           ((u64)'\0' << 48) |         \
+                           ((u64)'\0' << 54))
+
 /* Header at the very beginning of the WIM file.  This is the in-memory
  * representation and does not include all fields; see `struct wim_header_disk'
- * for the on-disk structure. */
+ * for the on-disk structure.  */
 struct wim_header {
+
+       /* Magic characters: either WIM_MAGIC or PWM_MAGIC.  */
+       le64 magic;
+
        /* Bitwise OR of one or more of the WIM_HDR_FLAG_* defined below. */
        u32 flags;
 
index aa541f6..c9e07ca 100644 (file)
@@ -8,11 +8,8 @@
 #define WIM_INTEGRITY_NOT_OK -1
 #define WIM_INTEGRITY_NONEXISTENT -2
 
-struct resource_entry;
-
 extern int
-write_integrity_table(int fd,
-                     struct resource_entry *integrity_res_entry,
+write_integrity_table(WIMStruct *wim,
                      off_t new_lookup_table_end,
                      off_t old_lookup_table_end,
                      wimlib_progress_func_t progress_func);
index ff6a12a..e953297 100644 (file)
@@ -59,17 +59,20 @@ enum resource_location {
        RESOURCE_NONEXISTENT = 0,
 
        /* The stream resource is located in a WIM file.  The WIMStruct for the
-        * WIM file will be pointed to by the @wim member. */
+        * WIM file will be pointed to by the @wim member.  The compression type
+        * of the resource will be cached in @compression_type, and the pipable
+        * status of the resource will be cached in @pipable.  */
        RESOURCE_IN_WIM,
 
-#ifndef __WIN32__
        /* The stream resource is located in an external file.  The name of the
-        * file will be provided by @file_on_disk member. */
+        * file will be provided by @file_on_disk member.
+        *
+        * Note: On Windows @file_on_disk may actually specify a named data
+        * stream.  */
        RESOURCE_IN_FILE_ON_DISK,
-#endif
 
        /* The stream resource is directly attached in an in-memory buffer
-        * pointed to by @attached_buffer. */
+        * pointed to by @attached_buffer.  */
        RESOURCE_IN_ATTACHED_BUFFER,
 
 #ifdef WITH_FUSE
@@ -82,15 +85,11 @@ enum resource_location {
        /* The stream resource is located in an NTFS volume.  It is identified
         * by volume, filename, data stream name, and by whether it is a reparse
         * point or not. @ntfs_loc points to a structure containing this
-        * information. */
+        * information.  */
        RESOURCE_IN_NTFS_VOLUME,
 #endif
 
 #ifdef __WIN32__
-       /* Resource must be accessed using Win32 API (may be a named data
-        * stream) */
-       RESOURCE_WIN32,
-
        /* Windows only: the file is on disk in the file named @file_on_disk,
         * but the file is encrypted and must be read using special functions.
         * */
@@ -154,23 +153,33 @@ struct wim_lookup_table_entry {
         * same as WIMLIB_COMPRESSION_TYPE_NONE.  */
        u16 compression_type : 2;
 
+       /* If resource_location == RESOURCE_IN_WIM, this flag will be set if the
+        * WIM is pipable and therefore the stream is in a slightly different
+        * format.  See comment above write_pipable_wim().  */
+       u16 is_pipable : 1;
+
+       /* Set to 1 when a metadata entry has its checksum changed; in such
+        * cases the hash is no longer valid to verify the data if the metadata
+        * resource is read again.  */
+       u16 dont_check_metadata_hash : 1;
+
        /* (On-disk field)
         * Number of times this lookup table entry is referenced by dentries.
         * Unfortunately, this field is not always set correctly in Microsoft's
         * WIMs, so we have no choice but to fix it if more references to the
-        * lookup table entry are found than stated here. */
+        * lookup table entry are found than stated here.  */
        u32 refcnt;
 
        union {
                /* (On-disk field) SHA1 message digest of the stream referenced
-                * by this lookup table entry */
+                * by this lookup table entry */
                u8  hash[SHA1_HASH_SIZE];
 
                /* First 4 or 8 bytes of the SHA1 message digest, used for
                 * inserting the entry into the hash table.  Since the SHA1
                 * message digest can be considered random, we don't really need
                 * the full 20 byte hash just to insert the entry in a hash
-                * table. */
+                * table.  */
                size_t hash_short;
 
                /* Unhashed entries only (unhashed == 1): these variables make
@@ -223,7 +232,19 @@ struct wim_lookup_table_entry {
                tchar *extracted_file;
        };
 
+       /* Temporary fields  */
        union {
+               /* Used temporarily during WIM file writing  */
+               struct {
+                       struct hlist_node hash_list_2;
+               };
+
+               /* Used temporarily during WIM file writing (after above)  */
+               struct {
+                       struct list_head msg_list;
+                       struct list_head being_compressed_list;
+               };
+
                /* When a WIM file is written, @output_resource_entry is filled
                 * in with the resource entry for the output WIM.  This will not
                 * necessarily be the same as the @resource_entry since:
@@ -233,27 +254,36 @@ struct wim_lookup_table_entry {
                 */
                struct resource_entry output_resource_entry;
 
-               struct {
-                       struct list_head msg_list;
-                       struct list_head being_compressed_list;
-               };
-               struct list_head lte_dentry_list;
 
-               struct {
-                       struct hlist_node hash_list_2;
-
-                       struct list_head write_streams_list;
+               /* Used temporarily during extraction  */
+               union {
+                       /* out_refcnt tracks number of slots filled */
+                       struct wim_dentry *inline_lte_dentries[4];
+                       struct {
+                               struct wim_dentry **lte_dentries;
+                               unsigned long alloc_lte_dentries;
+                       };
                };
        };
 
        /* Temporary list fields */
        union {
-               struct list_head unhashed_list;
-               struct list_head swm_stream_list;
+               /* Links streams when writing lookup table.  */
                struct list_head lookup_table_list;
+
+               /* Links streams being extracted.  */
                struct list_head extraction_list;
+
+               /* Links streams being exported.  */
                struct list_head export_stream_list;
        };
+
+       /* Links streams that are still unhashed after being been added
+        * to a WIM.  */
+       struct list_head unhashed_list;
+
+       /* Links streams being written to the WIM.  */
+       struct list_head write_streams_list;
 };
 
 static inline u64
@@ -284,12 +314,9 @@ wim_resource_compression_type(const struct wim_lookup_table_entry *lte)
 static inline bool
 lte_filename_valid(const struct wim_lookup_table_entry *lte)
 {
-       return 0
+       return     lte->resource_location == RESOURCE_IN_FILE_ON_DISK
        #ifdef __WIN32__
-               || lte->resource_location == RESOURCE_WIN32
                || lte->resource_location == RESOURCE_WIN32_ENCRYPTED
-       #else
-               || lte->resource_location == RESOURCE_IN_FILE_ON_DISK
        #endif
        #ifdef WITH_FUSE
                || lte->resource_location == RESOURCE_IN_STAGING_FILE
@@ -301,15 +328,12 @@ extern struct wim_lookup_table *
 new_lookup_table(size_t capacity) _malloc_attribute;
 
 extern int
-read_lookup_table(WIMStruct *wim);
+read_wim_lookup_table(WIMStruct *wim);
 
 extern int
-write_lookup_table(WIMStruct *wim, int image, struct resource_entry *out_res_entry);
-
-extern int
-write_lookup_table_from_stream_list(struct list_head *stream_list,
-                                   int out_fd,
-                                   struct resource_entry *out_res_entry);
+write_wim_lookup_table(WIMStruct *wim, int image, int write_flags,
+                      struct resource_entry *out_res_entry,
+                      struct list_head *stream_list_override);
 
 extern void
 free_lookup_table(struct wim_lookup_table *table);
@@ -384,8 +408,12 @@ lte_zero_real_refcnt(struct wim_lookup_table_entry *entry, void *ignore);
 extern int
 lte_free_extracted_file(struct wim_lookup_table_entry *lte, void *ignore);
 
+extern void
+lte_init_wim(struct wim_lookup_table_entry *lte, WIMStruct *wim);
+
 extern int
-inode_resolve_ltes(struct wim_inode *inode, struct wim_lookup_table *table);
+inode_resolve_ltes(struct wim_inode *inode, struct wim_lookup_table *table,
+                  bool force);
 
 extern void
 inode_unresolve_ltes(struct wim_inode *inode);
index 577e932..f37e2f4 100644 (file)
@@ -64,12 +64,13 @@ make_reparse_buffer(const struct reparse_data * restrict rpdata,
 extern int
 wim_inode_get_reparse_data(const struct wim_inode * restrict inode,
                           u8 * restrict rpbuf,
-                          u16 * restrict rpbuflen_ret);
+                          u16 * restrict rpbuflen_ret,
+                          struct wim_lookup_table_entry *lte_override);
 
 #ifndef __WIN32__
 ssize_t
 wim_inode_readlink(const struct wim_inode * restrict inode, char * restrict buf,
-                  size_t buf_len);
+                  size_t buf_len, struct wim_lookup_table_entry *lte);
 
 extern int
 wim_inode_set_symlink(struct wim_inode *inode, const char *target,
index 60da5bf..e030e33 100644 (file)
@@ -2,27 +2,39 @@
 #define _WIMLIB_RESOURCE_H
 
 #include "wimlib/types.h"
+#include "wimlib/endianness.h"
 #include "wimlib/callback.h"
+#include "wimlib/file_io.h"
+#include "wimlib/sha1.h"
 
 struct wim_lookup_table_entry;
 struct wim_image_metadata;
 
-/* Metadata for a resource in a WIM file. */
+/* Description of the location, size, and compression status of a WIM resource
+ * (stream).  This is the in-memory version of `struct resource_entry_disk'.  */
 struct resource_entry {
-       /* Size, in bytes, of the resource in the WIM file. */
+       /* Size, in bytes, of the resource as it appears in the WIM file.  If
+        * the resource is uncompressed, this will be the same as
+        * @original_size.  If the resource is compressed, this will be the
+        * compressed size of the resource, including all compressed chunks as
+        * well as the chunk table.
+        *
+        * Note: if the WIM is "pipable", this value does not include the stream
+        * header.  */
        u64 size  : 56;
 
-       /* Bitwise or of one or more of the WIM_RESHDR_FLAG_* flags. */
+       /* Bitwise OR of one or more of the WIM_RESHDR_FLAG_* flags.  */
        u64 flags : 8;
 
-       /* Offset, in bytes, of the resource in the WIM file. */
+       /* Offset, in bytes, of the resource from the start of the WIM file.  */
        u64 offset;
 
-       /* Uncompressed size of the resource in the WIM file.  Is the same as
-        * @size if the resource is uncompressed. */
+       /* Uncompressed size, in bytes, of the resource (stream).  */
        u64 original_size;
 };
 
+/* On-disk version of `struct resource_entry'.  See `struct resource_entry' for
+ * description of fields.  */
 struct resource_entry_disk {
        u8 size[7];
        u8 flags;
@@ -35,60 +47,42 @@ struct resource_entry_disk {
 /* I haven't seen this flag used in any of the WIMs I have examined.  I assume
  * it means that there are no references to the stream, so the space is free.
  * However, even after deleting files from a WIM mounted with `imagex.exe
- * /mountrw', I could not see this flag being used.  Either way, we don't
+ * /mountrw', I could not see this flag being used.  Either way, wimlib doesn't
  * actually use this flag for anything. */
 #define WIM_RESHDR_FLAG_FREE            0x01
 
-/* Indicates that the stream is a metadata resource for a WIM image. */
+/* Indicates that the stream is a metadata resource for a WIM image.  This flag
+ * is also set in the resource entry for the lookup table in the WIM header.  */
 #define WIM_RESHDR_FLAG_METADATA        0x02
 
-/* Indicates that the stream is compressed. */
+/* Indicates that the stream is compressed (using the WIM's set compression
+ * type).  */
 #define WIM_RESHDR_FLAG_COMPRESSED     0x04
 
 /* I haven't seen this flag used in any of the WIMs I have examined.  Perhaps it
  * means that a stream could possibly be split among multiple split WIM parts.
  * However, `imagex.exe /split' does not seem to create any WIMs like this.
- * Either way, we don't actually use this flag for anything.  */
+ * Either way, wimlib doesn't actually use this flag for anything.  */
 #define WIM_RESHDR_FLAG_SPANNED         0x08
 
-/* Nonzero if a struct resource_entry indicates a compressed resource. */
+/* Functions that operate directly on `struct resource_entry's.  */
+
 static inline int
 resource_is_compressed(const struct resource_entry *entry)
 {
        return (entry->flags & WIM_RESHDR_FLAG_COMPRESSED);
 }
 
-#if 1
-#  define copy_resource_entry(dst, src) memcpy(dst, src, sizeof(struct resource_entry))
-#  define zero_resource_entry(entry) memset(entry, 0, sizeof(struct resource_entry))
-#else
-static inline void
-copy_resource_entry(struct resource_entry *dst,
-                   const struct resource_entry *src)
+static inline void copy_resource_entry(struct resource_entry *dst,
+                                      const struct resource_entry *src)
 {
-       BUILD_BUG_ON(sizeof(struct resource_entry) != 24);
-       ((u64*)dst)[0] = ((u64*)src)[0];
-       ((u64*)dst)[1] = ((u64*)src)[1];
-       ((u64*)dst)[2] = ((u64*)src)[2];
+       memcpy(dst, src, sizeof(struct resource_entry));
 }
 
-static inline void
-zero_resource_entry(struct resource_entry *entry)
+static inline void zero_resource_entry(struct resource_entry *entry)
 {
-       BUILD_BUG_ON(sizeof(struct resource_entry) != 24);
-       ((u64*)entry)[0] = 0;
-       ((u64*)entry)[1] = 0;
-       ((u64*)entry)[2] = 0;
+       memset(entry, 0, sizeof(struct resource_entry));
 }
-#endif
-
-#define WIMLIB_RESOURCE_FLAG_RAW               0x1
-#define WIMLIB_RESOURCE_FLAG_RECOMPRESS                0x2
-
-extern int
-read_resource_prefix(const struct wim_lookup_table_entry *lte,
-                    u64 size, consume_data_callback_t cb, void *ctx_or_buf,
-                    int flags);
 
 extern void
 get_resource_entry(const struct resource_entry_disk *disk_entry,
@@ -98,6 +92,26 @@ extern void
 put_resource_entry(const struct resource_entry *entry,
                   struct resource_entry_disk *disk_entry);
 
+/* wimlib internal flags used when reading or writing resources.  */
+#define WIMLIB_WRITE_RESOURCE_FLAG_RECOMPRESS          0x00000001
+#define WIMLIB_WRITE_RESOURCE_FLAG_PIPABLE             0x00000002
+#define WIMLIB_WRITE_RESOURCE_MASK                     0x0000ffff
+
+#define WIMLIB_READ_RESOURCE_FLAG_RAW_FULL             0x80000000
+#define WIMLIB_READ_RESOURCE_FLAG_RAW_CHUNKS           0x40000000
+#define WIMLIB_READ_RESOURCE_FLAG_SEEK_ONLY            0x20000000
+#define WIMLIB_READ_RESOURCE_FLAG_RAW          (WIMLIB_READ_RESOURCE_FLAG_RAW_FULL |  \
+                                                WIMLIB_READ_RESOURCE_FLAG_RAW_CHUNKS)
+#define WIMLIB_READ_RESOURCE_MASK                      0xffff0000
+
+
+/* Functions to read a resource.  */
+
+extern int
+read_partial_wim_resource(const struct wim_lookup_table_entry *lte,
+                         u64 size, consume_data_callback_t cb,
+                         void *ctx_or_buf, int flags, u64 offset);
+
 extern int
 read_partial_wim_resource_into_buf(const struct wim_lookup_table_entry *lte,
                                   size_t size, u64 offset, void *buf);
@@ -105,9 +119,33 @@ extern int
 read_full_resource_into_buf(const struct wim_lookup_table_entry *lte, void *buf);
 
 extern int
-write_wim_resource(struct wim_lookup_table_entry *lte, int out_fd,
+read_full_resource_into_alloc_buf(const struct wim_lookup_table_entry *lte,
+                                 void **buf_ret);
+
+extern int
+res_entry_to_data(const struct resource_entry *res_entry,
+                 WIMStruct *wim, void **buf_ret);
+
+extern int
+read_resource_prefix(const struct wim_lookup_table_entry *lte,
+                    u64 size, consume_data_callback_t cb, void *ctx_or_buf,
+                    int flags);
+
+/* Functions to write a resource.  */
+
+extern int
+write_wim_resource(struct wim_lookup_table_entry *lte, struct filedes *out_fd,
                   int out_ctype, struct resource_entry *out_res_entry,
-                  int flags);
+                  int write_resource_flags);
+
+extern int
+write_wim_resource_from_buffer(const void *buf, size_t buf_size,
+                              int reshdr_flags, struct filedes *out_fd,
+                              int out_ctype,
+                              struct resource_entry *out_res_entry,
+                              u8 *hash_ret, int write_resource_flags);
+
+/* Functions to extract a resource.  */
 
 extern int
 extract_wim_resource(const struct wim_lookup_table_entry *lte,
@@ -117,19 +155,47 @@ extract_wim_resource(const struct wim_lookup_table_entry *lte,
 
 extern int
 extract_wim_resource_to_fd(const struct wim_lookup_table_entry *lte,
-                          int fd, u64 size);
+                          struct filedes *fd, u64 size);
+
+/* Miscellaneous resource functions.  */
 
 extern int
 sha1_resource(struct wim_lookup_table_entry *lte);
 
-extern int
-copy_resource(struct wim_lookup_table_entry *lte, void *wim);
+/* Functions to read/write metadata resources.  */
 
 extern int
 read_metadata_resource(WIMStruct *wim,
                       struct wim_image_metadata *image_metadata);
 
 extern int
-write_metadata_resource(WIMStruct *wim);
+write_metadata_resource(WIMStruct *wim, int image, int write_resource_flags);
+
+/* Definitions specific to pipable WIM resources.  */
+
+/* Arbitrary number to begin each stream in the pipable WIM, used for sanity
+ * checking.  */
+#define PWM_STREAM_MAGIC 0x2b9b9ba2443db9d8ULL
+
+/* Header that precedes each resource in a pipable WIM.  */
+struct pwm_stream_hdr {
+       le64 magic;                     /* +0   */
+       le64 uncompressed_size;         /* +8   */
+       u8 hash[SHA1_HASH_SIZE];        /* +16  */
+       le32 flags;                     /* +36  */
+                                       /* +40  */
+} _packed_attribute;
+
+/* Extra flag for the @flags field in `struct pipable_wim_stream_hdr': Indicates
+ * that the SHA1 message digest of the stream has not been calculated.
+ * Currently only used for the XML data.  */
+#define PWM_RESHDR_FLAG_UNHASHED         0x100
+
+/* Header that precedes each chunk of a compressed resource in a pipable WIM.
+ */
+struct pwm_chunk_hdr {
+       le32 compressed_size;
+} _packed_attribute;
+
 
 #endif /* _WIMLIB_RESOURCE_H */
index 1ef55dd..dfd31fa 100644 (file)
@@ -3,6 +3,7 @@
 
 #include "wimlib/header.h"
 #include "wimlib/types.h"
+#include "wimlib/file_io.h"
 
 struct wim_info;
 struct wim_lookup_table;
@@ -11,14 +12,15 @@ struct wim_image_metadata;
 /* The opaque structure exposed to the wimlib API. */
 struct WIMStruct {
 
-       /* File descriptor for the WIM file, opened for reading, or -1 if it has
-        * not been opened or there is no associated file backing it yet. */
-       int in_fd;
+       /* File descriptor for the WIM file, opened for reading.  in_fd.fd is -1
+        * if the WIM file has not been opened or there is no associated file
+        * backing it yet. */
+       struct filedes in_fd;
 
        /* File descriptor, opened either for writing only or for
         * reading+writing, for the WIM file (if any) currently being written.
         * */
-       int out_fd;
+       struct filedes out_fd;
 
        /* The name of the WIM file (if any) that has been opened. */
        tchar *filename;
@@ -60,23 +62,35 @@ struct WIMStruct {
        u8 compression_type : 2;
 };
 
+static inline bool wim_is_pipable(const WIMStruct *wim)
+{
+       return (wim->hdr.magic == PWM_MAGIC);
+}
+
+static inline bool wim_has_integrity_table(const WIMStruct *wim)
+{
+       return (wim->hdr.integrity.offset != 0);
+}
+
 extern void
 wim_recalculate_refcnts(WIMStruct *wim);
 
 extern int
-read_header(const tchar *filename, int in_fd, struct wim_header *hdr);
+init_wim_header(struct wim_header *hdr, int ctype);
 
 extern int
-write_header(const struct wim_header *hdr, int out_fd);
+read_wim_header(const tchar *filename, struct filedes *in_fd,
+               struct wim_header *hdr);
 
 extern int
-init_header(struct wim_header *hdr, int ctype);
+write_wim_header(const struct wim_header *hdr, struct filedes *out_fd);
 
 extern int
-write_header_flags(u32 hdr_flags, int out_fd);
+write_wim_header_at_offset(const struct wim_header *hdr, struct filedes *out_fd,
+                          off_t offset);
 
 extern int
-write_header_part_data(u16 part_number, u16 total_parts, int out_fd);
+write_wim_header_flags(u32 hdr_flags, struct filedes *out_fd);
 
 extern int
 rename_wim_path(WIMStruct *wim, const tchar *from, const tchar *to);
@@ -93,6 +107,16 @@ wim_checksum_unhashed_streams(WIMStruct *wim);
 extern int
 reopen_wim(WIMStruct *wim);
 
+/* Internal open flags (pass to open_wim_as_WIMStruct(), not wimlib_open_wim())
+ */
+#define WIMLIB_OPEN_FLAG_FROM_PIPE     0x80000000
+#define WIMLIB_OPEN_MASK_PUBLIC                0x7fffffff
+
+extern int
+open_wim_as_WIMStruct(const void *wim_filename_or_fd, int open_flags,
+                     WIMStruct **wim_ret,
+                     wimlib_progress_func_t progress_func);
+
 extern int
 close_wim(WIMStruct *wim);
 
index d84b0ff..4366c1a 100644 (file)
@@ -30,7 +30,8 @@ extern int
 win32_error_to_errno(DWORD err_code);
 
 extern int
-win32_get_vol_flags(const wchar_t *path, unsigned *vol_flags_ret);
+win32_get_vol_flags(const wchar_t *path, unsigned *vol_flags_ret,
+                   bool *supports_SetFileShortName_ret);
 
 extern HANDLE
 win32_open_existing_file(const wchar_t *path, DWORD dwDesiredAccess);
index 1722dab..7a7bae6 100644 (file)
@@ -5,20 +5,14 @@
 #include "wimlib/types.h"
 
 /* Internal use only */
-#define WIMLIB_WRITE_FLAG_NO_LOOKUP_TABLE      0x80000000
-#define WIMLIB_WRITE_FLAG_REUSE_INTEGRITY_TABLE 0x40000000
-#define WIMLIB_WRITE_FLAG_CHECKPOINT_AFTER_XML  0x20000000
-#define WIMLIB_WRITE_MASK_PUBLIC               0x1fffffff
-
-extern int
-begin_write(WIMStruct *wim, const tchar *path, int write_flags);
-
-extern void
-close_wim_writable(WIMStruct *wim);
-
-extern int
-finish_write(WIMStruct *wim, int image, int write_flags,
-            wimlib_progress_func_t progress_func);
+#define WIMLIB_WRITE_FLAG_NO_LOOKUP_TABLE              0x80000000
+#define WIMLIB_WRITE_FLAG_CHECKPOINT_AFTER_XML         0x40000000
+#define WIMLIB_WRITE_FLAG_REUSE_INTEGRITY_TABLE                0x20000000
+#define WIMLIB_WRITE_FLAG_HEADER_AT_END                        0x10000000
+#define WIMLIB_WRITE_FLAG_FILE_DESCRIPTOR              0x08000000
+#define WIMLIB_WRITE_FLAG_USE_EXISTING_TOTALBYTES      0x04000000
+#define WIMLIB_WRITE_FLAG_NO_METADATA                   0x02000000
+#define WIMLIB_WRITE_MASK_PUBLIC                       0x01ffffff
 
 #if defined(HAVE_SYS_FILE_H) && defined(HAVE_FLOCK)
 extern int
@@ -31,4 +25,18 @@ lock_wim(WIMStruct *wim, int fd)
 }
 #endif
 
+struct list_head;
+
+int
+write_wim_part(WIMStruct *wim,
+              const void *path_or_fd,
+              int image,
+              int write_flags,
+              unsigned num_threads,
+              wimlib_progress_func_t progress_func,
+              unsigned part_number,
+              unsigned total_parts,
+              struct list_head *stream_list_override,
+              const u8 *guid);
+
 #endif /* _WIMLIB_WRITE_H */
index 0f0d368..ce82b14 100644 (file)
@@ -2,20 +2,20 @@
 #define _WIMLIB_XML_H
 
 #include "wimlib/types.h"
+#include "wimlib/file_io.h"
 
-struct image_info;
+struct wim_info;
 struct resource_entry;
 
-/* A struct wim_info structure corresponds to the entire XML data for a WIM file. */
-struct wim_info {
-       u64 total_bytes;
-       int num_images;
-       /* Array of `struct image_info's, one for each image in the WIM that is
-        * mentioned in the XML data. */
-       struct image_info *images;
-};
+extern u64
+wim_info_get_total_bytes(const struct wim_info *info);
+
+extern u64
+wim_info_get_image_total_bytes(const struct wim_info *info, int image);
+
+extern unsigned
+wim_info_get_num_images(const struct wim_info *info);
 
-/* xml.c */
 extern int
 xml_export_image(const struct wim_info *old_wim_info, int image,
                 struct wim_info **new_wim_info_p,
@@ -40,13 +40,16 @@ free_wim_info(struct wim_info *info);
 extern void
 print_image_info(const struct wim_info *wim_info, int image);
 
+#define WIM_TOTALBYTES_USE_EXISTING  ((u64)0 - 1)
+#define WIM_TOTALBYTES_OMIT          ((u64)0 - 2)
+
 extern int
-read_xml_data(int in_fd, const struct resource_entry *res,
-             struct wim_info **info_ret);
+read_wim_xml_data(WIMStruct *wim);
 
 extern int
-write_xml_data(const struct wim_info *wim_info, int image, int out_fd,
-              u64 total_bytes, struct resource_entry *out_res_entry);
+write_wim_xml_data(WIMStruct *wim, int image,
+                  u64 total_bytes, struct resource_entry *out_res_entry,
+                  int write_resource_flags);
 
 extern void
 libxml_global_init(void);
@@ -54,18 +57,6 @@ libxml_global_init(void);
 extern void
 libxml_global_cleanup(void);
 
-static inline u64
-wim_info_get_total_bytes(const struct wim_info *info)
-{
-       return info->total_bytes;
-}
-
-static inline unsigned
-wim_info_get_num_images(const struct wim_info *info)
-{
-       return info->num_images;
-}
-
 #ifdef ENABLE_CUSTOM_MEMORY_ALLOCATOR
 extern void
 xml_set_memory_allocator(void *(*malloc_func)(size_t),
index f0b17f0..3ec85fc 100644 (file)
@@ -51,6 +51,7 @@ typedef wchar_t tchar;
 #  define tstrerror    _wcserror
 #  define taccess      _waccess
 #  define tstrdup      wcsdup
+#  define ttempnam      _wtempnam
 /* The following "tchar" functions do not have exact wide-character equivalents
  * on Windows so require parameter rearrangement or redirection to a replacement
  * function defined ourselves. */
@@ -109,6 +110,7 @@ typedef char tchar;
 #  define tstrtoul     strtoul
 #  define tmkdir       mkdir
 #  define tstrdup      strdup
+#  define ttempnam      tempnam
 #  define TSTRDUP      STRDUP
 #  define tstrerror_r  strerror_r
 #  define trename      rename
index 8a22681..840ea2c 100755 (executable)
@@ -1,13 +1,13 @@
 #!/bin/bash
 
-oldver='1\.4\.1'
-oldmonth=May
+oldver='1\.4\.2'
+oldmonth=June
 oldyear=2013
 
 newmajor=1
-newminor=4
-newpatch=2
-newmonth=June
+newminor=5
+newpatch=0
+newmonth=August
 newyear=2013
 
 
index 9e93c4c..e6e751a 100644 (file)
@@ -13,6 +13,8 @@
 #include <string.h>
 #include <assert.h>
 #include <stdio.h>
+#include <fcntl.h>
+#include <io.h>
 
 /* Replacement for glob() in Windows native builds that operates on wide
  * characters. */
@@ -260,3 +262,7 @@ win32_wbasename(wchar_t *path)
        return p;
 }
 
+void set_fd_to_binary_mode(int fd)
+{
+       _setmode(fd, _O_BINARY);
+}
index 5f21fd9..0d5f5ad 100644 (file)
@@ -45,6 +45,9 @@ win32_mbs_to_wcs(const char *mbs, size_t mbs_nbytes, size_t *num_wchars_ret);
 extern wchar_t *
 win32_wbasename(wchar_t *path);
 
+extern void
+set_fd_to_binary_mode(int fd);
+
 #include "wgetopt.h"
 
 #define optarg                 woptarg
index 4e05f5b..f66bcd5 100644 (file)
@@ -60,6 +60,9 @@
 #  define tglob                glob
 #  define OS_PREFERRED_PATH_SEPARATOR '/'
 #  define OS_PREFERRED_PATH_SEPARATOR_STRING "/"
+static inline void set_fd_to_binary_mode(int fd)
+{
+}
 #endif /* !__WIN32 */
 
 
@@ -90,17 +93,22 @@ static void usage(int cmd_type);
 static void usage_all(void);
 
 static bool imagex_be_quiet = false;
+static FILE *imagex_info_file;
+
+#define imagex_printf(format, ...) \
+               tfprintf(imagex_info_file, format, ##__VA_ARGS__)
 
 
 static const tchar *usage_strings[] = {
 [APPEND] =
 T(
 IMAGEX_PROGNAME" append (DIRECTORY | NTFS_VOLUME) WIMFILE [IMAGE_NAME]\n"
-"                     [DESCRIPTION] [--boot] [--check] [--flags EDITION_ID]\n"
-"                     [--verbose] [--dereference] [--config=FILE]\n"
-"                     [--threads=NUM_THREADS] [--rebuild] [--unix-data]\n"
-"                     [--source-list] [--no-acls] [--strict-acls]\n"
-"                     [--rpfix] [--norpfix]\n"
+"                    [DESCRIPTION] [--boot] [--check] [--nocheck]\n"
+"                    [--flags EDITION_ID] [--verbose] [--dereference]\n"
+"                    [--config=FILE] [--threads=NUM_THREADS] [--rebuild]\n"
+"                    [--unix-data] [--source-list] [--no-acls]\n"
+"                    [--strict-acls] [--rpfix] [--norpfix] [--pipable]\n"
+"                    [--not-pipable]\n"
 ),
 [APPLY] =
 T(
@@ -113,15 +121,17 @@ IMAGEX_PROGNAME" apply WIMFILE [IMAGE_NUM | IMAGE_NAME | all]\n"
 [CAPTURE] =
 T(
 IMAGEX_PROGNAME" capture (DIRECTORY | NTFS_VOLUME) WIMFILE [IMAGE_NAME]\n"
-"                      [DESCRIPTION] [--boot] [--check] [--compress=TYPE]\n"
-"                      [--flags EDITION_ID] [--verbose] [--dereference]\n"
-"                      [--config=FILE] [--threads=NUM_THREADS] [--unix-data]\n"
-"                      [--source-list] [--no-acls] [--strict-acls]\n"
-"                      [--rpfix] [--norpfix]\n"
+"                    [DESCRIPTION] [--boot] [--check] [--nocheck]\n"
+"                    [--compress=TYPE] [--flags EDITION_ID] [--verbose]\n"
+"                    [--dereference] [--config=FILE]\n"
+"                    [--threads=NUM_THREADS] [--unix-data] [--source-list]\n"
+"                    [--no-acls] [--strict-acls] [--rpfix] [--norpfix]\n"
+"                    [--pipable] [--not-pipable]\n"
 ),
 [DELETE] =
 T(
-IMAGEX_PROGNAME" delete WIMFILE (IMAGE_NUM | IMAGE_NAME | all) [--check] [--soft]\n"
+IMAGEX_PROGNAME" delete WIMFILE (IMAGE_NUM | IMAGE_NAME | all) [--check]\n"
+"                    [--soft]\n"
 ),
 [DIR] =
 T(
@@ -130,22 +140,24 @@ IMAGEX_PROGNAME" dir WIMFILE (IMAGE_NUM | IMAGE_NAME | all) [--path=PATH]\n"
 [EXPORT] =
 T(
 IMAGEX_PROGNAME" export SRC_WIMFILE (SRC_IMAGE_NUM | SRC_IMAGE_NAME | all ) \n"
-"              DEST_WIMFILE [DEST_IMAGE_NAME] [DEST_IMAGE_DESCRIPTION]\n"
-"              [--boot] [--check] [--compress=TYPE] [--ref=\"GLOB\"]\n"
-"              [--threads=NUM_THREADS] [--rebuild]\n"
+"                    DEST_WIMFILE [DEST_IMAGE_NAME] [DEST_IMAGE_DESCRIPTION]\n"
+"                    [--boot] [--check] [--nocheck] [--compress=TYPE]\n"
+"                    [--ref=\"GLOB\"] [--threads=NUM_THREADS] [--rebuild]\n"
+"                    [--pipable] [--not-pipable]\n"
 ),
 [EXTRACT] =
 T(
 IMAGEX_PROGNAME" extract WIMFILE (IMAGE_NUM | IMAGE_NAME) [PATH...]\n"
-"              [--check] [--ref=\"GLOB\"] [--verbose] [--unix-data]\n"
-"              [--no-acls] [--strict-acls] [--to-stdout] [--dest-dir=DIR]\n"
-"              [--include-invalid-names]\n"
+"                    [--check] [--ref=\"GLOB\"] [--verbose] [--unix-data]\n"
+"                    [--no-acls] [--strict-acls] [--to-stdout] [--dest-dir=DIR]\n"
+"                    [--include-invalid-names]\n"
 ),
 [INFO] =
 T(
 IMAGEX_PROGNAME" info WIMFILE [IMAGE_NUM | IMAGE_NAME] [NEW_NAME]\n"
-"                   [NEW_DESC] [--boot] [--check] [--header] [--lookup-table]\n"
-"                   [--xml] [--extract-xml FILE] [--metadata]\n"
+"                    [NEW_DESC] [--boot] [--check] [--nocheck] [--header]\n"
+"                    [--lookup-table] [--xml] [--extract-xml FILE]\n"
+"                    [--metadata]\n"
 ),
 [JOIN] =
 T(
@@ -160,13 +172,13 @@ IMAGEX_PROGNAME" mount WIMFILE (IMAGE_NUM | IMAGE_NAME) DIRECTORY\n"
 [MOUNTRW] =
 T(
 IMAGEX_PROGNAME" mountrw WIMFILE [IMAGE_NUM | IMAGE_NAME] DIRECTORY\n"
-"                      [--check] [--debug] [--streams-interface=INTERFACE]\n"
-"                      [--staging-dir=DIR] [--unix-data] [--allow-other]\n"
+"                    [--check] [--debug] [--streams-interface=INTERFACE]\n"
+"                    [--staging-dir=DIR] [--unix-data] [--allow-other]\n"
 ),
 [OPTIMIZE] =
 T(
-IMAGEX_PROGNAME" optimize WIMFILE [--check] [--recompress]\n"
-"                      [--threads=NUM_THREADS]\n"
+IMAGEX_PROGNAME" optimize WIMFILE [--check] [--nocheck] [--recompress]\n"
+"                    [--threads=NUM_THREADS] [--pipable] [--not-pipable]\n"
 ),
 [SPLIT] =
 T(
@@ -179,8 +191,8 @@ IMAGEX_PROGNAME" unmount DIRECTORY [--commit] [--check] [--rebuild] [--lazy]\n"
 [UPDATE] =
 T(
 IMAGEX_PROGNAME" update WIMFILE [IMAGE_NUM | IMAGE_NAME] [--check] [--rebuild]\n"
-"                       [--threads=NUM_THREADS] [DEFAULT_ADD_OPTIONS]\n"
-"                       [DEFAULT_DELETE_OPTIONS] [--command=STRING] [< CMDFILE]\n"
+"                    [--threads=NUM_THREADS] [DEFAULT_ADD_OPTIONS]\n"
+"                    [DEFAULT_DELETE_OPTIONS] [--command=STRING] [< CMDFILE]\n"
 ),
 };
 
@@ -220,8 +232,11 @@ enum {
        IMAGEX_LOOKUP_TABLE_OPTION,
        IMAGEX_METADATA_OPTION,
        IMAGEX_NORPFIX_OPTION,
+       IMAGEX_NOCHECK_OPTION,
        IMAGEX_NO_ACLS_OPTION,
+       IMAGEX_NOT_PIPABLE_OPTION,
        IMAGEX_PATH_OPTION,
+       IMAGEX_PIPABLE_OPTION,
        IMAGEX_REBUILD_OPTION,
        IMAGEX_RECOMPRESS_OPTION,
        IMAGEX_RECURSIVE_OPTION,
@@ -258,6 +273,8 @@ static const struct option apply_options[] = {
 static const struct option capture_or_append_options[] = {
        {T("boot"),        no_argument,       NULL, IMAGEX_BOOT_OPTION},
        {T("check"),       no_argument,       NULL, IMAGEX_CHECK_OPTION},
+       {T("no-check"),    no_argument,       NULL, IMAGEX_NOCHECK_OPTION},
+       {T("nocheck"),     no_argument,       NULL, IMAGEX_NOCHECK_OPTION},
        {T("compress"),    required_argument, NULL, IMAGEX_COMPRESS_OPTION},
        {T("config"),      required_argument, NULL, IMAGEX_CONFIG_OPTION},
        {T("dereference"), no_argument,       NULL, IMAGEX_DEREFERENCE_OPTION},
@@ -272,6 +289,8 @@ static const struct option capture_or_append_options[] = {
        {T("strict-acls"), no_argument,       NULL, IMAGEX_STRICT_ACLS_OPTION},
        {T("rpfix"),       no_argument,       NULL, IMAGEX_RPFIX_OPTION},
        {T("norpfix"),     no_argument,       NULL, IMAGEX_NORPFIX_OPTION},
+       {T("pipable"),     no_argument,       NULL, IMAGEX_PIPABLE_OPTION},
+       {T("not-pipable"), no_argument,       NULL, IMAGEX_NOT_PIPABLE_OPTION},
        {NULL, 0, NULL, 0},
 };
 static const struct option delete_options[] = {
@@ -286,12 +305,16 @@ static const struct option dir_options[] = {
 };
 
 static const struct option export_options[] = {
-       {T("boot"),       no_argument,       NULL, IMAGEX_BOOT_OPTION},
-       {T("check"),      no_argument,       NULL, IMAGEX_CHECK_OPTION},
-       {T("compress"),   required_argument, NULL, IMAGEX_COMPRESS_OPTION},
-       {T("ref"),        required_argument, NULL, IMAGEX_REF_OPTION},
-       {T("threads"),    required_argument, NULL, IMAGEX_THREADS_OPTION},
-       {T("rebuild"),    no_argument,       NULL, IMAGEX_REBUILD_OPTION},
+       {T("boot"),        no_argument,       NULL, IMAGEX_BOOT_OPTION},
+       {T("check"),       no_argument,       NULL, IMAGEX_CHECK_OPTION},
+       {T("nocheck"),     no_argument,       NULL, IMAGEX_NOCHECK_OPTION},
+       {T("no-check"),    no_argument,       NULL, IMAGEX_NOCHECK_OPTION},
+       {T("compress"),    required_argument, NULL, IMAGEX_COMPRESS_OPTION},
+       {T("ref"),         required_argument, NULL, IMAGEX_REF_OPTION},
+       {T("threads"),     required_argument, NULL, IMAGEX_THREADS_OPTION},
+       {T("rebuild"),     no_argument,       NULL, IMAGEX_REBUILD_OPTION},
+       {T("pipable"),     no_argument,       NULL, IMAGEX_PIPABLE_OPTION},
+       {T("not-pipable"), no_argument,       NULL, IMAGEX_NOT_PIPABLE_OPTION},
        {NULL, 0, NULL, 0},
 };
 
@@ -312,6 +335,8 @@ static const struct option extract_options[] = {
 static const struct option info_options[] = {
        {T("boot"),         no_argument,       NULL, IMAGEX_BOOT_OPTION},
        {T("check"),        no_argument,       NULL, IMAGEX_CHECK_OPTION},
+       {T("nocheck"),      no_argument,       NULL, IMAGEX_NOCHECK_OPTION},
+       {T("no-check"),     no_argument,       NULL, IMAGEX_NOCHECK_OPTION},
        {T("extract-xml"),  required_argument, NULL, IMAGEX_EXTRACT_XML_OPTION},
        {T("header"),       no_argument,       NULL, IMAGEX_HEADER_OPTION},
        {T("lookup-table"), no_argument,       NULL, IMAGEX_LOOKUP_TABLE_OPTION},
@@ -337,9 +362,13 @@ static const struct option mount_options[] = {
 };
 
 static const struct option optimize_options[] = {
-       {T("check"),      no_argument, NULL, IMAGEX_CHECK_OPTION},
-       {T("recompress"), no_argument, NULL, IMAGEX_RECOMPRESS_OPTION},
-       {T("threads"),    required_argument, NULL, IMAGEX_THREADS_OPTION},
+       {T("check"),       no_argument,       NULL, IMAGEX_CHECK_OPTION},
+       {T("nocheck"),     no_argument,       NULL, IMAGEX_NOCHECK_OPTION},
+       {T("no-check"),    no_argument,       NULL, IMAGEX_NOCHECK_OPTION},
+       {T("recompress"),  no_argument,       NULL, IMAGEX_RECOMPRESS_OPTION},
+       {T("threads"),     required_argument, NULL, IMAGEX_THREADS_OPTION},
+       {T("pipable"),     no_argument,       NULL, IMAGEX_PIPABLE_OPTION},
+       {T("not-pipable"), no_argument,       NULL, IMAGEX_NOT_PIPABLE_OPTION},
        {NULL, 0, NULL, 0},
 };
 
@@ -1053,11 +1082,11 @@ imagex_progress_func(enum wimlib_progress_msg msg,
                        const tchar *data_type;
 
                        data_type = get_data_type(info->write_streams.compression_type);
-                       tprintf(T("Writing %"TS" data using %u thread%"TS"\n"),
+                       imagex_printf(T("Writing %"TS" data using %u thread%"TS"\n"),
                                data_type, info->write_streams.num_threads,
                                (info->write_streams.num_threads == 1) ? T("") : T("s"));
                }
-               tprintf(T("\r%"PRIu64" %"TS" of %"PRIu64" %"TS" (uncompressed) "
+               imagex_printf(T("\r%"PRIu64" %"TS" of %"PRIu64" %"TS" (uncompressed) "
                        "written (%u%% done)"),
                        info->write_streams.completed_bytes >> unit_shift,
                        unit_name,
@@ -1065,23 +1094,23 @@ imagex_progress_func(enum wimlib_progress_msg msg,
                        unit_name,
                        percent_done);
                if (info->write_streams.completed_bytes >= info->write_streams.total_bytes)
-                       tputchar(T('\n'));
+                       imagex_printf(T("\n"));
                break;
        case WIMLIB_PROGRESS_MSG_SCAN_BEGIN:
-               tprintf(T("Scanning \"%"TS"\""), info->scan.source);
+               imagex_printf(T("Scanning \"%"TS"\""), info->scan.source);
                if (*info->scan.wim_target_path) {
-                       tprintf(T(" (loading as WIM path: "
+                       imagex_printf(T(" (loading as WIM path: "
                                  "\""WIMLIB_WIM_PATH_SEPARATOR_STRING"%"TS"\")...\n"),
                               info->scan.wim_target_path);
                } else {
-                       tprintf(T(" (loading as root of WIM image)...\n"));
+                       imagex_printf(T(" (loading as root of WIM image)...\n"));
                }
                break;
        case WIMLIB_PROGRESS_MSG_SCAN_DENTRY:
                if (info->scan.excluded)
-                       tprintf(T("Excluding \"%"TS"\" from capture\n"), info->scan.cur_path);
+                       imagex_printf(T("Excluding \"%"TS"\" from capture\n"), info->scan.cur_path);
                else
-                       tprintf(T("Scanning \"%"TS"\"\n"), info->scan.cur_path);
+                       imagex_printf(T("Scanning \"%"TS"\"\n"), info->scan.cur_path);
                break;
        /*case WIMLIB_PROGRESS_MSG_SCAN_END:*/
                /*break;*/
@@ -1089,7 +1118,7 @@ imagex_progress_func(enum wimlib_progress_msg msg,
                unit_shift = get_unit(info->integrity.total_bytes, &unit_name);
                percent_done = TO_PERCENT(info->integrity.completed_bytes,
                                          info->integrity.total_bytes);
-               tprintf(T("\rVerifying integrity of \"%"TS"\": %"PRIu64" %"TS" "
+               imagex_printf(T("\rVerifying integrity of \"%"TS"\": %"PRIu64" %"TS" "
                        "of %"PRIu64" %"TS" (%u%%) done"),
                        info->integrity.filename,
                        info->integrity.completed_bytes >> unit_shift,
@@ -1098,13 +1127,13 @@ imagex_progress_func(enum wimlib_progress_msg msg,
                        unit_name,
                        percent_done);
                if (info->integrity.completed_bytes == info->integrity.total_bytes)
-                       tputchar(T('\n'));
+                       imagex_printf(T("\n"));
                break;
        case WIMLIB_PROGRESS_MSG_CALC_INTEGRITY:
                unit_shift = get_unit(info->integrity.total_bytes, &unit_name);
                percent_done = TO_PERCENT(info->integrity.completed_bytes,
                                          info->integrity.total_bytes);
-               tprintf(T("\rCalculating integrity table for WIM: %"PRIu64" %"TS" "
+               imagex_printf(T("\rCalculating integrity table for WIM: %"PRIu64" %"TS" "
                          "of %"PRIu64" %"TS" (%u%%) done"),
                        info->integrity.completed_bytes >> unit_shift,
                        unit_name,
@@ -1112,10 +1141,10 @@ imagex_progress_func(enum wimlib_progress_msg msg,
                        unit_name,
                        percent_done);
                if (info->integrity.completed_bytes == info->integrity.total_bytes)
-                       tputchar(T('\n'));
+                       imagex_printf(T("\n"));
                break;
        case WIMLIB_PROGRESS_MSG_EXTRACT_IMAGE_BEGIN:
-               tprintf(T("Applying image %d (\"%"TS"\") from \"%"TS"\" "
+               imagex_printf(T("Applying image %d (\"%"TS"\") from \"%"TS"\" "
                          "to %"TS" \"%"TS"\"\n"),
                        info->extract.image,
                        info->extract.image_name,
@@ -1125,7 +1154,7 @@ imagex_progress_func(enum wimlib_progress_msg msg,
                        info->extract.target);
                break;
        case WIMLIB_PROGRESS_MSG_EXTRACT_TREE_BEGIN:
-               tprintf(T("Extracting "
+               imagex_printf(T("Extracting "
                          "\""WIMLIB_WIM_PATH_SEPARATOR_STRING"%"TS"\" from image %d (\"%"TS"\") "
                          "in \"%"TS"\" to \"%"TS"\"\n"),
                        info->extract.extract_root_wim_source_path,
@@ -1135,14 +1164,14 @@ imagex_progress_func(enum wimlib_progress_msg msg,
                        info->extract.target);
                break;
        /*case WIMLIB_PROGRESS_MSG_EXTRACT_DIR_STRUCTURE_BEGIN:*/
-               /*tprintf(T("Applying directory structure to %"TS"\n"),*/
+               /*imagex_printf(T("Applying directory structure to %"TS"\n"),*/
                        /*info->extract.target);*/
                /*break;*/
        case WIMLIB_PROGRESS_MSG_EXTRACT_STREAMS:
                percent_done = TO_PERCENT(info->extract.completed_bytes,
                                          info->extract.total_bytes);
                unit_shift = get_unit(info->extract.total_bytes, &unit_name);
-               tprintf(T("\rExtracting files: "
+               imagex_printf(T("\rExtracting files: "
                          "%"PRIu64" %"TS" of %"PRIu64" %"TS" (%u%%) done"),
                        info->extract.completed_bytes >> unit_shift,
                        unit_name,
@@ -1150,18 +1179,15 @@ imagex_progress_func(enum wimlib_progress_msg msg,
                        unit_name,
                        percent_done);
                if (info->extract.completed_bytes >= info->extract.total_bytes)
-                       tputchar(T('\n'));
-               break;
-       case WIMLIB_PROGRESS_MSG_EXTRACT_DENTRY:
-               tprintf(T("%"TS"\n"), info->extract.cur_path);
+                       imagex_printf(T("\n"));
                break;
        case WIMLIB_PROGRESS_MSG_APPLY_TIMESTAMPS:
                if (info->extract.extract_root_wim_source_path[0] == T('\0'))
-                       tprintf(T("Setting timestamps on all extracted files...\n"));
+                       imagex_printf(T("Setting timestamps on all extracted files...\n"));
                break;
        case WIMLIB_PROGRESS_MSG_EXTRACT_IMAGE_END:
                if (info->extract.extract_flags & WIMLIB_EXTRACT_FLAG_NTFS) {
-                       tprintf(T("Unmounting NTFS volume \"%"TS"\"...\n"),
+                       imagex_printf(T("Unmounting NTFS volume \"%"TS"\"...\n"),
                                info->extract.target);
                }
                break;
@@ -1169,7 +1195,7 @@ imagex_progress_func(enum wimlib_progress_msg msg,
                percent_done = TO_PERCENT(info->join.completed_bytes,
                                          info->join.total_bytes);
                unit_shift = get_unit(info->join.total_bytes, &unit_name);
-               tprintf(T("Writing resources from part %u of %u: "
+               imagex_printf(T("Writing resources from part %u of %u: "
                          "%"PRIu64 " %"TS" of %"PRIu64" %"TS" (%u%%) written\n"),
                        (info->join.completed_parts == info->join.total_parts) ?
                        info->join.completed_parts : info->join.completed_parts + 1,
@@ -1184,9 +1210,11 @@ imagex_progress_func(enum wimlib_progress_msg msg,
                percent_done = TO_PERCENT(info->split.completed_bytes,
                                          info->split.total_bytes);
                unit_shift = get_unit(info->split.total_bytes, &unit_name);
-               tprintf(T("Writing \"%"TS"\": %"PRIu64" %"TS" of "
+               imagex_printf(T("Writing \"%"TS"\" (part %u of %u): %"PRIu64" %"TS" of "
                          "%"PRIu64" %"TS" (%u%%) written\n"),
                        info->split.part_name,
+                       info->split.cur_part_number,
+                       info->split.total_parts,
                        info->split.completed_bytes >> unit_shift,
                        unit_name,
                        info->split.total_bytes >> unit_shift,
@@ -1195,19 +1223,20 @@ imagex_progress_func(enum wimlib_progress_msg msg,
                break;
        case WIMLIB_PROGRESS_MSG_SPLIT_END_PART:
                if (info->split.completed_bytes == info->split.total_bytes) {
-                       tprintf(T("Finished writing %u split WIM parts\n"),
-                               info->split.cur_part_number);
+                       imagex_printf(T("Finished writing part %u of %u WIM parts\n"),
+                               info->split.cur_part_number,
+                               info->split.total_parts);
                }
                break;
        case WIMLIB_PROGRESS_MSG_UPDATE_END_COMMAND:
                switch (info->update.command->op) {
                case WIMLIB_UPDATE_OP_DELETE:
-                       tprintf(T("Deleted WIM path "
+                       imagex_printf(T("Deleted WIM path "
                                  "\""WIMLIB_WIM_PATH_SEPARATOR_STRING "%"TS"\"\n"),
                                info->update.command->delete.wim_path);
                        break;
                case WIMLIB_UPDATE_OP_RENAME:
-                       tprintf(T("Renamed WIM path "
+                       imagex_printf(T("Renamed WIM path "
                                  "\""WIMLIB_WIM_PATH_SEPARATOR_STRING "%"TS"\" => "
                                  "\""WIMLIB_WIM_PATH_SEPARATOR_STRING "%"TS"\"\n"),
                                info->update.command->rename.wim_source_path,
@@ -1511,7 +1540,7 @@ parse_update_command_file(tchar **cmd_file_contents_p, size_t cmd_file_nchars,
 }
 
 /* Apply one image, or all images, from a WIM file into a directory, OR apply
- * one image from a WIM file to a NTFS volume. */
+ * one image from a WIM file to a NTFS volume.  */
 static int
 imagex_apply(int argc, tchar **argv)
 {
@@ -1522,6 +1551,7 @@ imagex_apply(int argc, tchar **argv)
        int ret;
        const tchar *wimfile;
        const tchar *target;
+       const tchar *image_num_or_name;
        int extract_flags = WIMLIB_EXTRACT_FLAG_SEQUENTIAL;
 
        const tchar *swm_glob = NULL;
@@ -1575,46 +1605,62 @@ imagex_apply(int argc, tchar **argv)
 
        wimfile = argv[0];
 
-       ret = wimlib_open_wim(wimfile, open_flags, &wim, imagex_progress_func);
-       if (ret)
-               goto out;
-
-       if (argc >= 3) {
-               /* Image explicitly specified.  */
-               image = wimlib_resolve_image(wim, argv[1]);
-               ret = verify_image_exists(image, argv[1], wimfile);
-               if (ret)
-                       goto out_wimlib_free;
+       if (!tstrcmp(wimfile, T("-"))) {
+               /* Attempt to apply pipable WIM from standard input.  */
+               if (argc < 3) {
+                       imagex_error(T("Imagex index or name must be explicitly "
+                                      "specified when applying pipable WIM on "
+                                      "standard input."));
+                       goto out_usage;
+               }
+               image_num_or_name = argv[1];
                target = argv[2];
+               wim = NULL;
+               num_additional_swms = 0;
+               additional_swms = NULL;
        } else {
-               /* No image specified; default to image 1, but only if the WIM
-                * contains exactly one image.  */
-               struct wimlib_wim_info info;
+               ret = wimlib_open_wim(wimfile, open_flags, &wim,
+                                     imagex_progress_func);
+               if (ret)
+                       goto out;
 
-               wimlib_get_wim_info(wim, &info);
-               if (info.image_count != 1) {
-                       imagex_error(T("\"%"TS"\" contains %d images; "
-                                      "Please select one (or all)."),
-                                    wimfile, info.image_count);
-                       wimlib_free(wim);
-                       goto out_usage;
+               if (argc >= 3) {
+                       /* Image explicitly specified.  */
+                       image_num_or_name = argv[1];
+                       image = wimlib_resolve_image(wim, image_num_or_name);
+                       ret = verify_image_exists(image, image_num_or_name, wimfile);
+                       if (ret)
+                               goto out_wimlib_free;
+                       target = argv[2];
+               } else {
+                       /* No image specified; default to image 1, but only if the WIM
+                        * contains exactly one image.  */
+                       struct wimlib_wim_info info;
+
+                       wimlib_get_wim_info(wim, &info);
+                       if (info.image_count != 1) {
+                               imagex_error(T("\"%"TS"\" contains %d images; "
+                                              "Please select one (or all)."),
+                                            wimfile, info.image_count);
+                               wimlib_free(wim);
+                               goto out_usage;
+                       }
+                       image = 1;
+                       target = argv[1];
                }
-               image = 1;
-               target = argv[1];
-       }
 
-       if (swm_glob) {
-               ret = open_swms_from_glob(swm_glob, wimfile, open_flags,
-                                         &additional_swms,
-                                         &num_additional_swms);
-               if (ret)
-                       goto out_wimlib_free;
-       } else {
-               additional_swms = NULL;
-               num_additional_swms = 0;
+               if (swm_glob) {
+                       ret = open_swms_from_glob(swm_glob, wimfile, open_flags,
+                                                 &additional_swms,
+                                                 &num_additional_swms);
+                       if (ret)
+                               goto out_wimlib_free;
+               } else {
+                       additional_swms = NULL;
+                       num_additional_swms = 0;
+               }
        }
 
-
 #ifndef __WIN32__
        {
                /* Interpret a regular file or block device target as a NTFS
@@ -1638,11 +1684,19 @@ imagex_apply(int argc, tchar **argv)
 #ifdef __WIN32__
        win32_acquire_restore_privileges();
 #endif
-       ret = wimlib_extract_image(wim, image, target, extract_flags,
-                                  additional_swms, num_additional_swms,
-                                  imagex_progress_func);
+       if (wim) {
+               ret = wimlib_extract_image(wim, image, target, extract_flags,
+                                          additional_swms, num_additional_swms,
+                                          imagex_progress_func);
+       } else {
+               set_fd_to_binary_mode(STDIN_FILENO);
+               ret = wimlib_extract_image_from_pipe(STDIN_FILENO,
+                                                    image_num_or_name,
+                                                    target, extract_flags,
+                                                    imagex_progress_func);
+       }
        if (ret == 0)
-               tprintf(T("Done applying WIM image.\n"));
+               imagex_printf(T("Done applying WIM image.\n"));
 #ifdef __WIN32__
        win32_release_restore_privileges();
 #endif
@@ -1674,6 +1728,7 @@ imagex_capture_or_append(int argc, tchar **argv)
        int write_flags = 0;
        int compression_type = WIMLIB_COMPRESSION_TYPE_XPRESS;
        const tchar *wimfile;
+       int wim_fd;
        const tchar *name;
        const tchar *desc;
        const tchar *flags_element = NULL;
@@ -1706,6 +1761,9 @@ imagex_capture_or_append(int argc, tchar **argv)
                        open_flags |= WIMLIB_OPEN_FLAG_CHECK_INTEGRITY;
                        write_flags |= WIMLIB_WRITE_FLAG_CHECK_INTEGRITY;
                        break;
+               case IMAGEX_NOCHECK_OPTION:
+                       write_flags |= WIMLIB_WRITE_FLAG_NO_CHECK_INTEGRITY;
+                       break;
                case IMAGEX_CONFIG_OPTION:
                        config_file = optarg;
                        break;
@@ -1749,6 +1807,12 @@ imagex_capture_or_append(int argc, tchar **argv)
                case IMAGEX_NORPFIX_OPTION:
                        add_image_flags |= WIMLIB_ADD_IMAGE_FLAG_NORPFIX;
                        break;
+               case IMAGEX_PIPABLE_OPTION:
+                       write_flags |= WIMLIB_WRITE_FLAG_PIPABLE;
+                       break;
+               case IMAGEX_NOT_PIPABLE_OPTION:
+                       write_flags |= WIMLIB_WRITE_FLAG_NOT_PIPABLE;
+                       break;
                default:
                        goto out_usage;
                }
@@ -1762,6 +1826,28 @@ imagex_capture_or_append(int argc, tchar **argv)
        source = argv[0];
        wimfile = argv[1];
 
+       if (!tstrcmp(wimfile, T("-"))) {
+       #if 0
+               if (!(write_flags & WIMLIB_WRITE_FLAG_PIPABLE)) {
+                       imagex_error("Can't write a non-pipable WIM to "
+                                    "standard output!  Specify --pipable\n"
+                                    "       if you want to create a pipable WIM "
+                                    "(but read the docs first).");
+                       goto out_err;
+               }
+       #else
+               write_flags |= WIMLIB_WRITE_FLAG_PIPABLE;
+       #endif
+               if (cmd == APPEND) {
+                       imagex_error(T("Using standard output for append does "
+                                      "not make sense."));
+                       goto out_err;
+               }
+               wim_fd = STDOUT_FILENO;
+               wimfile = NULL;
+               imagex_info_file = stderr;
+       }
+
        if (argc >= 3) {
                name = argv[2];
                name_defaulted = false;
@@ -1845,7 +1931,7 @@ imagex_capture_or_append(int argc, tchar **argv)
 
                if (tstat(source, &stbuf) == 0) {
                        if (S_ISBLK(stbuf.st_mode) || S_ISREG(stbuf.st_mode)) {
-                               tprintf(T("Capturing WIM image from NTFS "
+                               imagex_printf(T("Capturing WIM image from NTFS "
                                          "filesystem on \"%"TS"\"\n"), source);
                                add_image_flags |= WIMLIB_ADD_IMAGE_FLAG_NTFS;
                        }
@@ -1911,16 +1997,20 @@ imagex_capture_or_append(int argc, tchar **argv)
                }
        }
 
+       /* Write the new WIM or overwrite the existing WIM with the new image
+        * appended.  */
        if (cmd == APPEND) {
                ret = wimlib_overwrite(wim, write_flags, num_threads,
                                       imagex_progress_func);
+       } else if (wimfile) {
+               ret = wimlib_write(wim, wimfile, WIMLIB_ALL_IMAGES,
+                                  write_flags, num_threads,
+                                  imagex_progress_func);
        } else {
-               ret = wimlib_write(wim, wimfile, WIMLIB_ALL_IMAGES, write_flags,
-                                  num_threads, imagex_progress_func);
+               ret = wimlib_write_to_fd(wim, wim_fd, WIMLIB_ALL_IMAGES,
+                                        write_flags, num_threads,
+                                        imagex_progress_func);
        }
-       if (ret)
-               imagex_error(T("Failed to write the WIM file \"%"TS"\""),
-                            wimfile);
 out_release_privs:
 #ifdef __WIN32__
        win32_release_capture_privileges();
@@ -2115,6 +2205,7 @@ imagex_export(int argc, tchar **argv)
        const tchar *src_wimfile;
        const tchar *src_image_num_or_name;
        const tchar *dest_wimfile;
+       int dest_wim_fd;
        const tchar *dest_name;
        const tchar *dest_desc;
        WIMStruct *src_wim;
@@ -2137,6 +2228,9 @@ imagex_export(int argc, tchar **argv)
                        open_flags |= WIMLIB_OPEN_FLAG_CHECK_INTEGRITY;
                        write_flags |= WIMLIB_WRITE_FLAG_CHECK_INTEGRITY;
                        break;
+               case IMAGEX_NOCHECK_OPTION:
+                       write_flags |= WIMLIB_WRITE_FLAG_NO_CHECK_INTEGRITY;
+                       break;
                case IMAGEX_COMPRESS_OPTION:
                        compression_type = get_compression_type(optarg);
                        if (compression_type == WIMLIB_COMPRESSION_TYPE_INVALID)
@@ -2153,6 +2247,12 @@ imagex_export(int argc, tchar **argv)
                case IMAGEX_REBUILD_OPTION:
                        write_flags |= WIMLIB_WRITE_FLAG_REBUILD;
                        break;
+               case IMAGEX_PIPABLE_OPTION:
+                       write_flags |= WIMLIB_WRITE_FLAG_PIPABLE;
+                       break;
+               case IMAGEX_NOT_PIPABLE_OPTION:
+                       write_flags |= WIMLIB_WRITE_FLAG_NOT_PIPABLE;
+                       break;
                default:
                        goto out_usage;
                }
@@ -2172,10 +2272,30 @@ imagex_export(int argc, tchar **argv)
        if (ret)
                goto out;
 
-       /* Determine if the destination is an existing file or not.
-        * If so, we try to append the exported image(s) to it; otherwise, we
-        * create a new WIM containing the exported image(s). */
-       if (tstat(dest_wimfile, &stbuf) == 0) {
+       /* Determine if the destination is an existing file or not.  If so, we
+        * try to append the exported image(s) to it; otherwise, we create a new
+        * WIM containing the exported image(s).  Furthermore, determine if we
+        * need to write a pipable WIM directly to standard output.  */
+
+       if (tstrcmp(dest_wimfile, T("-")) == 0) {
+       #if 0
+               if (!(write_flags & WIMLIB_WRITE_FLAG_PIPABLE)) {
+                       imagex_error("Can't write a non-pipable WIM to "
+                                    "standard output!  Specify --pipable\n"
+                                    "       if you want to create a pipable WIM "
+                                    "(but read the docs first).");
+                       ret = -1;
+                       goto out_free_src_wim;
+               }
+       #else
+               write_flags |= WIMLIB_WRITE_FLAG_PIPABLE;
+       #endif
+               dest_wimfile = NULL;
+               dest_wim_fd = STDOUT_FILENO;
+               imagex_info_file = stderr;
+       }
+       errno = ENOENT;
+       if (dest_wimfile != NULL && tstat(dest_wimfile, &stbuf) == 0) {
                wim_is_new = false;
                /* Destination file exists. */
 
@@ -2254,13 +2374,19 @@ imagex_export(int argc, tchar **argv)
        if (ret)
                goto out_free_swms;
 
-       if (wim_is_new)
+       if (!wim_is_new)
+               ret = wimlib_overwrite(dest_wim, write_flags, num_threads,
+                                      imagex_progress_func);
+       else if (dest_wimfile)
                ret = wimlib_write(dest_wim, dest_wimfile, WIMLIB_ALL_IMAGES,
                                   write_flags, num_threads,
                                   imagex_progress_func);
        else
-               ret = wimlib_overwrite(dest_wim, write_flags, num_threads,
-                                      imagex_progress_func);
+               ret = wimlib_write_to_fd(dest_wim, dest_wim_fd,
+                                        WIMLIB_ALL_IMAGES, write_flags,
+                                        num_threads, imagex_progress_func);
+       if (ret)
+               imagex_error(T("Export failed."));
 out_free_swms:
        for (unsigned i = 0; i < num_additional_swms; i++)
                wimlib_free(additional_swms[i]);
@@ -2386,6 +2512,7 @@ imagex_extract(int argc, tchar **argv)
                        break;
                case IMAGEX_TO_STDOUT_OPTION:
                        extract_flags |= WIMLIB_EXTRACT_FLAG_TO_STDOUT;
+                       imagex_info_file = stderr;
                        imagex_be_quiet = true;
                        break;
                case IMAGEX_INCLUDE_INVALID_NAMES_OPTION:
@@ -2444,7 +2571,7 @@ imagex_extract(int argc, tchar **argv)
                                   imagex_progress_func);
        if (ret == 0) {
                if (!imagex_be_quiet)
-                       tprintf(T("Done extracting files.\n"));
+                       imagex_printf(T("Done extracting files.\n"));
        } else if (ret == WIMLIB_ERR_PATH_DOES_NOT_EXIST) {
                tfprintf(stderr, T("Note: You can use `"IMAGEX_PROGNAME" dir' to see what "
                                   "files and directories\n"
@@ -2495,6 +2622,8 @@ print_wim_information(const tchar *wimfile, const struct wimlib_wim_info *info)
                info->has_integrity_table ? T("yes") : T("no"));
        tprintf(T("Relative path junction: %"TS"\n"),
                info->has_rpfix ? T("yes") : T("no"));
+       tprintf(T("Pipable:        %"TS"\n"),
+               info->pipable ? T("yes") : T("no"));
        tputchar(T('\n'));
 }
 
@@ -2548,6 +2677,7 @@ imagex_info(int argc, tchar **argv)
        int c;
        bool boot         = false;
        bool check        = false;
+       bool nocheck      = false;
        bool header       = false;
        bool lookup_table = false;
        bool xml          = false;
@@ -2572,6 +2702,9 @@ imagex_info(int argc, tchar **argv)
                case IMAGEX_CHECK_OPTION:
                        check = true;
                        break;
+               case IMAGEX_NOCHECK_OPTION:
+                       nocheck = true;
+                       break;
                case IMAGEX_HEADER_OPTION:
                        header = true;
                        short_header = false;
@@ -2607,6 +2740,11 @@ imagex_info(int argc, tchar **argv)
        new_name          = (argc >= 3) ? argv[2] : NULL;
        new_desc          = (argc >= 4) ? argv[3] : NULL;
 
+       if (check && nocheck) {
+               imagex_error(T("Can't specify both --check and --nocheck"));
+               goto out_err;
+       }
+
        if (check)
                open_flags |= WIMLIB_OPEN_FLAG_CHECK_INTEGRITY;
 
@@ -2730,11 +2868,11 @@ imagex_info(int argc, tchar **argv)
 
                if (boot) {
                        if (image == info.boot_index) {
-                               tprintf(T("Image %d is already marked as "
+                               imagex_printf(T("Image %d is already marked as "
                                          "bootable.\n"), image);
                                boot = false;
                        } else {
-                               tprintf(T("Marking image %d as bootable.\n"),
+                               imagex_printf(T("Marking image %d as bootable.\n"),
                                        image);
                                info.boot_index = image;
                                ret = wimlib_set_wim_info(wim, &info,
@@ -2746,11 +2884,11 @@ imagex_info(int argc, tchar **argv)
                if (new_name) {
                        if (!tstrcmp(wimlib_get_image_name(wim, image), new_name))
                        {
-                               tprintf(T("Image %d is already named \"%"TS"\".\n"),
+                               imagex_printf(T("Image %d is already named \"%"TS"\".\n"),
                                        image, new_name);
                                new_name = NULL;
                        } else {
-                               tprintf(T("Changing the name of image %d to "
+                               imagex_printf(T("Changing the name of image %d to "
                                          "\"%"TS"\".\n"), image, new_name);
                                ret = wimlib_set_image_name(wim, image, new_name);
                                if (ret)
@@ -2761,11 +2899,11 @@ imagex_info(int argc, tchar **argv)
                        const tchar *old_desc;
                        old_desc = wimlib_get_image_description(wim, image);
                        if (old_desc && !tstrcmp(old_desc, new_desc)) {
-                               tprintf(T("The description of image %d is already "
+                               imagex_printf(T("The description of image %d is already "
                                          "\"%"TS"\".\n"), image, new_desc);
                                new_desc = NULL;
                        } else {
-                               tprintf(T("Changing the description of image %d "
+                               imagex_printf(T("Changing the description of image %d "
                                          "to \"%"TS"\".\n"), image, new_desc);
                                ret = wimlib_set_image_descripton(wim, image,
                                                                  new_desc);
@@ -2775,19 +2913,23 @@ imagex_info(int argc, tchar **argv)
                }
 
                /* Only call wimlib_overwrite() if something actually needs to
-                * be changed. */
+                * be changed.  */
                if (boot || new_name || new_desc ||
-                   (check && !info.has_integrity_table))
+                   (check && !info.has_integrity_table) ||
+                   (nocheck && info.has_integrity_table))
                {
                        int write_flags = 0;
 
                        if (check)
                                write_flags |= WIMLIB_WRITE_FLAG_CHECK_INTEGRITY;
+                       if (nocheck)
+                               write_flags |= WIMLIB_WRITE_FLAG_NO_CHECK_INTEGRITY;
                        ret = wimlib_overwrite(wim, write_flags, 1,
                                               imagex_progress_func);
                } else {
-                       tprintf(T("The file \"%"TS"\" was not modified because nothing "
-                                 "needed to be done.\n"), wimfile);
+                       imagex_printf(T("The file \"%"TS"\" was not modified "
+                                       "because nothing needed to be done.\n"),
+                                     wimfile);
                        ret = 0;
                }
        }
@@ -2798,6 +2940,7 @@ out:
 
 out_usage:
        usage(INFO);
+out_err:
        ret = -1;
        goto out;
 }
@@ -2992,6 +3135,9 @@ imagex_optimize(int argc, tchar **argv)
                        open_flags |= WIMLIB_OPEN_FLAG_CHECK_INTEGRITY;
                        write_flags |= WIMLIB_WRITE_FLAG_CHECK_INTEGRITY;
                        break;
+               case IMAGEX_NOCHECK_OPTION:
+                       write_flags |= WIMLIB_WRITE_FLAG_NO_CHECK_INTEGRITY;
+                       break;
                case IMAGEX_RECOMPRESS_OPTION:
                        write_flags |= WIMLIB_WRITE_FLAG_RECOMPRESS;
                        break;
@@ -3000,6 +3146,12 @@ imagex_optimize(int argc, tchar **argv)
                        if (num_threads == UINT_MAX)
                                goto out_err;
                        break;
+               case IMAGEX_PIPABLE_OPTION:
+                       write_flags |= WIMLIB_WRITE_FLAG_PIPABLE;
+                       break;
+               case IMAGEX_NOT_PIPABLE_OPTION:
+                       write_flags |= WIMLIB_WRITE_FLAG_NOT_PIPABLE;
+                       break;
                default:
                        goto out_usage;
                }
@@ -3025,8 +3177,10 @@ imagex_optimize(int argc, tchar **argv)
 
        ret = wimlib_overwrite(wim, write_flags, num_threads,
                               imagex_progress_func);
-       if (ret)
+       if (ret) {
+               imagex_error(T("Optimization of \"%"TS"\" failed."), wimfile);
                goto out_wimlib_free;
+       }
 
        new_size = file_get_size(wimfile);
        tprintf(T("\"%"TS"\" optimized size: "), wimfile);
@@ -3470,7 +3624,7 @@ usage_all(void)
 {
        tfputs(T("Usage:\n"), stdout);
        for (int i = 0; i < ARRAY_LEN(usage_strings); i++)
-               tprintf(T("    %"TS), usage_strings[i]);
+               tprintf(T("    %"TS"\n"), usage_strings[i]);
        static const tchar *extra =
        T(
 "    "IMAGEX_PROGNAME" --help\n"
@@ -3498,6 +3652,8 @@ main(int argc, char **argv)
        int ret;
        int init_flags = 0;
 
+       imagex_info_file = stdout;
+
 #ifndef __WIN32__
        if (getenv("WIMLIB_IMAGEX_USE_UTF8")) {
                init_flags |= WIMLIB_INIT_FLAG_ASSUME_UTF8;
index b700e3a..81761f9 100644 (file)
@@ -1,6 +1,6 @@
 Summary:   Library to extract, create, modify, and mount WIM files
 Name:      wimlib
-Version:   1.4.2
+Version:   1.5.0
 Release:   1
 License:   GPLv3+
 Group:     System/Libraries
index 55c4c75..7ea2f06 100644 (file)
@@ -1,6 +1,6 @@
 Summary:   Library to extract, create, modify, and mount WIM files
 Name:      wimlib
-Version:   1.4.2
+Version:   1.5.0
 Release:   1
 License:   GPLv3+
 Group:     System/Libraries
index d40dfa0..746c06a 100644 (file)
@@ -68,7 +68,7 @@ add_new_dentry_tree(WIMStruct *wim, struct wim_dentry *root_dentry,
        return ret;
 }
 
-/* Append an empty image to the WIMStruct. */
+/* API function documented in wimlib.h  */
 WIMLIBAPI int
 wimlib_add_empty_image(WIMStruct *wim, const tchar *name, int *new_idx_ret)
 {
@@ -148,8 +148,7 @@ capture_sources_to_add_cmds(const struct wimlib_capture_source *sources,
        return add_cmds;
 }
 
-/* Adds an image to the WIMStruct from multiple on-disk directory trees, or a
- * NTFS volume. */
+/* API function documented in wimlib.h  */
 WIMLIBAPI int
 wimlib_add_image_multisource(WIMStruct *wim,
                             const struct wimlib_capture_source *sources,
@@ -200,7 +199,7 @@ out:
        return ret;
 }
 
-/* Adds an image to the WIMStruct from an on-disk directory tree or NTFS volume. */
+/* API function documented in wimlib.h  */
 WIMLIBAPI int
 wimlib_add_image(WIMStruct *wim,
                 const tchar *source,
index cc01ba4..1ef8823 100644 (file)
@@ -147,7 +147,7 @@ make_huffman_decode_table(u16 *decode_table,  unsigned num_syms,
                left <<= 1;
                left -= len_counts[len];
                if (unlikely(left < 0)) { /* over-subscribed */
-                       ERROR("Invalid Huffman code (over-subscribed)");
+                       DEBUG("Invalid Huffman code (over-subscribed)");
                        return -1;
                }
        }
@@ -159,7 +159,7 @@ make_huffman_decode_table(u16 *decode_table,  unsigned num_syms,
                               table_num_entries * sizeof(decode_table[0]));
                        return 0;
                } else {
-                       ERROR("Invalid Huffman code (incomplete set)");
+                       DEBUG("Invalid Huffman code (incomplete set)");
                        return -1;
                }
        }
@@ -417,7 +417,7 @@ read_huffsym_near_end_of_input(struct input_bitstream *istream,
                bitstream_remove_bits(istream, key_size);
                do {
                        if (bitsleft == 0) {
-                               ERROR("Input stream exhausted");
+                               DEBUG("Input stream exhausted");
                                return -1;
                        }
                        key_bits = sym + bitstream_peek_bits(istream, 1);
index e73efab..2ad5a27 100644 (file)
@@ -32,9 +32,7 @@
 #include "wimlib/wim.h"
 #include "wimlib/xml.h"
 
-/*
- * Deletes an image from the WIM.
- */
+/* API function documented in wimlib.h  */
 WIMLIBAPI int
 wimlib_delete_image(WIMStruct *wim, int image)
 {
index 02db923..c9a84d7 100644 (file)
@@ -791,8 +791,12 @@ get_dentry_utf16le(WIMStruct *wim, const utf16lechar *path)
        return cur_dentry;
 }
 
-/* Returns the dentry corresponding to the @path, or NULL if there is no such
- * dentry. */
+/*
+ * Returns the dentry in the currently selected WIM image named by @path
+ * starting from the root of the WIM image, or NULL if there is no such dentry.
+ *
+ * On Windows, the search is done case-insensitively.
+ */
 struct wim_dentry *
 get_dentry(WIMStruct *wim, const tchar *path)
 {
@@ -1188,11 +1192,13 @@ do_free_dentry(struct wim_dentry *dentry, void *_lookup_table)
 /*
  * Unlinks and frees a dentry tree.
  *
- * @root:              The root of the tree.
- * @lookup_table:      The lookup table for dentries.  If non-NULL, the
- *                     reference counts in the lookup table for the lookup
- *                     table entries corresponding to the dentries will be
- *                     decremented.
+ * @root:
+ *     The root of the tree.
+ *
+ * @lookup_table:
+ *     The lookup table for dentries.  If non-NULL, the reference counts in the
+ *     lookup table for the lookup table entries corresponding to the dentries
+ *     will be decremented.
  */
 void
 free_dentry_tree(struct wim_dentry *root, struct wim_lookup_table *lookup_table)
@@ -1487,6 +1493,15 @@ inode_add_ads_with_data(struct wim_inode *inode, const tchar *name,
        return 0;
 }
 
+bool
+inode_has_named_stream(const struct wim_inode *inode)
+{
+       for (u16 i = 0; i < inode->i_num_ads; i++)
+               if (ads_entry_is_named_stream(&inode->i_ads_entries[i]))
+                       return true;
+       return false;
+}
+
 /* Set the unnamed stream of a WIM inode, given a data buffer containing the
  * stream contents. */
 int
@@ -1527,6 +1542,15 @@ inode_remove_ads(struct wim_inode *inode, u16 idx,
        inode->i_num_ads--;
 }
 
+bool
+inode_has_unix_data(const struct wim_inode *inode)
+{
+       for (u16 i = 0; i < inode->i_num_ads; i++)
+               if (ads_entry_is_unix_data(&inode->i_ads_entries[i]))
+                       return true;
+       return false;
+}
+
 #ifndef __WIN32__
 int
 inode_get_unix_data(const struct wim_inode *inode,
@@ -1602,19 +1626,24 @@ inode_set_unix_data(struct wim_inode *inode, uid_t uid, gid_t gid, mode_t mode,
 /*
  * Reads the alternate data stream entries of a WIM dentry.
  *
- * @p: Pointer to buffer that starts with the first alternate stream entry.
+ * @p:
+ *     Pointer to buffer that starts with the first alternate stream entry.
  *
- * @inode:     Inode to load the alternate data streams into.
- *             @inode->i_num_ads must have been set to the number of
- *             alternate data streams that are expected.
+ * @inode:
+ *     Inode to load the alternate data streams into.  @inode->i_num_ads must
+ *     have been set to the number of alternate data streams that are expected.
  *
- * @remaining_size:    Number of bytes of data remaining in the buffer pointed
- *                     to by @p.
+ * @remaining_size:
+ *     Number of bytes of data remaining in the buffer pointed to by @p.
  *
+ * On success, inode->i_ads_entries is set to an array of `struct
+ * wim_ads_entry's of length inode->i_num_ads.  On failure, @inode is not
+ * modified.
  *
- * Return 0 on success or nonzero on failure.  On success, inode->i_ads_entries
- * is set to an array of `struct wim_ads_entry's of length inode->i_num_ads.  On
- * failure, @inode is not modified.
+ * Return values:
+ *     WIMLIB_ERR_SUCCESS (0)
+ *     WIMLIB_ERR_INVALID_METADATA_RESOURCE
+ *     WIMLIB_ERR_NOMEM
  */
 static int
 read_ads_entries(const u8 * restrict p, struct wim_inode * restrict inode,
@@ -1718,7 +1747,7 @@ out_of_memory:
        goto out_free_ads_entries;
 out_invalid:
        ERROR("An alternate data stream entry is invalid");
-       ret = WIMLIB_ERR_INVALID_DENTRY;
+       ret = WIMLIB_ERR_INVALID_METADATA_RESOURCE;
 out_free_ads_entries:
        if (ads_entries) {
                for (u16 i = 0; i < num_ads; i++)
@@ -1739,7 +1768,7 @@ out:
  * @metadata_resource_len:
  *             Length of the metadata resource buffer, in bytes.
  *
- * @offset:    Offset of the dentry within the metadata resource.
+ * @offset:    Offset of the dentry within the metadata resource.
  *
  * @dentry:    A `struct wim_dentry' that will be filled in by this function.
  *
@@ -1749,9 +1778,10 @@ out:
  * this was a special "end of directory" dentry and not a real dentry.  If
  * nonzero, this was a real dentry.
  *
- * Possible errors include:
- *     WIMLIB_ERR_NOMEM
- *     WIMLIB_ERR_INVALID_DENTRY
+ * Return values:
+ *     WIMLIB_ERR_SUCCESS (0)
+ *     WIMLIB_ERR_INVALID_METADATA_RESOURCE
+ *     WIMLIB_ERR_NOMEM
  */
 int
 read_dentry(const u8 * restrict metadata_resource, u64 metadata_resource_len,
@@ -1785,7 +1815,7 @@ read_dentry(const u8 * restrict metadata_resource, u64 metadata_resource_len,
                ERROR("Directory entry starting at %"PRIu64" ends past the "
                      "end of the metadata resource (size %"PRIu64")",
                      offset, metadata_resource_len);
-               return WIMLIB_ERR_INVALID_DENTRY;
+               return WIMLIB_ERR_INVALID_METADATA_RESOURCE;
        }
        dentry->length = le64_to_cpu(disk_dentry->length);
 
@@ -1807,7 +1837,7 @@ read_dentry(const u8 * restrict metadata_resource, u64 metadata_resource_len,
                      "%"PRIu64" ends past the end of the metadata resource "
                      "(size %"PRIu64")",
                      offset, dentry->length, metadata_resource_len);
-               return WIMLIB_ERR_INVALID_DENTRY;
+               return WIMLIB_ERR_INVALID_METADATA_RESOURCE;
        }
 
        /* Make sure the dentry length is at least as large as the number of
@@ -1815,7 +1845,7 @@ read_dentry(const u8 * restrict metadata_resource, u64 metadata_resource_len,
        if (dentry->length < sizeof(struct wim_dentry_on_disk)) {
                ERROR("Directory entry has invalid length of %"PRIu64" bytes",
                      dentry->length);
-               return WIMLIB_ERR_INVALID_DENTRY;
+               return WIMLIB_ERR_INVALID_METADATA_RESOURCE;
        }
 
        /* Allocate a `struct wim_inode' for this `struct wim_dentry'. */
@@ -1861,7 +1891,7 @@ read_dentry(const u8 * restrict metadata_resource, u64 metadata_resource_len,
        if ((short_name_nbytes & 1) | (file_name_nbytes & 1))
        {
                ERROR("Dentry name is not valid UTF-16LE (odd number of bytes)!");
-               ret = WIMLIB_ERR_INVALID_DENTRY;
+               ret = WIMLIB_ERR_INVALID_METADATA_RESOURCE;
                goto out_free_inode;
        }
 
@@ -1878,7 +1908,7 @@ read_dentry(const u8 * restrict metadata_resource, u64 metadata_resource_len,
                ERROR("Unexpected end of directory entry! (Expected "
                      "at least %"PRIu64" bytes, got %"PRIu64" bytes.)",
                      calculated_size, dentry->length);
-               ret = WIMLIB_ERR_INVALID_DENTRY;
+               ret = WIMLIB_ERR_INVALID_METADATA_RESOURCE;
                goto out_free_inode;
        }
 
@@ -1932,7 +1962,7 @@ read_dentry(const u8 * restrict metadata_resource, u64 metadata_resource_len,
         * be included in the dentry->length field for some reason.
         */
        if (inode->i_num_ads != 0) {
-               ret = WIMLIB_ERR_INVALID_DENTRY;
+               ret = WIMLIB_ERR_INVALID_METADATA_RESOURCE;
                if (offset + dentry->length > metadata_resource_len ||
                    (ret = read_ads_entries(&metadata_resource[offset + dentry->length],
                                            inode,
@@ -1977,18 +2007,22 @@ dentry_get_file_type_string(const struct wim_dentry *dentry)
 /* Reads the children of a dentry, and all their children, ..., etc. from the
  * metadata resource and into the dentry tree.
  *
- * @metadata_resource: An array that contains the uncompressed metadata
- *                     resource for the WIM file.
+ * @metadata_resource:
+ *     An array that contains the uncompressed metadata resource for the WIM
+ *     file.
  *
- * @metadata_resource_len:  The length of the uncompressed metadata resource, in
- *                         bytes.
+ * @metadata_resource_len:
+ *     The length of the uncompressed metadata resource, in bytes.
  *
- * @dentry:    A pointer to a `struct wim_dentry' that is the root of the directory
- *             tree and has already been read from the metadata resource.  It
- *             does not need to be the real root because this procedure is
- *             called recursively.
+ * @dentry:
+ *     A pointer to a `struct wim_dentry' that is the root of the directory
+ *     tree and has already been read from the metadata resource.  It does not
+ *     need to be the real root because this procedure is called recursively.
  *
- * Returns zero on success; nonzero on failure.
+ * Return values:
+ *     WIMLIB_ERR_SUCCESS (0)
+ *     WIMLIB_ERR_INVALID_METADATA_RESOURCE
+ *     WIMLIB_ERR_NOMEM
  */
 int
 read_dentry_tree(const u8 * restrict metadata_resource,
@@ -2019,7 +2053,7 @@ read_dentry_tree(const u8 * restrict metadata_resource,
                              "of \"%"TS"\" coincide with children of \"%"TS"\"",
                              dentry_full_path(dentry),
                              dentry_full_path(parent));
-                       return WIMLIB_ERR_INVALID_DENTRY;
+                       return WIMLIB_ERR_INVALID_METADATA_RESOURCE;
                }
        }
 
@@ -2113,7 +2147,7 @@ write_dentry(const struct wim_dentry * restrict dentry, u8 * restrict p)
        wimlib_assert(((uintptr_t)p & 7) == 0); /* 8 byte aligned */
        orig_p = p;
 
-       inode = dentry->d_inode;
+       inode = dentry->d_inode;
        disk_dentry = (struct wim_dentry_on_disk*)p;
 
        disk_dentry->attributes = cpu_to_le32(inode->i_attributes);
@@ -2321,7 +2355,7 @@ init_wimlib_dentry(struct wimlib_dir_entry *wdentry,
                wdentry->num_named_streams++;
                if (lte) {
                        lte_to_wimlib_resource_entry(lte, &wdentry->streams[
-                                                               wdentry->num_named_streams].resource);
+                                                               wdentry->num_named_streams].resource);
                }
        #if TCHAR_IS_UTF16LE
                wdentry->streams[wdentry->num_named_streams].stream_name =
@@ -2332,7 +2366,7 @@ init_wimlib_dentry(struct wimlib_dir_entry *wdentry,
                ret = utf16le_to_tstr(inode->i_ads_entries[i].stream_name,
                                      inode->i_ads_entries[i].stream_name_nbytes,
                                      (tchar**)&wdentry->streams[
-                                               wdentry->num_named_streams].stream_name,
+                                               wdentry->num_named_streams].stream_name,
                                      &dummy);
                if (ret)
                        return ret;
@@ -2385,8 +2419,8 @@ do_iterate_dir_tree(WIMStruct *wim,
 
 
        wdentry = CALLOC(1, sizeof(struct wimlib_dir_entry) +
-                                 (1 + dentry->d_inode->i_num_ads) *
-                                       sizeof(struct wimlib_stream_entry));
+                                 (1 + dentry->d_inode->i_num_ads) *
+                                       sizeof(struct wimlib_stream_entry));
        if (!wdentry)
                goto out;
 
@@ -2437,6 +2471,7 @@ image_do_iterate_dir_tree(WIMStruct *wim)
        return do_iterate_dir_tree(wim, dentry, ctx->flags, ctx->cb, ctx->user_ctx);
 }
 
+/* API function documented in wimlib.h  */
 WIMLIBAPI int
 wimlib_iterate_dir_tree(WIMStruct *wim, int image, const tchar *path,
                        int flags,
index 9f8bfa3..17c4f7d 100644 (file)
@@ -95,9 +95,7 @@ inode_move_ltes_to_table(struct wim_inode *inode,
        }
 }
 
-/*
- * Exports an image, or all the images, from a WIM file, into another WIM file.
- */
+/* API function documented in wimlib.h  */
 WIMLIBAPI int
 wimlib_export_image(WIMStruct *src_wim,
                    int src_image,
@@ -255,6 +253,7 @@ wimlib_export_image(WIMStruct *src_wim,
                 * the flag is set on the source WIM. */
                dest_wim->hdr.flags |= WIM_HDR_FLAG_RP_FIX;
        }
+       DEBUG("Successfully exported image.");
        ret = 0;
        goto out;
 out_xml_delete_image:
index 0f5a38d..95071bd 100644 (file)
  * along with wimlib; if not, see http://www.gnu.org/licenses/.
  */
 
+/*
+ * This file provides the API functions wimlib_extract_image(),
+ * wimlib_extract_files(), and wimlib_extract_image_from_pipe().  Internally,
+ * all end up calling extract_tree() zero or more times to extract a tree of
+ * files from the currently selected WIM image to the specified target directory
+ * or NTFS volume.
+ *
+ * Although wimlib supports multiple extraction modes/backends (NTFS-3g, UNIX,
+ * Win32), this file does not itself have code to extract files or directories
+ * to any specific target; instead, it handles generic functionality and relies
+ * on lower-level callback functions declared in `struct apply_operations' to do
+ * the actual extraction.
+ */
+
 #ifdef HAVE_CONFIG_H
 #  include "config.h"
 #endif
 
-#ifdef __WIN32__
-#  include "wimlib/win32_common.h" /* For GetFullPathName() */
-#endif
-
 #include "wimlib/apply.h"
 #include "wimlib/dentry.h"
 #include "wimlib/encoding.h"
 #include "wimlib/endianness.h"
 #include "wimlib/error.h"
 #include "wimlib/lookup_table.h"
+#include "wimlib/metadata.h"
 #include "wimlib/paths.h"
+#include "wimlib/reparse.h"
 #include "wimlib/resource.h"
+#include "wimlib/security.h"
 #include "wimlib/swm.h"
 #ifdef __WIN32__
 #  include "wimlib/win32.h" /* for realpath() equivalent */
 #endif
 #include "wimlib/xml.h"
+#include "wimlib/wim.h"
 
 #include <errno.h>
-#include <limits.h>
-#ifdef WITH_NTFS_3G
-#  include <ntfs-3g/volume.h> /* for ntfs_mount(), ntfs_umount() */
-#endif
+#include <fcntl.h>
 #include <stdlib.h>
 #include <sys/stat.h>
 #include <unistd.h>
 
-#define MAX_EXTRACT_LONG_PATH_WARNINGS 5
+#define WIMLIB_EXTRACT_FLAG_MULTI_IMAGE 0x80000000
+#define WIMLIB_EXTRACT_FLAG_FROM_PIPE   0x40000000
+#define WIMLIB_EXTRACT_MASK_PUBLIC      0x3fffffff
 
+/* Given a WIM dentry in the tree to be extracted, resolve all streams in the
+ * corresponding inode and set 'out_refcnt' in each to 0.  */
 static int
-do_apply_op(struct wim_dentry *dentry, struct apply_args *args,
-           int (*apply_dentry_func)(const tchar *, size_t,
-                                    struct wim_dentry *, struct apply_args *))
+dentry_resolve_and_zero_lte_refcnt(struct wim_dentry *dentry, void *_ctx)
 {
-       tchar *p;
-       size_t extraction_path_nchars;
-       struct wim_dentry *d;
-       LIST_HEAD(ancestor_list);
-       const tchar *target;
+       struct apply_ctx *ctx = _ctx;
+       struct wim_inode *inode = dentry->d_inode;
+       struct wim_lookup_table_entry *lte;
+       int ret;
+       bool force = false;
+
+       if (dentry->extraction_skipped)
+               return 0;
+
+       /* Special case:  when extracting from a pipe, the WIM lookup table is
+        * initially empty, so "resolving" an inode's streams is initially not
+        * possible.  However, we still need to keep track of which streams,
+        * identified by SHA1 message digests, need to be extracted, so we
+        * "resolve" the inode's streams anyway by allocating new entries.  */
+       if (ctx->extract_flags & WIMLIB_EXTRACT_FLAG_FROM_PIPE)
+               force = true;
+       ret = inode_resolve_ltes(inode, ctx->wim->lookup_table, force);
+       if (ret)
+               return ret;
+       for (unsigned i = 0; i <= inode->i_num_ads; i++) {
+               lte = inode_stream_lte_resolved(inode, i);
+               if (lte)
+                       lte->out_refcnt = 0;
+       }
+       return 0;
+}
+
+static inline bool
+is_linked_extraction(const struct apply_ctx *ctx)
+{
+       return 0 != (ctx->extract_flags & (WIMLIB_EXTRACT_FLAG_HARDLINK |
+                                          WIMLIB_EXTRACT_FLAG_SYMLINK));
+}
+
+static inline bool
+can_extract_named_data_streams(const struct apply_ctx *ctx)
+{
+       return ctx->supported_features.named_data_streams &&
+               !is_linked_extraction(ctx);
+}
+
+static int
+ref_stream_to_extract(struct wim_lookup_table_entry *lte,
+                     struct wim_dentry *dentry, struct apply_ctx *ctx)
+{
+       if (!lte)
+               return 0;
+
+       if (likely(!is_linked_extraction(ctx)) || (lte->out_refcnt == 0 &&
+                                                  lte->extracted_file == NULL))
+       {
+               ctx->progress.extract.total_bytes += wim_resource_size(lte);
+               ctx->progress.extract.num_streams++;
+       }
+
+       if (lte->out_refcnt == 0) {
+               list_add_tail(&lte->extraction_list, &ctx->stream_list);
+               ctx->num_streams_remaining++;
+       }
+
+       if (ctx->extract_flags & WIMLIB_EXTRACT_FLAG_SEQUENTIAL) {
+               struct wim_dentry **lte_dentries;
+
+               /* Append dentry to this stream's array of dentries referencing
+                * it.  Use inline array to avoid memory allocation until the
+                * number of dentries becomes too large.  */
+               if (lte->out_refcnt < ARRAY_LEN(lte->inline_lte_dentries)) {
+                       lte_dentries = lte->inline_lte_dentries;
+               } else {
+                       struct wim_dentry **prev_lte_dentries;
+                       size_t alloc_lte_dentries;
+
+                       if (lte->out_refcnt == ARRAY_LEN(lte->inline_lte_dentries)) {
+                               prev_lte_dentries = NULL;
+                               alloc_lte_dentries = ARRAY_LEN(lte->inline_lte_dentries);
+                       } else {
+                               prev_lte_dentries = lte->lte_dentries;
+                               alloc_lte_dentries = lte->alloc_lte_dentries;
+                       }
+
+                       if (lte->out_refcnt == alloc_lte_dentries) {
+                               alloc_lte_dentries *= 2;
+                               lte_dentries = REALLOC(prev_lte_dentries,
+                                                      alloc_lte_dentries *
+                                                       sizeof(lte_dentries[0]));
+                               if (!lte_dentries)
+                                       return WIMLIB_ERR_NOMEM;
+                               if (prev_lte_dentries == NULL) {
+                                       memcpy(lte_dentries,
+                                              lte->inline_lte_dentries,
+                                              sizeof(lte->inline_lte_dentries));
+                               }
+                               lte->lte_dentries = lte_dentries;
+                               lte->alloc_lte_dentries = alloc_lte_dentries;
+                       }
+                       lte_dentries = lte->lte_dentries;
+               }
+               lte_dentries[lte->out_refcnt] = dentry;
+       }
+       lte->out_refcnt++;
+       return 0;
+}
+
+/* Given a WIM dentry in the tree to be extracted, iterate through streams that
+ * need to be extracted.  For each one, add it to the list of streams to be
+ * extracted (ctx->stream_list) if not already done so, and also update the
+ * progress information (ctx->progress) with the stream.  Furthermore, if doing
+ * a sequential extraction, build a mapping from each the stream to the dentries
+ * referencing it.  */
+static int
+dentry_add_streams_to_extract(struct wim_dentry *dentry, void *_ctx)
+{
+       struct apply_ctx *ctx = _ctx;
+       struct wim_inode *inode = dentry->d_inode;
+       int ret;
+
+       /* Don't process dentries marked as skipped.  */
+       if (dentry->extraction_skipped)
+               return 0;
+
+       /* Don't process additional hard links.  */
+       if (inode->i_visited && ctx->supported_features.hard_links)
+               return 0;
+
+       /* The unnamed data stream will always be extracted, except in an
+        * unlikely case.  */
+       if (!inode_is_encrypted_directory(inode)) {
+               ret = ref_stream_to_extract(inode_unnamed_lte_resolved(inode),
+                                           dentry, ctx);
+               if (ret)
+                       return ret;
+       }
+
+       /* Named data streams will be extracted only if supported in the current
+        * extraction mode and volume, and to avoid complications, if not doing
+        * a linked extraction.  */
+       if (can_extract_named_data_streams(ctx)) {
+               for (u16 i = 0; i < inode->i_num_ads; i++) {
+                       if (!ads_entry_is_named_stream(&inode->i_ads_entries[i]))
+                               continue;
+                       ret = ref_stream_to_extract(inode->i_ads_entries[i].lte,
+                                                   dentry, ctx);
+                       if (ret)
+                               return ret;
+               }
+       }
+       inode->i_visited = 1;
+       return 0;
+}
+
+/* Inform library user of progress of stream extraction following the successful
+ * extraction of a copy of the stream specified by @lte.  */
+static void
+update_extract_progress(struct apply_ctx *ctx,
+                       const struct wim_lookup_table_entry *lte)
+{
+       wimlib_progress_func_t progress_func = ctx->progress_func;
+       union wimlib_progress_info *progress = &ctx->progress;
+
+       progress->extract.completed_bytes += wim_resource_size(lte);
+       if (progress_func &&
+           progress->extract.completed_bytes >= ctx->next_progress)
+       {
+               progress_func(WIMLIB_PROGRESS_MSG_EXTRACT_STREAMS, progress);
+               if (progress->extract.completed_bytes >=
+                   progress->extract.total_bytes)
+               {
+                       ctx->next_progress = ~0ULL;
+               } else {
+                       ctx->next_progress += progress->extract.total_bytes / 128;
+                       if (ctx->next_progress > progress->extract.total_bytes)
+                               ctx->next_progress = progress->extract.total_bytes;
+               }
+       }
+}
+
+#ifndef __WIN32__
+/* Extract a symbolic link (not directly as reparse data), handling fixing up
+ * the target of absolute symbolic links and updating the extract progress.
+ *
+ * @inode must specify the WIM inode for a symbolic link or junction reparse
+ * point.
+ *
+ * @lte_override overrides the resource used as the reparse data for the
+ * symbolic link.  */
+static int
+extract_symlink(const tchar *path, struct apply_ctx *ctx,
+               struct wim_inode *inode,
+               struct wim_lookup_table_entry *lte_override)
+{
+       ssize_t bufsize = ctx->ops->path_max;
+       tchar target[bufsize];
+       tchar *buf = target;
+       tchar *fixed_target;
+       ssize_t sret;
+       int ret;
+
+       /* If absolute symbolic link fixups requested, reserve space in the link
+        * target buffer for the absolute path of the target directory.  */
+       if (ctx->extract_flags & WIMLIB_EXTRACT_FLAG_RPFIX)
+       {
+               buf += ctx->realtarget_nchars;
+               bufsize -= ctx->realtarget_nchars;
+       }
+
+       /* Translate the WIM inode's reparse data into the link target.  */
+       sret = wim_inode_readlink(inode, buf, bufsize - 1, lte_override);
+       if (sret < 0) {
+               errno = -sret;
+               return WIMLIB_ERR_READLINK;
+       }
+       buf[sret] = '\0';
+
+       if ((ctx->extract_flags & WIMLIB_EXTRACT_FLAG_RPFIX) &&
+           buf[0] == '/')
+       {
+               /* Fix absolute symbolic link target to point into the
+                * actual extraction destination.  */
+               tmemcpy(target, ctx->realtarget, ctx->realtarget_nchars);
+               fixed_target = target;
+       } else {
+               /* Keep same link target.  */
+               fixed_target = buf;
+       }
+
+       /* Call into the apply_operations to create the symbolic link.  */
+       DEBUG("Creating symlink \"%"TS"\" => \"%"TS"\"",
+             path, fixed_target);
+       ret = ctx->ops->create_symlink(fixed_target, path, ctx);
+       if (ret) {
+               ERROR_WITH_ERRNO("Failed to create symlink "
+                                "\"%"TS"\" => \"%"TS"\"", path, fixed_target);
+               return ret;
+       }
+
+       /* Account for reparse data consumed.  */
+       update_extract_progress(ctx,
+                               (lte_override ? lte_override :
+                                     inode_unnamed_lte_resolved(inode)));
+       return 0;
+}
+#endif /* !__WIN32__ */
+
+/* Create a file, directory, or symbolic link.  */
+static int
+extract_inode(const tchar *path, struct apply_ctx *ctx, struct wim_inode *inode)
+{
+       int ret;
+
+#ifndef __WIN32__
+       if (ctx->supported_features.symlink_reparse_points &&
+           !ctx->supported_features.reparse_points &&
+           inode_is_symlink(inode))
+       {
+               ret = extract_symlink(path, ctx, inode, NULL);
+       } else
+#endif /* !__WIN32__ */
+       if (inode->i_attributes & FILE_ATTRIBUTE_DIRECTORY) {
+               ret = ctx->ops->create_directory(path, ctx);
+               if (ret) {
+                       ERROR_WITH_ERRNO("Failed to create the directory "
+                                        "\"%"TS"\"", path);
+               }
+       } else {
+               ret = ctx->ops->create_file(path, ctx);
+               if (ret) {
+                       ERROR_WITH_ERRNO("Failed to create the file "
+                                        "\"%"TS"\"", path);
+               }
+       }
+       return ret;
+}
+
+static int
+extract_hardlink(const tchar *oldpath, const tchar *newpath,
+                struct apply_ctx *ctx)
+{
+       int ret;
+
+       DEBUG("Creating hardlink \"%"TS"\" => \"%"TS"\"", newpath, oldpath);
+       ret = ctx->ops->create_hardlink(oldpath, newpath, ctx);
+       if (ret) {
+               ERROR_WITH_ERRNO("Failed to create hardlink "
+                                "\"%"TS"\" => \"%"TS"\"",
+                                newpath, oldpath);
+       }
+       return ret;
+}
+
+#ifdef __WIN32__
+static int
+try_extract_rpfix(u8 *rpbuf,
+                 u16 *rpbuflen_p,
+                 const wchar_t *extract_root_realpath,
+                 unsigned extract_root_realpath_nchars)
+{
+       struct reparse_data rpdata;
+       wchar_t *target;
        size_t target_nchars;
+       size_t stripped_nchars;
+       wchar_t *stripped_target;
+       wchar_t stripped_target_nchars;
+       int ret;
+
+       utf16lechar *new_target;
+       utf16lechar *new_print_name;
+       size_t new_target_nchars;
+       size_t new_print_name_nchars;
+       utf16lechar *p;
+
+       ret = parse_reparse_data(rpbuf, *rpbuflen_p, &rpdata);
+       if (ret)
+               return ret;
+
+       if (extract_root_realpath[0] == L'\0' ||
+           extract_root_realpath[1] != L':' ||
+           extract_root_realpath[2] != L'\\')
+               return WIMLIB_ERR_REPARSE_POINT_FIXUP_FAILED;
+
+       ret = parse_substitute_name(rpdata.substitute_name,
+                                   rpdata.substitute_name_nbytes,
+                                   rpdata.rptag);
+       if (ret < 0)
+               return 0;
+       stripped_nchars = ret;
+       target = rpdata.substitute_name;
+       target_nchars = rpdata.substitute_name_nbytes / sizeof(utf16lechar);
+       stripped_target = target + stripped_nchars;
+       stripped_target_nchars = target_nchars - stripped_nchars;
+
+       new_target = alloca((6 + extract_root_realpath_nchars +
+                            stripped_target_nchars) * sizeof(utf16lechar));
+
+       p = new_target;
+       if (stripped_nchars == 6) {
+               /* Include \??\ prefix if it was present before */
+               p = wmempcpy(p, L"\\??\\", 4);
+       }
+
+       /* Print name excludes the \??\ if present. */
+       new_print_name = p;
+       if (stripped_nchars != 0) {
+               /* Get drive letter from real path to extract root, if a drive
+                * letter was present before. */
+               *p++ = extract_root_realpath[0];
+               *p++ = extract_root_realpath[1];
+       }
+       /* Copy the rest of the extract root */
+       p = wmempcpy(p, extract_root_realpath + 2, extract_root_realpath_nchars - 2);
+
+       /* Append the stripped target */
+       p = wmempcpy(p, stripped_target, stripped_target_nchars);
+       new_target_nchars = p - new_target;
+       new_print_name_nchars = p - new_print_name;
+
+       if (new_target_nchars * sizeof(utf16lechar) >= REPARSE_POINT_MAX_SIZE ||
+           new_print_name_nchars * sizeof(utf16lechar) >= REPARSE_POINT_MAX_SIZE)
+               return WIMLIB_ERR_REPARSE_POINT_FIXUP_FAILED;
+
+       rpdata.substitute_name = new_target;
+       rpdata.substitute_name_nbytes = new_target_nchars * sizeof(utf16lechar);
+       rpdata.print_name = new_print_name;
+       rpdata.print_name_nbytes = new_print_name_nchars * sizeof(utf16lechar);
+       return make_reparse_buffer(&rpdata, rpbuf, rpbuflen_p);
+}
+#endif /* __WIN32__ */
+
+/* Set reparse data on extracted file or directory that has
+ * FILE_ATTRIBUTE_REPARSE_POINT set.  */
+static int
+extract_reparse_data(const tchar *path, struct apply_ctx *ctx,
+                    struct wim_inode *inode,
+                    struct wim_lookup_table_entry *lte_override)
+{
+       int ret;
+       u8 rpbuf[REPARSE_POINT_MAX_SIZE];
+       u16 rpbuflen;
+
+       ret = wim_inode_get_reparse_data(inode, rpbuf, &rpbuflen, lte_override);
+       if (ret)
+               goto error;
 
 #ifdef __WIN32__
-       if (args->target_lowlevel_path) {
-               target = args->target_lowlevel_path;
-               target_nchars = args->target_lowlevel_path_nchars;
-       } else
+       /* Fix up target of absolute symbolic link or junction points so
+        * that they point into the actual extraction target.  */
+       if ((ctx->extract_flags & WIMLIB_EXTRACT_FLAG_RPFIX) &&
+           (inode->i_reparse_tag == WIM_IO_REPARSE_TAG_SYMLINK ||
+            inode->i_reparse_tag == WIM_IO_REPARSE_TAG_MOUNT_POINT) &&
+           !inode->i_not_rpfixed)
+       {
+               ret = try_extract_rpfix(rpbuf, &rpbuflen, ctx->realtarget,
+                                       ctx->realtarget_nchars);
+               if (ret && !(ctx->extract_flags &
+                            WIMLIB_EXTRACT_FLAG_STRICT_SYMLINKS))
+               {
+                       WARNING("Reparse point fixup of \"%"TS"\" "
+                               "failed", path);
+                       ret = 0;
+               }
+               if (ret)
+                       goto error;
+       }
 #endif
+
+       ret = ctx->ops->set_reparse_data(path, rpbuf, rpbuflen, ctx);
+
+       /* On Windows, the SeCreateSymbolicLink privilege is required to create
+        * symbolic links.  To be more friendly towards non-Administrator users,
+        * we merely warn the user if symbolic links cannot be created due to
+        * insufficient permissions or privileges, unless
+        * WIMLIB_EXTRACT_FLAG_STRICT_SYMLINKS was provided.  */
+#ifdef __WIN32__
+       if (ret && inode_is_symlink(inode) &&
+           (errno == EACCES || errno == EPERM) &&
+           !(ctx->extract_flags & WIMLIB_EXTRACT_FLAG_STRICT_SYMLINKS))
        {
-               target = args->target;
-               target_nchars = args->target_nchars;
+               WARNING("Can't set reparse data on \"%"TS"\": "
+                       "Access denied!\n"
+                       "          You may be trying to "
+                       "extract a symbolic link without the\n"
+                       "          SeCreateSymbolicLink privilege, "
+                       "which by default non-Administrator\n"
+                       "          accounts do not have.",
+                       path);
+               ret = 0;
        }
+#endif
+       if (ret)
+               goto error;
 
-       extraction_path_nchars = target_nchars;
+       /* Account for reparse data consumed.  */
+       update_extract_progress(ctx,
+                               (lte_override ? lte_override :
+                                     inode_unnamed_lte_resolved(inode)));
+       return 0;
 
-       for (d = dentry; d != args->extract_root; d = d->parent) {
-               if (d->not_extracted)
-                       return 0;
-               extraction_path_nchars += d->extraction_name_nchars + 1;
-               list_add(&d->tmp_list, &ancestor_list);
+error:
+       ERROR_WITH_ERRNO("Failed to set reparse data on \"%"TS"\"", path);
+       return ret;
+}
+
+/*
+ * Extract zero or more streams to a file.
+ *
+ * This function operates slightly differently depending on whether @lte_spec is
+ * NULL or not.  When @lte_spec is NULL, the behavior is to extract the default
+ * file contents (unnamed stream), and, if named data streams are supported in
+ * the extract mode and volume, any named data streams.  When @lte_spec is NULL,
+ * the behavior is to extract only all copies of the stream @lte_spec, and in
+ * addition use @lte_spec to set the reparse data or create the symbolic link if
+ * appropriate.
+ *
+ * @path
+ *     Path to file to extract (as can be passed to apply_operations
+ *     functions).
+ * @ctx
+ *     Apply context.
+ * @dentry
+ *     WIM dentry that corresponds to the file being extracted.
+ * @lte_spec
+ *     If non-NULL, specifies the lookup table entry for a stream to extract,
+ *     and only that stream will be extracted (although there may be more than
+ *     one instance of it).
+ * @lte_override
+ *     Used only if @lte_spec != NULL; it is passed to the extraction functions
+ *     rather than @lte_spec, allowing the location of the stream to be
+ *     overridden.  (This is used when the WIM is being read from a nonseekable
+ *     file, such as a pipe, when streams need to be used more than once; each
+ *     such stream is extracted to a temporary file.)
+ */
+static int
+extract_streams(const tchar *path, struct apply_ctx *ctx,
+               struct wim_dentry *dentry,
+               struct wim_lookup_table_entry *lte_spec,
+               struct wim_lookup_table_entry *lte_override)
+{
+       struct wim_inode *inode = dentry->d_inode;
+       struct wim_lookup_table_entry *lte;
+       int ret;
+
+       if (dentry->was_hardlinked)
+               return 0;
+
+#ifdef ENABLE_DEBUG
+       if (lte_spec) {
+               char sha1_str[100];
+               char *p = sha1_str;
+               for (unsigned i = 0; i < SHA1_HASH_SIZE; i++)
+                       p += sprintf(p, "%02x", lte_override->hash[i]);
+               DEBUG("Extracting stream SHA1=%s to \"%"TS"\"",
+                     sha1_str, path, inode->i_ino);
+       } else {
+               DEBUG("Extracting streams to \"%"TS"\"", path, inode->i_ino);
        }
+#endif
 
-       tchar extraction_path[extraction_path_nchars + 1];
-       p = tmempcpy(extraction_path, target, target_nchars);
+       /* Unnamed data stream.  */
+       lte = inode_unnamed_lte_resolved(inode);
+       if (lte && (!lte_spec || lte == lte_spec)) {
+               if (lte_spec)
+                       lte = lte_override;
+               if (!(inode->i_attributes & (FILE_ATTRIBUTE_DIRECTORY |
+                                            FILE_ATTRIBUTE_REPARSE_POINT)))
+               {
+                       if ((inode->i_attributes & FILE_ATTRIBUTE_ENCRYPTED) &&
+                           ctx->supported_features.encrypted_files)
+                               ret = ctx->ops->extract_encrypted_stream(path, lte, ctx);
+                       else
+                               ret = ctx->ops->extract_unnamed_stream(path, lte, ctx);
+                       if (ret)
+                               goto error;
+                       update_extract_progress(ctx, lte);
+               }
+               else if (inode->i_attributes & FILE_ATTRIBUTE_REPARSE_POINT)
+               {
+                       ret = 0;
+                       if (ctx->supported_features.reparse_points)
+                               ret = extract_reparse_data(path, ctx, inode, lte);
+               #ifndef __WIN32__
+                       else if ((inode_is_symlink(inode) &&
+                                 ctx->supported_features.symlink_reparse_points))
+                               ret = extract_symlink(path, ctx, inode, lte);
+               #endif
+                       if (ret)
+                               return ret;
+               }
+       }
 
+       /* Named data streams.  */
+       if (can_extract_named_data_streams(ctx)) {
+               for (u16 i = 0; i < inode->i_num_ads; i++) {
+                       struct wim_ads_entry *entry = &inode->i_ads_entries[i];
+
+                       if (!ads_entry_is_named_stream(entry))
+                               continue;
+                       lte = entry->lte;
+                       if (!lte)
+                               continue;
+                       if (lte_spec && lte_spec != lte)
+                               continue;
+                       if (lte_spec)
+                               lte = lte_override;
+                       ret = ctx->ops->extract_named_stream(path, entry->stream_name,
+                                                            entry->stream_name_nbytes / 2,
+                                                            lte, ctx);
+                       if (ret)
+                               goto error;
+                       update_extract_progress(ctx, lte);
+               }
+       }
+       return 0;
 
-       list_for_each_entry(d, &ancestor_list, tmp_list) {
-               *p++ = OS_PREFERRED_PATH_SEPARATOR;
-               p = tmempcpy(p, d->extraction_name, d->extraction_name_nchars);
+error:
+       ERROR_WITH_ERRNO("Failed to extract data of \"%"TS"\"", path);
+       return ret;
+}
+
+/* Set attributes on an extracted file or directory if supported by the
+ * extraction mode.  */
+static int
+extract_file_attributes(const tchar *path, struct apply_ctx *ctx,
+                       struct wim_dentry *dentry)
+{
+       int ret;
+
+       if (ctx->ops->set_file_attributes) {
+               if (dentry == ctx->extract_root && ctx->root_dentry_is_special)
+                       return 0;
+               ret = ctx->ops->set_file_attributes(path,
+                                                   dentry->d_inode->i_attributes,
+                                                   ctx);
+               if (ret) {
+                       ERROR_WITH_ERRNO("Failed to set attributes on "
+                                        "\"%"TS"\"", path);
+                       return ret;
+               }
        }
-       *p = T('\0');
+       return 0;
+}
 
-#ifdef __WIN32__
-       /* Warn the user if the path exceeds MAX_PATH */
 
-       /* + 1 for '\0', -4 for \\?\.  */
-       if (extraction_path_nchars + 1 - 4 > MAX_PATH) {
-               if (dentry->needs_extraction &&
-                   args->num_long_paths < MAX_EXTRACT_LONG_PATH_WARNINGS)
+/* Set or remove the short (DOS) name on an extracted file or directory if
+ * supported by the extraction mode.  Since DOS names are unimportant and it's
+ * easy to run into problems setting them on Windows (SetFileShortName()
+ * requires SE_RESTORE privilege, which only the Administrator can request, and
+ * also requires DELETE access to the file), failure is ignored unless
+ * WIMLIB_EXTRACT_FLAG_STRICT_SHORT_NAMES is set.  */
+static int
+extract_short_name(const tchar *path, struct apply_ctx *ctx,
+                  struct wim_dentry *dentry)
+{
+       int ret;
+
+       /* The root of the dentry tree being extracted may not be extracted to
+        * its original name, so its short name should be ignored.  */
+       if (dentry == ctx->extract_root)
+               return 0;
+
+       if (ctx->supported_features.short_names) {
+               ret = ctx->ops->set_short_name(path,
+                                              dentry->short_name,
+                                              dentry->short_name_nbytes / 2,
+                                              ctx);
+               if (ret && (ctx->extract_flags &
+                           WIMLIB_EXTRACT_FLAG_STRICT_SHORT_NAMES))
                {
-                       WARNING("Path \"%ls\" exceeds MAX_PATH and will not be accessible "
-                               "to most Windows software", extraction_path);
-                       if (++args->num_long_paths == MAX_EXTRACT_LONG_PATH_WARNINGS)
-                               WARNING("Suppressing further warnings about long paths");
+                       ERROR_WITH_ERRNO("Failed to set short name of "
+                                        "\"%"TS"\"", path);
+                       return ret;
                }
        }
-#endif
-       return (*apply_dentry_func)(extraction_path, extraction_path_nchars,
-                                   dentry, args);
+       return 0;
 }
 
+/* Set security descriptor, UNIX data, or neither on an extracted file, taking
+ * into account the current extraction mode and flags.  */
+static int
+extract_security(const tchar *path, struct apply_ctx *ctx,
+                struct wim_dentry *dentry)
+{
+       int ret;
+       struct wim_inode *inode = dentry->d_inode;
+
+       if (ctx->extract_flags & WIMLIB_EXTRACT_FLAG_NO_ACLS)
+               return 0;
+
+       if ((ctx->extract_root == dentry) && ctx->root_dentry_is_special)
+               return 0;
+
+#ifndef __WIN32__
+       if (ctx->extract_flags & WIMLIB_EXTRACT_FLAG_UNIX_DATA) {
+               struct wimlib_unix_data data;
+
+               ret = inode_get_unix_data(inode, &data, NULL);
+               if (ret < 0)
+                       ret = 0;
+               else if (ret == 0)
+                       ret = ctx->ops->set_unix_data(path, &data, ctx);
+               if (ret) {
+                       if (ctx->extract_flags & WIMLIB_EXTRACT_FLAG_STRICT_ACLS) {
+                               ERROR_WITH_ERRNO("Failed to set UNIX owner, "
+                                                "group, and/or mode on "
+                                                "\"%"TS"\"", path);
+                               return ret;
+                       } else {
+                               WARNING_WITH_ERRNO("Failed to set UNIX owner, "
+                                                  "group, and/or/mode on "
+                                                  "\"%"TS"\"", path);
+                       }
+               }
+       }
+       else
+#endif /* __WIN32__ */
+       if (ctx->supported_features.security_descriptors &&
+           inode->i_security_id != -1)
+       {
+               const struct wim_security_data *sd;
+               const u8 *desc;
+               size_t desc_size;
+
+               sd = wim_const_security_data(ctx->wim);
+               desc = sd->descriptors[inode->i_security_id];
+               desc_size = sd->sizes[inode->i_security_id];
+
+               ret = ctx->ops->set_security_descriptor(path, desc,
+                                                       desc_size, ctx,
+                                                       !!(ctx->extract_flags &
+                                                          WIMLIB_EXTRACT_FLAG_STRICT_ACLS));
+               if (ret) {
+                       if (ctx->extract_flags & WIMLIB_EXTRACT_FLAG_STRICT_ACLS) {
+                               ERROR_WITH_ERRNO("Failed to set security "
+                                                "descriptor on \"%"TS"\"", path);
+                               return ret;
+                       } else {
+                               if (errno != EACCES) {
+                                       WARNING_WITH_ERRNO("Failed to set "
+                                                          "security descriptor "
+                                                          "on \"%"TS"\"", path);
+                               }
+                       }
+               }
+       }
+       return 0;
+}
 
-/* Extracts a file, directory, or symbolic link from the WIM archive. */
+/* Set timestamps on an extracted file.  Failure is warning-only unless
+ * WIMLIB_EXTRACT_FLAG_STRICT_TIMESTAMPS is set.  */
 static int
-apply_dentry_normal(struct wim_dentry *dentry, void *arg)
+extract_timestamps(const tchar *path, struct apply_ctx *ctx,
+                  struct wim_dentry *dentry)
 {
-#ifdef __WIN32__
-       return do_apply_op(dentry, arg, win32_do_apply_dentry);
-#else
-       return do_apply_op(dentry, arg, unix_do_apply_dentry);
-#endif
+       struct wim_inode *inode = dentry->d_inode;
+       int ret;
+
+       if ((ctx->extract_root == dentry) && ctx->root_dentry_is_special)
+               return 0;
+
+       if (ctx->ops->set_timestamps) {
+               ret = ctx->ops->set_timestamps(path,
+                                              inode->i_creation_time,
+                                              inode->i_last_write_time,
+                                              inode->i_last_access_time,
+                                              ctx);
+               if (ret) {
+                       if (ctx->extract_flags & WIMLIB_EXTRACT_FLAG_STRICT_TIMESTAMPS) {
+                               ERROR_WITH_ERRNO("Failed to set timestamps "
+                                                "on \"%"TS"\"", path);
+                               return ret;
+                       } else {
+                               WARNING_WITH_ERRNO("Failed to set timestamps "
+                                                  "on \"%"TS"\"", path);
+                       }
+               }
+       }
+       return 0;
 }
 
+/* Check whether the extraction of a dentry should be skipped completely.  */
+static bool
+dentry_is_supported(struct wim_dentry *dentry,
+                   const struct wim_features *supported_features)
+{
+       struct wim_inode *inode = dentry->d_inode;
+
+       if (inode->i_attributes & FILE_ATTRIBUTE_REPARSE_POINT) {
+               if (supported_features->reparse_points)
+                       return true;
+               if (supported_features->symlink_reparse_points &&
+                   inode_is_symlink(inode))
+                       return true;
+               return false;
+       }
+       return true;
+}
+
+/* Given a WIM dentry to extract, build the path to which to extract it, in the
+ * format understood by the callbacks in the apply_operations being used.
+ *
+ * Write the resulting path into @path, which must have room for at least
+ * ctx->ops->max_path characters including the null-terminator.
+ *
+ * Return %true if successful; %false if this WIM dentry doesn't actually need
+ * to be extracted or if the calculated path exceeds ctx->ops->max_path
+ * characters.
+ *
+ * This function clobbers the tmp_list member of @dentry and its ancestors up
+ * until the extraction root.  */
+static bool
+build_extraction_path(tchar path[], struct wim_dentry *dentry,
+                     struct apply_ctx *ctx)
+{
+       size_t path_nchars;
+       LIST_HEAD(ancestor_list);
+       tchar *p = path;
+       const tchar *target_prefix;
+       size_t target_prefix_nchars;
+       struct wim_dentry *d;
+
+       if (dentry->extraction_skipped)
+               return false;
+
+       path_nchars = ctx->ops->path_prefix_nchars;
+
+       if (ctx->ops->requires_realtarget_in_paths) {
+               target_prefix        = ctx->realtarget;
+               target_prefix_nchars = ctx->realtarget_nchars;
+       } else if (ctx->ops->requires_target_in_paths) {
+               target_prefix        = ctx->target;
+               target_prefix_nchars = ctx->target_nchars;
+       } else {
+               target_prefix        = NULL;
+               target_prefix_nchars = 0;
+       }
+       path_nchars += target_prefix_nchars;
+
+       for (d = dentry; d != ctx->extract_root; d = d->parent) {
+               path_nchars += d->extraction_name_nchars + 1;
+               list_add(&d->tmp_list, &ancestor_list);
+       }
+
+       path_nchars++; /* null terminator */
+
+       if (path_nchars > ctx->ops->path_max) {
+               WARNING("\"%"TS"\": Path too long to extract",
+                       dentry_full_path(dentry));
+               return false;
+       }
+
+       p = tmempcpy(p, ctx->ops->path_prefix, ctx->ops->path_prefix_nchars);
+       p = tmempcpy(p, target_prefix, target_prefix_nchars);
+       list_for_each_entry(d, &ancestor_list, tmp_list) {
+               *p++ = ctx->ops->path_separator;
+               p = tmempcpy(p, d->extraction_name, d->extraction_name_nchars);
+       }
+       *p++ = T('\0');
+       wimlib_assert(p - path == path_nchars);
+       return true;
+}
+
+static unsigned
+get_num_path_components(const tchar *path, tchar path_separator)
+{
+       unsigned num_components = 0;
+
+       while (*path) {
+               while (*path == path_separator)
+                       path++;
+               if (*path)
+                       num_components++;
+               while (*path && *path != path_separator)
+                       path++;
+       }
+       return num_components;
+}
 
-/* Apply timestamps to an extracted file or directory */
 static int
-apply_dentry_timestamps_normal(struct wim_dentry *dentry, void *arg)
+extract_multiimage_symlink(const tchar *oldpath, const tchar *newpath,
+                          struct apply_ctx *ctx, struct wim_dentry *dentry)
 {
-#ifdef __WIN32__
-       return do_apply_op(dentry, arg, win32_do_apply_dentry_timestamps);
-#else
-       return do_apply_op(dentry, arg, unix_do_apply_dentry_timestamps);
-#endif
+       size_t num_raw_path_components;
+       const struct wim_dentry *d;
+       size_t num_target_path_components;
+       tchar *p;
+       const tchar *p_old;
+       int ret;
+
+       num_raw_path_components = 0;
+       for (d = dentry; d != ctx->extract_root; d = d->parent)
+               num_raw_path_components++;
+
+       if (ctx->ops->requires_realtarget_in_paths)
+               num_target_path_components = get_num_path_components(ctx->realtarget,
+                                                                    ctx->ops->path_separator);
+       else if (ctx->ops->requires_target_in_paths)
+               num_target_path_components = get_num_path_components(ctx->target,
+                                                                    ctx->ops->path_separator);
+       else
+               num_target_path_components = 0;
+
+       if (ctx->extract_flags & WIMLIB_EXTRACT_FLAG_MULTI_IMAGE) {
+               wimlib_assert(num_target_path_components > 0);
+               num_raw_path_components++;
+               num_target_path_components--;
+       }
+
+       p_old = oldpath;
+       while (*p_old == ctx->ops->path_separator)
+               p_old++;
+       while (--num_target_path_components) {
+               while (*p_old != ctx->ops->path_separator)
+                       p_old++;
+               while (*p_old == ctx->ops->path_separator)
+                       p_old++;
+       }
+
+       tchar symlink_target[tstrlen(p_old) + 3 * num_raw_path_components + 1];
+
+       p = &symlink_target[0];
+       while (num_raw_path_components--) {
+               *p++ = '.';
+               *p++ = '.';
+               *p++ = ctx->ops->path_separator;
+       }
+       tstrcpy(p, p_old);
+       DEBUG("Creating symlink \"%"TS"\" => \"%"TS"\"",
+             newpath, symlink_target);
+       ret = ctx->ops->create_symlink(symlink_target, newpath, ctx);
+       if (ret) {
+               ERROR_WITH_ERRNO("Failed to create symlink "
+                                "\"%"TS"\" => \"%"TS"\"",
+                                newpath, symlink_target);
+       }
+       return ret;
+}
+
+/* Create the "skeleton" of an extracted file or directory.  Don't yet extract
+ * data streams, reparse data (including symbolic links), timestamps, and
+ * security descriptors.  Basically, everything that doesn't require reading
+ * non-metadata resources from the WIM file and isn't delayed until the final
+ * pass.  */
+static int
+do_dentry_extract_skeleton(tchar path[], struct wim_dentry *dentry,
+                          struct apply_ctx *ctx)
+{
+       struct wim_inode *inode = dentry->d_inode;
+       int ret;
+       const tchar *oldpath;
+
+       if (unlikely(is_linked_extraction(ctx))) {
+               struct wim_lookup_table_entry *unnamed_lte;
+
+               unnamed_lte = inode_unnamed_lte_resolved(dentry->d_inode);
+               if (unnamed_lte && unnamed_lte->extracted_file) {
+                       oldpath = unnamed_lte->extracted_file;
+                       if (ctx->extract_flags & WIMLIB_EXTRACT_FLAG_HARDLINK)
+                               goto hardlink;
+                       else
+                               goto symlink;
+               }
+       }
+
+       /* Create hard link if this dentry corresponds to an already-extracted
+        * inode.  */
+       if (inode->i_extracted_file) {
+               oldpath = inode->i_extracted_file;
+               goto hardlink;
+       }
+
+       /* Skip symlinks unless they can be extracted as reparse points rather
+        * than created directly.  */
+       if (inode_is_symlink(inode) && !ctx->supported_features.reparse_points)
+               return 0;
+
+       /* Create this file or directory unless it's the extraction root, which
+        * was already created if necessary.  */
+       if (dentry != ctx->extract_root) {
+               ret = extract_inode(path, ctx, inode);
+               if (ret)
+                       return ret;
+       }
+
+       /* Create empty named data streams.  */
+       if (can_extract_named_data_streams(ctx)) {
+               for (u16 i = 0; i < inode->i_num_ads; i++) {
+                       struct wim_ads_entry *entry = &inode->i_ads_entries[i];
+
+                       if (!ads_entry_is_named_stream(entry))
+                               continue;
+                       if (entry->lte)
+                               continue;
+                       ret = ctx->ops->extract_named_stream(path,
+                                                            entry->stream_name,
+                                                            entry->stream_name_nbytes / 2,
+                                                            entry->lte, ctx);
+                       if (ret) {
+                               ERROR_WITH_ERRNO("\"%"TS"\": failed to create "
+                                                "empty named data stream",
+                                                path);
+                               return ret;
+                       }
+               }
+       }
+
+       /* Set file attributes (if supported).  */
+       ret = extract_file_attributes(path, ctx, dentry);
+       if (ret)
+               return ret;
+
+       /* Set or remove file short name (if supported).  */
+       ret = extract_short_name(path, ctx, dentry);
+       if (ret)
+               return ret;
+
+       /* If inode has multiple links and hard links are supported in this
+        * extraction mode and volume, save the path to the extracted file in
+        * case it's needed to create a hard link.  */
+       if (unlikely(is_linked_extraction(ctx))) {
+               struct wim_lookup_table_entry *unnamed_lte;
+
+               unnamed_lte = inode_unnamed_lte_resolved(dentry->d_inode);
+               if (unnamed_lte) {
+                       unnamed_lte->extracted_file = TSTRDUP(path);
+                       if (!unnamed_lte->extracted_file)
+                               return WIMLIB_ERR_NOMEM;
+               }
+       } else if (inode->i_nlink > 1 && ctx->supported_features.hard_links) {
+               inode->i_extracted_file = TSTRDUP(path);
+               if (!inode->i_extracted_file)
+                       return WIMLIB_ERR_NOMEM;
+       }
+       return 0;
+
+symlink:
+       ret = extract_multiimage_symlink(oldpath, path, ctx, dentry);
+       if (ret)
+               return ret;
+       dentry->was_hardlinked = 1;
+       return 0;
+
+hardlink:
+       ret = extract_hardlink(oldpath, path, ctx);
+       if (ret)
+               return ret;
+       dentry->was_hardlinked = 1;
+       return 0;
 }
 
-static bool
-dentry_is_dot_or_dotdot(const struct wim_dentry *dentry)
+static int
+dentry_extract_skeleton(struct wim_dentry *dentry, void *_ctx)
 {
-       const utf16lechar *file_name = dentry->file_name;
-       return file_name != NULL &&
-               file_name[0] == cpu_to_le16('.') &&
-               (file_name[1] == cpu_to_le16('\0') ||
-                (file_name[1] == cpu_to_le16('.') &&
-                 file_name[2] == cpu_to_le16('\0')));
+       struct apply_ctx *ctx = _ctx;
+       tchar path[ctx->ops->path_max];
+
+       if (!build_extraction_path(path, dentry, ctx))
+               return 0;
+       return do_dentry_extract_skeleton(path, dentry, ctx);
 }
 
-/* Extract a dentry if it hasn't already been extracted and either
- * WIMLIB_EXTRACT_FLAG_NO_STREAMS is not specified, or the dentry is a directory
- * and/or has no unnamed stream. */
+/* Create a file or directory, then immediately extract all streams.  This
+ * assumes that WIMLIB_EXTRACT_FLAG_SEQUENTIAL is not specified, since the WIM
+ * may not be read sequentially by this function.  */
 static int
-maybe_apply_dentry(struct wim_dentry *dentry, void *arg)
+dentry_extract(struct wim_dentry *dentry, void *_ctx)
 {
-       struct apply_args *args = arg;
+       struct apply_ctx *ctx = _ctx;
+       tchar path[ctx->ops->path_max];
        int ret;
 
-       if (!dentry->needs_extraction)
+       if (!build_extraction_path(path, dentry, ctx))
                return 0;
 
-       if (args->extract_flags & WIMLIB_EXTRACT_FLAG_NO_STREAMS &&
-           !dentry_is_directory(dentry) &&
-           inode_unnamed_lte_resolved(dentry->d_inode) != NULL)
-               return 0;
+       ret = do_dentry_extract_skeleton(path, dentry, ctx);
+       if (ret)
+               return ret;
 
-       if ((args->extract_flags & WIMLIB_EXTRACT_FLAG_VERBOSE) &&
-            args->progress_func) {
-               ret = calculate_dentry_full_path(dentry);
-               if (ret)
-                       return ret;
-               args->progress.extract.cur_path = dentry->_full_path;
-               args->progress_func(WIMLIB_PROGRESS_MSG_EXTRACT_DENTRY,
-                                   &args->progress);
-       }
-       ret = args->apply_dentry(dentry, args);
-       if (ret == 0)
-               dentry->needs_extraction = 0;
-       return ret;
+       return extract_streams(path, ctx, dentry, NULL, NULL);
 }
 
-static void
-calculate_bytes_to_extract(struct list_head *stream_list,
-                          int extract_flags,
-                          union wimlib_progress_info *progress)
+/* Extract all instances of the stream @lte that are being extracted in this
+ * call of extract_tree().  @can_seek specifies whether the WIM file descriptor
+ * is seekable or not (e.g. is a pipe).  If not and the stream needs to be
+ * extracted multiple times, it is extracted to a temporary file first.
+ *
+ * This is intended for use with sequential extraction of a WIM image
+ * (WIMLIB_EXTRACT_FLAG_SEQUENTIAL specified).  */
+static int
+extract_stream_instances(struct wim_lookup_table_entry *lte,
+                        struct apply_ctx *ctx, bool can_seek)
 {
-       struct wim_lookup_table_entry *lte;
-       u64 total_bytes = 0;
-       u64 num_streams = 0;
+       struct wim_dentry **lte_dentries;
+       struct wim_lookup_table_entry *lte_tmp = NULL;
+       struct wim_lookup_table_entry *lte_override;
+       tchar *stream_tmp_filename = NULL;
+       tchar path[ctx->ops->path_max];
+       unsigned i;
+       int ret;
 
-       /* For each stream to be extracted... */
-       list_for_each_entry(lte, stream_list, extraction_list) {
-               if (extract_flags &
-                   (WIMLIB_EXTRACT_FLAG_SYMLINK | WIMLIB_EXTRACT_FLAG_HARDLINK))
-               {
-                       /* In the symlink or hard link extraction mode, each
-                        * stream will be extracted one time regardless of how
-                        * many dentries share the stream. */
-                       wimlib_assert(!(extract_flags & WIMLIB_EXTRACT_FLAG_NTFS));
-                       if (!lte->extracted_file) {
-                               num_streams++;
-                               total_bytes += wim_resource_size(lte);
-                       }
-               } else {
-                       num_streams += lte->out_refcnt;
-                       total_bytes += lte->out_refcnt * wim_resource_size(lte);
+       if (lte->out_refcnt <= ARRAY_LEN(lte->inline_lte_dentries))
+               lte_dentries = lte->inline_lte_dentries;
+       else
+               lte_dentries = lte->lte_dentries;
+
+       if (likely(can_seek || lte->out_refcnt < 2)) {
+               lte_override = lte;
+       } else {
+               /* Need to extract stream to temporary file.  */
+               struct filedes fd;
+               int raw_fd;
+
+               stream_tmp_filename = ttempnam(NULL, T("wimlib"));
+               if (!stream_tmp_filename) {
+                       ERROR_WITH_ERRNO("Failed to create temporary filename");
+                       ret = WIMLIB_ERR_OPEN;
+                       goto out;
+               }
+
+               lte_tmp = memdup(lte, sizeof(struct wim_lookup_table_entry));
+               if (!lte_tmp) {
+                       ret = WIMLIB_ERR_NOMEM;
+                       goto out_free_stream_tmp_filename;
+               }
+               lte_tmp->resource_location = RESOURCE_IN_FILE_ON_DISK;
+               lte_tmp->file_on_disk = stream_tmp_filename;
+               lte_override = lte_tmp;
+
+               raw_fd = topen(stream_tmp_filename,
+                              O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0600);
+               if (raw_fd < 0) {
+                       ERROR_WITH_ERRNO("Failed to open temporary file");
+                       ret = WIMLIB_ERR_OPEN;
+                       goto out_free_lte_tmp;
                }
+               filedes_init(&fd, raw_fd);
+               ret = extract_wim_resource_to_fd(lte, &fd,
+                                                wim_resource_size(lte));
+               if (filedes_close(&fd) && !ret)
+                       ret = WIMLIB_ERR_WRITE;
+               if (ret)
+                       goto out_unlink_stream_tmp_file;
        }
-       progress->extract.num_streams = num_streams;
-       progress->extract.total_bytes = total_bytes;
-       progress->extract.completed_bytes = 0;
-}
 
-static void
-maybe_add_stream_for_extraction(struct wim_lookup_table_entry *lte,
-                               struct list_head *stream_list)
-{
-       if (++lte->out_refcnt == 1) {
-               INIT_LIST_HEAD(&lte->lte_dentry_list);
-               list_add_tail(&lte->extraction_list, stream_list);
+       /* Extract all instances of the stream, reading either from the stream
+        * in the WIM file or from the temporary file containing the stream.
+        * dentry->tmp_flag is used to ensure that each dentry is processed only
+        * once regardless of how many times this stream appears in the streams
+        * of the corresponding inode.  */
+       for (i = 0; i < lte->out_refcnt; i++) {
+               struct wim_dentry *dentry = lte_dentries[i];
+
+               if (dentry->tmp_flag)
+                       continue;
+               if (!build_extraction_path(path, dentry, ctx))
+                       continue;
+               ret = extract_streams(path, ctx, dentry,
+                                     lte, lte_override);
+               if (ret)
+                       goto out_clear_tmp_flags;
+               dentry->tmp_flag = 1;
        }
+       ret = 0;
+out_clear_tmp_flags:
+       for (i = 0; i < lte->out_refcnt; i++)
+               lte_dentries[i]->tmp_flag = 0;
+out_unlink_stream_tmp_file:
+       if (stream_tmp_filename)
+               tunlink(stream_tmp_filename);
+out_free_lte_tmp:
+       FREE(lte_tmp);
+out_free_stream_tmp_filename:
+       FREE(stream_tmp_filename);
+out:
+       return ret;
 }
 
-struct find_streams_ctx {
-       struct list_head stream_list;
-       int extract_flags;
-};
-
+/* Extracts a list of streams (ctx.stream_list), assuming that the directory
+ * structure and empty files were already created.  This relies on the
+ * per-`struct wim_lookup_table_entry' list of dentries that reference each
+ * stream that was constructed earlier.  Streams are extracted exactly in the
+ * order of the stream list; however, unless the WIM's file descriptor is
+ * detected to be non-seekable, streams may be read from the WIM file more than
+ * one time if multiple copies need to be extracted.  */
 static int
-dentry_find_streams_to_extract(struct wim_dentry *dentry, void *_ctx)
+extract_stream_list(struct apply_ctx *ctx)
 {
-       struct find_streams_ctx *ctx = _ctx;
-       struct wim_inode *inode = dentry->d_inode;
        struct wim_lookup_table_entry *lte;
-       bool dentry_added = false;
-       struct list_head *stream_list = &ctx->stream_list;
-       int extract_flags = ctx->extract_flags;
-
-       if (!dentry->needs_extraction)
-               return 0;
+       bool can_seek;
+       int ret;
 
-       lte = inode_unnamed_lte_resolved(inode);
-       if (lte) {
-               if (!inode->i_visited)
-                       maybe_add_stream_for_extraction(lte, stream_list);
-               list_add_tail(&dentry->extraction_stream_list, &lte->lte_dentry_list);
-               dentry_added = true;
-       }
-
-       /* Determine whether to include alternate data stream entries or not.
-        *
-        * UNIX:  Include them if extracting using NTFS-3g.
-        *
-        * Windows: Include them undconditionally, although if the filesystem is
-        * not NTFS we won't actually be able to extract them. */
-#if defined(WITH_NTFS_3G)
-       if (extract_flags & WIMLIB_EXTRACT_FLAG_NTFS)
-#elif defined(__WIN32__)
-       if (1)
-#else
-       if (0)
-#endif
-       {
-               for (unsigned i = 0; i < inode->i_num_ads; i++) {
-                       if (inode->i_ads_entries[i].stream_name_nbytes != 0) {
-                               lte = inode->i_ads_entries[i].lte;
-                               if (lte) {
-                                       if (!inode->i_visited) {
-                                               maybe_add_stream_for_extraction(lte,
-                                                                               stream_list);
-                                       }
-                                       if (!dentry_added) {
-                                               list_add_tail(&dentry->extraction_stream_list,
-                                                             &lte->lte_dentry_list);
-                                               dentry_added = true;
-                                       }
-                               }
-                       }
-               }
+       can_seek = !(lseek(ctx->wim->in_fd.fd, 0, SEEK_CUR) == (off_t)-1 &&
+                    errno == ESPIPE);
+       list_for_each_entry(lte, &ctx->stream_list, extraction_list) {
+               ret = extract_stream_instances(lte, ctx, can_seek);
+               if (ret)
+                       return ret;
        }
-       inode->i_visited = 1;
        return 0;
 }
 
+/* Read the header from a stream in a pipable WIM.  */
 static int
-dentry_resolve_and_zero_lte_refcnt(struct wim_dentry *dentry, void *_lookup_table)
+read_pwm_stream_header(WIMStruct *pwm, struct wim_lookup_table_entry *lte,
+                      bool allow_header)
 {
-       struct wim_inode *inode = dentry->d_inode;
-       struct wim_lookup_table *lookup_table = _lookup_table;
-       struct wim_lookup_table_entry *lte;
+       struct pwm_stream_hdr stream_hdr;
        int ret;
 
-       ret = inode_resolve_ltes(inode, lookup_table);
+       ret = full_read(&pwm->in_fd, &stream_hdr, sizeof(stream_hdr));
        if (ret)
-               return ret;
-       for (unsigned i = 0; i <= inode->i_num_ads; i++) {
-               lte = inode_stream_lte_resolved(inode, i);
-               if (lte)
-                       lte->out_refcnt = 0;
+               goto read_error;
+
+       if (allow_header && stream_hdr.magic == PWM_MAGIC) {
+               u8 buf[WIM_HEADER_DISK_SIZE - sizeof(stream_hdr)];
+               ret = full_read(&pwm->in_fd, buf, sizeof(buf));
+               if (ret)
+                       goto read_error;
+               lte->resource_location = RESOURCE_NONEXISTENT;
+               return 0;
+       }
+
+       if (stream_hdr.magic != PWM_STREAM_MAGIC) {
+               ERROR("Data read on pipe is invalid (expected stream header).");
+               return WIMLIB_ERR_INVALID_PIPABLE_WIM;
        }
+
+       lte->resource_entry.original_size = le64_to_cpu(stream_hdr.uncompressed_size);
+       copy_hash(lte->hash, stream_hdr.hash);
+       lte->resource_entry.flags = le32_to_cpu(stream_hdr.flags);
+       lte->resource_entry.offset = pwm->in_fd.offset;
+       lte->resource_location = RESOURCE_IN_WIM;
+       lte->wim = pwm;
+       if (lte->resource_entry.flags & WIM_RESHDR_FLAG_COMPRESSED) {
+               lte->compression_type = pwm->compression_type;
+               lte->resource_entry.size = 0;
+       } else {
+               lte->compression_type = WIMLIB_COMPRESSION_TYPE_NONE;
+               lte->resource_entry.size = lte->resource_entry.original_size;
+       }
+       lte->is_pipable = 1;
        return 0;
+
+read_error:
+       ERROR_WITH_ERRNO("Error reading pipable WIM from pipe");
+       return ret;
+}
+
+/* Skip over an unneeded stream in a pipable WIM being read from a pipe.  */
+static int
+skip_pwm_stream(struct wim_lookup_table_entry *lte)
+{
+       return read_partial_wim_resource(lte, wim_resource_size(lte),
+                                        NULL, NULL,
+                                        WIMLIB_READ_RESOURCE_FLAG_SEEK_ONLY,
+                                        0);
 }
 
 static int
-find_streams_for_extraction(struct wim_dentry *root,
-                           struct list_head *stream_list,
-                           struct wim_lookup_table *lookup_table,
-                           int extract_flags)
+extract_streams_from_pipe(struct apply_ctx *ctx)
 {
-       struct find_streams_ctx ctx;
+       struct wim_lookup_table_entry *found_lte;
+       struct wim_lookup_table_entry *needed_lte;
+       struct wim_lookup_table *lookup_table;
        int ret;
 
-       INIT_LIST_HEAD(&ctx.stream_list);
-       ctx.extract_flags = extract_flags;
-       ret = for_dentry_in_tree(root, dentry_resolve_and_zero_lte_refcnt, lookup_table);
-       if (ret)
-               return ret;
-       for_dentry_in_tree(root, dentry_find_streams_to_extract, &ctx);
-       list_transfer(&ctx.stream_list, stream_list);
-       return 0;
-}
+       ret = WIMLIB_ERR_NOMEM;
+       found_lte = new_lookup_table_entry();
+       if (!found_lte)
+               goto out;
 
-struct apply_operations {
-       int (*apply_dentry)(struct wim_dentry *dentry, void *arg);
-       int (*apply_dentry_timestamps)(struct wim_dentry *dentry, void *arg);
-};
+       lookup_table = ctx->wim->lookup_table;
 
-static const struct apply_operations normal_apply_operations = {
-       .apply_dentry = apply_dentry_normal,
-       .apply_dentry_timestamps = apply_dentry_timestamps_normal,
-};
+       while (ctx->num_streams_remaining) {
+               ret = read_pwm_stream_header(ctx->wim, found_lte, true);
+               if (ret)
+                       goto out_free_found_lte;
 
-#ifdef WITH_NTFS_3G
-static const struct apply_operations ntfs_apply_operations = {
-       .apply_dentry = apply_dentry_ntfs,
-       .apply_dentry_timestamps = apply_dentry_timestamps_ntfs,
-};
-#endif
+               if ((found_lte->resource_location != RESOURCE_NONEXISTENT)
+                   && !(found_lte->resource_entry.flags & WIM_RESHDR_FLAG_METADATA)
+                   && (needed_lte = __lookup_resource(lookup_table, found_lte->hash))
+                   && (needed_lte->out_refcnt))
+               {
+                       copy_resource_entry(&needed_lte->resource_entry,
+                                           &found_lte->resource_entry);
+                       needed_lte->resource_location = found_lte->resource_location;
+                       needed_lte->wim               = found_lte->wim;
+                       needed_lte->compression_type  = found_lte->compression_type;
+                       needed_lte->is_pipable        = found_lte->is_pipable;
+
+                       ret = extract_stream_instances(needed_lte, ctx, false);
+                       if (ret)
+                               goto out_free_found_lte;
+                       ctx->num_streams_remaining--;
+               } else if (found_lte->resource_location != RESOURCE_NONEXISTENT) {
+                       ret = skip_pwm_stream(found_lte);
+                       if (ret)
+                               goto out_free_found_lte;
+               }
+       }
+       ret = 0;
+out_free_found_lte:
+       free_lookup_table_entry(found_lte);
+out:
+       return ret;
+}
 
+/* Finish extracting a file, directory, or symbolic link by setting file
+ * security and timestamps.  */
 static int
-apply_stream_list(struct list_head *stream_list,
-                 struct apply_args *args,
-                 const struct apply_operations *ops,
-                 wimlib_progress_func_t progress_func)
+dentry_extract_final(struct wim_dentry *dentry, void *_ctx)
 {
-       uint64_t bytes_per_progress = args->progress.extract.total_bytes / 100;
-       uint64_t next_progress = bytes_per_progress;
-       struct wim_lookup_table_entry *lte;
-       struct wim_dentry *dentry;
+       struct apply_ctx *ctx = _ctx;
        int ret;
+       tchar path[ctx->ops->path_max];
 
-       /* This complicated loop is essentially looping through the dentries,
-        * although dentries may be visited more than once (if a dentry contains
-        * two different nonempty streams) or not at all (if a dentry contains
-        * no non-empty streams).
-        *
-        * The outer loop is over the distinct streams to be extracted so that
-        * sequential reading of the WIM can be implemented. */
-
-       /* For each distinct stream to be extracted */
-       list_for_each_entry(lte, stream_list, extraction_list) {
-               /* For each dentry to be extracted that is a name for an inode
-                * containing the stream */
-               list_for_each_entry(dentry, &lte->lte_dentry_list, extraction_stream_list) {
-                       /* Extract the dentry if it was not already
-                        * extracted */
-                       ret = maybe_apply_dentry(dentry, args);
-                       if (ret)
-                               return ret;
-                       if (progress_func &&
-                           args->progress.extract.completed_bytes >= next_progress)
-                       {
-                               progress_func(WIMLIB_PROGRESS_MSG_EXTRACT_STREAMS,
-                                             &args->progress);
-                               if (args->progress.extract.completed_bytes >=
-                                   args->progress.extract.total_bytes)
-                               {
-                                       next_progress = ~0ULL;
-                               } else {
-                                       next_progress =
-                                               min (args->progress.extract.completed_bytes +
-                                                    bytes_per_progress,
-                                                    args->progress.extract.total_bytes);
-                               }
-                       }
-               }
-       }
-       return 0;
+       if (!build_extraction_path(path, dentry, ctx))
+               return 0;
+
+       ret = extract_security(path, ctx, dentry);
+       if (ret)
+               return ret;
+
+       return extract_timestamps(path, ctx, dentry);
 }
 
+/* Sorts a list of streams in ascending order of their offset in the WIM file in
+ * order to prepare for sequential extraction.  */
 static int
 sort_stream_list_by_wim_position(struct list_head *stream_list)
 {
@@ -429,7 +1375,7 @@ sort_stream_list_by_wim_position(struct list_head *stream_list)
 }
 
 /*
- * Extract a dentry to standard output.
+ * Extract a WIM dentry to standard output.
  *
  * This obviously doesn't make sense in all cases.  We return an error if the
  * dentry does not correspond to a regular file.  Otherwise we extract the
@@ -450,7 +1396,9 @@ extract_dentry_to_stdout(struct wim_dentry *dentry)
 
                lte = inode_unnamed_lte_resolved(dentry->d_inode);
                if (lte) {
-                       ret = extract_wim_resource_to_fd(lte, STDOUT_FILENO,
+                       struct filedes _stdout;
+                       filedes_init(&_stdout, STDOUT_FILENO);
+                       ret = extract_wim_resource_to_fd(lte, &_stdout,
                                                         wim_resource_size(lte));
                }
        }
@@ -504,6 +1452,24 @@ file_name_valid(utf16lechar *name, size_t num_chars, bool fix)
        return true;
 }
 
+static bool
+dentry_is_dot_or_dotdot(const struct wim_dentry *dentry)
+{
+       const utf16lechar *file_name = dentry->file_name;
+       return file_name != NULL &&
+               file_name[0] == cpu_to_le16('.') &&
+               (file_name[1] == cpu_to_le16('\0') ||
+                (file_name[1] == cpu_to_le16('.') &&
+                 file_name[2] == cpu_to_le16('\0')));
+}
+
+static int
+dentry_mark_skipped(struct wim_dentry *dentry, void *_ignore)
+{
+       dentry->extraction_skipped = 1;
+       return 0;
+}
+
 /*
  * dentry_calculate_extraction_path-
  *
@@ -514,7 +1480,7 @@ file_name_valid(utf16lechar *name, size_t num_chars, bool fix)
  * UNIX, converted into the platform's multibyte encoding).  However, if the
  * file name contains characters that are not valid on the current platform or
  * has some other format that is not valid, leave dentry->extraction_name as
- * NULL and clear dentry->needs_extraction to indicate that this dentry should
+ * NULL and set dentry->extraction_skipped to indicate that this dentry should
  * not be extracted, unless the appropriate flag
  * WIMLIB_EXTRACT_FLAG_REPLACE_INVALID_FILENAMES is set in the extract flags, in
  * which case a substitute filename will be created and set instead.
@@ -525,37 +1491,42 @@ file_name_valid(utf16lechar *name, size_t num_chars, bool fix)
 static int
 dentry_calculate_extraction_path(struct wim_dentry *dentry, void *_args)
 {
-       struct apply_args *args = _args;
+       struct apply_ctx *ctx = _args;
        int ret;
 
-       dentry->needs_extraction = 1;
-
-       if (dentry == args->extract_root)
+       if (dentry == ctx->extract_root || dentry->extraction_skipped)
                return 0;
 
+       if (!dentry_is_supported(dentry, &ctx->supported_features))
+               goto skip_dentry;
+
        if (dentry_is_dot_or_dotdot(dentry)) {
                /* WIM files shouldn't contain . or .. entries.  But if they are
                 * there, don't attempt to extract them. */
-               WARNING("Skipping extraction of unexpected . or .. file \"%"TS"\"",
-                       dentry_full_path(dentry));
+               WARNING("Skipping extraction of unexpected . or .. file "
+                       "\"%"TS"\"", dentry_full_path(dentry));
                goto skip_dentry;
        }
 
 #ifdef __WIN32__
-       struct wim_dentry *other;
-       list_for_each_entry(other, &dentry->case_insensitive_conflict_list,
-                           case_insensitive_conflict_list)
+       if (!ctx->ops->supports_case_sensitive_filenames)
        {
-               if (other->needs_extraction) {
-                       if (args->extract_flags & WIMLIB_EXTRACT_FLAG_ALL_CASE_CONFLICTS)
-                       {
-                               WARNING("\"%"TS"\" has the same case-insensitive "
-                                       "name as \"%"TS"\"; extracting dummy name instead",
+               struct wim_dentry *other;
+               list_for_each_entry(other, &dentry->case_insensitive_conflict_list,
+                                   case_insensitive_conflict_list)
+               {
+                       if (ctx->extract_flags &
+                           WIMLIB_EXTRACT_FLAG_ALL_CASE_CONFLICTS) {
+                               WARNING("\"%"TS"\" has the same "
+                                       "case-insensitive name as "
+                                       "\"%"TS"\"; extracting "
+                                       "dummy name instead",
                                        dentry_full_path(dentry),
                                        dentry_full_path(other));
                                goto out_replace;
                        } else {
-                               WARNING("Not extracting \"%"TS"\": has same case-insensitive "
+                               WARNING("Not extracting \"%"TS"\": "
+                                       "has same case-insensitive "
                                        "name as \"%"TS"\"",
                                        dentry_full_path(dentry),
                                        dentry_full_path(other));
@@ -563,7 +1534,9 @@ dentry_calculate_extraction_path(struct wim_dentry *dentry, void *_args)
                        }
                }
        }
-#endif
+#else  /* __WIN32__ */
+       wimlib_assert(ctx->ops->supports_case_sensitive_filenames);
+#endif /* !__WIN32__ */
 
        if (file_name_valid(dentry->file_name, dentry->file_name_nbytes / 2, false)) {
 #ifdef __WIN32__
@@ -577,7 +1550,7 @@ dentry_calculate_extraction_path(struct wim_dentry *dentry, void *_args)
                                       &dentry->extraction_name_nchars);
 #endif
        } else {
-               if (args->extract_flags & WIMLIB_EXTRACT_FLAG_REPLACE_INVALID_FILENAMES)
+               if (ctx->extract_flags & WIMLIB_EXTRACT_FLAG_REPLACE_INVALID_FILENAMES)
                {
                        WARNING("\"%"TS"\" has an invalid filename "
                                "that is not supported on this platform; "
@@ -592,78 +1565,323 @@ dentry_calculate_extraction_path(struct wim_dentry *dentry, void *_args)
                }
        }
 
-out_replace:
+out_replace:
+       {
+               utf16lechar utf16_name_copy[dentry->file_name_nbytes / 2];
+
+               memcpy(utf16_name_copy, dentry->file_name, dentry->file_name_nbytes);
+               file_name_valid(utf16_name_copy, dentry->file_name_nbytes / 2, true);
+
+               tchar *tchar_name;
+               size_t tchar_nchars;
+       #ifdef __WIN32__
+               tchar_name = utf16_name_copy;
+               tchar_nchars = dentry->file_name_nbytes / 2;
+       #else
+               ret = utf16le_to_tstr(utf16_name_copy,
+                                     dentry->file_name_nbytes,
+                                     &tchar_name, &tchar_nchars);
+               if (ret)
+                       return ret;
+       #endif
+               size_t fixed_name_num_chars = tchar_nchars;
+               tchar fixed_name[tchar_nchars + 50];
+
+               tmemcpy(fixed_name, tchar_name, tchar_nchars);
+               fixed_name_num_chars += tsprintf(fixed_name + tchar_nchars,
+                                                T(" (invalid filename #%lu)"),
+                                                ++ctx->invalid_sequence);
+       #ifndef __WIN32__
+               FREE(tchar_name);
+       #endif
+               dentry->extraction_name = memdup(fixed_name,
+                                                2 * fixed_name_num_chars + 2);
+               if (!dentry->extraction_name)
+                       return WIMLIB_ERR_NOMEM;
+               dentry->extraction_name_nchars = fixed_name_num_chars;
+       }
+       return 0;
+
+skip_dentry:
+       for_dentry_in_tree(dentry, dentry_mark_skipped, NULL);
+       return 0;
+}
+
+/* Clean up dentry and inode structure after extraction.  */
+static int
+dentry_reset_needs_extraction(struct wim_dentry *dentry, void *_ignore)
+{
+       struct wim_inode *inode = dentry->d_inode;
+
+       dentry->extraction_skipped = 0;
+       dentry->was_hardlinked = 0;
+       inode->i_visited = 0;
+       FREE(inode->i_extracted_file);
+       inode->i_extracted_file = NULL;
+       if ((void*)dentry->extraction_name != (void*)dentry->file_name)
+               FREE(dentry->extraction_name);
+       dentry->extraction_name = NULL;
+       return 0;
+}
+
+/* Tally features necessary to extract a dentry and the corresponding inode.  */
+static int
+dentry_tally_features(struct wim_dentry *dentry, void *_features)
+{
+       struct wim_features *features = _features;
+       struct wim_inode *inode = dentry->d_inode;
+
+       if (inode->i_attributes & FILE_ATTRIBUTE_ARCHIVE)
+               features->archive_files++;
+       if (inode->i_attributes & FILE_ATTRIBUTE_HIDDEN)
+               features->hidden_files++;
+       if (inode->i_attributes & FILE_ATTRIBUTE_SYSTEM)
+               features->system_files++;
+       if (inode->i_attributes & FILE_ATTRIBUTE_COMPRESSED)
+               features->compressed_files++;
+       if (inode->i_attributes & FILE_ATTRIBUTE_ENCRYPTED)
+               features->encrypted_files++;
+       if (inode->i_attributes & FILE_ATTRIBUTE_NOT_CONTENT_INDEXED)
+               features->not_context_indexed_files++;
+       if (inode->i_attributes & FILE_ATTRIBUTE_SPARSE_FILE)
+               features->sparse_files++;
+       if (inode_has_named_stream(inode))
+               features->named_data_streams++;
+       if (inode->i_visited)
+               features->hard_links++;
+       if (inode->i_attributes & FILE_ATTRIBUTE_REPARSE_POINT) {
+               features->reparse_points++;
+               if (inode_is_symlink(inode))
+                       features->symlink_reparse_points++;
+               else
+                       features->other_reparse_points++;
+       }
+       if (inode->i_security_id != -1)
+               features->security_descriptors++;
+       if (dentry->short_name_nbytes)
+               features->short_names++;
+       if (inode_has_unix_data(inode))
+               features->unix_data++;
+       inode->i_visited = 1;
+       return 0;
+}
+
+static int
+dentry_clear_inode_visited(struct wim_dentry *dentry, void *_ignore)
+{
+       dentry->d_inode->i_visited = 0;
+       return 0;
+}
+
+/* Tally the features necessary to extract a dentry tree.  */
+static void
+dentry_tree_get_features(struct wim_dentry *root, struct wim_features *features)
+{
+       memset(features, 0, sizeof(struct wim_features));
+       for_dentry_in_tree(root, dentry_tally_features, features);
+       for_dentry_in_tree(root, dentry_clear_inode_visited, NULL);
+}
+
+static int
+do_feature_check(const struct wim_features *required_features,
+                const struct wim_features *supported_features,
+                int extract_flags,
+                const struct apply_operations *ops)
+{
+       if (required_features->archive_files && !supported_features->archive_files)
+       {
+               WARNING(
+          "%lu files are marked as archived, but this attribute\n"
+"          is not supported in this extraction mode or volume.",
+                       required_features->archive_files);
+       }
+
+       if (required_features->hidden_files && !supported_features->hidden_files)
+       {
+               WARNING(
+          "%lu files are marked as hidden, but this attribute\n"
+"          is not supported in this extraction mode or volume.",
+                       required_features->hidden_files);
+       }
+
+       if (required_features->system_files && !supported_features->system_files)
+       {
+               WARNING(
+          "%lu files are marked as system files, but this attribute\n"
+"          is not supported in this extraction mode or volume.",
+                       required_features->system_files);
+       }
+
+       if (required_features->compressed_files && !supported_features->compressed_files)
+       {
+               WARNING(
+          "%lu files are marked as being transparently compressed, but\n"
+"          transparent compression is not supported in this extraction\n"
+"          mode or volume.  These files will be extracted as uncompressed.",
+                       required_features->compressed_files);
+       }
+
+       if (required_features->encrypted_files && !supported_features->encrypted_files)
+       {
+               WARNING(
+          "%lu files are marked as being encrypted, but encryption is not\n"
+"          supported in this extraction mode or volume.  These files will be\n"
+"          extracted as raw encrypted data instead.",
+                       required_features->encrypted_files);
+       }
+
+       if (required_features->not_context_indexed_files &&
+           !supported_features->not_context_indexed_files)
+       {
+               WARNING(
+          "%lu files are marked as not content indexed, but this attribute\n"
+"          is not supported in this extraction mode or volume.",
+                       required_features->not_context_indexed_files);
+       }
+
+       if (required_features->sparse_files && !supported_features->sparse_files)
+       {
+               WARNING(
+          "%lu files are marked as sparse, but creating sparse files is not\n"
+"          supported in this extraction mode or volume.  These files will be\n"
+"          extracted as non-sparse.",
+                       required_features->not_context_indexed_files);
+       }
+
+       if (required_features->named_data_streams &&
+           !supported_features->named_data_streams)
+       {
+               WARNING(
+          "%lu files contain one or more alternate (named) data streams,\n"
+"          which are not supported in this extraction mode or volume.\n"
+"          Alternate data streams will NOT be extracted.",
+                       required_features->named_data_streams);
+       }
+
+       if (unlikely(extract_flags & (WIMLIB_EXTRACT_FLAG_HARDLINK |
+                                     WIMLIB_EXTRACT_FLAG_SYMLINK)) &&
+           required_features->named_data_streams &&
+           supported_features->named_data_streams)
+       {
+               WARNING(
+          "%lu files contain one or more alternate (named) data streams,\n"
+"          which are not supported in linked extraction mode.\n"
+"          Alternate data streams will NOT be extracted.",
+                       required_features->named_data_streams);
+       }
+
+       if (required_features->hard_links && !supported_features->hard_links)
+       {
+               WARNING(
+          "%lu files are hard links, but hard links are not supported in\n"
+"          this extraction mode or volume.  Hard links will be extracted as\n"
+"          duplicate copies of the linked files.",
+                       required_features->hard_links);
+       }
+
+       if (required_features->reparse_points && !supported_features->reparse_points)
+       {
+               if (supported_features->symlink_reparse_points) {
+                       if (required_features->other_reparse_points) {
+                               WARNING(
+          "%lu files are reparse points that are neither symbolic links\n"
+"          nor junction points and are not supported in this extraction mode\n"
+"          or volume.  These reparse points will not be extracted.",
+                                       required_features->other_reparse_points);
+                       }
+               } else {
+                       WARNING(
+          "%lu files are reparse points, which are not supported in this\n"
+"          extraction mode or volume and will not be extracted.",
+                               required_features->reparse_points);
+               }
+       }
+
+       if (required_features->security_descriptors &&
+           !supported_features->security_descriptors)
+       {
+               WARNING(
+          "%lu files have Windows NT security descriptors, but extracting\n"
+"          security descriptors is not supported in this extraction mode\n"
+"          or volume.  No security descriptors will be extracted.",
+                       required_features->security_descriptors);
+       }
+
+       if (required_features->short_names && !supported_features->short_names)
+       {
+               WARNING(
+          "%lu files have short (DOS) names, but extracting short names\n"
+"          is not supported in this extraction mode or volume.  Short names\n"
+"          will not be extracted.\n",
+                       required_features->short_names);
+       }
+
+       if ((extract_flags & WIMLIB_EXTRACT_FLAG_UNIX_DATA) &&
+           required_features->unix_data && !supported_features->unix_data)
+       {
+               ERROR("UNIX data not supported in this extraction mode "
+                     "or volume", ops->name);
+               return WIMLIB_ERR_UNSUPPORTED;
+       }
+       if ((extract_flags & WIMLIB_EXTRACT_FLAG_STRICT_SHORT_NAMES) &&
+           required_features->short_names && !supported_features->short_names)
+       {
+               ERROR("Short names are not supported in this extraction "
+                     "mode or volume", ops->name);
+               return WIMLIB_ERR_UNSUPPORTED;
+       }
+       if ((extract_flags & WIMLIB_EXTRACT_FLAG_STRICT_TIMESTAMPS) &&
+           !ops->set_timestamps)
+       {
+               ERROR("Timestamps are not supported in this extraction "
+                     "mode or volume", ops->name);
+               return WIMLIB_ERR_UNSUPPORTED;
+       }
+       if (((extract_flags & (WIMLIB_EXTRACT_FLAG_STRICT_ACLS |
+                              WIMLIB_EXTRACT_FLAG_UNIX_DATA))
+            == WIMLIB_EXTRACT_FLAG_STRICT_ACLS) &&
+           required_features->security_descriptors &&
+           !supported_features->security_descriptors)
+       {
+               ERROR("Security descriptors not supported in this extraction "
+                     "mode or volume.");
+               return WIMLIB_ERR_UNSUPPORTED;
+       }
+
+       if ((extract_flags & WIMLIB_EXTRACT_FLAG_HARDLINK) &&
+           !supported_features->hard_links)
        {
-               utf16lechar utf16_name_copy[dentry->file_name_nbytes / 2];
-
-               memcpy(utf16_name_copy, dentry->file_name, dentry->file_name_nbytes);
-               file_name_valid(utf16_name_copy, dentry->file_name_nbytes / 2, true);
-
-               tchar *tchar_name;
-               size_t tchar_nchars;
-       #ifdef __WIN32__
-               tchar_name = utf16_name_copy;
-               tchar_nchars = dentry->file_name_nbytes / 2;
-       #else
-               ret = utf16le_to_tstr(utf16_name_copy,
-                                     dentry->file_name_nbytes,
-                                     &tchar_name, &tchar_nchars);
-               if (ret)
-                       return ret;
-       #endif
-               size_t fixed_name_num_chars = tchar_nchars;
-               tchar fixed_name[tchar_nchars + 50];
-
-               tmemcpy(fixed_name, tchar_name, tchar_nchars);
-               fixed_name_num_chars += tsprintf(fixed_name + tchar_nchars,
-                                                T(" (invalid filename #%lu)"),
-                                                ++args->invalid_sequence);
-       #ifndef __WIN32__
-               FREE(tchar_name);
-       #endif
-               dentry->extraction_name = memdup(fixed_name, 2 * fixed_name_num_chars + 2);
-               if (!dentry->extraction_name)
-                       return WIMLIB_ERR_NOMEM;
-               dentry->extraction_name_nchars = fixed_name_num_chars;
+               ERROR("Hard link extraction mode requested, but "
+                     "extraction mode or volume does not support hard links!");
+               return WIMLIB_ERR_UNSUPPORTED;
        }
-       return 0;
-skip_dentry:
-       dentry->needs_extraction = 0;
-       dentry->not_extracted = 1;
-       return 0;
-}
-
-static int
-dentry_reset_needs_extraction(struct wim_dentry *dentry, void *_ignore)
-{
-       struct wim_inode *inode = dentry->d_inode;
 
-       dentry->needs_extraction = 0;
-       dentry->not_extracted = 0;
-       inode->i_visited = 0;
-       inode->i_dos_name_extracted = 0;
-       FREE(inode->i_extracted_file);
-       inode->i_extracted_file = NULL;
-       if ((void*)dentry->extraction_name != (void*)dentry->file_name)
-               FREE(dentry->extraction_name);
-       dentry->extraction_name = NULL;
+       if ((extract_flags & WIMLIB_EXTRACT_FLAG_SYMLINK) &&
+           !supported_features->symlink_reparse_points)
+       {
+               ERROR("Symbolic link extraction mode requested, but "
+                     "extraction mode or volume does not support symbolic "
+                     "links!");
+               return WIMLIB_ERR_UNSUPPORTED;
+       }
        return 0;
 }
 
-#define WINDOWS_NT_MAX_PATH 32768
-
 /*
  * extract_tree - Extract a file or directory tree from the currently selected
  *               WIM image.
  *
  * @wim:       WIMStruct for the WIM file, with the desired image selected
  *             (as wim->current_image).
+ *
  * @wim_source_path:
  *             "Canonical" (i.e. no leading or trailing slashes, path
- *             separators forwald slashes) path inside the WIM image to
+ *             separators WIM_PATH_SEPARATOR) path inside the WIM image to
  *             extract.  An empty string means the full image.
+ *
  * @target:
  *             Filesystem path to extract the file or directory tree to.
+ *             (Or, with WIMLIB_EXTRACT_FLAG_NTFS: the name of a NTFS volume.)
  *
  * @extract_flags:
  *             WIMLIB_EXTRACT_FLAG_*.  Also, the private flag
@@ -673,7 +1891,7 @@ dentry_reset_needs_extraction(struct wim_dentry *dentry, void *_ignore)
  *
  * @progress_func:
  *             If non-NULL, progress function for the extraction.  The messages
- *             we may in this function are:
+ *             that may be sent in this function are:
  *
  *             WIMLIB_PROGRESS_MSG_EXTRACT_TREE_BEGIN or
  *                     WIMLIB_PROGRESS_MSG_EXTRACT_IMAGE_BEGIN;
@@ -685,195 +1903,281 @@ dentry_reset_needs_extraction(struct wim_dentry *dentry, void *_ignore)
  *             WIMLIB_PROGRESS_MSG_EXTRACT_TREE_END or
  *                     WIMLIB_PROGRESS_MSG_EXTRACT_IMAGE_END.
  *
- * Returns 0 on success; nonzero on failure.
+ * Returns 0 on success; a positive WIMLIB_ERR_* code on failure.
  */
 static int
 extract_tree(WIMStruct *wim, const tchar *wim_source_path, const tchar *target,
             int extract_flags, wimlib_progress_func_t progress_func)
 {
-       int ret;
-       struct list_head stream_list;
-       struct apply_args args;
-       const struct apply_operations *ops;
        struct wim_dentry *root;
+       struct wim_features required_features;
+       struct apply_ctx ctx;
+       int ret;
+       struct wim_lookup_table_entry *lte;
 
-       memset(&args, 0, sizeof(args));
-
-
-       args.wim                    = wim;
-       args.target                 = target;
-       args.target_nchars          = tstrlen(target);
-       args.extract_flags          = extract_flags;
-       args.progress_func          = progress_func;
-
-#ifdef __WIN32__
-       /* Work around defective behavior in Windows where paths longer than 260
-        * characters are not supported by default; instead they need to be
-        * turned into absolute paths and prefixed with "\\?\".  */
-       args.target_lowlevel_path = MALLOC(WINDOWS_NT_MAX_PATH * sizeof(wchar_t));
-       if (!args.target_lowlevel_path)
-       {
-               ret = WIMLIB_ERR_NOMEM;
-               goto out;
-       }
-       args.target_lowlevel_path_nchars =
-               GetFullPathName(args.target, WINDOWS_NT_MAX_PATH - 4,
-                               &args.target_lowlevel_path[4], NULL);
-
-       if (args.target_lowlevel_path_nchars == 0 ||
-           args.target_lowlevel_path_nchars >= WINDOWS_NT_MAX_PATH - 4)
-       {
-               WARNING("Can't get full path name for \"%ls\"", args.target);
-               FREE(args.target_lowlevel_path);
-               args.target_lowlevel_path = NULL;
-       } else {
-               wmemcpy(args.target_lowlevel_path, L"\\\\?\\", 4);
-               args.target_lowlevel_path_nchars += 4;
-       }
-#endif
-
+       /* Start initializing the apply_ctx.  */
+       memset(&ctx, 0, sizeof(struct apply_ctx));
+       ctx.wim = wim;
+       ctx.extract_flags = extract_flags;
+       ctx.target = target;
+       ctx.target_nchars = tstrlen(target);
+       ctx.progress_func = progress_func;
        if (progress_func) {
-               args.progress.extract.wimfile_name = wim->filename;
-               args.progress.extract.image = wim->current_image;
-               args.progress.extract.extract_flags = (extract_flags &
-                                                      WIMLIB_EXTRACT_MASK_PUBLIC);
-               args.progress.extract.image_name = wimlib_get_image_name(wim,
-                                                                        wim->current_image);
-               args.progress.extract.extract_root_wim_source_path = wim_source_path;
-               args.progress.extract.target = target;
+               ctx.progress.extract.wimfile_name = wim->filename;
+               ctx.progress.extract.image = wim->current_image;
+               ctx.progress.extract.extract_flags = (extract_flags &
+                                                     WIMLIB_EXTRACT_MASK_PUBLIC);
+               ctx.progress.extract.image_name = wimlib_get_image_name(wim,
+                                                                       wim->current_image);
+               ctx.progress.extract.extract_root_wim_source_path = wim_source_path;
+               ctx.progress.extract.target = target;
        }
+       INIT_LIST_HEAD(&ctx.stream_list);
 
-#ifdef WITH_NTFS_3G
-       if (extract_flags & WIMLIB_EXTRACT_FLAG_NTFS) {
-               args.vol = ntfs_mount(target, 0);
-               if (!args.vol) {
-                       ERROR_WITH_ERRNO("Failed to mount NTFS volume `%"TS"'",
-                                        target);
-                       ret = WIMLIB_ERR_NTFS_3G;
-                       goto out_free_target_lowlevel_path;
-               }
-               ops = &ntfs_apply_operations;
-       } else
-#endif
-               ops = &normal_apply_operations;
-
+       /* Translate the path to extract into the corresponding
+        * `struct wim_dentry', which will be the root of the
+        * "dentry tree" to extract.  */
        root = get_dentry(wim, wim_source_path);
        if (!root) {
                ERROR("Path \"%"TS"\" does not exist in WIM image %d",
                      wim_source_path, wim->current_image);
                ret = WIMLIB_ERR_PATH_DOES_NOT_EXIST;
-               goto out_ntfs_umount;
+               goto out;
        }
-       args.extract_root = root;
 
-       /* Calculate the actual filename component of each extracted dentry, and
-        * in the process set the dentry->needs_extraction flag on dentries that
-        * will be extracted. */
-       ret = for_dentry_in_tree(root, dentry_calculate_extraction_path, &args);
+       ctx.extract_root = root;
+
+       /* Select the appropriate apply_operations based on the
+        * platform and extract_flags.  */
+#ifdef __WIN32__
+       ctx.ops = &win32_apply_ops;
+#else
+       ctx.ops = &unix_apply_ops;
+#endif
+
+#ifdef WITH_NTFS_3G
+       if (extract_flags & WIMLIB_EXTRACT_FLAG_NTFS)
+               ctx.ops = &ntfs_3g_apply_ops;
+#endif
+
+       /* Call the start_extract() callback.  This gives the apply_operations
+        * implementation a chance to do any setup needed to access the volume.
+        * Furthermore, it's expected to set the supported features of this
+        * extraction mode (ctx.supported_features), which are determined at
+        * runtime as they may vary depending on the actual volume.  These
+        * features are then compared with the actual features extracting this
+        * dentry tree requires.  Some mismatches will merely produce warnings
+        * and the unsupported data will be ignored; others will produce errors.
+        */
+       ret = ctx.ops->start_extract(target, &ctx);
+       if (ret)
+               goto out;
+
+       dentry_tree_get_features(root, &required_features);
+       ret = do_feature_check(&required_features, &ctx.supported_features,
+                              extract_flags, ctx.ops);
+       if (ret)
+               goto out_finish_or_abort_extract;
+
+       /* Figure out whether the root dentry is being extracted to the root of
+        * a volume and therefore needs to be treated "specially", for example
+        * not being explicitly created and not having attributes set.  */
+       if (ctx.ops->target_is_root && ctx.ops->root_directory_is_special)
+               ctx.root_dentry_is_special = ctx.ops->target_is_root(target);
+
+       /* Calculate the actual filename component of each extracted dentry.  In
+        * the process, set the dentry->extraction_skipped flag on dentries that
+        * are being skipped for some reason (e.g. invalid filename).  */
+       ret = for_dentry_in_tree(root, dentry_calculate_extraction_path, &ctx);
        if (ret)
                goto out_dentry_reset_needs_extraction;
 
-       /* Build a list of the streams that need to be extracted */
-       ret = find_streams_for_extraction(root,
-                                         &stream_list,
-                                         wim->lookup_table, extract_flags);
+       /* Build the list of the streams that need to be extracted and
+        * initialize ctx.progress.extract with stream information.  */
+       ret = for_dentry_in_tree(ctx.extract_root,
+                                dentry_resolve_and_zero_lte_refcnt, &ctx);
        if (ret)
                goto out_dentry_reset_needs_extraction;
 
-       /* Calculate the number of bytes of data that will be extracted */
-       calculate_bytes_to_extract(&stream_list, extract_flags,
-                                  &args.progress);
+       ret = for_dentry_in_tree(ctx.extract_root,
+                                dentry_add_streams_to_extract, &ctx);
+       if (ret)
+               goto out_teardown_stream_list;
+
+       if (extract_flags & WIMLIB_EXTRACT_FLAG_FROM_PIPE) {
+               /* When extracting from a pipe, the number of bytes of data to
+                * extract can't be determined in the normal way (examining the
+                * lookup table), since at this point all we have is a set of
+                * SHA1 message digests of streams that need to be extracted.
+                * However, we can get a reasonably accurate estimate by taking
+                * <TOTALBYTES> from the corresponding <IMAGE> in the WIM XML
+                * data.  This does assume that a full image is being extracted,
+                * but currently there is no API for doing otherwise.  */
+               ctx.progress.extract.total_bytes =
+                       wim_info_get_image_total_bytes(wim->wim_info,
+                                                      wim->current_image);
+       }
 
-       if (extract_flags & WIMLIB_EXTRACT_FLAG_TO_STDOUT) {
+       /* Handle the special case of extracting a file to standard
+        * output.  In that case, "root" should be a single file, not a
+        * directory tree.  (If not, extract_dentry_to_stdout() will
+        * return an error.)  */
+       if (extract_flags & WIMLIB_EXTRACT_FLAG_TO_STDOUT &&
+           !(extract_flags & WIMLIB_EXTRACT_FLAG_FROM_PIPE)) {
                ret = extract_dentry_to_stdout(root);
-               goto out_dentry_reset_needs_extraction;
+               goto out_teardown_stream_list;
        }
 
-       if (progress_func) {
-               progress_func(*wim_source_path ? WIMLIB_PROGRESS_MSG_EXTRACT_TREE_BEGIN :
-                             WIMLIB_PROGRESS_MSG_EXTRACT_IMAGE_BEGIN,
-                             &args.progress);
+       /* If a sequential extraction was specified, sort the streams to be
+        * extracted by their position in the WIM file so that the WIM file can
+        * be read sequentially.  */
+       if ((extract_flags & (WIMLIB_EXTRACT_FLAG_SEQUENTIAL |
+                             WIMLIB_EXTRACT_FLAG_FROM_PIPE))
+                                       == WIMLIB_EXTRACT_FLAG_SEQUENTIAL)
+       {
+               ret = sort_stream_list_by_wim_position(&ctx.stream_list);
+               if (ret)
+                       goto out_teardown_stream_list;
        }
 
-       /* If a sequential extraction was specified, sort the streams to be
-        * extracted by their position in the WIM file, so that the WIM file can
-        * be read sequentially. */
-       if (extract_flags & WIMLIB_EXTRACT_FLAG_SEQUENTIAL) {
-               ret = sort_stream_list_by_wim_position(&stream_list);
-               if (ret != 0) {
-                       WARNING("Falling back to non-sequential extraction");
-                       extract_flags &= ~WIMLIB_EXTRACT_FLAG_SEQUENTIAL;
+       if (ctx.ops->realpath_works_on_nonexisting_files &&
+           ((extract_flags & WIMLIB_EXTRACT_FLAG_RPFIX) ||
+            ctx.ops->requires_realtarget_in_paths))
+       {
+               ctx.realtarget = realpath(target, NULL);
+               if (!ctx.realtarget) {
+                       ret = WIMLIB_ERR_NOMEM;
+                       goto out_teardown_stream_list;
                }
+               ctx.realtarget_nchars = tstrlen(ctx.realtarget);
        }
 
        if (progress_func) {
-               progress_func(WIMLIB_PROGRESS_MSG_EXTRACT_DIR_STRUCTURE_BEGIN,
-                             &args.progress);
+               progress_func(*wim_source_path ? WIMLIB_PROGRESS_MSG_EXTRACT_TREE_BEGIN :
+                                                WIMLIB_PROGRESS_MSG_EXTRACT_IMAGE_BEGIN,
+                             &ctx.progress);
        }
 
-       /* Make the directory structure and extract empty files */
-       args.extract_flags |= WIMLIB_EXTRACT_FLAG_NO_STREAMS;
-       args.apply_dentry = ops->apply_dentry;
-       ret = for_dentry_in_tree(root, maybe_apply_dentry, &args);
-       args.extract_flags &= ~WIMLIB_EXTRACT_FLAG_NO_STREAMS;
-       if (ret)
-               goto out_dentry_reset_needs_extraction;
-
-       if (progress_func) {
-               progress_func(WIMLIB_PROGRESS_MSG_EXTRACT_DIR_STRUCTURE_END,
-                             &args.progress);
+       if (!ctx.root_dentry_is_special)
+       {
+               tchar path[ctx.ops->path_max];
+               if (build_extraction_path(path, root, &ctx))
+               {
+                       ret = extract_inode(path, &ctx, root->d_inode);
+                       if (ret)
+                               goto out_free_realtarget;
+               }
        }
 
-       if (extract_flags & WIMLIB_EXTRACT_FLAG_RPFIX) {
-               args.target_realpath = realpath(target, NULL);
-               if (!args.target_realpath) {
+       /* If we need to fix up the targets of absolute symbolic links
+        * (WIMLIB_EXTRACT_FLAG_RPFIX) or the extraction mode requires paths to
+        * be absolute, use realpath() (or its replacement on Windows) to get
+        * the absolute path to the extraction target.  Note that this requires
+        * the target directory to exist, unless
+        * realpath_works_on_nonexisting_files is set in the apply_operations.
+        * */
+       if (!ctx.realtarget &&
+           (((extract_flags & WIMLIB_EXTRACT_FLAG_RPFIX) &&
+             required_features.symlink_reparse_points) ||
+            ctx.ops->requires_realtarget_in_paths))
+       {
+               ctx.realtarget = realpath(target, NULL);
+               if (!ctx.realtarget) {
                        ret = WIMLIB_ERR_NOMEM;
-                       goto out_dentry_reset_needs_extraction;
+                       goto out_free_realtarget;
                }
-               args.target_realpath_len = tstrlen(args.target_realpath);
+               ctx.realtarget_nchars = tstrlen(ctx.realtarget);
        }
 
-       /* Extract non-empty files */
-       ret = apply_stream_list(&stream_list, &args, ops, progress_func);
-       if (ret)
-               goto out_free_target_realpath;
+       /* Finally, the important part: extract the tree of files.  */
+       if (extract_flags & (WIMLIB_EXTRACT_FLAG_SEQUENTIAL |
+                            WIMLIB_EXTRACT_FLAG_FROM_PIPE)) {
+               /* Sequential extraction requested, so two passes are needed
+                * (one for directory structure, one for streams.)  */
+               if (progress_func)
+                       progress_func(WIMLIB_PROGRESS_MSG_EXTRACT_DIR_STRUCTURE_BEGIN,
+                                     &ctx.progress);
+               ret = for_dentry_in_tree(root, dentry_extract_skeleton, &ctx);
+               if (ret)
+                       goto out_free_realtarget;
+               if (progress_func)
+                       progress_func(WIMLIB_PROGRESS_MSG_EXTRACT_DIR_STRUCTURE_END,
+                                     &ctx.progress);
+               if (extract_flags & WIMLIB_EXTRACT_FLAG_FROM_PIPE)
+                       ret = extract_streams_from_pipe(&ctx);
+               else
+                       ret = extract_stream_list(&ctx);
+               if (ret)
+                       goto out_free_realtarget;
+       } else {
+               /* Sequential extraction was not requested, so we can make do
+                * with one pass where we both create the files and extract
+                * streams.   */
+               if (progress_func)
+                       progress_func(WIMLIB_PROGRESS_MSG_EXTRACT_DIR_STRUCTURE_BEGIN,
+                                     &ctx.progress);
+               ret = for_dentry_in_tree(root, dentry_extract, &ctx);
+               if (ret)
+                       goto out_free_realtarget;
+               if (progress_func)
+                       progress_func(WIMLIB_PROGRESS_MSG_EXTRACT_DIR_STRUCTURE_END,
+                                     &ctx.progress);
+       }
 
-       if (progress_func) {
-               progress_func(WIMLIB_PROGRESS_MSG_APPLY_TIMESTAMPS,
-                             &args.progress);
+       /* If the total number of bytes to extract was miscalculated, just jump
+        * to the calculated number in order to avoid confusing the progress
+        * function.  This should only occur when extracting from a pipe.  */
+       if (ctx.progress.extract.completed_bytes != ctx.progress.extract.total_bytes)
+       {
+               DEBUG("Calculated %"PRIu64" bytes to extract, but actually "
+                     "extracted %"PRIu64,
+                     ctx.progress.extract.total_bytes,
+                     ctx.progress.extract.completed_bytes);
+       }
+       if (progress_func &&
+           ctx.progress.extract.completed_bytes < ctx.progress.extract.total_bytes)
+       {
+               ctx.progress.extract.completed_bytes = ctx.progress.extract.total_bytes;
+               progress_func(WIMLIB_PROGRESS_MSG_EXTRACT_STREAMS, &ctx.progress);
        }
 
-       /* Apply timestamps */
-       ret = for_dentry_in_tree_depth(root,
-                                      ops->apply_dentry_timestamps, &args);
+       /* Apply security descriptors and timestamps.  This is done at the end,
+        * and in a depth-first manner, to prevent timestamps from getting
+        * changed by subsequent extract operations and to minimize the chance
+        * of the restored security descriptors getting in our way.  */
+       if (progress_func)
+               progress_func(WIMLIB_PROGRESS_MSG_APPLY_TIMESTAMPS,
+                             &ctx.progress);
+       ret = for_dentry_in_tree_depth(root, dentry_extract_final, &ctx);
        if (ret)
-               goto out_free_target_realpath;
+               goto out_free_realtarget;
 
        if (progress_func) {
                progress_func(*wim_source_path ? WIMLIB_PROGRESS_MSG_EXTRACT_TREE_END :
                              WIMLIB_PROGRESS_MSG_EXTRACT_IMAGE_END,
-                             &args.progress);
+                             &ctx.progress);
        }
-out_free_target_realpath:
-       FREE(args.target_realpath);
+
+       ret = 0;
+out_free_realtarget:
+       FREE(ctx.realtarget);
+out_teardown_stream_list:
+       /* Free memory allocated as part of the mapping from each
+        * wim_lookup_table_entry to the dentries that reference it.  */
+       if (ctx.extract_flags & WIMLIB_EXTRACT_FLAG_SEQUENTIAL)
+               list_for_each_entry(lte, &ctx.stream_list, extraction_list)
+                       if (lte->out_refcnt > ARRAY_LEN(lte->inline_lte_dentries))
+                               FREE(lte->lte_dentries);
 out_dentry_reset_needs_extraction:
        for_dentry_in_tree(root, dentry_reset_needs_extraction, NULL);
-out_ntfs_umount:
-#ifdef WITH_NTFS_3G
-       /* Unmount the NTFS volume */
-       if (extract_flags & WIMLIB_EXTRACT_FLAG_NTFS) {
-               if (ntfs_umount(args.vol, FALSE) != 0) {
-                       ERROR_WITH_ERRNO("Failed to unmount NTFS volume `%"TS"'",
-                                        args.target);
-                       if (ret == 0)
-                               ret = WIMLIB_ERR_NTFS_3G;
-               }
+out_finish_or_abort_extract:
+       if (ret) {
+               if (ctx.ops->abort_extract)
+                       ctx.ops->abort_extract(&ctx);
+       } else {
+               if (ctx.ops->finish_extract)
+                       ret = ctx.ops->finish_extract(&ctx);
        }
-#endif
-out_free_target_lowlevel_path:
-#ifdef __WIN32__
-       FREE(args.target_lowlevel_path);
-#endif
 out:
        return ret;
 }
@@ -884,7 +2188,6 @@ static int
 check_extract_command(struct wimlib_extract_command *cmd, int wim_header_flags)
 {
        int extract_flags;
-       bool is_entire_image = (cmd->wim_source_path[0] == T('\0'));
 
        /* Empty destination path? */
        if (cmd->fs_dest_path[0] == T('\0'))
@@ -892,77 +2195,61 @@ check_extract_command(struct wimlib_extract_command *cmd, int wim_header_flags)
 
        extract_flags = cmd->extract_flags;
 
-       /* Specified both symlink and hardlink modes? */
+       /* Check for invalid flag combinations  */
        if ((extract_flags &
             (WIMLIB_EXTRACT_FLAG_SYMLINK |
              WIMLIB_EXTRACT_FLAG_HARDLINK)) == (WIMLIB_EXTRACT_FLAG_SYMLINK |
                                                 WIMLIB_EXTRACT_FLAG_HARDLINK))
                return WIMLIB_ERR_INVALID_PARAM;
 
-#ifdef __WIN32__
-       /* Wanted UNIX data on Windows? */
-       if (extract_flags & WIMLIB_EXTRACT_FLAG_UNIX_DATA) {
-               ERROR("Extracting UNIX data is not supported on Windows");
+       if ((extract_flags &
+            (WIMLIB_EXTRACT_FLAG_NO_ACLS |
+             WIMLIB_EXTRACT_FLAG_STRICT_ACLS)) == (WIMLIB_EXTRACT_FLAG_NO_ACLS |
+                                                   WIMLIB_EXTRACT_FLAG_STRICT_ACLS))
                return WIMLIB_ERR_INVALID_PARAM;
-       }
-       /* Wanted linked extraction on Windows?  (XXX This is possible, just not
-        * implemented yet.) */
-       if (extract_flags & (WIMLIB_EXTRACT_FLAG_SYMLINK |
-                            WIMLIB_EXTRACT_FLAG_HARDLINK))
-       {
-               ERROR("Linked extraction modes are not supported on Windows");
+
+       if ((extract_flags &
+            (WIMLIB_EXTRACT_FLAG_RPFIX |
+             WIMLIB_EXTRACT_FLAG_NORPFIX)) == (WIMLIB_EXTRACT_FLAG_RPFIX |
+                                               WIMLIB_EXTRACT_FLAG_NORPFIX))
                return WIMLIB_ERR_INVALID_PARAM;
-       }
-#endif
 
        if (extract_flags & WIMLIB_EXTRACT_FLAG_NTFS) {
-               /* NTFS-3g extraction mode requested */
-#ifdef WITH_NTFS_3G
-               if ((extract_flags & (WIMLIB_EXTRACT_FLAG_SYMLINK |
-                                     WIMLIB_EXTRACT_FLAG_HARDLINK))) {
-                       ERROR("Cannot specify symlink or hardlink flags when applying\n"
-                             "        directly to a NTFS volume");
-                       return WIMLIB_ERR_INVALID_PARAM;
-               }
-               if (!is_entire_image &&
-                   (extract_flags & WIMLIB_EXTRACT_FLAG_NTFS))
-               {
-                       ERROR("When applying directly to a NTFS volume you can "
-                             "only extract a full image, not part of one");
-                       return WIMLIB_ERR_INVALID_PARAM;
-               }
-               if (extract_flags & WIMLIB_EXTRACT_FLAG_UNIX_DATA) {
-                       ERROR("Cannot restore UNIX-specific data in "
-                             "the NTFS extraction mode");
-                       return WIMLIB_ERR_INVALID_PARAM;
-               }
-#else
-               ERROR("wimlib was compiled without support for NTFS-3g, so");
-               ERROR("we cannot apply a WIM image directly to a NTFS volume");
+#ifndef WITH_NTFS_3G
+               ERROR("wimlib was compiled without support for NTFS-3g, so\n"
+                     "        we cannot apply a WIM image directly to a NTFS volume.");
                return WIMLIB_ERR_UNSUPPORTED;
 #endif
        }
 
        if ((extract_flags & (WIMLIB_EXTRACT_FLAG_RPFIX |
-                             WIMLIB_EXTRACT_FLAG_NORPFIX)) ==
-               (WIMLIB_EXTRACT_FLAG_RPFIX | WIMLIB_EXTRACT_FLAG_NORPFIX))
-       {
-               ERROR("Cannot specify RPFIX and NORPFIX flags at the same time!");
-               return WIMLIB_ERR_INVALID_PARAM;
-       }
-
-       if ((extract_flags & (WIMLIB_EXTRACT_FLAG_RPFIX |
                              WIMLIB_EXTRACT_FLAG_NORPFIX)) == 0)
        {
                /* Do reparse point fixups by default if the WIM header says
                 * they are enabled and we are extracting a full image. */
-               if ((wim_header_flags & WIM_HDR_FLAG_RP_FIX) && is_entire_image)
+               if (wim_header_flags & WIM_HDR_FLAG_RP_FIX)
                        extract_flags |= WIMLIB_EXTRACT_FLAG_RPFIX;
        }
 
-       if (!is_entire_image && (extract_flags & WIMLIB_EXTRACT_FLAG_RPFIX)) {
-               ERROR("Cannot specify --rpfix when not extracting entire image");
-               return WIMLIB_ERR_INVALID_PARAM;
+       /* TODO: Since UNIX data entries are stored in the file resources, in a
+        * completely sequential extraction they may come up before the
+        * corresponding file or symbolic link data.  This needs to be handled
+        * better.  */
+       if ((extract_flags & (WIMLIB_EXTRACT_FLAG_UNIX_DATA |
+                             WIMLIB_EXTRACT_FLAG_SEQUENTIAL))
+                                   == (WIMLIB_EXTRACT_FLAG_UNIX_DATA |
+                                       WIMLIB_EXTRACT_FLAG_SEQUENTIAL))
+       {
+               if (extract_flags & WIMLIB_EXTRACT_FLAG_FROM_PIPE) {
+                       WARNING("Setting UNIX file/owner group may "
+                               "be impossible on some\n"
+                               "          symbolic links "
+                               "when applying from a pipe.");
+               } else {
+                       extract_flags &= ~WIMLIB_EXTRACT_FLAG_SEQUENTIAL;
+                       WARNING("Disabling sequential extraction for "
+                               "UNIX data mode");
+               }
        }
 
        cmd->extract_flags = extract_flags;
@@ -970,7 +2257,8 @@ check_extract_command(struct wimlib_extract_command *cmd, int wim_header_flags)
 }
 
 
-/* Internal function to execute extraction commands for a WIM image. */
+/* Internal function to execute extraction commands for a WIM image.  The paths
+ * in the extract commands are expected to be already "canonicalized".  */
 static int
 do_wimlib_extract_files(WIMStruct *wim,
                        int image,
@@ -988,7 +2276,8 @@ do_wimlib_extract_files(WIMStruct *wim,
                return ret;
 
        /* Make sure there are no streams in the WIM that have not been
-        * checksummed yet. */
+        * checksummed yet.  Needed at least because 'unhashed_list' aliases
+        * 'extraction_list' in `struct wim_lookup_table_entry'.  */
        ret = wim_checksum_unhashed_streams(wim);
        if (ret)
                return ret;
@@ -1024,7 +2313,7 @@ do_wimlib_extract_files(WIMStruct *wim,
        return 0;
 }
 
-/* Extract files or directories from a WIM image. */
+/* API function documented in wimlib.h  */
 WIMLIBAPI int
 wimlib_extract_files(WIMStruct *wim,
                     int image,
@@ -1170,12 +2459,18 @@ extract_all_images(WIMStruct *wim,
        const tchar *image_name;
        struct stat stbuf;
 
+       extract_flags |= WIMLIB_EXTRACT_FLAG_MULTI_IMAGE;
+
+#ifdef WITH_NTFS_3G
+       if (extract_flags & WIMLIB_EXTRACT_FLAG_NTFS) {
+               ERROR("Cannot extract multiple images in NTFS extraction mode.");
+               return WIMLIB_ERR_INVALID_PARAM;
+       }
+#endif
+
        if (tstat(target, &stbuf)) {
-               if (errno == ENOENT)
-               {
-                       if (tmkdir(target, S_IRWXU | S_IRGRP | S_IXGRP |
-                                          S_IROTH | S_IXOTH))
-                       {
+               if (errno == ENOENT) {
+                       if (tmkdir(target, 0755)) {
                                ERROR_WITH_ERRNO("Failed to create directory \"%"TS"\"", target);
                                return WIMLIB_ERR_MKDIR;
                        }
@@ -1207,31 +2502,31 @@ extract_all_images(WIMStruct *wim,
        return 0;
 }
 
-/* Extracts a single image or all images from a WIM file to a directory or NTFS
- * volume. */
-WIMLIBAPI int
-wimlib_extract_image(WIMStruct *wim,
-                    int image,
-                    const tchar *target,
-                    int extract_flags,
-                    WIMStruct **additional_swms,
-                    unsigned num_additional_swms,
-                    wimlib_progress_func_t progress_func)
+static int
+do_wimlib_extract_image(WIMStruct *wim,
+                       int image,
+                       const tchar *target,
+                       int extract_flags,
+                       WIMStruct **additional_swms,
+