X-Git-Url: https://wimlib.net/git/?p=wimlib;a=blobdiff_plain;f=src%2Fextract.c;h=496d9c9870664ba050902139a942363197956605;hp=c312429cc7e9555e48ee43e514d9269219e5b24e;hb=011502c8756394066172b3164567a8b9835cc280;hpb=16f3bfa6b5f10408653e16ae2c5de66995b87605

diff --git a/src/extract.c b/src/extract.c
index c312429c..496d9c98 100644
--- a/src/extract.c
+++ b/src/extract.c
@@ -6,22 +6,20 @@
  */
 
 /*
- * Copyright (C) 2012, 2013, 2014 Eric Biggers
+ * Copyright (C) 2012-2016 Eric Biggers
  *
- * This file is part of wimlib, a library for working with WIM files.
+ * This file is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU Lesser General Public License as published by the Free
+ * Software Foundation; either version 3 of the License, or (at your option) any
+ * later version.
  *
- * wimlib is free software; you can redistribute it and/or modify it under the
- * terms of the GNU General Public License as published by the Free
- * Software Foundation; either version 3 of the License, or (at your option)
- * any later version.
- *
- * wimlib is distributed in the hope that it will be useful, but WITHOUT ANY
- * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
- * A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * This file is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
  * details.
  *
- * You should have received a copy of the GNU General Public License
- * along with wimlib; if not, see http://www.gnu.org/licenses/.
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this file; if not, see http://www.gnu.org/licenses/.
  */
 
 /*
@@ -41,31 +39,30 @@
 #  include "config.h"
 #endif
 
+#include <errno.h>
+#include <fcntl.h>
+#include <stdlib.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
 #include "wimlib/apply.h"
+#include "wimlib/assert.h"
+#include "wimlib/blob_table.h"
 #include "wimlib/dentry.h"
 #include "wimlib/encoding.h"
 #include "wimlib/endianness.h"
 #include "wimlib/error.h"
-#include "wimlib/lookup_table.h"
 #include "wimlib/metadata.h"
 #include "wimlib/pathlist.h"
 #include "wimlib/paths.h"
+#include "wimlib/pattern.h"
 #include "wimlib/reparse.h"
 #include "wimlib/resource.h"
 #include "wimlib/security.h"
 #include "wimlib/unix_data.h"
-#ifdef __WIN32__
-#  include "wimlib/win32.h" /* for realpath() equivalent */
-#endif
-#include "wimlib/xml.h"
-#include "wimlib/wildcard.h"
 #include "wimlib/wim.h"
-
-#include <errno.h>
-#include <fcntl.h>
-#include <stdlib.h>
-#include <sys/stat.h>
-#include <unistd.h>
+#include "wimlib/win32.h" /* for realpath() equivalent */
+#include "wimlib/xml.h"
 
 #define WIMLIB_EXTRACT_FLAG_FROM_PIPE   0x80000000
 #define WIMLIB_EXTRACT_FLAG_IMAGEMODE   0x40000000
@@ -88,217 +85,314 @@
 	 WIMLIB_EXTRACT_FLAG_STRICT_GLOB		|	\
 	 WIMLIB_EXTRACT_FLAG_NO_ATTRIBUTES		|	\
 	 WIMLIB_EXTRACT_FLAG_NO_PRESERVE_DIR_STRUCTURE  |	\
-	 WIMLIB_EXTRACT_FLAG_WIMBOOT)
+	 WIMLIB_EXTRACT_FLAG_WIMBOOT			|	\
+	 WIMLIB_EXTRACT_FLAG_COMPACT_XPRESS4K		|	\
+	 WIMLIB_EXTRACT_FLAG_COMPACT_XPRESS8K		|	\
+	 WIMLIB_EXTRACT_FLAG_COMPACT_XPRESS16K		|	\
+	 WIMLIB_EXTRACT_FLAG_COMPACT_LZX			\
+	 )
+
+/* Send WIMLIB_PROGRESS_MSG_EXTRACT_FILE_STRUCTURE or
+ * WIMLIB_PROGRESS_MSG_EXTRACT_METADATA.  */
+int
+do_file_extract_progress(struct apply_ctx *ctx, enum wimlib_progress_msg msg)
+{
+	ctx->count_until_file_progress = 500;  /* Arbitrary value to limit calls  */
+	return extract_progress(ctx, msg);
+}
 
-/* Check whether the extraction of a dentry should be skipped completely.  */
-static bool
-dentry_is_supported(struct wim_dentry *dentry,
-		    const struct wim_features *supported_features)
+static int
+start_file_phase(struct apply_ctx *ctx, u64 end_file_count, enum wimlib_progress_msg msg)
 {
-	struct wim_inode *inode = dentry->d_inode;
+	ctx->progress.extract.current_file_count = 0;
+	ctx->progress.extract.end_file_count = end_file_count;
+	return do_file_extract_progress(ctx, msg);
+}
 
-	if (inode->i_attributes & FILE_ATTRIBUTE_REPARSE_POINT) {
-		return supported_features->reparse_points ||
-			(inode_is_symlink(inode) &&
-			 supported_features->symlink_reparse_points);
-	}
-	if (inode->i_attributes & FILE_ATTRIBUTE_ENCRYPTED) {
-		if (inode->i_attributes & FILE_ATTRIBUTE_DIRECTORY)
-			return supported_features->encrypted_directories != 0;
-		else
-			return supported_features->encrypted_files != 0;
-	}
-	return true;
+int
+start_file_structure_phase(struct apply_ctx *ctx, u64 end_file_count)
+{
+	return start_file_phase(ctx, end_file_count, WIMLIB_PROGRESS_MSG_EXTRACT_FILE_STRUCTURE);
+}
+
+int
+start_file_metadata_phase(struct apply_ctx *ctx, u64 end_file_count)
+{
+	return start_file_phase(ctx, end_file_count, WIMLIB_PROGRESS_MSG_EXTRACT_METADATA);
+}
+
+static int
+end_file_phase(struct apply_ctx *ctx, enum wimlib_progress_msg msg)
+{
+	ctx->progress.extract.current_file_count = ctx->progress.extract.end_file_count;
+	return do_file_extract_progress(ctx, msg);
+}
+
+int
+end_file_structure_phase(struct apply_ctx *ctx)
+{
+	return end_file_phase(ctx, WIMLIB_PROGRESS_MSG_EXTRACT_FILE_STRUCTURE);
 }
 
+int
+end_file_metadata_phase(struct apply_ctx *ctx)
+{
+	return end_file_phase(ctx, WIMLIB_PROGRESS_MSG_EXTRACT_METADATA);
+}
 
-#define PWM_ALLOW_WIM_HDR 0x00001
+#define PWM_FOUND_WIM_HDR (-1)
 
-/* Read the header from a stream in a pipable WIM.  */
+/* Read the header for a blob in a pipable WIM.  If @pwm_hdr_ret is not NULL,
+ * also look for a pipable WIM header and return PWM_FOUND_WIM_HDR if found.  */
 static int
-read_pwm_stream_header(WIMStruct *pwm, struct wim_lookup_table_entry *lte,
-		       struct wim_resource_spec *rspec,
-		       int flags, struct wim_header_disk *hdr_ret)
+read_pwm_blob_header(WIMStruct *pwm, u8 hash_ret[SHA1_HASH_SIZE],
+		     struct wim_reshdr *reshdr_ret,
+		     struct wim_header_disk *pwm_hdr_ret)
 {
-	union {
-		struct pwm_stream_hdr stream_hdr;
-		struct wim_header_disk pwm_hdr;
-	} buf;
-	struct wim_reshdr reshdr;
 	int ret;
+	struct pwm_blob_hdr blob_hdr;
+	u64 magic;
 
-	ret = full_read(&pwm->in_fd, &buf.stream_hdr, sizeof(buf.stream_hdr));
-	if (ret)
+	ret = full_read(&pwm->in_fd, &blob_hdr, sizeof(blob_hdr));
+	if (unlikely(ret))
 		goto read_error;
 
-	if ((flags & PWM_ALLOW_WIM_HDR) &&
-	    le64_to_cpu(buf.stream_hdr.magic) == PWM_MAGIC)
-	{
-		BUILD_BUG_ON(sizeof(buf.pwm_hdr) < sizeof(buf.stream_hdr));
-		ret = full_read(&pwm->in_fd, &buf.stream_hdr + 1,
-				sizeof(buf.pwm_hdr) - sizeof(buf.stream_hdr));
+	magic = le64_to_cpu(blob_hdr.magic);
 
-		if (ret)
+	if (magic == PWM_MAGIC && pwm_hdr_ret != NULL) {
+		memcpy(pwm_hdr_ret, &blob_hdr, sizeof(blob_hdr));
+		ret = full_read(&pwm->in_fd,
+				(u8 *)pwm_hdr_ret + sizeof(blob_hdr),
+				sizeof(*pwm_hdr_ret) - sizeof(blob_hdr));
+		if (unlikely(ret))
 			goto read_error;
-		lte->resource_location = RESOURCE_NONEXISTENT;
-		memcpy(hdr_ret, &buf.pwm_hdr, sizeof(buf.pwm_hdr));
-		return 0;
+		return PWM_FOUND_WIM_HDR;
 	}
 
-	if (le64_to_cpu(buf.stream_hdr.magic) != PWM_STREAM_MAGIC) {
-		ERROR("Data read on pipe is invalid (expected stream header).");
+	if (unlikely(magic != PWM_BLOB_MAGIC)) {
+		ERROR("Data read on pipe is invalid (expected blob header)");
 		return WIMLIB_ERR_INVALID_PIPABLE_WIM;
 	}
 
-	copy_hash(lte->hash, buf.stream_hdr.hash);
+	copy_hash(hash_ret, blob_hdr.hash);
+
+	reshdr_ret->size_in_wim = 0; /* Not available  */
+	reshdr_ret->flags = le32_to_cpu(blob_hdr.flags);
+	reshdr_ret->offset_in_wim = pwm->in_fd.offset;
+	reshdr_ret->uncompressed_size = le64_to_cpu(blob_hdr.uncompressed_size);
+
+	if (unlikely(reshdr_ret->uncompressed_size == 0)) {
+		ERROR("Data read on pipe is invalid (resource is of 0 size)");
+		return WIMLIB_ERR_INVALID_PIPABLE_WIM;
+	}
 
-	reshdr.size_in_wim = 0;
-	reshdr.flags = le32_to_cpu(buf.stream_hdr.flags);
-	reshdr.offset_in_wim = pwm->in_fd.offset;
-	reshdr.uncompressed_size = le64_to_cpu(buf.stream_hdr.uncompressed_size);
-	wim_res_hdr_to_spec(&reshdr, pwm, rspec);
-	lte_bind_wim_resource_spec(lte, rspec);
-	lte->flags = rspec->flags;
-	lte->size = rspec->uncompressed_size;
-	lte->offset_in_res = 0;
 	return 0;
 
 read_error:
-	ERROR_WITH_ERRNO("Error reading pipable WIM from pipe");
+	if (ret == WIMLIB_ERR_UNEXPECTED_END_OF_FILE)
+		ERROR("The pipe ended before all needed data was sent!");
+	else
+		ERROR_WITH_ERRNO("Error reading pipable WIM from pipe");
 	return ret;
 }
 
 static int
-load_streams_from_pipe(struct apply_ctx *ctx,
-		       const struct read_stream_list_callbacks *cbs)
+read_blobs_from_pipe(struct apply_ctx *ctx, const struct read_blob_callbacks *cbs)
 {
-	struct wim_lookup_table_entry *found_lte = NULL;
-	struct wim_resource_spec *rspec = NULL;
-	struct wim_lookup_table *lookup_table;
 	int ret;
+	u8 hash[SHA1_HASH_SIZE];
+	struct wim_reshdr reshdr;
+	struct wim_header_disk pwm_hdr;
+	struct wim_resource_descriptor rdesc;
+	struct blob_descriptor *blob;
 
-	ret = WIMLIB_ERR_NOMEM;
-	found_lte = new_lookup_table_entry();
-	if (!found_lte)
-		goto out;
-
-	rspec = MALLOC(sizeof(struct wim_resource_spec));
-	if (!rspec)
-		goto out;
-
-	lookup_table = ctx->wim->lookup_table;
-	memcpy(ctx->progress.extract.guid, ctx->wim->hdr.guid, WIM_GUID_LEN);
+	copy_guid(ctx->progress.extract.guid, ctx->wim->hdr.guid);
 	ctx->progress.extract.part_number = ctx->wim->hdr.part_number;
 	ctx->progress.extract.total_parts = ctx->wim->hdr.total_parts;
 	ret = extract_progress(ctx, WIMLIB_PROGRESS_MSG_EXTRACT_SPWM_PART_BEGIN);
 	if (ret)
-		goto out;
+		return ret;
 
-	while (ctx->num_streams_remaining) {
-		struct wim_header_disk pwm_hdr;
-		struct wim_lookup_table_entry *needed_lte;
+	while (ctx->num_blobs_remaining) {
 
-		if (found_lte->resource_location != RESOURCE_NONEXISTENT)
-			lte_unbind_wim_resource_spec(found_lte);
-		ret = read_pwm_stream_header(ctx->wim, found_lte, rspec,
-					     PWM_ALLOW_WIM_HDR, &pwm_hdr);
-		if (ret)
-			goto out;
+		ret = read_pwm_blob_header(ctx->wim, hash, &reshdr, &pwm_hdr);
 
-		if ((found_lte->resource_location != RESOURCE_NONEXISTENT)
-		    && !(found_lte->flags & WIM_RESHDR_FLAG_METADATA)
-		    && (needed_lte = lookup_stream(lookup_table, found_lte->hash))
-		    && (needed_lte->out_refcnt))
-		{
-			needed_lte->offset_in_res = found_lte->offset_in_res;
-			needed_lte->flags = found_lte->flags;
-			needed_lte->size = found_lte->size;
+		if (ret == PWM_FOUND_WIM_HDR) {
+			u16 part_number = le16_to_cpu(pwm_hdr.part_number);
+			u16 total_parts = le16_to_cpu(pwm_hdr.total_parts);
 
-			lte_unbind_wim_resource_spec(found_lte);
-			lte_bind_wim_resource_spec(needed_lte, rspec);
+			if (part_number == ctx->progress.extract.part_number &&
+			    total_parts == ctx->progress.extract.total_parts &&
+			    guids_equal(pwm_hdr.guid, ctx->progress.extract.guid))
+				continue;
 
-			ret = (*cbs->begin_stream)(needed_lte, 0,
-						   cbs->begin_stream_ctx);
-			if (ret) {
-				lte_unbind_wim_resource_spec(needed_lte);
-				goto out;
-			}
+			copy_guid(ctx->progress.extract.guid, pwm_hdr.guid);
+			ctx->progress.extract.part_number = part_number;
+			ctx->progress.extract.total_parts = total_parts;
+			ret = extract_progress(ctx, WIMLIB_PROGRESS_MSG_EXTRACT_SPWM_PART_BEGIN);
+			if (ret)
+				return ret;
 
-			ret = extract_stream(needed_lte, needed_lte->size,
-					     cbs->consume_chunk,
-					     cbs->consume_chunk_ctx);
+			continue;
+		}
 
-			ret = (*cbs->end_stream)(needed_lte, ret,
-						 cbs->end_stream_ctx);
-			lte_unbind_wim_resource_spec(needed_lte);
-			if (ret)
-				goto out;
-			ctx->num_streams_remaining--;
-		} else if (found_lte->resource_location != RESOURCE_NONEXISTENT) {
-			ret = skip_wim_stream(found_lte);
+		if (ret)
+			return ret;
+
+		if (!(reshdr.flags & WIM_RESHDR_FLAG_METADATA)
+		    && (blob = lookup_blob(ctx->wim->blob_table, hash))
+		    && (blob->out_refcnt))
+		{
+			wim_reshdr_to_desc_and_blob(&reshdr, ctx->wim, &rdesc, blob);
+			ret = read_blob_with_sha1(blob, cbs);
+			blob_unset_is_located_in_wim_resource(blob);
 			if (ret)
-				goto out;
+				return ret;
+			ctx->num_blobs_remaining--;
 		} else {
-			u16 part_number = le16_to_cpu(pwm_hdr.part_number);
-			u16 total_parts = le16_to_cpu(pwm_hdr.total_parts);
-
-			if (part_number != ctx->progress.extract.part_number ||
-			    total_parts != ctx->progress.extract.total_parts ||
-			    memcmp(pwm_hdr.guid, ctx->progress.extract.guid,
-				   WIM_GUID_LEN))
-			{
-				ctx->progress.extract.part_number = part_number;
-				ctx->progress.extract.total_parts = total_parts;
-				memcpy(ctx->progress.extract.guid,
-				       pwm_hdr.guid, WIM_GUID_LEN);
-				ret = extract_progress(ctx,
-						       WIMLIB_PROGRESS_MSG_EXTRACT_SPWM_PART_BEGIN);
-				if (ret)
-					goto out;
-			}
+			wim_reshdr_to_desc(&reshdr, ctx->wim, &rdesc);
+			ret = skip_wim_resource(&rdesc);
+			if (ret)
+				return ret;
 		}
 	}
-	ret = 0;
+
+	return 0;
+}
+
+static int
+handle_pwm_metadata_resource(WIMStruct *pwm, int image, bool is_needed)
+{
+	struct blob_descriptor *blob;
+	struct wim_reshdr reshdr;
+	struct wim_resource_descriptor *rdesc;
+	int ret;
+
+	ret = WIMLIB_ERR_NOMEM;
+	blob = new_blob_descriptor();
+	if (!blob)
+		goto out;
+
+	ret = read_pwm_blob_header(pwm, blob->hash, &reshdr, NULL);
+	if (ret)
+		goto out;
+
+	ret = WIMLIB_ERR_INVALID_PIPABLE_WIM;
+	if (!(reshdr.flags & WIM_RESHDR_FLAG_METADATA)) {
+		ERROR("Expected metadata resource, but found non-metadata "
+		      "resource");
+		goto out;
+	}
+
+	ret = WIMLIB_ERR_NOMEM;
+	rdesc = MALLOC(sizeof(*rdesc));
+	if (!rdesc)
+		goto out;
+
+	wim_reshdr_to_desc_and_blob(&reshdr, pwm, rdesc, blob);
+	pwm->refcnt++;
+
+	ret = WIMLIB_ERR_NOMEM;
+	pwm->image_metadata[image - 1] = new_unloaded_image_metadata(blob);
+	if (!pwm->image_metadata[image - 1])
+		goto out;
+	blob = NULL;
+
+	/* If the metadata resource is for the image being extracted, then parse
+	 * it and save the metadata in memory.  Otherwise, skip over it.  */
+	if (is_needed)
+		ret = select_wim_image(pwm, image);
+	else
+		ret = skip_wim_resource(rdesc);
 out:
-	if (found_lte->resource_location != RESOURCE_IN_WIM)
-		FREE(rspec);
-	free_lookup_table_entry(found_lte);
+	free_blob_descriptor(blob);
 	return ret;
 }
 
+/* Creates a temporary file opened for writing.  The open file descriptor is
+ * returned in @fd_ret and its name is returned in @name_ret (dynamically
+ * allocated).  */
 static int
-begin_extract_stream_with_progress(struct wim_lookup_table_entry *lte,
-				   u32 flags, void *_ctx)
+create_temporary_file(struct filedes *fd_ret, tchar **name_ret)
+{
+	tchar *name;
+	int raw_fd;
+
+#ifdef __WIN32__
+retry:
+	name = _wtempnam(NULL, L"wimlib");
+	if (!name) {
+		ERROR_WITH_ERRNO("Failed to create temporary filename");
+		return WIMLIB_ERR_NOMEM;
+	}
+	raw_fd = _wopen(name, O_WRONLY | O_CREAT | O_EXCL | O_BINARY |
+			_O_SHORT_LIVED, 0600);
+	if (raw_fd < 0 && errno == EEXIST) {
+		FREE(name);
+		goto retry;
+	}
+#else /* __WIN32__ */
+	const char *tmpdir = getenv("TMPDIR");
+	if (!tmpdir)
+		tmpdir = P_tmpdir;
+	name = MALLOC(strlen(tmpdir) + 1 + 6 + 6 + 1);
+	if (!name)
+		return WIMLIB_ERR_NOMEM;
+	sprintf(name, "%s/wimlibXXXXXX", tmpdir);
+	raw_fd = mkstemp(name);
+#endif /* !__WIN32__ */
+
+	if (raw_fd < 0) {
+		ERROR_WITH_ERRNO("Failed to create temporary file "
+				 "\"%"TS"\"", name);
+		FREE(name);
+		return WIMLIB_ERR_OPEN;
+	}
+
+	filedes_init(fd_ret, raw_fd);
+	*name_ret = name;
+	return 0;
+}
+
+static int
+begin_extract_blob_wrapper(struct blob_descriptor *blob, void *_ctx)
 {
 	struct apply_ctx *ctx = _ctx;
 
-	ctx->cur_stream = lte;
+	ctx->cur_blob = blob;
+	ctx->cur_blob_offset = 0;
+
+	if (unlikely(blob->out_refcnt > MAX_OPEN_FILES))
+		return create_temporary_file(&ctx->tmpfile_fd, &ctx->tmpfile_name);
 
-	return (*ctx->saved_cbs->begin_stream)(lte, flags,
-					       ctx->saved_cbs->begin_stream_ctx);
+	return call_begin_blob(blob, ctx->saved_cbs);
 }
 
 static int
-consume_chunk_with_progress(const void *chunk, size_t size, void *_ctx)
+extract_chunk_wrapper(const void *chunk, size_t size, void *_ctx)
 {
 	struct apply_ctx *ctx = _ctx;
 	union wimlib_progress_info *progress = &ctx->progress;
 	int ret;
 
+	ctx->cur_blob_offset += size;
+
 	if (likely(ctx->supported_features.hard_links)) {
 		progress->extract.completed_bytes +=
-			(u64)size * ctx->cur_stream->out_refcnt;
+			(u64)size * ctx->cur_blob->out_refcnt;
+		if (ctx->cur_blob_offset == ctx->cur_blob->size)
+			progress->extract.completed_streams += ctx->cur_blob->out_refcnt;
 	} else {
-		const struct stream_owner *owners = stream_owners(ctx->cur_stream);
-		for (u32 i = 0; i < ctx->cur_stream->out_refcnt; i++) {
-			const struct wim_inode *inode = owners[i].inode;
+		const struct blob_extraction_target *targets =
+			blob_extraction_targets(ctx->cur_blob);
+		for (u32 i = 0; i < ctx->cur_blob->out_refcnt; i++) {
+			const struct wim_inode *inode = targets[i].inode;
 			const struct wim_dentry *dentry;
 
-			list_for_each_entry(dentry,
-					    &inode->i_extraction_aliases,
-					    d_extraction_alias_node)
-			{
+			inode_for_each_extraction_alias(dentry, inode) {
 				progress->extract.completed_bytes += size;
+				if (ctx->cur_blob_offset == ctx->cur_blob->size)
+					progress->extract.completed_streams++;
 			}
 		}
 	}
@@ -308,54 +402,105 @@ consume_chunk_with_progress(const void *chunk, size_t size, void *_ctx)
 		if (ret)
 			return ret;
 
-		if (progress->extract.completed_bytes >=
-		    progress->extract.total_bytes)
-		{
-			ctx->next_progress = UINT64_MAX;
-		} else {
-			ctx->next_progress += progress->extract.total_bytes / 128;
-			if (ctx->next_progress > progress->extract.total_bytes)
-				ctx->next_progress = progress->extract.total_bytes;
+		set_next_progress(progress->extract.completed_bytes,
+				  progress->extract.total_bytes,
+				  &ctx->next_progress);
+	}
+
+	if (unlikely(filedes_valid(&ctx->tmpfile_fd))) {
+		/* Just extracting to temporary file for now.  */
+		ret = full_write(&ctx->tmpfile_fd, chunk, size);
+		if (ret) {
+			ERROR_WITH_ERRNO("Error writing data to "
+					 "temporary file \"%"TS"\"",
+					 ctx->tmpfile_name);
 		}
+		return ret;
 	}
-	return (*ctx->saved_cbs->consume_chunk)(chunk, size,
-						ctx->saved_cbs->consume_chunk_ctx);
+
+	return call_consume_chunk(chunk, size, ctx->saved_cbs);
+}
+
+/* Copy the blob's data from the temporary file to each of its targets.
+ *
+ * This is executed only in the very uncommon case that a blob is being
+ * extracted to more than MAX_OPEN_FILES targets!  */
+static int
+extract_from_tmpfile(const tchar *tmpfile_name,
+		     const struct blob_descriptor *orig_blob,
+		     const struct read_blob_callbacks *cbs)
+{
+	struct blob_descriptor tmpfile_blob;
+	const struct blob_extraction_target *targets = blob_extraction_targets(orig_blob);
+	int ret;
+
+	memcpy(&tmpfile_blob, orig_blob, sizeof(struct blob_descriptor));
+	tmpfile_blob.blob_location = BLOB_IN_FILE_ON_DISK;
+	tmpfile_blob.file_on_disk = (tchar *)tmpfile_name;
+	tmpfile_blob.out_refcnt = 1;
+
+	for (u32 i = 0; i < orig_blob->out_refcnt; i++) {
+		tmpfile_blob.inline_blob_extraction_targets[0] = targets[i];
+		ret = read_blob_with_cbs(&tmpfile_blob, cbs);
+		if (ret)
+			return ret;
+	}
+	return 0;
+}
+
+static int
+end_extract_blob_wrapper(struct blob_descriptor *blob, int status, void *_ctx)
+{
+	struct apply_ctx *ctx = _ctx;
+
+	if (unlikely(filedes_valid(&ctx->tmpfile_fd))) {
+		filedes_close(&ctx->tmpfile_fd);
+		if (!status)
+			status = extract_from_tmpfile(ctx->tmpfile_name, blob,
+						      ctx->saved_cbs);
+		filedes_invalidate(&ctx->tmpfile_fd);
+		tunlink(ctx->tmpfile_name);
+		FREE(ctx->tmpfile_name);
+		return status;
+	}
+
+	return call_end_blob(blob, status, ctx->saved_cbs);
 }
 
 /*
- * Read the list of single-instance streams to extract and feed their data into
- * the specified callback functions.
+ * Read the list of blobs to extract and feed their data into the specified
+ * callback functions.
  *
- * This handles checksumming each stream.
+ * This handles checksumming each blob.
  *
  * This also handles sending WIMLIB_PROGRESS_MSG_EXTRACT_STREAMS.
  *
- * This also works if the WIM is being read from a pipe, whereas attempting to
- * read streams directly (e.g. with read_full_stream_into_buf()) will not.
+ * This also works if the WIM is being read from a pipe.
+ *
+ * This also will split up blobs that will need to be extracted to more than
+ * MAX_OPEN_FILES locations, as measured by the 'out_refcnt' of each blob.
+ * Therefore, the apply_operations implementation need not worry about running
+ * out of file descriptors, unless it might open more than one file descriptor
+ * per 'blob_extraction_target' (e.g. Win32 currently might because the
+ * destination file system might not support hard links).
  */
 int
-extract_stream_list(struct apply_ctx *ctx,
-		    const struct read_stream_list_callbacks *cbs)
+extract_blob_list(struct apply_ctx *ctx, const struct read_blob_callbacks *cbs)
 {
-	struct read_stream_list_callbacks wrapper_cbs = {
-		.begin_stream      = begin_extract_stream_with_progress,
-		.begin_stream_ctx  = ctx,
-		.consume_chunk     = consume_chunk_with_progress,
-		.consume_chunk_ctx = ctx,
-		.end_stream        = cbs->end_stream,
-		.end_stream_ctx    = cbs->end_stream_ctx,
+	struct read_blob_callbacks wrapper_cbs = {
+		.begin_blob	= begin_extract_blob_wrapper,
+		.consume_chunk	= extract_chunk_wrapper,
+		.end_blob	= end_extract_blob_wrapper,
+		.ctx		= ctx,
 	};
-	if (ctx->progfunc) {
-		ctx->saved_cbs = cbs;
-		cbs = &wrapper_cbs;
-	}
+	ctx->saved_cbs = cbs;
 	if (ctx->extract_flags & WIMLIB_EXTRACT_FLAG_FROM_PIPE) {
-		return load_streams_from_pipe(ctx, cbs);
+		return read_blobs_from_pipe(ctx, &wrapper_cbs);
 	} else {
-		return read_stream_list(&ctx->stream_list,
-					offsetof(struct wim_lookup_table_entry,
-						 extraction_list),
-					cbs, VERIFY_STREAM_HASHES);
+		return read_blob_list(&ctx->blob_list,
+				      offsetof(struct blob_descriptor,
+					       extraction_list),
+				      &wrapper_cbs, VERIFY_BLOB_HASHES);
 	}
 }
 
@@ -366,38 +511,39 @@ extract_stream_list(struct apply_ctx *ctx,
  * unnamed data stream only.  */
 static int
 extract_dentry_to_stdout(struct wim_dentry *dentry,
-			 const struct wim_lookup_table *lookup_table)
+			 const struct blob_table *blob_table)
 {
 	struct wim_inode *inode = dentry->d_inode;
-	struct wim_lookup_table_entry *lte;
+	struct blob_descriptor *blob;
 	struct filedes _stdout;
 
 	if (inode->i_attributes & (FILE_ATTRIBUTE_REPARSE_POINT |
-				   FILE_ATTRIBUTE_DIRECTORY))
+				   FILE_ATTRIBUTE_DIRECTORY |
+				   FILE_ATTRIBUTE_ENCRYPTED))
 	{
 		ERROR("\"%"TS"\" is not a regular file and therefore cannot be "
 		      "extracted to standard output", dentry_full_path(dentry));
 		return WIMLIB_ERR_NOT_A_REGULAR_FILE;
 	}
 
-	lte = inode_unnamed_lte(inode, lookup_table);
-	if (!lte) {
-		const u8 *hash = inode_unnamed_stream_hash(inode);
+	blob = inode_get_blob_for_unnamed_data_stream(inode, blob_table);
+	if (!blob) {
+		const u8 *hash = inode_get_hash_of_unnamed_data_stream(inode);
 		if (!is_zero_hash(hash))
-			return stream_not_found_error(inode, hash);
+			return blob_not_found_error(inode, hash);
 		return 0;
 	}
 
 	filedes_init(&_stdout, STDOUT_FILENO);
-	return extract_full_stream_to_fd(lte, &_stdout);
+	return extract_blob_to_fd(blob, &_stdout);
 }
 
 static int
 extract_dentries_to_stdout(struct wim_dentry **dentries, size_t num_dentries,
-			   const struct wim_lookup_table *lookup_table)
+			   const struct blob_table *blob_table)
 {
 	for (size_t i = 0; i < num_dentries; i++) {
-		int ret = extract_dentry_to_stdout(dentries[i], lookup_table);
+		int ret = extract_dentry_to_stdout(dentries[i], blob_table);
 		if (ret)
 			return ret;
 	}
@@ -416,14 +562,14 @@ remove_duplicate_trees(struct wim_dentry **trees, size_t num_trees)
 {
 	size_t i, j = 0;
 	for (i = 0; i < num_trees; i++) {
-		if (!trees[i]->tmp_flag) {
+		if (!trees[i]->d_tmp_flag) {
 			/* Found distinct dentry.  */
-			trees[i]->tmp_flag = 1;
+			trees[i]->d_tmp_flag = 1;
 			trees[j++] = trees[i];
 		}
 	}
 	for (i = 0; i < j; i++)
-		trees[i]->tmp_flag = 0;
+		trees[i]->d_tmp_flag = 0;
 	return j;
 }
 
@@ -437,23 +583,23 @@ remove_contained_trees(struct wim_dentry **trees, size_t num_trees)
 {
 	size_t i, j = 0;
 	for (i = 0; i < num_trees; i++)
-		trees[i]->tmp_flag = 1;
+		trees[i]->d_tmp_flag = 1;
 	for (i = 0; i < num_trees; i++) {
 		struct wim_dentry *d = trees[i];
 		while (!dentry_is_root(d)) {
 			d = d->d_parent;
-			if (d->tmp_flag)
+			if (d->d_tmp_flag)
 				goto tree_contained;
 		}
 		trees[j++] = trees[i];
 		continue;
 
 	tree_contained:
-		trees[i]->tmp_flag = 0;
+		trees[i]->d_tmp_flag = 0;
 	}
 
 	for (i = 0; i < j; i++)
-		trees[i]->tmp_flag = 0;
+		trees[i]->d_tmp_flag = 0;
 	return j;
 }
 
@@ -474,8 +620,10 @@ dentry_reset_extraction_list_node(struct wim_dentry *dentry)
 static int
 dentry_delete_from_list(struct wim_dentry *dentry, void *_ignore)
 {
-	list_del(&dentry->d_extraction_list_node);
-	dentry_reset_extraction_list_node(dentry);
+	if (will_extract_dentry(dentry)) {
+		list_del(&dentry->d_extraction_list_node);
+		dentry_reset_extraction_list_node(dentry);
+	}
 	return 0;
 }
 
@@ -536,7 +684,8 @@ destroy_dentry_list(struct list_head *dentry_list)
 		inode = dentry->d_inode;
 		dentry_reset_extraction_list_node(dentry);
 		inode->i_visited = 0;
-		if ((void *)dentry->d_extraction_name != (void *)dentry->file_name)
+		inode->i_can_externally_back = 0;
+		if ((void *)dentry->d_extraction_name != (void *)dentry->d_name)
 			FREE(dentry->d_extraction_name);
 		dentry->d_extraction_name = NULL;
 		dentry->d_extraction_name_nchars = 0;
@@ -544,13 +693,13 @@ destroy_dentry_list(struct list_head *dentry_list)
 }
 
 static void
-destroy_stream_list(struct list_head *stream_list)
+destroy_blob_list(struct list_head *blob_list)
 {
-	struct wim_lookup_table_entry *lte;
+	struct blob_descriptor *blob;
 
-	list_for_each_entry(lte, stream_list, extraction_list)
-		if (lte->out_refcnt > ARRAY_LEN(lte->inline_stream_owners))
-			FREE(lte->stream_owners);
+	list_for_each_entry(blob, blob_list, extraction_list)
+		if (blob->out_refcnt > ARRAY_LEN(blob->inline_blob_extraction_targets))
+			FREE(blob->blob_extraction_targets);
 }
 
 #ifdef __WIN32__
@@ -567,19 +716,20 @@ file_name_valid(utf16lechar *name, size_t num_chars, bool fix)
 	if (num_chars == 0)
 		return true;
 	for (i = 0; i < num_chars; i++) {
-		switch (name[i]) {
+		switch (le16_to_cpu(name[i])) {
 	#ifdef __WIN32__
-		case cpu_to_le16('\\'):
-		case cpu_to_le16(':'):
-		case cpu_to_le16('*'):
-		case cpu_to_le16('?'):
-		case cpu_to_le16('"'):
-		case cpu_to_le16('<'):
-		case cpu_to_le16('>'):
-		case cpu_to_le16('|'):
+		case '\x01'...'\x1F':
+		case '\\':
+		case ':':
+		case '*':
+		case '?':
+		case '"':
+		case '<':
+		case '>':
+		case '|':
 	#endif
-		case cpu_to_le16('/'):
-		case cpu_to_le16('\0'):
+		case '/':
+		case '\0':
 			if (fix)
 				name[i] = replacement_char;
 			else
@@ -587,16 +737,6 @@ file_name_valid(utf16lechar *name, size_t num_chars, bool fix)
 		}
 	}
 
-#ifdef __WIN32__
-	if (name[num_chars - 1] == cpu_to_le16(' ') ||
-	    name[num_chars - 1] == cpu_to_le16('.'))
-	{
-		if (fix)
-			name[num_chars - 1] = replacement_char;
-		else
-			return false;
-	}
-#endif
 	return true;
 }
 
@@ -606,18 +746,17 @@ dentry_calculate_extraction_name(struct wim_dentry *dentry,
 {
 	int ret;
 
-	if (!dentry_is_supported(dentry, &ctx->supported_features))
-		goto skip_dentry;
-
 	if (dentry_is_root(dentry))
 		return 0;
 
+#ifdef WITH_NTFS_3G
 	if (ctx->extract_flags & WIMLIB_EXTRACT_FLAG_NTFS) {
-		dentry->d_extraction_name = dentry->file_name;
-		dentry->d_extraction_name_nchars = dentry->file_name_nbytes /
+		dentry->d_extraction_name = dentry->d_name;
+		dentry->d_extraction_name_nchars = dentry->d_name_nbytes /
 						   sizeof(utf16lechar);
 		return 0;
 	}
+#endif
 
 	if (!ctx->supported_features.case_sensitive_filenames) {
 		struct wim_dentry *other;
@@ -646,12 +785,13 @@ dentry_calculate_extraction_name(struct wim_dentry *dentry,
 		}
 	}
 
-	if (file_name_valid(dentry->file_name, dentry->file_name_nbytes / 2, false)) {
-		ret = utf16le_get_tstr(dentry->file_name,
-				       dentry->file_name_nbytes,
+	if (file_name_valid(dentry->d_name, dentry->d_name_nbytes / 2, false)) {
+		size_t nbytes = 0;
+		ret = utf16le_get_tstr(dentry->d_name,
+				       dentry->d_name_nbytes,
 				       (const tchar **)&dentry->d_extraction_name,
-				       &dentry->d_extraction_name_nchars);
-		dentry->d_extraction_name_nchars /= sizeof(tchar);
+				       &nbytes);
+		dentry->d_extraction_name_nchars = nbytes / sizeof(tchar);
 		return ret;
 	} else {
 		if (ctx->extract_flags & WIMLIB_EXTRACT_FLAG_REPLACE_INVALID_FILENAMES)
@@ -671,16 +811,16 @@ dentry_calculate_extraction_name(struct wim_dentry *dentry,
 
 out_replace:
 	{
-		utf16lechar utf16_name_copy[dentry->file_name_nbytes / 2];
+		utf16lechar utf16_name_copy[dentry->d_name_nbytes / 2];
 
-		memcpy(utf16_name_copy, dentry->file_name, dentry->file_name_nbytes);
-		file_name_valid(utf16_name_copy, dentry->file_name_nbytes / 2, true);
+		memcpy(utf16_name_copy, dentry->d_name, dentry->d_name_nbytes);
+		file_name_valid(utf16_name_copy, dentry->d_name_nbytes / 2, true);
 
 		const tchar *tchar_name;
 		size_t tchar_nchars;
 
 		ret = utf16le_get_tstr(utf16_name_copy,
-				       dentry->file_name_nbytes,
+				       dentry->d_name_nbytes,
 				       &tchar_name, &tchar_nchars);
 		if (ret)
 			return ret;
@@ -697,8 +837,7 @@ out_replace:
 
 		utf16le_put_tstr(tchar_name);
 
-		dentry->d_extraction_name = memdup(fixed_name,
-						   2 * fixed_name_num_chars + 2);
+		dentry->d_extraction_name = TSTRDUP(fixed_name);
 		if (!dentry->d_extraction_name)
 			return WIMLIB_ERR_NOMEM;
 		dentry->d_extraction_name_nchars = fixed_name_num_chars;
@@ -755,34 +894,35 @@ dentry_list_calculate_extraction_names(struct list_head *dentry_list,
 
 static int
 dentry_resolve_streams(struct wim_dentry *dentry, int extract_flags,
-		       struct wim_lookup_table *lookup_table)
+		       struct blob_table *blob_table)
 {
 	struct wim_inode *inode = dentry->d_inode;
-	struct wim_lookup_table_entry *lte;
+	struct blob_descriptor *blob;
 	int ret;
 	bool force = false;
 
-	/* Special case:  when extracting from a pipe, the WIM lookup table is
+	/* Special case:  when extracting from a pipe, the WIM blob table is
 	 * initially empty, so "resolving" an inode's streams is initially not
-	 * possible.  However, we still need to keep track of which streams,
-	 * identified by SHA1 message digests, need to be extracted, so we
-	 * "resolve" the inode's streams anyway by allocating new entries.  */
+	 * possible.  However, we still need to keep track of which blobs,
+	 * identified by SHA-1 message digests, need to be extracted, so we
+	 * "resolve" the inode's streams anyway by allocating a 'struct
+	 * blob_descriptor' for each one.  */
 	if (extract_flags & WIMLIB_EXTRACT_FLAG_FROM_PIPE)
 		force = true;
-	ret = inode_resolve_streams(inode, lookup_table, force);
+	ret = inode_resolve_streams(inode, blob_table, force);
 	if (ret)
 		return ret;
-	for (u32 i = 0; i <= inode->i_num_ads; i++) {
-		lte = inode_stream_lte_resolved(inode, i);
-		if (lte)
-			lte->out_refcnt = 0;
+	for (unsigned i = 0; i < inode->i_num_streams; i++) {
+		blob = stream_blob_resolved(&inode->i_streams[i]);
+		if (blob)
+			blob->out_refcnt = 0;
 	}
 	return 0;
 }
 
 /*
  * For each dentry to be extracted, resolve all streams in the corresponding
- * inode and set 'out_refcnt' in each to 0.
+ * inode and set 'out_refcnt' in all referenced blob_descriptors to 0.
  *
  * Possible error codes: WIMLIB_ERR_RESOURCE_NOT_FOUND, WIMLIB_ERR_NOMEM.
  */
@@ -796,7 +936,7 @@ dentry_list_resolve_streams(struct list_head *dentry_list,
 	list_for_each_entry(dentry, dentry_list, d_extraction_list_node) {
 		ret = dentry_resolve_streams(dentry,
 					     ctx->extract_flags,
-					     ctx->wim->lookup_table);
+					     ctx->wim->blob_table);
 		if (ret)
 			return ret;
 	}
@@ -804,73 +944,130 @@ dentry_list_resolve_streams(struct list_head *dentry_list,
 }
 
 static int
-ref_stream(struct wim_lookup_table_entry *lte, u32 stream_idx,
-	   struct wim_dentry *dentry, struct apply_ctx *ctx)
+ref_stream(struct wim_inode_stream *strm, struct wim_dentry *dentry,
+	   struct apply_ctx *ctx)
 {
 	struct wim_inode *inode = dentry->d_inode;
-	struct stream_owner *stream_owners;
+	struct blob_descriptor *blob = stream_blob_resolved(strm);
+	struct blob_extraction_target *targets;
 
-	if (!lte)
+	if (!blob)
 		return 0;
 
-	/* Tally the size only for each extraction of the stream (not hard
-	 * links).  */
+	/* Tally the size only for each actual extraction of the stream (not
+	 * additional hard links to the inode).  */
 	if (inode->i_visited && ctx->supported_features.hard_links)
 		return 0;
 
-	ctx->progress.extract.total_bytes += lte->size;
-	ctx->progress.extract.num_streams++;
+	ctx->progress.extract.total_bytes += blob->size;
+	ctx->progress.extract.total_streams++;
 
 	if (inode->i_visited)
 		return 0;
 
-	/* Add stream to the dentry_list only one time, even if it's going
-	 * to be extracted to multiple inodes.  */
-	if (lte->out_refcnt == 0) {
-		list_add_tail(&lte->extraction_list, &ctx->stream_list);
-		ctx->num_streams_remaining++;
+	/* Add each blob to 'ctx->blob_list' only one time, regardless of how
+	 * many extraction targets it will have.  */
+	if (blob->out_refcnt == 0) {
+		list_add_tail(&blob->extraction_list, &ctx->blob_list);
+		ctx->num_blobs_remaining++;
 	}
 
-	/* If inode not yet been visited, append it to the stream_owners array.  */
-	if (lte->out_refcnt < ARRAY_LEN(lte->inline_stream_owners)) {
-		stream_owners = lte->inline_stream_owners;
+	/* Set this stream as an extraction target of 'blob'.  */
+
+	if (blob->out_refcnt < ARRAY_LEN(blob->inline_blob_extraction_targets)) {
+		targets = blob->inline_blob_extraction_targets;
 	} else {
-		struct stream_owner *prev_stream_owners;
-		size_t alloc_stream_owners;
+		struct blob_extraction_target *prev_targets;
+		size_t alloc_blob_extraction_targets;
 
-		if (lte->out_refcnt == ARRAY_LEN(lte->inline_stream_owners)) {
-			prev_stream_owners = NULL;
-			alloc_stream_owners = ARRAY_LEN(lte->inline_stream_owners);
+		if (blob->out_refcnt == ARRAY_LEN(blob->inline_blob_extraction_targets)) {
+			prev_targets = NULL;
+			alloc_blob_extraction_targets = ARRAY_LEN(blob->inline_blob_extraction_targets);
 		} else {
-			prev_stream_owners = lte->stream_owners;
-			alloc_stream_owners = lte->alloc_stream_owners;
+			prev_targets = blob->blob_extraction_targets;
+			alloc_blob_extraction_targets = blob->alloc_blob_extraction_targets;
 		}
 
-		if (lte->out_refcnt == alloc_stream_owners) {
-			alloc_stream_owners *= 2;
-			stream_owners = REALLOC(prev_stream_owners,
-					       alloc_stream_owners *
-						sizeof(stream_owners[0]));
-			if (!stream_owners)
+		if (blob->out_refcnt == alloc_blob_extraction_targets) {
+			alloc_blob_extraction_targets *= 2;
+			targets = REALLOC(prev_targets,
+					  alloc_blob_extraction_targets *
+					  sizeof(targets[0]));
+			if (!targets)
 				return WIMLIB_ERR_NOMEM;
-			if (!prev_stream_owners) {
-				memcpy(stream_owners,
-				       lte->inline_stream_owners,
-				       sizeof(lte->inline_stream_owners));
+			if (!prev_targets) {
+				memcpy(targets,
+				       blob->inline_blob_extraction_targets,
+				       sizeof(blob->inline_blob_extraction_targets));
 			}
-			lte->stream_owners = stream_owners;
-			lte->alloc_stream_owners = alloc_stream_owners;
+			blob->blob_extraction_targets = targets;
+			blob->alloc_blob_extraction_targets = alloc_blob_extraction_targets;
 		}
-		stream_owners = lte->stream_owners;
+		targets = blob->blob_extraction_targets;
 	}
-	stream_owners[lte->out_refcnt].inode = inode;
-	if (stream_idx == 0) {
-		stream_owners[lte->out_refcnt].stream_name = NULL;
-	} else {
-		stream_owners[lte->out_refcnt].stream_name =
-			inode->i_ads_entries[stream_idx - 1].stream_name;
+	targets[blob->out_refcnt].inode = inode;
+	targets[blob->out_refcnt].stream = strm;
+	blob->out_refcnt++;
+	return 0;
+}
+
+static int
+ref_stream_if_needed(struct wim_dentry *dentry, struct wim_inode *inode,
+		     struct wim_inode_stream *strm, struct apply_ctx *ctx)
+{
+	bool need_stream = false;
+	switch (strm->stream_type) {
+	case STREAM_TYPE_DATA:
+		if (stream_is_named(strm)) {
+			/* Named data stream  */
+			if (ctx->supported_features.named_data_streams)
+				need_stream = true;
+		} else if (!(inode->i_attributes & (FILE_ATTRIBUTE_DIRECTORY |
+						    FILE_ATTRIBUTE_ENCRYPTED))
+			   && !(inode_is_symlink(inode)
+				&& !ctx->supported_features.reparse_points
+				&& ctx->supported_features.symlink_reparse_points))
+		{
+			/*
+			 * Unnamed data stream.  Skip if any of the following is true:
+			 *
+			 * - file is a directory
+			 * - file is encrypted
+			 * - backend needs to create the file as UNIX symlink
+			 * - backend will extract the stream as externally
+			 *   backed from the WIM archive itself
+			 */
+			if (ctx->apply_ops->will_back_from_wim) {
+				int ret = (*ctx->apply_ops->will_back_from_wim)(dentry, ctx);
+				if (ret > 0) /* Error?  */
+					return ret;
+				if (ret < 0) /* Won't externally back?  */
+					need_stream = true;
+			} else {
+				need_stream = true;
+			}
+		}
+		break;
+	case STREAM_TYPE_REPARSE_POINT:
+		wimlib_assert(inode->i_attributes & FILE_ATTRIBUTE_REPARSE_POINT);
+		if (ctx->supported_features.reparse_points ||
+		    (inode_is_symlink(inode) &&
+		     ctx->supported_features.symlink_reparse_points))
+			need_stream = true;
+		break;
+	case STREAM_TYPE_EFSRPC_RAW_DATA:
+		wimlib_assert(inode->i_attributes & FILE_ATTRIBUTE_ENCRYPTED);
+		if (inode->i_attributes & FILE_ATTRIBUTE_DIRECTORY) {
+			if (ctx->supported_features.encrypted_directories)
+				need_stream = true;
+		} else {
+			if (ctx->supported_features.encrypted_files)
+				need_stream = true;
+		}
+		break;
 	}
-	lte->out_refcnt++;
+	if (need_stream)
+		return ref_stream(strm, dentry, ctx);
 	return 0;
 }
 
@@ -878,51 +1075,25 @@ static int
 dentry_ref_streams(struct wim_dentry *dentry, struct apply_ctx *ctx)
 {
 	struct wim_inode *inode = dentry->d_inode;
-	int ret;
-
-	/* The unnamed data stream will always be extracted, except in an
-	 * unlikely case.  */
-	if (!inode_is_encrypted_directory(inode)) {
-		u16 stream_idx;
-		struct wim_lookup_table_entry *stream;
-
-		stream = inode_unnamed_stream_resolved(inode, &stream_idx);
-		ret = ref_stream(stream, stream_idx, dentry, ctx);
+	for (unsigned i = 0; i < inode->i_num_streams; i++) {
+		int ret = ref_stream_if_needed(dentry, inode,
+					       &inode->i_streams[i], ctx);
 		if (ret)
 			return ret;
 	}
-
-	/* Named data streams will be extracted only if supported in the current
-	 * extraction mode and volume, and to avoid complications, if not doing
-	 * a linked extraction.  */
-	if (ctx->supported_features.named_data_streams) {
-		for (u16 i = 0; i < inode->i_num_ads; i++) {
-			if (!ads_entry_is_named_stream(&inode->i_ads_entries[i]))
-				continue;
-			ret = ref_stream(inode->i_ads_entries[i].lte, i + 1,
-					 dentry, ctx);
-			if (ret)
-				return ret;
-		}
-	}
 	inode->i_visited = 1;
 	return 0;
 }
 
 /*
- * For each dentry to be extracted, iterate through the data streams of the
- * corresponding inode.  For each such stream that is not to be ignored due to
- * the supported features or extraction flags, add it to the list of streams to
- * be extracted (ctx->stream_list) if not already done so.
+ * Given a list of dentries to be extracted, build the list of blobs that need
+ * to be extracted, and for each blob determine the streams to which that blob
+ * will be extracted.
  *
- * Also builds a mapping from each stream to the inodes referencing it.
- *
- * This also initializes the extract progress info with byte and stream
+ * This also initializes the extract progress info with byte and blob
  * information.
  *
  * ctx->supported_features must be filled in.
- *
- * Possible error codes: WIMLIB_ERR_NOMEM.
  */
 static int
 dentry_list_ref_streams(struct list_head *dentry_list, struct apply_ctx *ctx)
@@ -944,30 +1115,28 @@ static void
 dentry_list_build_inode_alias_lists(struct list_head *dentry_list)
 {
 	struct wim_dentry *dentry;
-	struct wim_inode *inode;
+
+	list_for_each_entry(dentry, dentry_list, d_extraction_list_node)
+		dentry->d_inode->i_first_extraction_alias = NULL;
 
 	list_for_each_entry(dentry, dentry_list, d_extraction_list_node) {
-		inode = dentry->d_inode;
-		if (!inode->i_visited)
-			INIT_LIST_HEAD(&inode->i_extraction_aliases);
-		list_add_tail(&dentry->d_extraction_alias_node,
-			      &inode->i_extraction_aliases);
-		inode->i_visited = 1;
+		dentry->d_next_extraction_alias = dentry->d_inode->i_first_extraction_alias;
+		dentry->d_inode->i_first_extraction_alias = dentry;
 	}
-	list_for_each_entry(dentry, dentry_list, d_extraction_list_node)
-		dentry->d_inode->i_visited = 0;
 }
 
 static void
 inode_tally_features(const struct wim_inode *inode,
 		     struct wim_features *features)
 {
-	if (inode->i_attributes & FILE_ATTRIBUTE_ARCHIVE)
-		features->archive_files++;
+	if (inode->i_attributes & FILE_ATTRIBUTE_READONLY)
+		features->readonly_files++;
 	if (inode->i_attributes & FILE_ATTRIBUTE_HIDDEN)
 		features->hidden_files++;
 	if (inode->i_attributes & FILE_ATTRIBUTE_SYSTEM)
 		features->system_files++;
+	if (inode->i_attributes & FILE_ATTRIBUTE_ARCHIVE)
+		features->archive_files++;
 	if (inode->i_attributes & FILE_ATTRIBUTE_COMPRESSED)
 		features->compressed_files++;
 	if (inode->i_attributes & FILE_ATTRIBUTE_ENCRYPTED) {
@@ -980,7 +1149,7 @@ inode_tally_features(const struct wim_inode *inode,
 		features->not_context_indexed_files++;
 	if (inode->i_attributes & FILE_ATTRIBUTE_SPARSE_FILE)
 		features->sparse_files++;
-	if (inode_has_named_stream(inode))
+	if (inode_has_named_data_stream(inode))
 		features->named_data_streams++;
 	if (inode->i_attributes & FILE_ATTRIBUTE_REPARSE_POINT) {
 		features->reparse_points++;
@@ -989,7 +1158,7 @@ inode_tally_features(const struct wim_inode *inode,
 		else
 			features->other_reparse_points++;
 	}
-	if (inode->i_security_id != -1)
+	if (inode_has_security_descriptor(inode))
 		features->security_descriptors++;
 	if (inode_has_unix_data(inode))
 		features->unix_data++;
@@ -1031,11 +1200,25 @@ do_feature_check(const struct wim_features *required_features,
 		 const struct wim_features *supported_features,
 		 int extract_flags)
 {
+	/* Encrypted files.  */
+	if (required_features->encrypted_files &&
+	    !supported_features->encrypted_files)
+		WARNING("Ignoring EFS-encrypted data of %lu files",
+			required_features->encrypted_files);
+
+	/* Named data streams.  */
+	if (required_features->named_data_streams &&
+	    !supported_features->named_data_streams)
+		WARNING("Ignoring named data streams of %lu files",
+			required_features->named_data_streams);
+
 	/* File attributes.  */
 	if (!(extract_flags & WIMLIB_EXTRACT_FLAG_NO_ATTRIBUTES)) {
-		/* Note: Don't bother the user about FILE_ATTRIBUTE_ARCHIVE.
-		 * We're an archive program, so theoretically we can do what we
-		 * want with it.  */
+
+		if (required_features->readonly_files &&
+		    !supported_features->readonly_files)
+			WARNING("Ignoring FILE_ATTRIBUTE_READONLY of %lu files",
+				required_features->readonly_files);
 
 		if (required_features->hidden_files &&
 		    !supported_features->hidden_files)
@@ -1047,6 +1230,10 @@ do_feature_check(const struct wim_features *required_features,
 			WARNING("Ignoring FILE_ATTRIBUTE_SYSTEM of %lu files",
 				required_features->system_files);
 
+		/* Note: Don't bother the user about FILE_ATTRIBUTE_ARCHIVE.
+		 * We're an archive program, so theoretically we can do what we
+		 * want with it.  */
+
 		if (required_features->compressed_files &&
 		    !supported_features->compressed_files)
 			WARNING("Ignoring FILE_ATTRIBUTE_COMPRESSED of %lu files",
@@ -1068,18 +1255,6 @@ do_feature_check(const struct wim_features *required_features,
 				required_features->encrypted_directories);
 	}
 
-	/* Encrypted files.  */
-	if (required_features->encrypted_files &&
-	    !supported_features->encrypted_files)
-		WARNING("Ignoring %lu encrypted files",
-			required_features->encrypted_files);
-
-	/* Named data streams.  */
-	if (required_features->named_data_streams &&
-	    (!supported_features->named_data_streams))
-		WARNING("Ignoring named data streams of %lu files",
-			required_features->named_data_streams);
-
 	/* Hard links.  */
 	if (required_features->hard_links && !supported_features->hard_links)
 		WARNING("Extracting %lu hard links as independent files",
@@ -1099,12 +1274,11 @@ do_feature_check(const struct wim_features *required_features,
 	{
 		if (supported_features->symlink_reparse_points) {
 			if (required_features->other_reparse_points) {
-				WARNING("Ignoring %lu non-symlink/junction "
-					"reparse point files",
+				WARNING("Ignoring reparse data of %lu non-symlink/junction files",
 					required_features->other_reparse_points);
 			}
 		} else {
-			WARNING("Ignoring %lu reparse point files",
+			WARNING("Ignoring reparse data of %lu files",
 				required_features->reparse_points);
 		}
 	}
@@ -1188,7 +1362,7 @@ extract_trees(WIMStruct *wim, struct wim_dentry **trees, size_t num_trees,
 
 	if (extract_flags & WIMLIB_EXTRACT_FLAG_TO_STDOUT) {
 		ret = extract_dentries_to_stdout(trees, num_trees,
-						 wim->lookup_table);
+						 wim->blob_table);
 		goto out;
 	}
 
@@ -1226,7 +1400,9 @@ extract_trees(WIMStruct *wim, struct wim_dentry **trees, size_t num_trees,
 									 wim->current_image);
 		ctx->progress.extract.target = target;
 	}
-	INIT_LIST_HEAD(&ctx->stream_list);
+	INIT_LIST_HEAD(&ctx->blob_list);
+	filedes_invalidate(&ctx->tmpfile_fd);
+	ctx->apply_ops = ops;
 
 	ret = (*ops->get_supported_features)(target, &ctx->supported_features);
 	if (ret)
@@ -1247,21 +1423,26 @@ extract_trees(WIMStruct *wim, struct wim_dentry **trees, size_t num_trees,
 	if (ret)
 		goto out_cleanup;
 
+	if (unlikely(list_empty(&dentry_list))) {
+		WARNING("There is nothing to extract!");
+		goto out_cleanup;
+	}
+
 	ret = dentry_list_resolve_streams(&dentry_list, ctx);
 	if (ret)
 		goto out_cleanup;
 
+	dentry_list_build_inode_alias_lists(&dentry_list);
+
 	ret = dentry_list_ref_streams(&dentry_list, ctx);
 	if (ret)
 		goto out_cleanup;
 
-	dentry_list_build_inode_alias_lists(&dentry_list);
-
 	if (extract_flags & WIMLIB_EXTRACT_FLAG_FROM_PIPE) {
 		/* When extracting from a pipe, the number of bytes of data to
 		 * extract can't be determined in the normal way (examining the
-		 * lookup table), since at this point all we have is a set of
-		 * SHA1 message digests of streams that need to be extracted.
+		 * blob table), since at this point all we have is a set of
+		 * SHA-1 message digests of blobs that need to be extracted.
 		 * However, we can get a reasonably accurate estimate by taking
 		 * <TOTALBYTES> from the corresponding <IMAGE> in the WIM XML
 		 * data.  This does assume that a full image is being extracted,
@@ -1269,12 +1450,12 @@ extract_trees(WIMStruct *wim, struct wim_dentry **trees, size_t num_trees,
 		 * subtract <HARDLINKBYTES> from this if hard links are
 		 * supported by the extraction mode.)  */
 		ctx->progress.extract.total_bytes =
-			wim_info_get_image_total_bytes(wim->wim_info,
-						       wim->current_image);
+			xml_get_image_total_bytes(wim->xml_info,
+						  wim->current_image);
 		if (ctx->supported_features.hard_links) {
 			ctx->progress.extract.total_bytes -=
-				wim_info_get_image_hard_link_bytes(wim->wim_info,
-								   wim->current_image);
+				xml_get_image_hard_link_bytes(wim->xml_info,
+							      wim->current_image);
 		}
 	}
 
@@ -1304,7 +1485,7 @@ extract_trees(WIMStruct *wim, struct wim_dentry **trees, size_t num_trees,
 				       WIMLIB_PROGRESS_MSG_EXTRACT_IMAGE_END :
 				       WIMLIB_PROGRESS_MSG_EXTRACT_TREE_END));
 out_cleanup:
-	destroy_stream_list(&ctx->stream_list);
+	destroy_blob_list(&ctx->blob_list);
 	destroy_dentry_list(&dentry_list);
 	FREE(ctx);
 out:
@@ -1314,23 +1495,20 @@ out:
 static int
 mkdir_if_needed(const tchar *target)
 {
-	struct stat stbuf;
-	if (tstat(target, &stbuf)) {
-		if (errno == ENOENT) {
-			if (tmkdir(target, 0755)) {
-				ERROR_WITH_ERRNO("Failed to create directory "
-						 "\"%"TS"\"", target);
-				return WIMLIB_ERR_MKDIR;
-			}
-		} else {
-			ERROR_WITH_ERRNO("Failed to stat \"%"TS"\"", target);
-			return WIMLIB_ERR_STAT;
-		}
-	} else if (!S_ISDIR(stbuf.st_mode)) {
-		ERROR("\"%"TS"\" is not a directory", target);
-		return WIMLIB_ERR_NOTDIR;
-	}
-	return 0;
+	if (!tmkdir(target, 0755))
+		return 0;
+
+	if (errno == EEXIST)
+		return 0;
+
+#ifdef __WIN32__
+	/* _wmkdir() fails with EACCES if called on a drive root directory.  */
+	if (errno == EACCES)
+		return 0;
+#endif
+
+	ERROR_WITH_ERRNO("Failed to create directory \"%"TS"\"", target);
+	return WIMLIB_ERR_MKDIR;
 }
 
 /* Make sure the extraction flags make sense, and update them if needed.  */
@@ -1361,12 +1539,44 @@ check_extract_flags(const WIMStruct *wim, int *extract_flags_p)
 	}
 #endif
 
-#ifndef __WIN32__
 	if (extract_flags & WIMLIB_EXTRACT_FLAG_WIMBOOT) {
+#ifdef __WIN32__
+		if (!wim->filename)
+			return WIMLIB_ERR_NO_FILENAME;
+#else
 		ERROR("WIMBoot extraction is only supported on Windows!");
 		return WIMLIB_ERR_UNSUPPORTED;
-	}
 #endif
+	}
+
+	if (extract_flags & (WIMLIB_EXTRACT_FLAG_COMPACT_XPRESS4K |
+			     WIMLIB_EXTRACT_FLAG_COMPACT_XPRESS8K |
+			     WIMLIB_EXTRACT_FLAG_COMPACT_XPRESS16K |
+			     WIMLIB_EXTRACT_FLAG_COMPACT_LZX))
+	{
+	#ifdef __WIN32__
+		int count = 0;
+		count += ((extract_flags & WIMLIB_EXTRACT_FLAG_COMPACT_XPRESS4K) != 0);
+		count += ((extract_flags & WIMLIB_EXTRACT_FLAG_COMPACT_XPRESS8K) != 0);
+		count += ((extract_flags & WIMLIB_EXTRACT_FLAG_COMPACT_XPRESS16K) != 0);
+		count += ((extract_flags & WIMLIB_EXTRACT_FLAG_COMPACT_LZX) != 0);
+		if (count != 1) {
+			ERROR("Only one compression format can be specified "
+			      "for compact-mode extraction!");
+			return WIMLIB_ERR_INVALID_PARAM;
+		}
+		if (extract_flags & WIMLIB_EXTRACT_FLAG_WIMBOOT) {
+			ERROR("Compact-mode extraction and WIMBoot-mode "
+			      "extraction are mutually exclusive!");
+			return WIMLIB_ERR_INVALID_PARAM;
+		}
+	#else
+		ERROR("Compact-mode extraction (System Compression) "
+		      "is only supported on Windows!");
+		return WIMLIB_ERR_UNSUPPORTED;
+	#endif
+	}
+
 
 	if ((extract_flags & (WIMLIB_EXTRACT_FLAG_RPFIX |
 			      WIMLIB_EXTRACT_FLAG_NORPFIX |
@@ -1383,22 +1593,6 @@ check_extract_flags(const WIMStruct *wim, int *extract_flags_p)
 	return 0;
 }
 
-static u32
-get_wildcard_flags(int extract_flags)
-{
-	u32 wildcard_flags = 0;
-
-	if (extract_flags & WIMLIB_EXTRACT_FLAG_STRICT_GLOB)
-		wildcard_flags |= WILDCARD_FLAG_ERROR_IF_NO_MATCH;
-	else
-		wildcard_flags |= WILDCARD_FLAG_WARN_IF_NO_MATCH;
-
-	if (default_ignore_case)
-		wildcard_flags |= WILDCARD_FLAG_CASE_INSENSITIVE;
-
-	return wildcard_flags;
-}
-
 struct append_dentry_ctx {
 	struct wim_dentry **dentries;
 	size_t num_dentries;
@@ -1427,6 +1621,31 @@ append_dentry_cb(struct wim_dentry *dentry, void *_ctx)
 	return 0;
 }
 
+/* Append dentries matched by a path which can contain wildcard characters.  */
+static int
+append_matched_dentries(WIMStruct *wim, const tchar *orig_pattern,
+			int extract_flags, struct append_dentry_ctx *ctx)
+{
+	const size_t count_before = ctx->num_dentries;
+	tchar *pattern;
+	int ret;
+
+	pattern = canonicalize_wim_path(orig_pattern);
+	if (!pattern)
+		return WIMLIB_ERR_NOMEM;
+	ret = expand_path_pattern(wim_get_current_root_dentry(wim), pattern,
+				  append_dentry_cb, ctx);
+	FREE(pattern);
+	if (ret || ctx->num_dentries > count_before)
+		return ret;
+	if (extract_flags & WIMLIB_EXTRACT_FLAG_STRICT_GLOB) {
+		ERROR("No matches for path pattern \"%"TS"\"", orig_pattern);
+		return WIMLIB_ERR_PATH_DOES_NOT_EXIST;
+	}
+	WARNING("No matches for path pattern \"%"TS"\"", orig_pattern);
+	return 0;
+}
+
 static int
 do_wimlib_extract_paths(WIMStruct *wim, int image, const tchar *target,
 			const tchar * const *paths, size_t num_paths,
@@ -1448,7 +1667,7 @@ do_wimlib_extract_paths(WIMStruct *wim, int image, const tchar *target,
 	if (ret)
 		return ret;
 
-	ret = wim_checksum_unhashed_streams(wim);
+	ret = wim_checksum_unhashed_blobs(wim);
 	if (ret)
 		return ret;
 
@@ -1469,20 +1688,10 @@ do_wimlib_extract_paths(WIMStruct *wim, int image, const tchar *target,
 			.num_alloc_dentries = 0,
 		};
 
-		u32 wildcard_flags = get_wildcard_flags(extract_flags);
-
 		for (size_t i = 0; i < num_paths; i++) {
-			tchar *path = canonicalize_wim_path(paths[i]);
-			if (path == NULL) {
-				ret = WIMLIB_ERR_NOMEM;
-				trees = append_dentry_ctx.dentries;
-				goto out_free_trees;
-			}
-			ret = expand_wildcard(wim, path,
-					      append_dentry_cb,
-					      &append_dentry_ctx,
-					      wildcard_flags);
-			FREE(path);
+			ret = append_matched_dentries(wim, paths[i],
+						      extract_flags,
+						      &append_dentry_ctx);
 			if (ret) {
 				trees = append_dentry_ctx.dentries;
 				goto out_free_trees;
@@ -1554,7 +1763,8 @@ image_name_ok_as_dir(const tchar *image_name)
 	return image_name && *image_name &&
 		!tstrpbrk(image_name, filename_forbidden_chars) &&
 		tstrcmp(image_name, T(".")) &&
-		tstrcmp(image_name, T(".."));
+		tstrcmp(image_name, T("..")) &&
+		tstrlen(image_name) <= 128;
 }
 
 /* Extracts all images from the WIM to the directory @target, with the images
@@ -1562,9 +1772,8 @@ image_name_ok_as_dir(const tchar *image_name)
 static int
 extract_all_images(WIMStruct *wim, const tchar *target, int extract_flags)
 {
-	size_t image_name_max_len = max(xml_get_max_image_name_len(wim), 20);
 	size_t output_path_len = tstrlen(target);
-	tchar buf[output_path_len + 1 + image_name_max_len + 1];
+	tchar buf[output_path_len + 1 + 128 + 1];
 	int ret;
 	int image;
 	const tchar *image_name;
@@ -1670,8 +1879,8 @@ wimlib_extract_image_from_pipe_with_progress(int pipe_fd,
 
 	/* Read the WIM header from the pipe and get a WIMStruct to represent
 	 * the pipable WIM.  Caveats:  Unlike getting a WIMStruct with
-	 * wimlib_open_wim(), getting a WIMStruct in this way will result in
-	 * an empty lookup table, no XML data read, and no filename set.  */
+	 * wimlib_open_wim(), getting a WIMStruct in this way will result in an
+	 * empty blob table, no XML data read, and no filename set.  */
 	ret = open_wim_as_WIMStruct(&pipe_fd, WIMLIB_OPEN_FLAG_FROM_PIPE, &pwm,
 				    progfunc, progctx);
 	if (ret)
@@ -1703,27 +1912,24 @@ wimlib_extract_image_from_pipe_with_progress(int pipe_fd,
 	 * write_pipable_wim() for more details about the format of pipable
 	 * WIMs.)  */
 	{
-		struct wim_lookup_table_entry xml_lte;
-		struct wim_resource_spec xml_rspec;
-		ret = read_pwm_stream_header(pwm, &xml_lte, &xml_rspec, 0, NULL);
+		u8 hash[SHA1_HASH_SIZE];
+
+		ret = read_pwm_blob_header(pwm, hash,
+					   &pwm->hdr.xml_data_reshdr, NULL);
 		if (ret)
 			goto out_wimlib_free;
 
-		if (!(xml_lte.flags & WIM_RESHDR_FLAG_METADATA))
-		{
-			ERROR("Expected XML data, but found non-metadata "
-			      "stream.");
+		if (!(pwm->hdr.xml_data_reshdr.flags & WIM_RESHDR_FLAG_METADATA)) {
+			ERROR("Expected XML data, but found non-metadata resource.");
 			ret = WIMLIB_ERR_INVALID_PIPABLE_WIM;
 			goto out_wimlib_free;
 		}
 
-		wim_res_spec_to_hdr(&xml_rspec, &pwm->hdr.xml_data_reshdr);
-
 		ret = read_wim_xml_data(pwm);
 		if (ret)
 			goto out_wimlib_free;
 
-		if (wim_info_get_num_images(pwm->wim_info) != pwm->hdr.image_count) {
+		if (xml_get_image_count(pwm->xml_info) != pwm->hdr.image_count) {
 			ERROR("Image count in XML data is not the same as in WIM header.");
 			ret = WIMLIB_ERR_IMAGE_COUNT;
 			goto out_wimlib_free;
@@ -1756,51 +1962,9 @@ wimlib_extract_image_from_pipe_with_progress(int pipe_fd,
 
 	/* Load the needed metadata resource.  */
 	for (i = 1; i <= pwm->hdr.image_count; i++) {
-		struct wim_lookup_table_entry *metadata_lte;
-		struct wim_image_metadata *imd;
-		struct wim_resource_spec *metadata_rspec;
-
-		metadata_lte = new_lookup_table_entry();
-		if (metadata_lte == NULL) {
-			ret = WIMLIB_ERR_NOMEM;
-			goto out_wimlib_free;
-		}
-		metadata_rspec = MALLOC(sizeof(struct wim_resource_spec));
-		if (metadata_rspec == NULL) {
-			ret = WIMLIB_ERR_NOMEM;
-			free_lookup_table_entry(metadata_lte);
-			goto out_wimlib_free;
-		}
-
-		ret = read_pwm_stream_header(pwm, metadata_lte, metadata_rspec, 0, NULL);
-		imd = pwm->image_metadata[i - 1];
-		imd->metadata_lte = metadata_lte;
-		if (ret) {
-			FREE(metadata_rspec);
-			goto out_wimlib_free;
-		}
-
-		if (!(metadata_lte->flags & WIM_RESHDR_FLAG_METADATA)) {
-			ERROR("Expected metadata resource, but found "
-			      "non-metadata stream.");
-			ret = WIMLIB_ERR_INVALID_PIPABLE_WIM;
+		ret = handle_pwm_metadata_resource(pwm, i, i == image);
+		if (ret)
 			goto out_wimlib_free;
-		}
-
-		if (i == image) {
-			/* Metadata resource is for the image being extracted.
-			 * Parse it and save the metadata in memory.  */
-			ret = read_metadata_resource(pwm, imd);
-			if (ret)
-				goto out_wimlib_free;
-			imd->modified = 1;
-		} else {
-			/* Metadata resource is not for the image being
-			 * extracted.  Skip over it.  */
-			ret = skip_wim_stream(metadata_lte);
-			if (ret)
-				goto out_wimlib_free;
-		}
 	}
 	/* Extract the image.  */
 	extract_flags |= WIMLIB_EXTRACT_FLAG_FROM_PIPE;