Add randomized testing program
authorEric Biggers <ebiggers3@gmail.com>
Sun, 17 Apr 2016 14:44:54 +0000 (09:44 -0500)
committerEric Biggers <ebiggers3@gmail.com>
Sun, 17 Apr 2016 15:16:41 +0000 (10:16 -0500)
Introduce the ability to configure the library with test-specific code,
add in-library code for directory tree generation and comparison, and add
the "wlfuzz" randomized test runner which uses the new functionality.

Makefile.am
configure.ac
include/wimlib/dentry.h
include/wimlib/inode.h
include/wimlib/scan.h
include/wimlib/test_support.h [new file with mode: 0644]
src/error.c
src/test_support.c [new file with mode: 0644]
src/update_image.c
tests/wlfuzz.c [new file with mode: 0644]

index a74a444..3615d49 100644 (file)
@@ -181,6 +181,16 @@ libwim_la_SOURCES += src/unix_apply.c              \
 PLATFORM_LIBS =
 endif
 
+if ENABLE_TEST_SUPPORT
+libwim_la_SOURCES += src/test_support.c                \
+                    include/wimlib/test_support.h
+if WINDOWS_NATIVE_BUILD
+# Workaround for "multiple definition" error when math symbols are present in
+# both libmsvcrt.a and ntdll.a
+AM_LDFLAGS += -Wl,--allow-multiple-definition
+endif
+endif
+
 libwim_la_CFLAGS =             \
        $(AM_CFLAGS)            \
        $(PTHREAD_CFLAGS)       \
@@ -339,4 +349,9 @@ else
 TESTS = $(dist_check_SCRIPTS)
 endif
 
+# Extra test programs (not run by 'make check')
+EXTRA_PROGRAMS = tests/wlfuzz
+tests_wlfuzz_SOURCES = tests/wlfuzz.c
+tests_wlfuzz_LDADD = $(top_builddir)/libwim.la
+
 ##############################################################################
index 60f07ce..be629c6 100644 (file)
@@ -268,6 +268,19 @@ AC_ARG_WITH(pkgconfigdir,
             [pkgconfigdir='${libdir}/pkgconfig'])
 AC_SUBST(pkgconfigdir)
 
+AC_MSG_CHECKING([whether to enable supporting code for tests])
+AC_ARG_ENABLE([test-support],
+             [AS_HELP_STRING([--enable-test-support],
+                             [Enable supporting code for tests (developers only)])],
+             [ENABLE_TEST_SUPPORT=$enableval],
+             [ENABLE_TEST_SUPPORT=no])
+AC_MSG_RESULT([$ENABLE_TEST_SUPPORT])
+if test "$ENABLE_TEST_SUPPORT" = "yes" ; then
+       AC_DEFINE([ENABLE_TEST_SUPPORT], [1],
+                 [Define to 1 to enable supporting code for tests])
+fi
+AM_CONDITIONAL([ENABLE_TEST_SUPPORT], [test "$ENABLE_TEST_SUPPORT" = "yes"])
+
 ###############################################################################
 
 AC_SUBST([PKGCONFIG_PRIVATE_REQUIRES], [$PKGCONFIG_PRIVATE_REQUIRES])
index fc745d5..9f4f466 100644 (file)
@@ -119,6 +119,10 @@ struct wim_dentry {
         * that needs to be extracted as part of the current extraction
         * operation, or NULL if this is the last alias.  */
        struct wim_dentry *d_next_extraction_alias;
+
+#ifdef ENABLE_TEST_SUPPORT
+       struct wim_dentry *d_corresponding;
+#endif
 };
 
 static inline bool
index b6203fb..db311c9 100644 (file)
@@ -226,6 +226,10 @@ struct wim_inode {
 
        /* Next stream ID to be assigned  */
        u32 i_next_stream_id;
+
+#ifdef ENABLE_TEST_SUPPORT
+       struct wim_inode *i_corresponding;
+#endif
 };
 
 /* Optional extra data for a WIM inode  */
index 13cd26d..a43baec 100644 (file)
@@ -114,6 +114,12 @@ unix_build_dentry_tree(struct wim_dentry **root_ret,
 #define platform_default_scan_tree unix_build_dentry_tree
 #endif
 
+#ifdef ENABLE_TEST_SUPPORT
+extern int
+generate_dentry_tree(struct wim_dentry **root_ret,
+                    const tchar *root_disk_path, struct scan_params *params);
+#endif
+
 #define WIMLIB_ADD_FLAG_ROOT   0x80000000
 
 static inline int
diff --git a/include/wimlib/test_support.h b/include/wimlib/test_support.h
new file mode 100644 (file)
index 0000000..8f06834
--- /dev/null
@@ -0,0 +1,25 @@
+#ifndef _WIMLIB_TEST_SUPPORT_H
+#define _WIMLIB_TEST_SUPPORT_H
+
+#ifdef ENABLE_TEST_SUPPORT
+
+#include "wimlib/types.h"
+
+#define WIMLIB_ERR_IMAGES_ARE_DIFFERENT                        200
+
+#define WIMLIB_ADD_FLAG_GENERATE_TEST_DATA             0x08000000
+
+#define WIMLIB_CMP_FLAG_SHORT_NAMES_NOT_PRESERVED      0x00000001
+#define WIMLIB_CMP_FLAG_ATTRIBUTES_NOT_PRESERVED       0x00000002
+#define WIMLIB_CMP_FLAG_SECURITY_NOT_PRESERVED         0x00000004
+#define WIMLIB_CMP_FLAG_ADS_NOT_PRESERVED              0x00000008
+#define WIMLIB_CMP_FLAG_COMPRESSION_NOT_PRESERVED      0x00000010
+#define WIMLIB_CMP_FLAG_IMAGE2_SHOULD_HAVE_SYMLINKS    0x00000020
+
+extern int
+wimlib_compare_images(WIMStruct *wim1, int image1,
+                     WIMStruct *wim2, int image2, int cmp_flags);
+
+#endif /* ENABLE_TEST_SUPPORT */
+
+#endif /* _WIMLIB_TEST_SUPPORT_H */
index f427c82..db2a107 100644 (file)
@@ -42,6 +42,7 @@
 
 #include "wimlib.h"
 #include "wimlib/error.h"
+#include "wimlib/test_support.h"
 #include "wimlib/util.h"
 #include "wimlib/win32.h"
 
@@ -346,6 +347,10 @@ static const tchar * const error_strings[] = {
                = T("A file being added to a WIM image was concurrently modified"),
        [WIMLIB_ERR_SNAPSHOT_FAILURE]
                = T("Unable to create a filesystem snapshot"),
+#ifdef ENABLE_TEST_SUPPORT
+       [WIMLIB_ERR_IMAGES_ARE_DIFFERENT]
+               = T("A difference was detected between the two images being compared"),
+#endif
 };
 
 WIMLIBAPI const tchar *
diff --git a/src/test_support.c b/src/test_support.c
new file mode 100644 (file)
index 0000000..0db98e6
--- /dev/null
@@ -0,0 +1,1071 @@
+/*
+ * test_support.c - Supporting code for tests
+ */
+
+/*
+ * Copyright (C) 2015-2016 Eric Biggers
+ *
+ * This file is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU Lesser General Public License as published by the Free
+ * Software Foundation; either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * This file is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this file; if not, see http://www.gnu.org/licenses/.
+ */
+
+/*
+ * This file contains specialized test code which is only compiled when the
+ * library is configured with --enable-test-support.  The major features are:
+ *
+ *     - Random directory tree generation
+ *     - Directory tree comparison
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include "config.h"
+#endif
+
+#ifdef ENABLE_TEST_SUPPORT
+
+#include <ctype.h>
+#include <math.h>
+
+#include "wimlib.h"
+#include "wimlib/endianness.h"
+#include "wimlib/encoding.h"
+#include "wimlib/metadata.h"
+#include "wimlib/dentry.h"
+#include "wimlib/inode.h"
+#include "wimlib/reparse.h"
+#include "wimlib/scan.h"
+#include "wimlib/security_descriptor.h"
+#include "wimlib/test_support.h"
+
+/*----------------------------------------------------------------------------*
+ *                            File tree generation                            *
+ *----------------------------------------------------------------------------*/
+
+struct generation_context {
+       struct scan_params *params;
+       struct wim_dentry *used_short_names[256];
+       bool metadata_only;
+};
+
+static u32
+rand32(void)
+{
+       static u64 state = 0x55DB93D0AB838771;
+
+       /* A simple linear congruential generator  */
+       state = (state * 25214903917 + 11) & ((1ULL << 48) - 1);
+       return state >> 16;
+}
+
+static bool
+randbool(void)
+{
+       return (rand32() & 1) != 0;
+}
+
+static u8
+rand8(void)
+{
+       return (u8)rand32();
+}
+
+static u16
+rand16(void)
+{
+       return (u16)rand32();
+}
+
+static u64
+rand64(void)
+{
+       return ((u64)rand32() << 32) | rand32();
+}
+
+static u64
+generate_random_timestamp(void)
+{
+       /* When setting timestamps on Windows:
+        * - 0 is a special value meaning "not specified"
+        * - if the high bit is set you get STATUS_INVALID_PARAMETER  */
+       return (1 + rand64()) & ~(1ULL << 63);
+}
+
+static const struct {
+       u8 num_subauthorities;
+       u64 identifier_authority;
+       u32 subauthorities[6];
+} common_sids[] = {
+       { 1, 0, {0}}, /* NULL_SID  */
+       { 1, 1, {0}}, /* WORLD_SID */
+       { 1, 2, {0}}, /* LOCAL_SID */
+       { 1, 3, {0}}, /* CREATOR_OWNER_SID */
+       { 1, 3, {1}}, /* CREATOR_GROUP_SID */
+       { 1, 3, {2}}, /* CREATOR_OWNER_SERVER_SID */
+       { 1, 3, {3}}, /* CREATOR_GROUP_SERVER_SID */
+       // { 0, 5, {}},  /* NT_AUTHORITY_SID */
+       { 1, 5, {1}}, /* DIALUP_SID */
+       { 1, 5, {2}}, /* NETWORK_SID */
+       { 1, 5, {3}}, /* BATCH_SID */
+       { 1, 5, {4}}, /* INTERACTIVE_SID */
+       { 1, 5, {6}}, /* SERVICE_SID */
+       { 1, 5, {7}}, /* ANONYMOUS_LOGON_SID */
+       { 1, 5, {8}}, /* PROXY_SID */
+       { 1, 5, {9}}, /* SERVER_LOGON_SID */
+       { 1, 5, {10}}, /* SELF_SID */
+       { 1, 5, {11}}, /* AUTHENTICATED_USER_SID */
+       { 1, 5, {12}}, /* RESTRICTED_CODE_SID */
+       { 1, 5, {13}}, /* TERMINAL_SERVER_SID */
+       { 1, 5, {18}}, /* NT AUTHORITY\SYSTEM */
+       { 1, 5, {19}}, /* NT AUTHORITY\LOCAL SERVICE */
+       { 1, 5, {20}}, /* NT AUTHORITY\NETWORK SERVICE */
+       { 5 ,80, {956008885, 3418522649, 1831038044, 1853292631, 2271478464}}, /* trusted installer  */
+       { 2 ,5, {32, 544} } /* BUILTIN\ADMINISTRATORS  */
+};
+
+/* Generate a SID and return its size in bytes.  */
+static size_t
+generate_random_sid(wimlib_SID *sid, struct generation_context *ctx)
+{
+       u32 r = rand32();
+
+       sid->revision = 1;
+
+       if (r & 1) {
+               /* Common SID  */
+               r = (r >> 1) % ARRAY_LEN(common_sids);
+
+               sid->sub_authority_count = common_sids[r].num_subauthorities;
+               for (int i = 0; i < 6; i++) {
+                       sid->identifier_authority[i] =
+                               common_sids[r].identifier_authority >> (40 - i * 8);
+               }
+               for (int i = 0; i < common_sids[r].num_subauthorities; i++)
+                       sid->sub_authority[i] = cpu_to_le32(common_sids[r].subauthorities[i]);
+       } else {
+               /* Random SID  */
+
+               sid->sub_authority_count = 1 + ((r >> 1) % 15);
+
+               for (int i = 0; i < 6; i++)
+                       sid->identifier_authority[i] = rand8();
+
+               for (int i = 0; i < sid->sub_authority_count; i++)
+                       sid->sub_authority[i] = cpu_to_le32(rand32());
+       }
+       return (u8 *)&sid->sub_authority[sid->sub_authority_count] - (u8 *)sid;
+}
+
+/* Generate an ACL and return its size in bytes.  */
+static size_t
+generate_random_acl(wimlib_ACL *acl, bool dacl, struct generation_context *ctx)
+{
+       u8 *p;
+       u16 ace_count;
+
+       ace_count = rand32() % 16;
+
+       acl->revision = 2;
+       acl->sbz1 = 0;
+       acl->ace_count = cpu_to_le16(ace_count);
+       acl->sbz2 = 0;
+
+       p = (u8 *)(acl + 1);
+
+       for (int i = 0; i < ace_count; i++) {
+               wimlib_ACCESS_ALLOWED_ACE *ace = (wimlib_ACCESS_ALLOWED_ACE *)p;
+
+               /* ACCESS_ALLOWED, ACCESS_DENIED, or SYSTEM_AUDIT; format is the
+                * same for all  */
+               if (dacl)
+                       ace->hdr.type = rand32() % 2;
+               else
+                       ace->hdr.type = 2;
+               ace->hdr.flags = rand8();
+               ace->mask = cpu_to_le32(rand32() & 0x001F01FF);
+
+               p += offsetof(wimlib_ACCESS_ALLOWED_ACE, sid) +
+                       generate_random_sid(&ace->sid, ctx);
+               ace->hdr.size = cpu_to_le16(p - (u8 *)ace);
+       }
+
+       acl->acl_size = cpu_to_le16(p - (u8 *)acl);
+       return p - (u8 *)acl;
+}
+
+/* Generate a security descriptor and return its size in bytes.  */
+static size_t
+generate_random_security_descriptor(void *_desc, struct generation_context *ctx)
+{
+       wimlib_SECURITY_DESCRIPTOR_RELATIVE *desc = _desc;
+       u16 control;
+       u8 *p;
+
+       control = rand16();
+
+       control &= (wimlib_SE_DACL_AUTO_INHERITED |
+                   wimlib_SE_SACL_AUTO_INHERITED);
+
+       control |= wimlib_SE_SELF_RELATIVE |
+                  wimlib_SE_DACL_PRESENT |
+                  wimlib_SE_SACL_PRESENT;
+
+       desc->revision = 1;
+       desc->sbz1 = 0;
+       desc->control = cpu_to_le16(control);
+
+       p = (u8 *)(desc + 1);
+
+       desc->owner_offset = cpu_to_le32(p - (u8 *)desc);
+       p += generate_random_sid((wimlib_SID *)p, ctx);
+
+       desc->group_offset = cpu_to_le32(p - (u8 *)desc);
+       p += generate_random_sid((wimlib_SID *)p, ctx);
+
+       if ((control & wimlib_SE_DACL_PRESENT) && randbool()) {
+               desc->dacl_offset = cpu_to_le32(p - (u8 *)desc);
+               p += generate_random_acl((wimlib_ACL *)p, true, ctx);
+       } else {
+               desc->dacl_offset = cpu_to_le32(0);
+       }
+
+       if ((control & wimlib_SE_SACL_PRESENT) && randbool()) {
+               desc->sacl_offset = cpu_to_le32(p - (u8 *)desc);
+               p += generate_random_acl((wimlib_ACL *)p, false, ctx);
+       } else {
+               desc->sacl_offset = cpu_to_le32(0);
+       }
+
+       return p - (u8 *)desc;
+}
+
+static int
+set_random_metadata(struct wim_inode *inode, struct generation_context *ctx)
+{
+       u32 v = rand32();
+       u32 attrib = (v & (FILE_ATTRIBUTE_READONLY |
+                          FILE_ATTRIBUTE_HIDDEN |
+                          FILE_ATTRIBUTE_SYSTEM |
+                          FILE_ATTRIBUTE_ARCHIVE |
+                          FILE_ATTRIBUTE_NOT_CONTENT_INDEXED |
+                          FILE_ATTRIBUTE_COMPRESSED));
+
+       /* File attributes  */
+       inode->i_attributes |= attrib;
+
+       /* Timestamps  */
+       inode->i_creation_time = generate_random_timestamp();
+       inode->i_last_access_time = generate_random_timestamp();
+       inode->i_last_write_time = generate_random_timestamp();
+
+       /* Security descriptor  */
+       if (randbool()) {
+               char desc[8192] _aligned_attribute(8);
+               size_t size;
+
+               size = generate_random_security_descriptor(desc, ctx);
+
+               wimlib_assert(size <= sizeof(desc));
+
+               inode->i_security_id = sd_set_add_sd(ctx->params->sd_set,
+                                                    desc, size);
+               if (unlikely(inode->i_security_id < 0))
+                       return WIMLIB_ERR_NOMEM;
+       }
+
+       return 0;
+
+}
+
+/* Choose a random size for generated file data.  We want to usually generate
+ * empty, small, or medium files, but occasionally generate large files.  */
+static size_t
+select_stream_size(struct generation_context *ctx)
+{
+       if (ctx->metadata_only)
+               return 0;
+
+       switch (rand32() % 2048) {
+       default:
+               /* Empty  */
+               return 0;
+       case 600 ... 799:
+               /* Microscopic  */
+               return rand32() % 64;
+       case 800 ... 1319:
+               /* Tiny  */
+               return rand32() % 4096;
+       case 1320 ... 1799:
+               /* Small  */
+               return rand32() % 32768;
+       case 1800 ... 2046:
+               /* Medium  */
+               return rand32() % 262144;
+       case 2047:
+               /* Large  */
+               return rand32() % 134217728;
+       }
+}
+
+/* Fill 'buffer' with 'size' bytes of "interesting" file data.  */
+static void
+generate_data(u8 *buffer, size_t size, struct generation_context *ctx)
+{
+       size_t mask = -1;
+       size_t num_byte_fills = rand32() % 256;
+
+       memset(buffer, rand32() % 256, size);
+
+       for (size_t i = 0; i < num_byte_fills; i++) {
+               u8 b = rand8();
+
+               size_t count = ((double)size / (double)num_byte_fills) *
+                               ((double)rand32() / 2e9);
+               size_t offset = rand32() & ~mask;
+
+               while (count--) {
+                       buffer[(offset +
+                               ((rand32()) & mask)) % size] = b;
+               }
+
+
+               if (rand32() % 4 == 0)
+                       mask = (size_t)-1 << rand32() % 4;
+       }
+
+       if (rand32() % 8 == 0) {
+               double magnitude = rand32() % 128;
+               double scale = 1.0 / (1 + (rand32() % 256));
+
+               for (size_t i = 0; i < size; i++)
+                       buffer[i] += (int)(magnitude * cos(i * scale));
+       }
+}
+
+static int
+add_stream(struct wim_inode *inode, struct generation_context *ctx,
+          int stream_type, const utf16lechar *stream_name,
+          void *buffer, size_t size)
+{
+       struct blob_descriptor *blob = NULL;
+       struct wim_inode_stream *strm;
+
+       if (size) {
+               blob = new_blob_descriptor();
+               if (!blob)
+                       goto err_nomem;
+               blob->attached_buffer = buffer;
+               blob->blob_location = BLOB_IN_ATTACHED_BUFFER;
+               blob->size = size;
+       }
+
+       strm = inode_add_stream(inode, stream_type, stream_name, blob);
+       if (unlikely(!strm))
+               goto err_nomem;
+
+       prepare_unhashed_blob(blob, inode, strm->stream_id,
+                             ctx->params->unhashed_blobs);
+       return 0;
+
+err_nomem:
+       free_blob_descriptor(blob);
+       return WIMLIB_ERR_NOMEM;
+}
+
+static int
+set_random_reparse_point(struct wim_inode *inode, struct generation_context *ctx)
+{
+       void *buffer = NULL;
+       size_t rpdatalen = select_stream_size(ctx) % (REPARSE_DATA_MAX_SIZE + 1);
+
+       if (rpdatalen) {
+               buffer = MALLOC(rpdatalen);
+               if (!buffer)
+                       return WIMLIB_ERR_NOMEM;
+               generate_data(buffer, rpdatalen, ctx);
+       }
+
+       inode->i_attributes |= FILE_ATTRIBUTE_REPARSE_POINT;
+       inode->i_rp_reserved = rand16();
+
+       if (rpdatalen >= GUID_SIZE && randbool()) {
+               /* Non-Microsoft reparse tag (16-byte GUID required)  */
+               u8 *guid = buffer;
+               guid[6] = (guid[6] & 0x0F) | 0x40;
+               guid[8] = (guid[8] & 0x3F) | 0x80;
+               inode->i_reparse_tag = 0x00000100;
+       } else {
+               /* Microsoft reparse tag  */
+               inode->i_reparse_tag = 0x80000000;
+       }
+
+       return add_stream(inode, ctx, STREAM_TYPE_REPARSE_POINT, NO_STREAM_NAME,
+                         buffer, rpdatalen);
+}
+
+static int
+add_random_data_stream(struct wim_inode *inode, struct generation_context *ctx,
+                      const utf16lechar *stream_name)
+{
+       void *buffer = NULL;
+       size_t size;
+
+       size = select_stream_size(ctx);
+       if (size) {
+               buffer = MALLOC(size);
+               if (!buffer)
+                       return WIMLIB_ERR_NOMEM;
+               generate_data(buffer, size, ctx);
+       }
+
+       return add_stream(inode, ctx, STREAM_TYPE_DATA, stream_name,
+                         buffer, size);
+}
+
+static int
+set_random_streams(struct wim_inode *inode, struct generation_context *ctx,
+                  bool reparse_ok)
+{
+       int ret;
+       u32 r;
+
+       /* Reparse point (sometimes)  */
+       if (reparse_ok && rand32() % 8 == 0) {
+               ret = set_random_reparse_point(inode, ctx);
+               if (ret)
+                       return ret;
+       }
+
+       /* Unnamed data stream (nondirectories only)  */
+       if (!(inode->i_attributes & FILE_ATTRIBUTE_DIRECTORY)) {
+               ret = add_random_data_stream(inode, ctx, NO_STREAM_NAME);
+               if (ret)
+                       return ret;
+       }
+
+       /* Named data streams (sometimes)  */
+       r = rand32() % 256;
+       if (r > 230) {
+               utf16lechar stream_name[2] = {cpu_to_le16('a'), '\0'};
+               r -= 230;
+               while (r--) {
+                       ret = add_random_data_stream(inode, ctx, stream_name);
+                       if (ret)
+                               return ret;
+                       stream_name[0] += cpu_to_le16(1);
+               }
+       }
+
+       return 0;
+}
+
+static int
+generate_random_file_name(tchar name[], int max_len,
+                         struct generation_context *ctx)
+{
+       int length;
+       switch (rand32() % 8) {
+       default:
+               /* short name  */
+               length = 1 + (rand32() % 6);
+               break;
+       case 2:
+       case 3:
+       case 4:
+               /* medium-length name  */
+               length = 7 + (rand32() % 8);
+               break;
+       case 5:
+       case 6:
+               /* long name  */
+               length = 15 + (rand32() % 15);
+               break;
+       case 7:
+               /* very long name  */
+               length = 30 + (rand32() % 90);
+               break;
+       }
+       length = min(length, max_len);
+       for (int i = 0; i < length; i++)
+               name[i] = 'a' + (rand32() % 26);
+       name[length] = 0;
+       return length;
+}
+
+static u64
+select_inode_number(struct generation_context *ctx)
+{
+       const struct wim_inode_table *table = ctx->params->inode_table;
+       const struct hlist_head *head;
+       const struct wim_inode *inode;
+
+       head = &table->array[rand32() % table->capacity];
+       hlist_for_each_entry(inode, head, i_hlist_node)
+               if (randbool())
+                       return inode->i_ino;
+
+       return rand32();
+}
+
+static u32
+select_num_children(u32 depth, struct generation_context *ctx)
+{
+       const double b = 1.01230;
+       u32 r = rand32() % 500;
+       return ((pow(b, pow(b, r)) - 1) / pow(depth, 1.5)) +
+               (2 - exp(0.04/depth));
+}
+
+static bool
+is_name_forbidden_in_win32_namespace(const utf16lechar *name)
+{
+       static const utf16lechar forbidden_names[][5] = {
+               { cpu_to_le16('C'), cpu_to_le16('O'), cpu_to_le16('N'), },
+               { cpu_to_le16('P'), cpu_to_le16('R'), cpu_to_le16('N'), },
+               { cpu_to_le16('A'), cpu_to_le16('U'), cpu_to_le16('X'), },
+               { cpu_to_le16('N'), cpu_to_le16('U'), cpu_to_le16('L'), },
+               { cpu_to_le16('C'), cpu_to_le16('O'), cpu_to_le16('M'), cpu_to_le16('1'), },
+               { cpu_to_le16('C'), cpu_to_le16('O'), cpu_to_le16('M'), cpu_to_le16('2'), },
+               { cpu_to_le16('C'), cpu_to_le16('O'), cpu_to_le16('M'), cpu_to_le16('3'), },
+               { cpu_to_le16('C'), cpu_to_le16('O'), cpu_to_le16('M'), cpu_to_le16('4'), },
+               { cpu_to_le16('C'), cpu_to_le16('O'), cpu_to_le16('M'), cpu_to_le16('5'), },
+               { cpu_to_le16('C'), cpu_to_le16('O'), cpu_to_le16('M'), cpu_to_le16('6'), },
+               { cpu_to_le16('C'), cpu_to_le16('O'), cpu_to_le16('M'), cpu_to_le16('7'), },
+               { cpu_to_le16('C'), cpu_to_le16('O'), cpu_to_le16('M'), cpu_to_le16('8'), },
+               { cpu_to_le16('C'), cpu_to_le16('O'), cpu_to_le16('M'), cpu_to_le16('9'), },
+               { cpu_to_le16('L'), cpu_to_le16('P'), cpu_to_le16('T'), cpu_to_le16('1'), },
+               { cpu_to_le16('L'), cpu_to_le16('P'), cpu_to_le16('T'), cpu_to_le16('2'), },
+               { cpu_to_le16('L'), cpu_to_le16('P'), cpu_to_le16('T'), cpu_to_le16('3'), },
+               { cpu_to_le16('L'), cpu_to_le16('P'), cpu_to_le16('T'), cpu_to_le16('4'), },
+               { cpu_to_le16('L'), cpu_to_le16('P'), cpu_to_le16('T'), cpu_to_le16('5'), },
+               { cpu_to_le16('L'), cpu_to_le16('P'), cpu_to_le16('T'), cpu_to_le16('6'), },
+               { cpu_to_le16('L'), cpu_to_le16('P'), cpu_to_le16('T'), cpu_to_le16('7'), },
+               { cpu_to_le16('L'), cpu_to_le16('P'), cpu_to_le16('T'), cpu_to_le16('8'), },
+               { cpu_to_le16('L'), cpu_to_le16('P'), cpu_to_le16('T'), cpu_to_le16('9'), },
+       };
+
+       if (!name)
+               return false;
+
+       for (size_t i = 0; i < ARRAY_LEN(forbidden_names); i++)
+               if (!cmp_utf16le_strings_z(forbidden_names[i], name, true))
+                       return true;
+
+       return false;
+}
+
+static int
+set_random_short_name(struct wim_dentry *dir, struct wim_dentry *child,
+                     struct generation_context *ctx)
+{
+       tchar name[12 + 1];
+       int ret;
+       const utf16lechar *short_name;
+       u32 hash;
+       struct wim_dentry **bucket;
+
+       /* If the long name is not allowed in the Win32 namespace, then it
+        * cannot be assigned a corresponding short name.  */
+       if (is_name_forbidden_in_win32_namespace(child->d_name))
+               return 0;
+
+retry:
+       /* Don't select a short name that is already used by a long name within
+        * the same directory.  */
+       do {
+               int len = generate_random_file_name(name, 12, ctx);
+
+               /* Legal short names on Windows take one of the following forms:
+                *
+                *    - 1 to 8 characters
+                *    - 1 to 8 characters, then a dot, then 1 to 3 characters */
+               if (len >= 9) {
+                       if (len == 9)
+                               len--;
+                       else
+                               name[8] = T('.');
+               }
+               name[len] = 0;
+       } while (get_dentry_child_with_name(dir, name,
+                                           WIMLIB_CASE_PLATFORM_DEFAULT));
+
+
+       /* Don't select a short name that is already used by another short name
+        * within the same directory.  */
+       hash = 0;
+       for (const tchar *p = name; *p; p++)
+               hash = (hash * 31) + totlower(*p);
+       ret = tstr_get_utf16le(name, &short_name);
+       if (ret)
+               return ret;
+       FREE(child->d_short_name);
+       child->d_short_name = utf16le_dup(short_name);
+       child->d_short_name_nbytes = utf16le_len_bytes(short_name);
+       tstr_put_utf16le(short_name);
+
+       if (!child->d_short_name)
+               return WIMLIB_ERR_NOMEM;
+
+       bucket = &ctx->used_short_names[hash % ARRAY_LEN(ctx->used_short_names)];
+
+       for (struct wim_dentry *d = *bucket; d != NULL;
+            d = d->d_next_extraction_alias)
+               if (!cmp_utf16le_strings_z(child->d_short_name,
+                                          d->d_short_name, true))
+                       goto retry;
+
+       if (is_name_forbidden_in_win32_namespace(child->d_short_name))
+               goto retry;
+
+       child->d_next_extraction_alias = *bucket;
+       *bucket = child;
+       return 0;
+}
+
+static bool
+inode_has_short_name(const struct wim_inode *inode)
+{
+       const struct wim_dentry *dentry;
+
+       inode_for_each_dentry(dentry, inode)
+               if (dentry_has_short_name(dentry))
+                       return true;
+
+       return false;
+}
+
+static int
+generate_dentry_tree_recursive(struct wim_dentry *dir, u32 depth,
+                              struct generation_context *ctx)
+{
+       u32 num_children = select_num_children(depth, ctx);
+       struct wim_dentry *child;
+       int ret;
+
+       memset(ctx->used_short_names, 0, sizeof(ctx->used_short_names));
+
+       /* Generate 'num_children' dentries within 'dir'.  Some may be
+        * directories themselves.  */
+
+       for (u32 i = 0; i < num_children; i++) {
+
+               /* Generate the next child dentry.  */
+
+               tchar name[128 + 1];
+               struct wim_dentry *duplicate;
+               struct wim_inode *inode;
+               u64 ino;
+               bool is_directory;
+
+               /* Choose a long filename that is unique within the directory.*/
+               do {
+                       generate_random_file_name(name, 128, ctx);
+               } while (get_dentry_child_with_name(dir, name,
+                                                   WIMLIB_CASE_PLATFORM_DEFAULT));
+
+               /* Decide whether to create a directory or not.
+                * If not a directory, also decide on the inode number (i.e. we
+                * may generate a "hard link" to an existing file).  */
+               is_directory = ((rand32() % 16) <= 6);
+               if (is_directory)
+                       ino = 0;
+               else
+                       ino = select_inode_number(ctx);
+
+               /* Create the dentry and add it to the directory.  */
+               ret = inode_table_new_dentry(ctx->params->inode_table, name,
+                                            ino, 0, is_directory, &child);
+               if (ret)
+                       return ret;
+
+               duplicate = dentry_add_child(dir, child);
+               wimlib_assert(!duplicate);
+
+               inode = child->d_inode;
+
+               if (inode->i_nlink > 1)  /* Existing inode?  */
+                       continue;
+
+               /* New inode; set attributes, metadata, and data.  */
+
+               if (is_directory)
+                       inode->i_attributes |= FILE_ATTRIBUTE_DIRECTORY;
+
+               ret = set_random_metadata(inode, ctx);
+               if (ret)
+                       return ret;
+
+               ret = set_random_streams(inode, ctx, true);
+               if (ret)
+                       return ret;
+
+               /* Recurse if it's a directory.  */
+               if (is_directory &&
+                   !(inode->i_attributes & FILE_ATTRIBUTE_REPARSE_POINT))
+               {
+                       ret = generate_dentry_tree_recursive(child, depth + 1,
+                                                            ctx);
+                       if (ret)
+                               return ret;
+               }
+       }
+
+       for_dentry_child(child, dir) {
+               /* sometimes generate a unique short name  */
+               if (randbool() && !inode_has_short_name(child->d_inode)) {
+                       ret = set_random_short_name(dir, child, ctx);
+                       if (ret)
+                               return ret;
+               }
+       }
+
+       return 0;
+}
+
+int
+generate_dentry_tree(struct wim_dentry **root_ret, const tchar *_ignored,
+                    struct scan_params *params)
+{
+       int ret;
+       struct wim_dentry *root = NULL;
+       struct generation_context ctx = {
+               .params = params,
+       };
+
+       ctx.metadata_only = ((rand32() % 8) != 0); /* usually metadata only  */
+
+       ret = inode_table_new_dentry(params->inode_table, NULL, 0, 0, true, &root);
+       if (!ret) {
+               root->d_inode->i_attributes = FILE_ATTRIBUTE_DIRECTORY;
+               ret = set_random_metadata(root->d_inode, &ctx);
+       }
+       if (!ret)
+               ret = set_random_streams(root->d_inode, &ctx, false);
+       if (!ret)
+               ret = generate_dentry_tree_recursive(root, 1, &ctx);
+       if (!ret)
+               *root_ret = root;
+       else
+               free_dentry_tree(root, params->blob_table);
+       return ret;
+}
+
+/*----------------------------------------------------------------------------*
+ *                            File tree comparison                            *
+ *----------------------------------------------------------------------------*/
+
+#define INDEX_NODE_TO_DENTRY(node)     \
+       ((node) ? avl_tree_entry((node), struct wim_dentry, d_index_node) : NULL)
+
+static struct wim_dentry *
+dentry_first_child(struct wim_dentry *dentry)
+{
+       return INDEX_NODE_TO_DENTRY(
+                       avl_tree_first_in_order(dentry->d_inode->i_children));
+}
+
+static struct wim_dentry *
+dentry_next_sibling(struct wim_dentry *dentry)
+{
+       return INDEX_NODE_TO_DENTRY(
+                       avl_tree_next_in_order(&dentry->d_index_node));
+}
+
+/*
+ * Verify that the dentries in the tree 'd1' exactly match the dentries in the
+ * tree 'd2', considering long and short filenames.  In addition, set
+ * 'd_corresponding' of each dentry to point to the corresponding dentry in the
+ * other tree, and set 'i_corresponding' of each inode to point to the
+ * unverified corresponding inode in the other tree.
+ */
+static int
+calc_corresponding_files_recursive(struct wim_dentry *d1, struct wim_dentry *d2,
+                                  int cmp_flags)
+{
+       struct wim_dentry *child1;
+       struct wim_dentry *child2;
+       int ret;
+
+       /* Compare long filenames, case sensitively.  */
+       if (cmp_utf16le_strings(d1->d_name, d1->d_name_nbytes / 2,
+                               d2->d_name, d2->d_name_nbytes / 2,
+                               false))
+       {
+               ERROR("Filename mismatch; path1=\"%"TS"\", path2=\"%"TS"\"",
+                     dentry_full_path(d1), dentry_full_path(d2));
+               return WIMLIB_ERR_IMAGES_ARE_DIFFERENT;
+       }
+
+       /* Compare short filenames, case insensitively.  */
+       if (!(d2->d_short_name_nbytes == 0 &&
+             (cmp_flags & WIMLIB_CMP_FLAG_SHORT_NAMES_NOT_PRESERVED)) &&
+           cmp_utf16le_strings(d1->d_short_name, d1->d_short_name_nbytes / 2,
+                               d2->d_short_name, d2->d_short_name_nbytes / 2,
+                               true))
+       {
+               ERROR("Short name mismatch; path=\"%"TS"\"",
+                     dentry_full_path(d1));
+               return WIMLIB_ERR_IMAGES_ARE_DIFFERENT;
+       }
+
+       /* Match up the dentries  */
+       d1->d_corresponding = d2;
+       d2->d_corresponding = d1;
+
+       /* Match up the inodes (may overwrite previous value)  */
+       d1->d_inode->i_corresponding = d2->d_inode;
+       d2->d_inode->i_corresponding = d1->d_inode;
+
+       /* Process children  */
+       child1 = dentry_first_child(d1);
+       child2 = dentry_first_child(d2);
+       while (child1 || child2) {
+
+               if (!child1 || !child2) {
+                       ERROR("Child count mismatch; "
+                             "path1=\"%"TS"\", path2=\"%"TS"\"",
+                             dentry_full_path(d1), dentry_full_path(d2));
+                       return WIMLIB_ERR_IMAGES_ARE_DIFFERENT;
+               }
+
+               /* Recurse on this pair of children.  */
+               ret = calc_corresponding_files_recursive(child1, child2,
+                                                        cmp_flags);
+               if (ret)
+                       return ret;
+
+               /* Continue to the next pair of children.  */
+               child1 = dentry_next_sibling(child1);
+               child2 = dentry_next_sibling(child2);
+       }
+       return 0;
+}
+
+/* Perform sanity checks on an image's inodes.  All assertions here should pass,
+ * even if the images being compared are different.  */
+static void
+assert_inodes_sane(const struct wim_image_metadata *imd)
+{
+       const struct wim_inode *inode;
+       const struct wim_dentry *dentry;
+       size_t link_count;
+
+       image_for_each_inode(inode, imd) {
+               link_count = 0;
+               inode_for_each_dentry(dentry, inode) {
+                       wimlib_assert(dentry->d_inode == inode);
+                       link_count++;
+               }
+               wimlib_assert(link_count > 0);
+               wimlib_assert(link_count == inode->i_nlink);
+               wimlib_assert(inode->i_corresponding != NULL);
+       }
+}
+
+static int
+check_hard_link(struct wim_dentry *dentry, void *_ignore)
+{
+       /* My inode is my corresponding dentry's inode's corresponding inode,
+        * and my inode's corresponding inode is my corresponding dentry's
+        * inode.  */
+       const struct wim_inode *a = dentry->d_inode;
+       const struct wim_inode *b = dentry->d_corresponding->d_inode;
+       if (a == b->i_corresponding && a->i_corresponding == b)
+               return 0;
+       ERROR("Hard link difference; path=%"TS"", dentry_full_path(dentry));
+       return WIMLIB_ERR_IMAGES_ARE_DIFFERENT;
+}
+
+static int
+cmp_inodes(const struct wim_inode *inode1, const struct wim_inode *inode2,
+          const struct wim_image_metadata *imd1,
+          const struct wim_image_metadata *imd2, int cmp_flags)
+{
+       const u32 attrib_diff = inode1->i_attributes ^ inode2->i_attributes;
+       bool reparse_point_should_preserved = true;
+
+       /* Compare attributes  */
+       if (cmp_flags & WIMLIB_CMP_FLAG_ATTRIBUTES_NOT_PRESERVED) {
+
+               /* In this mode, we expect that most attributes are not
+                * preserved.  However, FILE_ATTRIBUTE_DIRECTORY should always
+                * match.  */
+               if (attrib_diff & FILE_ATTRIBUTE_DIRECTORY)
+                       goto attrib_mismatch;
+
+               /* We may also expect FILE_ATTRIBUTE_REPARSE_POINT to be
+                * preserved for symlinks.  It also shouldn't be set if it
+                * wasn't set before.  */
+
+               if ((cmp_flags & WIMLIB_CMP_FLAG_IMAGE2_SHOULD_HAVE_SYMLINKS) &&
+                   inode_is_symlink(inode1))
+                       reparse_point_should_preserved = true;
+               else
+                       reparse_point_should_preserved = false;
+
+               if ((attrib_diff & FILE_ATTRIBUTE_REPARSE_POINT) &&
+                   (reparse_point_should_preserved ||
+                    (inode2->i_attributes & FILE_ATTRIBUTE_REPARSE_POINT)))
+                       goto attrib_mismatch;
+       } else {
+
+               /* Most attributes should be preserved.  */
+
+               /* Nothing other than COMPRESSED and NORMAL should have changed.
+                */
+               if (attrib_diff & ~(FILE_ATTRIBUTE_COMPRESSED |
+                                   FILE_ATTRIBUTE_NORMAL))
+                       goto attrib_mismatch;
+
+               /* COMPRESSED shouldn't have changed unless specifically
+                * excluded.  */
+               if ((attrib_diff & FILE_ATTRIBUTE_COMPRESSED) &&
+                   !(cmp_flags & WIMLIB_CMP_FLAG_COMPRESSION_NOT_PRESERVED))
+                       goto attrib_mismatch;
+
+               /* We allow NORMAL to change, but not if the file ended up with
+                * other attributes set as well.  */
+               if ((attrib_diff & FILE_ATTRIBUTE_NORMAL) &&
+                   (inode2->i_attributes & ~FILE_ATTRIBUTE_NORMAL))
+                       goto attrib_mismatch;
+       }
+
+       /* Compare security descriptors  */
+       if (inode_has_security_descriptor(inode1)) {
+               if (inode_has_security_descriptor(inode2)) {
+                       const void *desc1 = imd1->security_data->descriptors[inode1->i_security_id];
+                       const void *desc2 = imd2->security_data->descriptors[inode2->i_security_id];
+                       size_t size1 = imd1->security_data->sizes[inode1->i_security_id];
+                       size_t size2 = imd2->security_data->sizes[inode2->i_security_id];
+
+                       if (size1 != size2 || memcmp(desc1, desc2, size1)) {
+                               ERROR("Security descriptor of %"TS" differs!",
+                                     inode_any_full_path(inode1));
+                               return WIMLIB_ERR_IMAGES_ARE_DIFFERENT;
+                       }
+               } else if (!(cmp_flags & WIMLIB_CMP_FLAG_SECURITY_NOT_PRESERVED)) {
+                       ERROR("%"TS" has a security descriptor in the first image but "
+                             "not in the second image!", inode_any_full_path(inode1));
+                       return WIMLIB_ERR_IMAGES_ARE_DIFFERENT;
+               }
+       } else if (inode_has_security_descriptor(inode2)) {
+               /* okay --- consider it acceptable if a default security
+                * descriptor was assigned  */
+               /*ERROR("%"TS" has a security descriptor in the second image but "*/
+                     /*"not in the first image!", inode_any_full_path(inode1));*/
+               /*return WIMLIB_ERR_IMAGES_ARE_DIFFERENT;*/
+       }
+
+       /* Compare streams  */
+       for (unsigned i = 0; i < inode1->i_num_streams; i++) {
+               const struct wim_inode_stream *strm1 = &inode1->i_streams[i];
+               const struct wim_inode_stream *strm2;
+
+               if (strm1->stream_type == STREAM_TYPE_REPARSE_POINT &&
+                   !reparse_point_should_preserved)
+                       continue;
+
+               if (strm1->stream_type == STREAM_TYPE_UNKNOWN)
+                       continue;
+
+               /* Get the corresponding stream from the second file  */
+               strm2 = inode_get_stream(inode2, strm1->stream_type, strm1->stream_name);
+
+               if (!strm2) {
+                       /* Corresponding stream not found  */
+                       if (stream_is_named(strm1) &&
+                           (cmp_flags & WIMLIB_CMP_FLAG_ADS_NOT_PRESERVED))
+                               continue;
+                       ERROR("Stream of %"TS" is missing in second image; "
+                             "type %d, named=%d, empty=%d",
+                             inode_any_full_path(inode1),
+                             strm1->stream_type,
+                             stream_is_named(strm1),
+                             is_zero_hash(stream_hash(strm1)));
+                       return WIMLIB_ERR_IMAGES_ARE_DIFFERENT;
+               }
+
+               if (!hashes_equal(stream_hash(strm1), stream_hash(strm2))) {
+                       ERROR("Stream of %"TS" differs; type %d",
+                             inode_any_full_path(inode1), strm1->stream_type);
+                       return WIMLIB_ERR_IMAGES_ARE_DIFFERENT;
+               }
+       }
+
+       return 0;
+
+attrib_mismatch:
+       ERROR("Attribute mismatch; %"TS" has attributes 0x%08"PRIx32" "
+             "in first image but attributes 0x%08"PRIx32" in second image",
+             inode_any_full_path(inode1), inode1->i_attributes,
+             inode2->i_attributes);
+       return WIMLIB_ERR_IMAGES_ARE_DIFFERENT;
+}
+
+static int
+cmp_images(const struct wim_image_metadata *imd1,
+          const struct wim_image_metadata *imd2, int cmp_flags)
+{
+       struct wim_dentry *root1 = imd1->root_dentry;
+       struct wim_dentry *root2 = imd2->root_dentry;
+       const struct wim_inode *inode;
+       int ret;
+
+       ret = calc_corresponding_files_recursive(root1, root2, cmp_flags);
+       if (ret)
+               return ret;
+
+       /* Verify that the hard links match up between the two images.  */
+       assert_inodes_sane(imd1);
+       assert_inodes_sane(imd2);
+       ret = for_dentry_in_tree(root1, check_hard_link, NULL);
+       if (ret)
+               return ret;
+
+       /* Compare corresponding inodes.  */
+       image_for_each_inode(inode, imd1) {
+               ret = cmp_inodes(inode, inode->i_corresponding,
+                                imd1, imd2, cmp_flags);
+               if (ret)
+                       return ret;
+       }
+
+       return 0;
+}
+
+static int
+load_image(WIMStruct *wim, int image, struct wim_image_metadata **imd_ret)
+{
+       int ret = select_wim_image(wim, image);
+       if (!ret) {
+               *imd_ret = wim_get_current_image_metadata(wim);
+               mark_image_dirty(*imd_ret);
+       }
+       return ret;
+}
+
+WIMLIBAPI int
+wimlib_compare_images(WIMStruct *wim1, int image1,
+                     WIMStruct *wim2, int image2, int cmp_flags)
+{
+       int ret;
+       struct wim_image_metadata *imd1, *imd2;
+
+       ret = load_image(wim1, image1, &imd1);
+       if (!ret)
+               ret = load_image(wim2, image2, &imd2);
+       if (!ret)
+               ret = cmp_images(imd1, imd2, cmp_flags);
+       return ret;
+}
+
+#endif /* ENABLE_TEST_SUPPORT */
index fb53563..b6fb67c 100644 (file)
@@ -64,6 +64,7 @@
 #include "wimlib/paths.h"
 #include "wimlib/progress.h"
 #include "wimlib/scan.h"
+#include "wimlib/test_support.h"
 #include "wimlib/xml_windows.h"
 
 /* Saved specification of a "primitive" update operation that was performed.  */
@@ -803,6 +804,11 @@ execute_add_command(struct update_command_journal *j,
                scan_tree = ntfs_3g_build_dentry_tree;
 #endif
 
+#ifdef ENABLE_TEST_SUPPORT
+       if (add_flags & WIMLIB_ADD_FLAG_GENERATE_TEST_DATA)
+               scan_tree = generate_dentry_tree;
+#endif
+
        ret = get_capture_config(config_file, &config,
                                 add_flags, fs_source_path);
        if (ret)
@@ -1206,6 +1212,9 @@ check_add_command(struct wimlib_update_command *cmd,
                          WIMLIB_ADD_FLAG_NO_REPLACE |
                          WIMLIB_ADD_FLAG_TEST_FILE_EXCLUSION |
                          WIMLIB_ADD_FLAG_SNAPSHOT |
+               #ifdef ENABLE_TEST_SUPPORT
+                         WIMLIB_ADD_FLAG_GENERATE_TEST_DATA |
+               #endif
                          WIMLIB_ADD_FLAG_FILE_PATHS_UNNEEDED))
                return WIMLIB_ERR_INVALID_PARAM;
 
diff --git a/tests/wlfuzz.c b/tests/wlfuzz.c
new file mode 100644 (file)
index 0000000..5160786
--- /dev/null
@@ -0,0 +1,1046 @@
+/*
+ * wlfuzz.c - Randomized tests for wimlib
+ */
+
+/*
+ * Copyright (C) 2015-2016 Eric Biggers
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * This program is a randomized test runner for wimlib.  It must be linked
+ * against a build of the library compiled with --enable-test-support.
+ *
+ * Various types of tests are run. Most important is the "apply and capture"
+ * test, which works as follows:
+ *
+ *     1. Generate an in-memory WIM image containing a random directory tree
+ *     2. Persist the image into a WIM file
+ *     3. Apply the WIM image to somewhere
+ *     4. Re-capture the applied image
+ *     5. Compare the directory tree of the re-captured image to the original
+ *
+ * Note that this is an "apply and capture" test, not a "capture and apply"
+ * test.  By using the filesystem as the intermediary rather than as the
+ * starting point and ending point, the tests will run nearly unchanged
+ * regardless of filesystem type (e.g. UNIX, Windows, or NTFS-3G).  This style
+ * of test has been effective at finding bugs in wimlib as well as bugs in
+ * NTFS-3G where its behavior differs from that of Windows.
+ *
+ * Care is taken to exercise different options, such as different compression
+ * formats, when multiple are available.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include "config.h"
+#endif
+
+#ifndef ENABLE_TEST_SUPPORT
+#  error "This program requires that wimlib was configured with --enable-test-support."
+#endif
+
+#include <errno.h>
+#include <dirent.h>
+#include <fcntl.h>
+#include <inttypes.h>
+#include <limits.h>
+#include <stdarg.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#ifdef WITH_NTFS_3G
+#  include <sys/wait.h>
+#endif
+#include <unistd.h>
+
+#ifdef __WIN32__
+#  include <windows.h>
+#  include <winternl.h>
+#  include <ntstatus.h>
+#endif
+
+#include "wimlib.h"
+#include "wimlib_tchar.h"
+#include "wimlib/test_support.h"
+#include "wimlib/wof.h"
+
+#ifndef O_BINARY
+#  define O_BINARY 0
+#endif
+
+#define ARRAY_LEN(A)   (sizeof(A) / sizeof((A)[0]))
+
+#define TMP_TARGET_NAME        T("wlfuzz-tmp-target")
+#define MAX_NUM_WIMS           4
+
+static bool wimfile_in_use[MAX_NUM_WIMS];
+static int in_use_wimfile_indices[MAX_NUM_WIMS];
+static int num_wimfiles_in_use = 0;
+
+static void
+assertion_failed(int line, const char *format, ...)
+{
+       va_list va;
+
+       va_start(va, format);
+       fprintf(stderr, "ASSERTION FAILED at line %d: ", line);
+       vfprintf(stderr, format, va);
+       fputc('\n', stderr);
+       va_end(va);
+
+       exit(1);
+}
+
+#define ASSERT(expr, msg, ...)                                         \
+({                                                                     \
+       if (__builtin_expect(!(expr), 0))                               \
+               assertion_failed(__LINE__, (msg), ##__VA_ARGS__);       \
+})
+
+#define CHECK_RET(ret)                                                 \
+({                                                                     \
+       int r = (ret);                                                  \
+       ASSERT(!r, "%"TS, wimlib_get_error_string(r));                  \
+})
+
+static void
+change_to_temporary_directory(void)
+{
+#ifdef __WIN32__
+       ASSERT(SetCurrentDirectory(L"E:\\"),
+              "failed to change directory to E:\\");
+#else
+       const char *tmpdir = getenv("TMPDIR");
+       if (!tmpdir)
+               tmpdir = P_tmpdir;
+       ASSERT(!chdir(tmpdir),
+              "failed to change to temporary directory \"%s\": %m", tmpdir);
+#endif
+}
+
+static void __attribute__((unused))
+copy_file(const tchar *src, const tchar *dst)
+{
+       int in_fd = topen(src, O_RDONLY|O_BINARY);
+       int out_fd = topen(dst, O_WRONLY|O_TRUNC|O_CREAT|O_BINARY, 0644);
+       char buf[32768];
+       ssize_t bytes_read, bytes_written, i;
+
+       ASSERT(in_fd >= 0, "%"TS": open error: %m", src);
+       ASSERT(out_fd >= 0, "%"TS": open error: %m", dst);
+       while ((bytes_read = read(in_fd, buf, sizeof(buf))) > 0) {
+               for (i = 0; i < bytes_read; i += bytes_written) {
+                       bytes_written = write(out_fd, &buf[i], bytes_read - i);
+                       ASSERT(bytes_written > 0, "%"TS": write error: %m", dst);
+               }
+       }
+       ASSERT(bytes_read == 0, "%"TS": read error: %m", src);
+       close(in_fd);
+       close(out_fd);
+}
+
+#ifdef WITH_NTFS_3G
+static void
+create_ntfs_volume(const char *name)
+{
+       int fd;
+       int pid;
+       int status;
+       static const char buffer[1] = {0};
+
+       fd = open(name, O_WRONLY|O_TRUNC|O_CREAT|O_NOFOLLOW, 0644);
+       ASSERT(fd >= 0, "%s: open error: %m", name);
+
+       ASSERT(lseek(fd, 999999999, SEEK_SET) != -1, "%s: lseek error: %m", name);
+
+       ASSERT(write(fd, buffer, 1) == 1, "%s: write error: %m", name);
+
+       ASSERT(close(fd) == 0, "%s: close error: %m", name);
+
+       pid = fork();
+       ASSERT(pid >= 0, "fork error: %m");
+       if (pid == 0) {
+               close(STDOUT_FILENO);
+               close(STDERR_FILENO);
+               execlp("mkntfs", "mkntfs", "--force", "--fast",
+                      name, (char *)NULL);
+               ASSERT(false, "Failed to execute mkntfs: %m");
+       }
+
+       ASSERT(wait(&status) != -1, "wait error: %m");
+       ASSERT(WIFEXITED(status) && WEXITSTATUS(status) == 0,
+              "mkntfs error: exited with status %d", status);
+}
+#endif /* WITH_NTFS_3G */
+
+#ifdef __WIN32__
+
+extern WINAPI NTSTATUS NtQueryDirectoryFile (HANDLE FileHandle,
+                                            HANDLE Event,
+                                            PIO_APC_ROUTINE ApcRoutine,
+                                            PVOID ApcContext,
+                                            PIO_STATUS_BLOCK IoStatusBlock,
+                                            PVOID FileInformation,
+                                            ULONG Length,
+                                            FILE_INFORMATION_CLASS FileInformationClass,
+                                            BOOLEAN ReturnSingleEntry,
+                                            PUNICODE_STRING FileName,
+                                            BOOLEAN RestartScan);
+
+static void
+delete_directory_tree_recursive(HANDLE cur_dir, UNICODE_STRING *name)
+{
+       OBJECT_ATTRIBUTES attr = { .Length = sizeof(attr), };
+       IO_STATUS_BLOCK iosb;
+       FILE_BASIC_INFORMATION basic = { .FileAttributes = FILE_ATTRIBUTE_NORMAL, };
+       HANDLE h;
+       const size_t bufsize = 8192;
+       void *buf;
+       NTSTATUS status;
+       ULONG perms;
+       ULONG flags;
+
+       flags = FILE_DELETE_ON_CLOSE |
+                     FILE_OPEN_REPARSE_POINT |
+                     FILE_OPEN_FOR_BACKUP_INTENT |
+                     FILE_SYNCHRONOUS_IO_NONALERT |
+                     FILE_SEQUENTIAL_ONLY;
+
+       name->MaximumLength = name->Length;
+
+       attr.RootDirectory = cur_dir;
+       attr.ObjectName = name;
+
+       perms = DELETE | SYNCHRONIZE | FILE_LIST_DIRECTORY | FILE_TRAVERSE;
+retry:
+       status = NtOpenFile(&h, perms, &attr, &iosb, FILE_SHARE_VALID_FLAGS, flags);
+       if (!NT_SUCCESS(status)) {
+               if (status == STATUS_OBJECT_NAME_NOT_FOUND)
+                       return;
+               if (status == STATUS_CANNOT_DELETE && (perms & DELETE)) {
+                       perms &= ~DELETE;
+                       flags &= ~FILE_DELETE_ON_CLOSE;
+                       perms |= FILE_WRITE_ATTRIBUTES;
+                       goto retry;
+               }
+               ASSERT(false, "NtOpenFile() for deletion failed; status=0x%08"PRIx32, status);
+       }
+       if (perms & FILE_WRITE_ATTRIBUTES) {
+               status = NtSetInformationFile(h, &iosb, &basic,
+                                             sizeof(basic), FileBasicInformation);
+               NtClose(h);
+               if (!NT_SUCCESS(status)) {
+                       ASSERT(false, "NtSetInformationFile() for deletion "
+                              "failed; status=0x%08"PRIx32, status);
+               }
+               perms &= ~FILE_WRITE_ATTRIBUTES;
+               perms |= DELETE;
+               flags |= FILE_DELETE_ON_CLOSE;
+               goto retry;
+       }
+
+       buf = malloc(bufsize);
+       ASSERT(buf != NULL, "out of memory!");
+
+       while (NT_SUCCESS(status = NtQueryDirectoryFile(h, NULL, NULL, NULL,
+                                                       &iosb, buf, bufsize,
+                                                       FileNamesInformation,
+                                                       FALSE, NULL, FALSE)))
+       {
+               const FILE_NAMES_INFORMATION *info = buf;
+               for (;;) {
+                       if (!(info->FileNameLength == 2 && info->FileName[0] == L'.') &&
+                           !(info->FileNameLength == 4 && info->FileName[0] == L'.' &&
+                                                          info->FileName[1] == L'.'))
+                       {
+                               name->Buffer = (wchar_t *)info->FileName;
+                               name->Length = info->FileNameLength;
+                               delete_directory_tree_recursive(h, name);
+                       }
+                       if (info->NextEntryOffset == 0)
+                               break;
+                       info = (const FILE_NAMES_INFORMATION *)
+                                       ((const char *)info + info->NextEntryOffset);
+               }
+       }
+
+       ASSERT(status == STATUS_NO_MORE_FILES || /* end of directory  */
+              status == STATUS_INVALID_PARAMETER, /* not a directory  */
+              "NtQueryDirectoryFile() for deletion failed; "
+              "status=0x%08"PRIx32, status);
+
+       free(buf);
+       NtClose(h);
+}
+
+static void
+delete_directory_tree(const wchar_t *name)
+{
+       UNICODE_STRING uname;
+       void *buffer;
+
+       ASSERT(RtlDosPathNameToNtPathName_U(name, &uname, NULL, NULL),
+              "Unable to translate %ls to NT namespace path", name);
+       buffer = uname.Buffer;
+       delete_directory_tree_recursive(NULL, &uname);
+       HeapFree(GetProcessHeap(), 0, buffer);
+       ASSERT(GetFileAttributes(name) == 0xFFFFFFFF, "Deletion didn't work!");
+}
+
+#else /* __WIN32__ */
+
+static void
+delete_directory_tree_recursive(int dirfd, const char *name)
+{
+       int fd;
+       DIR *dir;
+       struct dirent *ent;
+
+       if (!unlinkat(dirfd, name, 0) || errno == ENOENT)
+               return;
+       ASSERT(errno == EISDIR, "%s: unlink error: %m", name);
+
+       fd = openat(dirfd, name, O_RDONLY | O_NOFOLLOW | O_DIRECTORY);
+       ASSERT(fd >= 0, "%m");
+
+       dir = fdopendir(fd);
+       ASSERT(dir != NULL, "%m");
+       while (errno = 0, (ent = readdir(dir)))
+               if (strcmp(ent->d_name, ".") && strcmp(ent->d_name, ".."))
+                       delete_directory_tree_recursive(fd, ent->d_name);
+       closedir(dir);
+
+       ASSERT(!unlinkat(dirfd, name, AT_REMOVEDIR), "%m");
+}
+
+static void
+delete_directory_tree(const tchar *name)
+{
+       delete_directory_tree_recursive(AT_FDCWD, name);
+}
+
+#endif /* !__WIN32__ */
+
+static uint32_t
+rand32(void)
+{
+       static uint64_t state;
+
+       /* A simple linear congruential generator  */
+       state = (state * 25214903917 + 11) & (((uint64_t)1 << 48) - 1);
+       return state >> 16;
+}
+
+static inline bool
+randbool(void)
+{
+       return rand32() & 1;
+}
+
+static tchar wimfile[32];
+
+static const tchar *
+get_wimfile(int index)
+{
+       tsprintf(wimfile, T("wim%d"), index);
+       return wimfile;
+}
+
+static int
+select_random_wimfile_index(void)
+{
+       return in_use_wimfile_indices[rand32() % num_wimfiles_in_use];
+}
+
+static const tchar *
+select_new_wimfile(void)
+{
+       int index = 0;
+
+       while (wimfile_in_use[index])
+               index++;
+
+       in_use_wimfile_indices[num_wimfiles_in_use++] = index;
+       wimfile_in_use[index] = true;
+
+       return get_wimfile(index);
+}
+
+static WIMStruct *
+open_wim(int index)
+{
+       const tchar *wimfile = get_wimfile(index);
+       WIMStruct *wim;
+       int open_flags = 0;
+
+       open_flags |= randbool() ? 0 : WIMLIB_OPEN_FLAG_CHECK_INTEGRITY;
+
+       printf("Opening %"TS" with flags 0x%08x\n", wimfile, open_flags);
+
+       CHECK_RET(wimlib_open_wim(wimfile, open_flags, &wim));
+
+       return wim;
+}
+
+static WIMStruct *
+open_random_wim(void)
+{
+       return open_wim(select_random_wimfile_index());
+}
+
+static int
+get_image_count(WIMStruct *wim)
+{
+       struct wimlib_wim_info info;
+
+       CHECK_RET(wimlib_get_wim_info(wim, &info));
+
+       return info.image_count;
+}
+
+#ifdef __WIN32__
+static bool
+is_wimboot_capable(WIMStruct *wim)
+{
+       struct wimlib_wim_info info;
+
+       CHECK_RET(wimlib_get_wim_info(wim, &info));
+
+       return info.wim_version == 0x10D00 &&
+               ((info.compression_type == WIMLIB_COMPRESSION_TYPE_XPRESS &&
+                 (info.chunk_size == 4096 || info.chunk_size == 8192 ||
+                  info.chunk_size == 16384 || info.chunk_size == 32768)) ||
+                (info.compression_type == WIMLIB_COMPRESSION_TYPE_LZX &&
+                 info.chunk_size == 32768));
+}
+#endif /* __WIN32__ */
+
+static void
+overwrite_wim(WIMStruct *wim)
+{
+       int write_flags = 0;
+       struct wimlib_wim_info info;
+
+       CHECK_RET(wimlib_get_wim_info(wim, &info));
+
+       switch (rand32() % 4) {
+       case 0:
+               write_flags |= WIMLIB_WRITE_FLAG_CHECK_INTEGRITY;
+               break;
+       case 1:
+               write_flags |= WIMLIB_WRITE_FLAG_NO_CHECK_INTEGRITY;
+               break;
+       }
+
+       switch (rand32() % 8) {
+       case 0:
+               write_flags |= WIMLIB_WRITE_FLAG_PIPABLE;
+               break;
+       case 1:
+               write_flags |= WIMLIB_WRITE_FLAG_NOT_PIPABLE;
+               break;
+       }
+
+       write_flags |= randbool() ? 0 : WIMLIB_WRITE_FLAG_RECOMPRESS;
+       write_flags |= randbool() ? 0 : WIMLIB_WRITE_FLAG_FSYNC;
+       write_flags |= randbool() ? 0 : WIMLIB_WRITE_FLAG_REBUILD;
+       write_flags |= randbool() ? 0 : WIMLIB_WRITE_FLAG_SOFT_DELETE;
+       write_flags |= randbool() ? 0 : WIMLIB_WRITE_FLAG_IGNORE_READONLY_FLAG;
+       write_flags |= randbool() ? 0 : WIMLIB_WRITE_FLAG_RETAIN_GUID;
+       write_flags |= randbool() ? 0 : WIMLIB_WRITE_FLAG_SEND_DONE_WITH_FILE_MESSAGES;
+       write_flags |= randbool() ? 0 : WIMLIB_WRITE_FLAG_NO_SOLID_SORT;
+
+       if (rand32() % 8 == 0 &&
+           !(write_flags & WIMLIB_WRITE_FLAG_PIPABLE) &&
+           (!info.pipable || (write_flags & WIMLIB_WRITE_FLAG_NOT_PIPABLE)))
+               write_flags |= WIMLIB_WRITE_FLAG_SOLID;
+
+       if (randbool() && !info.pipable &&
+           !(write_flags & (WIMLIB_WRITE_FLAG_RECOMPRESS |
+                            WIMLIB_WRITE_FLAG_PIPABLE)))
+               write_flags |= WIMLIB_WRITE_FLAG_UNSAFE_COMPACT;
+
+       printf("overwrite with flags: 0x%08x\n", write_flags);
+
+       CHECK_RET(wimlib_overwrite(wim, write_flags, 0));
+}
+
+static int
+get_random_write_flags(void)
+{
+       int write_flags = 0;
+
+       write_flags |= randbool() ? 0 : WIMLIB_WRITE_FLAG_CHECK_INTEGRITY;
+       write_flags |= randbool() ? 0 : WIMLIB_WRITE_FLAG_SEND_DONE_WITH_FILE_MESSAGES;
+       write_flags |= randbool() ? 0 : WIMLIB_WRITE_FLAG_NO_SOLID_SORT;
+       switch (rand32() % 8) {
+       case 0:
+               write_flags |= WIMLIB_WRITE_FLAG_PIPABLE;
+               break;
+       case 1:
+               write_flags |= WIMLIB_WRITE_FLAG_SOLID;
+               break;
+       }
+
+       return write_flags;
+}
+
+static void
+op__create_new_wim(void)
+{
+       printf(":::op__create_new_wim\n");
+
+       const tchar *wimfile;
+       WIMStruct *wim;
+       int write_flags;
+
+       if (num_wimfiles_in_use == MAX_NUM_WIMS)
+               return;
+
+       wimfile = select_new_wimfile();
+
+       CHECK_RET(wimlib_create_new_wim(WIMLIB_COMPRESSION_TYPE_NONE, &wim));
+
+       /* Select a random compression type and chunk size.  */
+       switch (rand32() % 8) {
+       default:
+               CHECK_RET(wimlib_set_output_compression_type(wim, WIMLIB_COMPRESSION_TYPE_NONE));
+               break;
+       case 3 ... 4:
+               CHECK_RET(wimlib_set_output_compression_type(wim, WIMLIB_COMPRESSION_TYPE_XPRESS));
+               CHECK_RET(wimlib_set_output_chunk_size(wim, 1 << (12 + rand32() % 5)));
+               break;
+       case 5 ... 6:
+               CHECK_RET(wimlib_set_output_compression_type(wim, WIMLIB_COMPRESSION_TYPE_LZX));
+               if (randbool())
+                       CHECK_RET(wimlib_set_output_chunk_size(wim, 1 << 15));
+               else
+                       CHECK_RET(wimlib_set_output_chunk_size(wim, 1 << (15 + rand32() % 7)));
+               break;
+       case 7:
+               CHECK_RET(wimlib_set_output_compression_type(wim, WIMLIB_COMPRESSION_TYPE_LZMS));
+               CHECK_RET(wimlib_set_output_chunk_size(wim, 1 << (15 + rand32() % 12)));
+               break;
+       }
+
+       /* Select random write flags.  */
+       write_flags = get_random_write_flags();
+
+       printf("Creating %"TS" with write flags 0x%08x\n", wimfile, write_flags);
+
+       CHECK_RET(wimlib_write(wim, wimfile, WIMLIB_ALL_IMAGES, write_flags, 0));
+
+       wimlib_free(wim);
+}
+
+static void
+op__add_empty_image_to_random_wim(void)
+{
+       printf(":::op__add_empty_image_to_random_wim\n");
+
+       WIMStruct *wim;
+       int new_idx;
+
+       if (num_wimfiles_in_use < 1)
+               return;
+
+       wim = open_random_wim();
+       CHECK_RET(wimlib_add_empty_image(wim, NULL, &new_idx));
+       printf("Adding empty image to %"TS" at index %d\n", wimfile, new_idx);
+       overwrite_wim(wim);
+       wimlib_free(wim);
+}
+
+static void
+op__delete_random_image_from_random_wim(void)
+{
+       printf(":::op__delete_random_image_from_random_wim\n");
+
+       WIMStruct *wim;
+       int image;
+       int image_count;
+
+       if (num_wimfiles_in_use == 0)
+               return;
+
+       wim = open_random_wim();
+       image_count = get_image_count(wim);
+       if (image_count != 0) {
+               image = 1 + (rand32() % image_count);
+               CHECK_RET(wimlib_delete_image(wim, image));
+               printf("Deleting image %d from %"TS"\n", image, wimfile);
+               overwrite_wim(wim);
+       }
+       wimlib_free(wim);
+}
+
+static void
+op__delete_random_wim(void)
+{
+       printf(":::op__delete_random_wim\n");
+
+       const tchar *wimfile;
+       int which;
+       int index;
+
+       if (num_wimfiles_in_use == 0)
+               return;
+
+       which = rand32() % num_wimfiles_in_use;
+       index = in_use_wimfile_indices[which];
+
+       wimfile = get_wimfile(index);
+
+       ASSERT(!tunlink(wimfile), "failed to unlink %"TS": %m", wimfile);
+
+       printf("Deleted %"TS"\n", wimfile);
+
+       for (int i = which; i < num_wimfiles_in_use - 1; i++)
+               in_use_wimfile_indices[i] = in_use_wimfile_indices[i + 1];
+       num_wimfiles_in_use--;
+       wimfile_in_use[index] = false;
+}
+
+static void
+op__verify_random_wim(void)
+{
+       printf(":::op__verify_random_wim\n");
+
+       WIMStruct *wim;
+
+       if (num_wimfiles_in_use == 0)
+               return;
+
+       wim = open_random_wim();
+       CHECK_RET(wimlib_verify_wim(wim, 0));
+       printf("Verified %"TS"\n", wimfile);
+       wimlib_free(wim);
+}
+
+static void
+op__overwrite_with_no_changes(void)
+{
+       printf(":::op__overwrite_with_no_changes\n");
+
+       WIMStruct *wim;
+
+       if (num_wimfiles_in_use == 0)
+               return;
+
+       wim = open_random_wim();
+       overwrite_wim(wim);
+       wimlib_free(wim);
+}
+
+static void
+op__export_random_image(void)
+{
+       printf(":::op__export_random_image\n");
+
+       int src_wimfile_index;
+       int dst_wimfile_index;
+       WIMStruct *src_wim;
+       WIMStruct *dst_wim;
+       int src_image_count;
+       int dst_image_count;
+       int src_image;
+       int dst_image;
+
+       if (num_wimfiles_in_use < 2)
+               return;
+
+       src_wimfile_index = select_random_wimfile_index();
+       do {
+               dst_wimfile_index = select_random_wimfile_index();
+       } while (dst_wimfile_index == src_wimfile_index);
+
+       src_wim = open_wim(src_wimfile_index);
+       dst_wim = open_wim(dst_wimfile_index);
+
+       src_image_count = get_image_count(src_wim);
+       dst_image_count = get_image_count(dst_wim);
+
+       /* Choose a random source image --- single or all.  */
+       src_image = WIMLIB_ALL_IMAGES;
+       if (src_image_count != 0 && randbool())
+               src_image = 1 + (rand32() % src_image_count);
+
+       printf("Exporting image %d of %d from wim %d into wim %d\n",
+              src_image, src_image_count, src_wimfile_index, dst_wimfile_index);
+       CHECK_RET(wimlib_export_image(src_wim, src_image, dst_wim, NULL, NULL, 0));
+
+       overwrite_wim(dst_wim);
+       wimlib_free(dst_wim);
+
+       dst_wim = open_wim(dst_wimfile_index);
+
+       /* Compare the images.  */
+       dst_image = dst_image_count;
+       for (int image = (src_image == WIMLIB_ALL_IMAGES ? 1 : src_image);
+            image <= (src_image == WIMLIB_ALL_IMAGES ? src_image_count : src_image);
+            image++)
+       {
+               CHECK_RET(wimlib_compare_images(src_wim, image, dst_wim, ++dst_image, 0));
+       }
+
+       wimlib_free(src_wim);
+       wimlib_free(dst_wim);
+}
+
+static void
+op__apply_and_capture_test(void)
+{
+       printf(":::op__apply_and_capture_test\n");
+
+       WIMStruct *wim;
+       int image;
+       int index;
+       int extract_flags = 0;
+       int add_flags = 0;
+       int cmp_flags = 0;
+
+       if (num_wimfiles_in_use == 0)
+               return;
+
+       /* Generate a random image.  */
+       index = select_random_wimfile_index();
+       wim = open_wim(index);
+
+       CHECK_RET(wimlib_add_image(wim, (void *)rand32, NULL, NULL,
+                                  WIMLIB_ADD_FLAG_GENERATE_TEST_DATA |
+                                  WIMLIB_ADD_FLAG_NORPFIX));
+       overwrite_wim(wim);
+       wimlib_free(wim);
+
+       /* Apply the generated image.  */
+       wim = open_wim(index);
+       image = get_image_count(wim);
+       printf("apply and capture wim%d; generated image is index %d\n",
+              index, image);
+       delete_directory_tree(TMP_TARGET_NAME);
+#ifdef WITH_NTFS_3G
+       if (rand32() & 1) {
+               printf("applying in NTFS mode\n");
+               extract_flags |= WIMLIB_EXTRACT_FLAG_NTFS;
+               extract_flags |= WIMLIB_EXTRACT_FLAG_STRICT_ACLS;
+               extract_flags |= WIMLIB_EXTRACT_FLAG_STRICT_SHORT_NAMES;
+               extract_flags |= WIMLIB_EXTRACT_FLAG_STRICT_TIMESTAMPS;
+               add_flags |= WIMLIB_ADD_FLAG_NTFS;
+               cmp_flags |= WIMLIB_CMP_FLAG_COMPRESSION_NOT_PRESERVED;
+               create_ntfs_volume(TMP_TARGET_NAME);
+       } else
+#endif
+       {
+#ifdef __WIN32__
+               printf("applying in Windows mode\n");
+#else /* __WIN32__ */
+               printf("applying in UNIX mode\n");
+               cmp_flags |= WIMLIB_CMP_FLAG_SHORT_NAMES_NOT_PRESERVED;
+               cmp_flags |= WIMLIB_CMP_FLAG_ATTRIBUTES_NOT_PRESERVED;
+               cmp_flags |= WIMLIB_CMP_FLAG_SECURITY_NOT_PRESERVED;
+               cmp_flags |= WIMLIB_CMP_FLAG_ADS_NOT_PRESERVED;
+               cmp_flags |= WIMLIB_CMP_FLAG_IMAGE2_SHOULD_HAVE_SYMLINKS;
+#endif /* !__WIN32__ */
+       }
+       add_flags |= WIMLIB_ADD_FLAG_NORPFIX;
+       CHECK_RET(wimlib_extract_image(wim, image, TMP_TARGET_NAME,
+                                      extract_flags));
+
+       /* Sometimes extract twice so that we test overwriting existing files.
+        */
+       if (!(extract_flags & WIMLIB_EXTRACT_FLAG_NTFS) && randbool()) {
+               CHECK_RET(wimlib_extract_image(wim, image, TMP_TARGET_NAME,
+                                              extract_flags));
+       }
+
+       /* Capture the applied image.  */
+       CHECK_RET(wimlib_add_image(wim, TMP_TARGET_NAME, NULL, NULL, add_flags));
+       overwrite_wim(wim);
+       wimlib_free(wim);
+
+       /* Compare the generated image with the captured image.  */
+       wim = open_wim(index);
+       CHECK_RET(wimlib_compare_images(wim, image, wim, image + 1, cmp_flags));
+       wimlib_free(wim);
+}
+
+#ifdef __WIN32__
+
+/* Enumerate and unregister all backing WIMs from the specified volume  */
+static void
+unregister_all_backing_wims(const tchar drive_letter)
+{
+       wchar_t volume[7];
+       HANDLE h;
+       void *overlay_list;
+       DWORD bytes_returned;
+       const struct wim_provider_overlay_entry *entry;
+       struct {
+               struct wof_external_info wof_info;
+               struct wim_provider_remove_overlay_input wim;
+       } in;
+
+       wsprintf(volume, L"\\\\.\\%lc:", drive_letter);
+
+       h = CreateFile(volume, GENERIC_READ | GENERIC_WRITE,
+                      FILE_SHARE_VALID_FLAGS, NULL, OPEN_EXISTING,
+                      FILE_FLAG_BACKUP_SEMANTICS, NULL);
+       ASSERT(h != INVALID_HANDLE_VALUE,
+              "Failed to open %ls; error=%u", volume, (unsigned)GetLastError());
+
+       overlay_list = malloc(32768);
+       ASSERT(overlay_list != NULL, "out of memory");
+
+       in.wof_info.version = WOF_CURRENT_VERSION;
+       in.wof_info.provider = WOF_PROVIDER_WIM;
+
+       if (!DeviceIoControl(h, FSCTL_ENUM_OVERLAY,
+                            &in, sizeof(struct wof_external_info),
+                            overlay_list, 32768, &bytes_returned, NULL))
+       {
+               ASSERT(GetLastError() == ERROR_INVALID_FUNCTION ||
+                      GetLastError() == ERROR_FILE_NOT_FOUND,
+                      "FSCTL_ENUM_OVERLAY failed; error=%u", GetLastError());
+               return;
+       }
+
+       entry = overlay_list;
+       for (;;) {
+               printf("Unregistering data source ID %"PRIu64"\n",
+                      entry->data_source_id);
+               in.wim.data_source_id = entry->data_source_id;
+               ASSERT(DeviceIoControl(h, FSCTL_REMOVE_OVERLAY, &in, sizeof(in),
+                                      NULL, 0, &bytes_returned, NULL),
+                      "FSCTL_REMOVE_OVERLAY failed; error=%u",
+                      (unsigned )GetLastError());
+               if (entry->next_entry_offset == 0)
+                       break;
+               entry = (const struct wim_provider_overlay_entry *)
+                       ((const uint8_t *)entry + entry->next_entry_offset);
+       }
+       free(overlay_list);
+       CloseHandle(h);
+}
+
+static void
+op__wimboot_test(void)
+{
+       int index;
+       int index2;
+       WIMStruct *wim;
+       WIMStruct *wim2;
+       int image_count;
+       int image;
+
+       if (num_wimfiles_in_use == 0)
+               return;
+
+       index = select_random_wimfile_index();
+
+       unregister_all_backing_wims(L'E');
+       copy_file(get_wimfile(index), L"wimboot.wim");
+
+       CHECK_RET(wimlib_open_wim(L"wimboot.wim", 0, &wim));
+
+       image_count = get_image_count(wim);
+       if (image_count == 0 || !is_wimboot_capable(wim)) {
+               wimlib_free(wim);
+               return;
+       }
+
+
+       image = 1 + (rand32() % image_count);
+
+       printf("WIMBOOT test; wim%d image %d\n", index, image);
+
+       delete_directory_tree(TMP_TARGET_NAME);
+
+       CHECK_RET(wimlib_extract_image(wim, image, TMP_TARGET_NAME,
+                                      WIMLIB_EXTRACT_FLAG_WIMBOOT));
+
+       if (randbool()) {
+               CHECK_RET(wimlib_extract_image(wim, image, TMP_TARGET_NAME,
+                                              WIMLIB_EXTRACT_FLAG_WIMBOOT));
+       }
+
+       index2 = select_random_wimfile_index();
+       wim2 = open_wim(index2);
+       image_count = get_image_count(wim2);
+
+       CHECK_RET(wimlib_add_image(wim2, TMP_TARGET_NAME, NULL, NULL,
+                                  WIMLIB_ADD_FLAG_NORPFIX));
+
+       overwrite_wim(wim2);
+       wimlib_free(wim2);
+
+       wim2 = open_wim(index2);
+
+       printf("comparing wimboot.wim:%d with wim%d:%d\n",
+              image, index2, image_count + 1);
+
+       CHECK_RET(wimlib_compare_images(wim, image, wim2, image_count + 1, 0));
+
+       wimlib_free(wim);
+       wimlib_free(wim2);
+}
+#endif /* __WIN32__ */
+
+static int
+is_solid_resource(const struct wimlib_resource_entry *resource, void *_ctx)
+{
+       return resource->packed;
+}
+
+static bool
+wim_contains_solid_resources(WIMStruct *wim)
+{
+       return wimlib_iterate_lookup_table(wim, 0, is_solid_resource, NULL);
+}
+
+static void
+op__split_test(void)
+{
+       printf(":::op__split_test\n");
+
+       WIMStruct *wim;
+       WIMStruct *swm;
+       WIMStruct *joined_wim;
+       uint64_t part_size;
+       int write_flags;
+       const tchar *globs[] = { T("tmp*.swm") };
+       int image_count;
+
+       if (num_wimfiles_in_use == 0)
+               return;
+
+       /* split, join, and compare  */
+
+       wim = open_random_wim();
+
+       if (wim_contains_solid_resources(wim)) {
+               /* Can't split a WIM containing solid resources  */
+               wimlib_free(wim);
+               return;
+       }
+
+       image_count = get_image_count(wim);
+
+       part_size = 10000 + (rand32() % 1000000);
+       write_flags = get_random_write_flags();
+       write_flags &= ~WIMLIB_WRITE_FLAG_SOLID;
+
+       printf("splitting WIM %"TS": part_size=%"PRIu64", write_flags=0x%08x\n",
+              wimfile, part_size, write_flags);
+
+       CHECK_RET(wimlib_split(wim, T("tmp.swm"), part_size, write_flags));
+
+       CHECK_RET(wimlib_open_wim(T("tmp.swm"), WIMLIB_OPEN_FLAG_CHECK_INTEGRITY,
+                                 &swm));
+
+       CHECK_RET(wimlib_reference_resource_files(swm, globs, 1,
+                                                 WIMLIB_REF_FLAG_GLOB_ENABLE |
+                                                       WIMLIB_REF_FLAG_GLOB_ERR_ON_NOMATCH,
+                                                 WIMLIB_OPEN_FLAG_CHECK_INTEGRITY));
+
+       CHECK_RET(wimlib_verify_wim(swm, 0));
+
+       CHECK_RET(wimlib_write(swm, T("joined.wim"), WIMLIB_ALL_IMAGES, write_flags, 0));
+       wimlib_free(swm);
+
+       CHECK_RET(wimlib_open_wim(T("joined.wim"), 0, &joined_wim));
+       for (int i = 1; i <= image_count; i++)
+               CHECK_RET(wimlib_compare_images(wim, 1, joined_wim, 1, 0));
+       CHECK_RET(wimlib_verify_wim(joined_wim, 0));
+       wimlib_free(joined_wim);
+       wimlib_free(wim);
+
+       tunlink(T("tmp.swm"));
+       for (int i = 2; ; i++) {
+               tchar name[32];
+               tsprintf(name, T("tmp%d.swm"), i);
+               if (tunlink(name))
+                       break;
+       }
+}
+
+static void
+op__set_compression_level(void)
+{
+       printf(":::op__set_compression_level\n");
+
+       unsigned int level = rand32() % 100;
+       printf("Changing compression levels to %d\n", level);
+       wimlib_set_default_compression_level(-1, level);
+}
+
+typedef void (*operation_func)(void);
+
+static const operation_func operation_table[] = {
+       op__create_new_wim,
+       op__add_empty_image_to_random_wim,
+       op__delete_random_image_from_random_wim,
+       op__delete_random_wim,
+       op__delete_random_wim,
+       op__verify_random_wim,
+       op__overwrite_with_no_changes,
+       op__export_random_image,
+       op__apply_and_capture_test,
+       op__apply_and_capture_test,
+       op__apply_and_capture_test,
+       op__apply_and_capture_test,
+       op__apply_and_capture_test,
+       op__split_test,
+       op__set_compression_level,
+#ifdef __WIN32__
+       op__wimboot_test,
+#endif
+};
+
+#ifdef __WIN32__
+extern int wmain(int argc, wchar_t **argv);
+#define main wmain
+#endif
+
+int
+main(int argc, tchar **argv)
+{
+       unsigned long long num_iterations;
+
+       if (argc < 2) {
+               num_iterations = ULLONG_MAX;
+               printf("Starting test runner\n");
+       } else {
+               num_iterations = tstrtoull(argv[1], NULL, 10);
+               printf("Starting test runner with %llu iterations\n",
+                      num_iterations);
+       }
+
+       CHECK_RET(wimlib_global_init(0));
+       wimlib_set_print_errors(true);
+
+       change_to_temporary_directory();
+
+       for (int i = 0; i < MAX_NUM_WIMS; i++)
+               ASSERT(!tunlink(get_wimfile(i)) || errno == ENOENT, "unlink: %m");
+
+       for (unsigned long long i = 0; i < num_iterations; i++) {
+               printf("--> iteration %llu\n", i);
+               (*operation_table[rand32() % ARRAY_LEN(operation_table)])();
+       }
+
+       wimlib_global_cleanup();
+       return 0;
+}