wlfuzz: updates for sparse files

diff --git a/include/wimlib/test_support.h b/include/wimlib/test_support.h
index 8f068348b7a636b750fb71f040f5d61f0941ac1e..8c19a01a7c8a926e26b146c82930442470dea45a 100644 (file)
--- a/include/wimlib/test_support.h
+++ b/include/wimlib/test_support.h
@@ -9,12 +9,9 @@
 
 #define WIMLIB_ADD_FLAG_GENERATE_TEST_DATA             0x08000000
 
 
 #define WIMLIB_ADD_FLAG_GENERATE_TEST_DATA             0x08000000
 

-#define WIMLIB_CMP_FLAG_SHORT_NAMES_NOT_PRESERVED      0x00000001
-#define WIMLIB_CMP_FLAG_ATTRIBUTES_NOT_PRESERVED       0x00000002
-#define WIMLIB_CMP_FLAG_SECURITY_NOT_PRESERVED         0x00000004
-#define WIMLIB_CMP_FLAG_ADS_NOT_PRESERVED              0x00000008
-#define WIMLIB_CMP_FLAG_COMPRESSION_NOT_PRESERVED      0x00000010
-#define WIMLIB_CMP_FLAG_IMAGE2_SHOULD_HAVE_SYMLINKS    0x00000020
+#define WIMLIB_CMP_FLAG_UNIX_MODE      0x00000001
+#define WIMLIB_CMP_FLAG_NTFS_3G_MODE   0x00000002
+#define WIMLIB_CMP_FLAG_WINDOWS_MODE   0x00000004

 
 extern int
 wimlib_compare_images(WIMStruct *wim1, int image1,
 
 extern int
 wimlib_compare_images(WIMStruct *wim1, int image1,

diff --git a/src/test_support.c b/src/test_support.c
index 49e5fed6903a28b72d98e19ed643292e56fd6b7f..f4bd9855b78a6bfa5e6ad8e718bb54fa6573857f 100644 (file)
--- a/src/test_support.c
+++ b/src/test_support.c
@@ -257,7 +257,8 @@ set_random_metadata(struct wim_inode *inode, struct generation_context *ctx)
                           FILE_ATTRIBUTE_SYSTEM |
                           FILE_ATTRIBUTE_ARCHIVE |
                           FILE_ATTRIBUTE_NOT_CONTENT_INDEXED |
                           FILE_ATTRIBUTE_SYSTEM |
                           FILE_ATTRIBUTE_ARCHIVE |
                           FILE_ATTRIBUTE_NOT_CONTENT_INDEXED |

-                          FILE_ATTRIBUTE_COMPRESSED));
+                          FILE_ATTRIBUTE_COMPRESSED |
+                          FILE_ATTRIBUTE_SPARSE_FILE));

 
        /* File attributes  */
        inode->i_attributes |= attrib;
 
        /* File attributes  */
        inode->i_attributes |= attrib;


@@ -323,8 +324,10 @@ generate_data(u8 *buffer, size_t size, struct generation_context *ctx)
        size_t mask = -1;
        size_t num_byte_fills = rand32() % 256;
 
        size_t mask = -1;
        size_t num_byte_fills = rand32() % 256;
 

+       /* Start by initializing to a random byte */

        memset(buffer, rand32() % 256, size);
 
        memset(buffer, rand32() % 256, size);
 

+       /* Add some random bytes in some random places */

        for (size_t i = 0; i < num_byte_fills; i++) {
                u8 b = rand8();
 
        for (size_t i = 0; i < num_byte_fills; i++) {
                u8 b = rand8();
 


@@ -342,6 +345,7 @@ generate_data(u8 *buffer, size_t size, struct generation_context *ctx)
                        mask = (size_t)-1 << rand32() % 4;
        }
 
                        mask = (size_t)-1 << rand32() % 4;
        }
 

+       /* Sometimes add a wave pattern */

        if (rand32() % 8 == 0) {
                double magnitude = rand32() % 128;
                double scale = 1.0 / (1 + (rand32() % 256));
        if (rand32() % 8 == 0) {
                double magnitude = rand32() % 128;
                double scale = 1.0 / (1 + (rand32() % 256));


@@ -349,6 +353,17 @@ generate_data(u8 *buffer, size_t size, struct generation_context *ctx)
                for (size_t i = 0; i < size; i++)
                        buffer[i] += (int)(magnitude * cos(i * scale));
        }
                for (size_t i = 0; i < size; i++)
                        buffer[i] += (int)(magnitude * cos(i * scale));
        }

+
+       /* Sometimes add some zero regions (holes) */
+       if (rand32() % 4 == 0) {
+               size_t num_holes = 1 + (rand32() % 16);
+               for (size_t i = 0; i < num_holes; i++) {
+                       size_t hole_offset = rand32() % size;
+                       size_t hole_len = min(size - hole_offset,
+                                             size / (1 + (rand32() % 16)));
+                       memset(&buffer[hole_offset], 0, hole_len);
+               }
+       }

 }
 
 static int
 }
 
 static int


@@ -902,7 +917,7 @@ calc_corresponding_files_recursive(struct wim_dentry *d1, struct wim_dentry *d2,
 
        /* Compare short filenames, case insensitively.  */
        if (!(d2->d_short_name_nbytes == 0 &&
 
        /* Compare short filenames, case insensitively.  */
        if (!(d2->d_short_name_nbytes == 0 &&

-             (cmp_flags & WIMLIB_CMP_FLAG_SHORT_NAMES_NOT_PRESERVED)) &&
+             (cmp_flags & WIMLIB_CMP_FLAG_UNIX_MODE)) &&

            cmp_utf16le_strings(d1->d_short_name, d1->d_short_name_nbytes / 2,
                                d2->d_short_name, d2->d_short_name_nbytes / 2,
                                true))
            cmp_utf16le_strings(d1->d_short_name, d1->d_short_name_nbytes / 2,
                                d2->d_short_name, d2->d_short_name_nbytes / 2,
                                true))


@@ -980,59 +995,108 @@ check_hard_link(struct wim_dentry *dentry, void *_ignore)
        return WIMLIB_ERR_IMAGES_ARE_DIFFERENT;
 }
 
        return WIMLIB_ERR_IMAGES_ARE_DIFFERENT;
 }
 

+static const struct {
+       u32 flag;
+       const char *name;
+} file_attr_flags[] = {
+       {FILE_ATTRIBUTE_READONLY,            "READONLY"},
+       {FILE_ATTRIBUTE_HIDDEN,              "HIDDEN"},
+       {FILE_ATTRIBUTE_SYSTEM,              "SYSTEM"},
+       {FILE_ATTRIBUTE_DIRECTORY,           "DIRECTORY"},
+       {FILE_ATTRIBUTE_ARCHIVE,             "ARCHIVE"},
+       {FILE_ATTRIBUTE_DEVICE,              "DEVICE"},
+       {FILE_ATTRIBUTE_NORMAL,              "NORMAL"},
+       {FILE_ATTRIBUTE_TEMPORARY,           "TEMPORARY"},
+       {FILE_ATTRIBUTE_SPARSE_FILE,         "SPARSE_FILE"},
+       {FILE_ATTRIBUTE_REPARSE_POINT,       "REPARSE_POINT"},
+       {FILE_ATTRIBUTE_COMPRESSED,          "COMPRESSED"},
+       {FILE_ATTRIBUTE_OFFLINE,             "OFFLINE"},
+       {FILE_ATTRIBUTE_NOT_CONTENT_INDEXED, "NOT_CONTENT_INDEXED"},
+       {FILE_ATTRIBUTE_ENCRYPTED,           "ENCRYPTED"},
+       {FILE_ATTRIBUTE_VIRTUAL,             "VIRTUAL"},
+};
+
+static int
+cmp_attributes(const struct wim_inode *inode1,
+              const struct wim_inode *inode2, int cmp_flags)
+{
+       const u32 changed = inode1->i_attributes ^ inode2->i_attributes;
+       const u32 set = inode2->i_attributes & ~inode1->i_attributes;
+       const u32 cleared = inode1->i_attributes & ~inode2->i_attributes;
+
+       /* NORMAL may change, but it must never be set along with other
+        * attributes. */
+       if ((inode2->i_attributes & FILE_ATTRIBUTE_NORMAL) &&
+           (inode2->i_attributes & ~FILE_ATTRIBUTE_NORMAL))
+               goto mismatch;
+
+       /* DIRECTORY must not change. */
+       if (changed & FILE_ATTRIBUTE_DIRECTORY)
+               goto mismatch;
+
+       /* REPARSE_POINT may be cleared in UNIX mode if the inode is not a
+        * symlink. */
+       if ((changed & FILE_ATTRIBUTE_REPARSE_POINT) &&
+           !((cleared & FILE_ATTRIBUTE_REPARSE_POINT) &&
+             (cmp_flags & WIMLIB_CMP_FLAG_UNIX_MODE) &&
+             !inode_is_symlink(inode1)))
+               goto mismatch;
+
+       /* SPARSE_FILE may be cleared in UNIX and NTFS-3G modes, or in Windows
+        * mode if the inode is a directory. */
+       if ((changed & FILE_ATTRIBUTE_SPARSE_FILE) &&
+           !((cleared & FILE_ATTRIBUTE_SPARSE_FILE) &&
+             ((cmp_flags & (WIMLIB_CMP_FLAG_UNIX_MODE |
+                            WIMLIB_CMP_FLAG_NTFS_3G_MODE)) ||
+              ((cmp_flags & WIMLIB_CMP_FLAG_WINDOWS_MODE) &&
+               (inode1->i_attributes & FILE_ATTRIBUTE_DIRECTORY)))))
+               goto mismatch;
+
+       /* COMPRESSED may change in UNIX and NTFS-3G modes.  (It *should* be
+        * preserved in NTFS-3G mode, but it's not implemented yet.) */
+       if ((changed & FILE_ATTRIBUTE_COMPRESSED) &&
+           !(cmp_flags & (WIMLIB_CMP_FLAG_UNIX_MODE |
+                          WIMLIB_CMP_FLAG_NTFS_3G_MODE)))
+               goto mismatch;
+
+       /* All other attributes can change in UNIX mode, but not in any other
+        * mode. */
+       if ((changed & ~(FILE_ATTRIBUTE_NORMAL |
+                        FILE_ATTRIBUTE_DIRECTORY |
+                        FILE_ATTRIBUTE_REPARSE_POINT |
+                        FILE_ATTRIBUTE_SPARSE_FILE |
+                        FILE_ATTRIBUTE_COMPRESSED)) &&
+           !(cmp_flags & WIMLIB_CMP_FLAG_UNIX_MODE))
+               goto mismatch;
+
+       return 0;
+
+mismatch:
+       ERROR("Attribute mismatch for %"TS": 0x%08"PRIx32" vs. 0x%08"PRIx32":",
+             inode_any_full_path(inode1), inode1->i_attributes,
+             inode2->i_attributes);
+       for (size_t i = 0; i < ARRAY_LEN(file_attr_flags); i++) {
+               u32 flag = file_attr_flags[i].flag;
+               if (changed & flag) {
+                       fprintf(stderr, "\tFILE_ATTRIBUTE_%s was %s\n",
+                               file_attr_flags[i].name,
+                               (set & flag) ? "set" : "cleared");
+               }
+       }
+       return WIMLIB_ERR_IMAGES_ARE_DIFFERENT;
+}
+

 static int
 cmp_inodes(const struct wim_inode *inode1, const struct wim_inode *inode2,
           const struct wim_image_metadata *imd1,
           const struct wim_image_metadata *imd2, int cmp_flags)
 {
 static int
 cmp_inodes(const struct wim_inode *inode1, const struct wim_inode *inode2,
           const struct wim_image_metadata *imd1,
           const struct wim_image_metadata *imd2, int cmp_flags)
 {

-       const u32 attrib_diff = inode1->i_attributes ^ inode2->i_attributes;
-       bool reparse_point_should_preserved = true;
+       int ret;

 
        /* Compare attributes  */
 
        /* Compare attributes  */

-       if (cmp_flags & WIMLIB_CMP_FLAG_ATTRIBUTES_NOT_PRESERVED) {
-
-               /* In this mode, we expect that most attributes are not
-                * preserved.  However, FILE_ATTRIBUTE_DIRECTORY should always
-                * match.  */
-               if (attrib_diff & FILE_ATTRIBUTE_DIRECTORY)
-                       goto attrib_mismatch;
-
-               /* We may also expect FILE_ATTRIBUTE_REPARSE_POINT to be
-                * preserved for symlinks.  It also shouldn't be set if it
-                * wasn't set before.  */
-
-               if ((cmp_flags & WIMLIB_CMP_FLAG_IMAGE2_SHOULD_HAVE_SYMLINKS) &&
-                   inode_is_symlink(inode1))
-                       reparse_point_should_preserved = true;
-               else
-                       reparse_point_should_preserved = false;
-
-               if ((attrib_diff & FILE_ATTRIBUTE_REPARSE_POINT) &&
-                   (reparse_point_should_preserved ||
-                    (inode2->i_attributes & FILE_ATTRIBUTE_REPARSE_POINT)))
-                       goto attrib_mismatch;
-       } else {
-
-               /* Most attributes should be preserved.  */
-
-               /* Nothing other than COMPRESSED and NORMAL should have changed.
-                */
-               if (attrib_diff & ~(FILE_ATTRIBUTE_COMPRESSED |
-                                   FILE_ATTRIBUTE_NORMAL))
-                       goto attrib_mismatch;
-
-               /* COMPRESSED shouldn't have changed unless specifically
-                * excluded.  */
-               if ((attrib_diff & FILE_ATTRIBUTE_COMPRESSED) &&
-                   !(cmp_flags & WIMLIB_CMP_FLAG_COMPRESSION_NOT_PRESERVED))
-                       goto attrib_mismatch;
-
-               /* We allow NORMAL to change, but not if the file ended up with
-                * other attributes set as well.  */
-               if ((attrib_diff & FILE_ATTRIBUTE_NORMAL) &&
-                   (inode2->i_attributes & ~FILE_ATTRIBUTE_NORMAL))
-                       goto attrib_mismatch;
-       }
+       ret = cmp_attributes(inode1, inode2, cmp_flags);
+       if (ret)
+               return ret;

 
        /* Compare security descriptors  */
        if (inode_has_security_descriptor(inode1)) {
 
        /* Compare security descriptors  */
        if (inode_has_security_descriptor(inode1)) {


@@ -1047,7 +1111,7 @@ cmp_inodes(const struct wim_inode *inode1, const struct wim_inode *inode2,
                                      inode_any_full_path(inode1));
                                return WIMLIB_ERR_IMAGES_ARE_DIFFERENT;
                        }
                                      inode_any_full_path(inode1));
                                return WIMLIB_ERR_IMAGES_ARE_DIFFERENT;
                        }

-               } else if (!(cmp_flags & WIMLIB_CMP_FLAG_SECURITY_NOT_PRESERVED)) {
+               } else if (!(cmp_flags & WIMLIB_CMP_FLAG_UNIX_MODE)) {

                        ERROR("%"TS" has a security descriptor in the first image but "
                              "not in the second image!", inode_any_full_path(inode1));
                        return WIMLIB_ERR_IMAGES_ARE_DIFFERENT;
                        ERROR("%"TS" has a security descriptor in the first image but "
                              "not in the second image!", inode_any_full_path(inode1));
                        return WIMLIB_ERR_IMAGES_ARE_DIFFERENT;


@@ -1066,7 +1130,8 @@ cmp_inodes(const struct wim_inode *inode1, const struct wim_inode *inode2,
                const struct wim_inode_stream *strm2;
 
                if (strm1->stream_type == STREAM_TYPE_REPARSE_POINT &&
                const struct wim_inode_stream *strm2;
 
                if (strm1->stream_type == STREAM_TYPE_REPARSE_POINT &&

-                   !reparse_point_should_preserved)
+                   (cmp_flags & WIMLIB_CMP_FLAG_UNIX_MODE &&
+                    !inode_is_symlink(inode1)))

                        continue;
 
                if (strm1->stream_type == STREAM_TYPE_UNKNOWN)
                        continue;
 
                if (strm1->stream_type == STREAM_TYPE_UNKNOWN)


@@ -1078,7 +1143,7 @@ cmp_inodes(const struct wim_inode *inode1, const struct wim_inode *inode2,
                if (!strm2) {
                        /* Corresponding stream not found  */
                        if (stream_is_named(strm1) &&
                if (!strm2) {
                        /* Corresponding stream not found  */
                        if (stream_is_named(strm1) &&

-                           (cmp_flags & WIMLIB_CMP_FLAG_ADS_NOT_PRESERVED))
+                           (cmp_flags & WIMLIB_CMP_FLAG_UNIX_MODE))

                                continue;
                        ERROR("Stream of %"TS" is missing in second image; "
                              "type %d, named=%d, empty=%d",
                                continue;
                        ERROR("Stream of %"TS" is missing in second image; "
                              "type %d, named=%d, empty=%d",


@@ -1097,13 +1162,6 @@ cmp_inodes(const struct wim_inode *inode1, const struct wim_inode *inode2,
        }
 
        return 0;
        }
 
        return 0;

-
-attrib_mismatch:
-       ERROR("Attribute mismatch; %"TS" has attributes 0x%08"PRIx32" "
-             "in first image but attributes 0x%08"PRIx32" in second image",
-             inode_any_full_path(inode1), inode1->i_attributes,
-             inode2->i_attributes);
-       return WIMLIB_ERR_IMAGES_ARE_DIFFERENT;

 }
 
 static int
 }
 
 static int

diff --git a/tests/wlfuzz.c b/tests/wlfuzz.c
index 51607869fb11913b3ddc0a0b9e7093e0658eb34f..3db4630ebbc5170724841c5e6658d85c7c931be7 100644 (file)
--- a/tests/wlfuzz.c
+++ b/tests/wlfuzz.c
@@ -739,20 +739,17 @@ op__apply_and_capture_test(void)
                extract_flags |= WIMLIB_EXTRACT_FLAG_STRICT_SHORT_NAMES;
                extract_flags |= WIMLIB_EXTRACT_FLAG_STRICT_TIMESTAMPS;
                add_flags |= WIMLIB_ADD_FLAG_NTFS;
                extract_flags |= WIMLIB_EXTRACT_FLAG_STRICT_SHORT_NAMES;
                extract_flags |= WIMLIB_EXTRACT_FLAG_STRICT_TIMESTAMPS;
                add_flags |= WIMLIB_ADD_FLAG_NTFS;

-               cmp_flags |= WIMLIB_CMP_FLAG_COMPRESSION_NOT_PRESERVED;
+               cmp_flags |= WIMLIB_CMP_FLAG_NTFS_3G_MODE;

                create_ntfs_volume(TMP_TARGET_NAME);
        } else
 #endif
        {
 #ifdef __WIN32__
                printf("applying in Windows mode\n");
                create_ntfs_volume(TMP_TARGET_NAME);
        } else
 #endif
        {
 #ifdef __WIN32__
                printf("applying in Windows mode\n");

+               cmp_flags |= WIMLIB_CMP_FLAG_WINDOWS_MODE;

 #else /* __WIN32__ */
                printf("applying in UNIX mode\n");
 #else /* __WIN32__ */
                printf("applying in UNIX mode\n");

-               cmp_flags |= WIMLIB_CMP_FLAG_SHORT_NAMES_NOT_PRESERVED;
-               cmp_flags |= WIMLIB_CMP_FLAG_ATTRIBUTES_NOT_PRESERVED;
-               cmp_flags |= WIMLIB_CMP_FLAG_SECURITY_NOT_PRESERVED;
-               cmp_flags |= WIMLIB_CMP_FLAG_ADS_NOT_PRESERVED;
-               cmp_flags |= WIMLIB_CMP_FLAG_IMAGE2_SHOULD_HAVE_SYMLINKS;
+               cmp_flags |= WIMLIB_CMP_FLAG_UNIX_MODE;

 #endif /* !__WIN32__ */
        }
        add_flags |= WIMLIB_ADD_FLAG_NORPFIX;
 #endif /* !__WIN32__ */
        }
        add_flags |= WIMLIB_ADD_FLAG_NORPFIX;


@@ -891,7 +888,8 @@ op__wimboot_test(void)
        printf("comparing wimboot.wim:%d with wim%d:%d\n",
               image, index2, image_count + 1);
 
        printf("comparing wimboot.wim:%d with wim%d:%d\n",
               image, index2, image_count + 1);
 

-       CHECK_RET(wimlib_compare_images(wim, image, wim2, image_count + 1, 0));
+       CHECK_RET(wimlib_compare_images(wim, image, wim2, image_count + 1,
+                                       WIMLIB_CMP_FLAG_WINDOWS_MODE));

 
        wimlib_free(wim);
        wimlib_free(wim2);
 
        wimlib_free(wim);
        wimlib_free(wim2);