From 8b74835779d25a217158ec2a3a3b5ade11cb9e4b Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Sun, 9 Apr 2023 11:39:36 -0700 Subject: [PATCH] Fix wlfuzz and enable in CI - Fix build failure on Linux by linking with the math library. - Don't hard-code the temporary directory to E:\ on Windows. - Relax some comparisons that don't work reliably on Windows: - Security descriptor - Last access time - Sparse file attribute (clear only) - Relax the timestamp comparisons when running on an ext4 filesystem, since ext4 doesn't support the full timestamp range that Windows does. Also, generate more timestamps that are close to the present date. - Make the command-line argument give the number of seconds to run, rather than the number of iterations. This makes it possible to run wlfuzz for 2 minutes in GitHub Actions, like the libFuzzer jobs. - Increase coverage by using a different random seed on each run. - Make wlfuzz initialize wimlib with STRICT_{CAPTURE,APPLY}_PRIVILIGES on Windows. I.e., don't allow running wlfuzz as non-Administrator. - Print security descriptors and timestamps when they differ. - Add GitHub Actions jobs that run wlfuzz on Linux and Windows. --- .github/workflows/ci.yml | 49 +++++++++ Makefile.am | 3 + include/wimlib/test_support.h | 4 + src/test_support.c | 195 +++++++++++++++++++++++++--------- src/timestamp.c | 2 + tests/wlfuzz.c | 122 ++++++++++++++------- 6 files changed, 285 insertions(+), 90 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 31a7aad6..de97a82e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -202,3 +202,52 @@ jobs: run: | tools/libFuzzer/fuzz.sh --time=120 ${{matrix.sanitizer}} \ ${{matrix.target}} + + fuzz-with-wlfuzz-linux: + name: Fuzz with wlfuzz (Linux, ${{matrix.sanitizer}}) + strategy: + matrix: + include: + - sanitizer: none + cflags: + - sanitizer: ASAN + cflags: -fsanitize=address -fno-sanitize-recover=address + - sanitizer: UBSAN + cflags: -fsanitize=undefined -fno-sanitize-recover=undefined + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y clang $DEPENDENCIES + - run: ./bootstrap + - run: ./configure --enable-test-support CC=clang CFLAGS="$DEF_CFLAGS ${{matrix.cflags}}" + - run: make -j8 tests/wlfuzz + - run: TMPDIR=$PWD/tmp.wlfuzz tests/wlfuzz 120 + + fuzz-with-wlfuzz-windows: + name: Fuzz with wlfuzz (Windows) + runs-on: windows-latest + defaults: + run: + shell: msys2 {0} + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 # Need tags for tools/get-version-number.sh + - uses: msys2/setup-msys2@v2 + with: + msystem: MINGW64 + update: true + install: > + autoconf + automake + git + libtool + make + mingw-w64-x86_64-gcc + pkgconf + - run: CFLAGS="$DEF_CFLAGS" ./tools/windows-build.sh -- --enable-test-support + - run: make tests/wlfuzz.exe + - run: TMPDIR=$PWD/tmp.wlfuzz tests/wlfuzz 120 diff --git a/Makefile.am b/Makefile.am index f11542f5..00ff3663 100644 --- a/Makefile.am +++ b/Makefile.am @@ -189,6 +189,9 @@ endif if ENABLE_TEST_SUPPORT libwim_la_SOURCES += src/test_support.c \ include/wimlib/test_support.h +if !WINDOWS_NATIVE_BUILD +PLATFORM_LIBS += -lm +endif endif libwim_la_CFLAGS = \ diff --git a/include/wimlib/test_support.h b/include/wimlib/test_support.h index c321b323..909b643c 100644 --- a/include/wimlib/test_support.h +++ b/include/wimlib/test_support.h @@ -13,6 +13,10 @@ #define WIMLIB_CMP_FLAG_UNIX_MODE 0x00000001 #define WIMLIB_CMP_FLAG_NTFS_3G_MODE 0x00000002 #define WIMLIB_CMP_FLAG_WINDOWS_MODE 0x00000004 +#define WIMLIB_CMP_FLAG_EXT4 0x00000008 + +WIMLIBAPI void +wimlib_seed_random(u64 seed); WIMLIBAPI int wimlib_compare_images(WIMStruct *wim1, int image1, diff --git a/src/test_support.c b/src/test_support.c index 96305f06..6a438fd9 100644 --- a/src/test_support.c +++ b/src/test_support.c @@ -38,6 +38,11 @@ #include #include #include +#ifdef _WIN32 +# include +# include +# undef ERROR +#endif #include "wimlib.h" #include "wimlib/endianness.h" @@ -50,6 +55,7 @@ #include "wimlib/scan.h" #include "wimlib/security_descriptor.h" #include "wimlib/test_support.h" +#include "wimlib/timestamp.h" #include "wimlib/unix_data.h" #include "wimlib/xattr.h" @@ -63,20 +69,26 @@ struct generation_context { bool metadata_only; }; +static u64 random_state; + +WIMLIBAPI void +wimlib_seed_random(u64 seed) +{ + random_state = seed; +} + static u32 rand32(void) { - static u64 state = 0x55DB93D0AB838771; - - /* A simple linear congruential generator */ - state = (state * 25214903917 + 11) & ((1ULL << 48) - 1); - return state >> 16; + /* A simple linear congruential generator */ + random_state = (random_state * 25214903917 + 11) % (1ULL << 48); + return random_state >> 16; } static bool randbool(void) { - return (rand32() & 1) != 0; + return rand32() % 2; } static u8 @@ -100,10 +112,18 @@ rand64(void) static u64 generate_random_timestamp(void) { - /* When setting timestamps on Windows: + u64 ts; + + if (randbool()) + ts = rand64(); + else + ts = time_t_to_wim_timestamp(rand64() % (1ULL << 34)); + /* + * When setting timestamps on Windows: * - 0 is a special value meaning "not specified" - * - if the high bit is set you get STATUS_INVALID_PARAMETER */ - return (1 + rand64()) & ~(1ULL << 63); + * - if the high bit is set you get STATUS_INVALID_PARAMETER + */ + return max(1, ts % (1ULL << 63)); } static inline bool @@ -1248,14 +1268,11 @@ cmp_attributes(const struct wim_inode *inode1, !inode_is_symlink(inode1))) goto mismatch; - /* SPARSE_FILE may be cleared in UNIX and NTFS-3G modes, or in Windows - * mode if the inode is a directory. */ + /* SPARSE_FILE may be cleared. This is true in UNIX and NTFS-3G modes. + * In Windows mode it should only be true for directories, but even on + * nondirectories it doesn't work 100% of the time for some reason. */ if ((changed & FILE_ATTRIBUTE_SPARSE_FILE) && - !((cleared & FILE_ATTRIBUTE_SPARSE_FILE) && - ((cmp_flags & (WIMLIB_CMP_FLAG_UNIX_MODE | - WIMLIB_CMP_FLAG_NTFS_3G_MODE)) || - ((cmp_flags & WIMLIB_CMP_FLAG_WINDOWS_MODE) && - (inode1->i_attributes & FILE_ATTRIBUTE_DIRECTORY))))) + !(cleared & FILE_ATTRIBUTE_SPARSE_FILE)) goto mismatch; /* COMPRESSED may change in UNIX and NTFS-3G modes. (It *should* be @@ -1292,6 +1309,73 @@ mismatch: return WIMLIB_ERR_IMAGES_ARE_DIFFERENT; } +static void +print_security_descriptor(const void *desc, size_t size, FILE *fp) +{ + print_byte_field(desc, size, fp); +#ifdef _WIN32 + wchar_t *str = NULL; + ConvertSecurityDescriptorToStringSecurityDescriptorW( + (void *)desc, + SDDL_REVISION_1, + OWNER_SECURITY_INFORMATION | + GROUP_SECURITY_INFORMATION | + DACL_SECURITY_INFORMATION | + SACL_SECURITY_INFORMATION, + &str, + NULL); + if (str) { + fprintf(fp, " [ %ls ]", str); + LocalFree(str); + } +#endif /* _WIN32 */ +} + +static int +cmp_security(const struct wim_inode *inode1, const struct wim_inode *inode2, + const struct wim_image_metadata *imd1, + const struct wim_image_metadata *imd2, int cmp_flags) +{ + /* + * Unfortunately this has to be disabled on Windows for now, since + * Windows changes security descriptors upon backup/restore in ways that + * are difficult to replicate... + */ + if (cmp_flags & WIMLIB_CMP_FLAG_WINDOWS_MODE) + return 0; + + if (inode_has_security_descriptor(inode1)) { + if (inode_has_security_descriptor(inode2)) { + const void *desc1 = imd1->security_data->descriptors[inode1->i_security_id]; + const void *desc2 = imd2->security_data->descriptors[inode2->i_security_id]; + size_t size1 = imd1->security_data->sizes[inode1->i_security_id]; + size_t size2 = imd2->security_data->sizes[inode2->i_security_id]; + + if (size1 != size2 || memcmp(desc1, desc2, size1)) { + ERROR("Security descriptor of %"TS" differs!", + inode_any_full_path(inode1)); + fprintf(stderr, "desc1="); + print_security_descriptor(desc1, size1, stderr); + fprintf(stderr, "\ndesc2="); + print_security_descriptor(desc2, size2, stderr); + fprintf(stderr, "\n"); + return WIMLIB_ERR_IMAGES_ARE_DIFFERENT; + } + } else if (!(cmp_flags & WIMLIB_CMP_FLAG_UNIX_MODE)) { + ERROR("%"TS" has a security descriptor in the first image but " + "not in the second image!", inode_any_full_path(inode1)); + return WIMLIB_ERR_IMAGES_ARE_DIFFERENT; + } + } else if (inode_has_security_descriptor(inode2)) { + /* okay --- consider it acceptable if a default security + * descriptor was assigned */ + /*ERROR("%"TS" has a security descriptor in the second image but "*/ + /*"not in the first image!", inode_any_full_path(inode1));*/ + /*return WIMLIB_ERR_IMAGES_ARE_DIFFERENT;*/ + } + return 0; +} + static int cmp_object_ids(const struct wim_inode *inode1, const struct wim_inode *inode2, int cmp_flags) @@ -1501,26 +1585,60 @@ cmp_xattrs(const struct wim_inode *inode1, const struct wim_inode *inode2, } } +/* + * ext4 only supports timestamps from years 1901 to 2446, more specifically the + * range [-0x80000000, 0x380000000) seconds relative to the start of UNIX epoch. + */ +static bool +in_ext4_range(u64 ts) +{ + return ts >= time_t_to_wim_timestamp(-0x80000000LL) && + ts < time_t_to_wim_timestamp(0x380000000LL); +} + +static bool +timestamps_differ(u64 ts1, u64 ts2, int cmp_flags) +{ + if (ts1 == ts2) + return false; + if ((cmp_flags & WIMLIB_CMP_FLAG_EXT4) && + (!in_ext4_range(ts1) || !in_ext4_range(ts2))) + return false; + return true; +} + static int cmp_timestamps(const struct wim_inode *inode1, const struct wim_inode *inode2, int cmp_flags) { - if (inode1->i_creation_time != inode2->i_creation_time && + if (timestamps_differ(inode1->i_creation_time, + inode2->i_creation_time, cmp_flags) && !(cmp_flags & WIMLIB_CMP_FLAG_UNIX_MODE)) { - ERROR("Creation time of %"TS" differs", - inode_any_full_path(inode1)); + ERROR("Creation time of %"TS" differs; %"PRIu64" != %"PRIu64, + inode_any_full_path(inode1), + inode1->i_creation_time, inode2->i_creation_time); return WIMLIB_ERR_IMAGES_ARE_DIFFERENT; } - if (inode1->i_last_write_time != inode2->i_last_write_time) { - ERROR("Last write time of %"TS" differs", - inode_any_full_path(inode1)); + if (timestamps_differ(inode1->i_last_write_time, + inode2->i_last_write_time, cmp_flags)) { + ERROR("Last write time of %"TS" differs; %"PRIu64" != %"PRIu64, + inode_any_full_path(inode1), + inode1->i_last_write_time, inode2->i_last_write_time); return WIMLIB_ERR_IMAGES_ARE_DIFFERENT; } - if (inode1->i_last_access_time != inode2->i_last_access_time) { - ERROR("Last access time of %"TS" differs", - inode_any_full_path(inode1)); + if (timestamps_differ(inode1->i_last_access_time, + inode2->i_last_access_time, cmp_flags) && + /* + * On Windows, sometimes a file's last access time will end up as + * the current time rather than the expected time. Maybe caused by + * some OS process scanning the files? + */ + !(cmp_flags & WIMLIB_CMP_FLAG_WINDOWS_MODE)) { + ERROR("Last access time of %"TS" differs; %"PRIu64" != %"PRIu64, + inode_any_full_path(inode1), + inode1->i_last_access_time, inode2->i_last_access_time); return WIMLIB_ERR_IMAGES_ARE_DIFFERENT; } @@ -1540,30 +1658,9 @@ cmp_inodes(const struct wim_inode *inode1, const struct wim_inode *inode2, return ret; /* Compare security descriptors */ - if (inode_has_security_descriptor(inode1)) { - if (inode_has_security_descriptor(inode2)) { - const void *desc1 = imd1->security_data->descriptors[inode1->i_security_id]; - const void *desc2 = imd2->security_data->descriptors[inode2->i_security_id]; - size_t size1 = imd1->security_data->sizes[inode1->i_security_id]; - size_t size2 = imd2->security_data->sizes[inode2->i_security_id]; - - if (size1 != size2 || memcmp(desc1, desc2, size1)) { - ERROR("Security descriptor of %"TS" differs!", - inode_any_full_path(inode1)); - return WIMLIB_ERR_IMAGES_ARE_DIFFERENT; - } - } else if (!(cmp_flags & WIMLIB_CMP_FLAG_UNIX_MODE)) { - ERROR("%"TS" has a security descriptor in the first image but " - "not in the second image!", inode_any_full_path(inode1)); - return WIMLIB_ERR_IMAGES_ARE_DIFFERENT; - } - } else if (inode_has_security_descriptor(inode2)) { - /* okay --- consider it acceptable if a default security - * descriptor was assigned */ - /*ERROR("%"TS" has a security descriptor in the second image but "*/ - /*"not in the first image!", inode_any_full_path(inode1));*/ - /*return WIMLIB_ERR_IMAGES_ARE_DIFFERENT;*/ - } + ret = cmp_security(inode1, inode2, imd1, imd2, cmp_flags); + if (ret) + return ret; /* Compare streams */ for (unsigned i = 0; i < inode1->i_num_streams; i++) { diff --git a/src/timestamp.c b/src/timestamp.c index ffd67ec5..70b2e5f3 100644 --- a/src/timestamp.c +++ b/src/timestamp.c @@ -95,6 +95,7 @@ wim_timestamp_to_timespec(u64 timestamp) .tv_nsec = (timestamp % TICKS_PER_SECOND) * NANOSECONDS_PER_TICK, }; } +#endif /* !_WIN32 */ /* UNIX timestamps to Windows NT timestamps */ @@ -104,6 +105,7 @@ time_t_to_wim_timestamp(time_t t) return ((u64)t + EPOCH_DISTANCE) * TICKS_PER_SECOND; } +#ifndef _WIN32 u64 timeval_to_wim_timestamp(const struct timeval *tv) { diff --git a/tests/wlfuzz.c b/tests/wlfuzz.c index eccedbaa..e3fe909d 100644 --- a/tests/wlfuzz.c +++ b/tests/wlfuzz.c @@ -3,7 +3,7 @@ */ /* - * Copyright (C) 2015-2021 Eric Biggers + * Copyright 2015-2023 Eric Biggers * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -61,6 +61,7 @@ #include #include #include +#include #ifdef WITH_NTFS_3G # include #endif @@ -70,6 +71,9 @@ # include # include # include +#else +# include +# include #endif #include "wimlib.h" @@ -89,6 +93,9 @@ static bool wimfile_in_use[MAX_NUM_WIMS]; static int in_use_wimfile_indices[MAX_NUM_WIMS]; static int num_wimfiles_in_use = 0; +#ifndef _WIN32 +static u32 filesystem_type; +#endif static void assertion_failed(int line, const char *format, ...) @@ -120,15 +127,22 @@ static void change_to_temporary_directory(void) { #ifdef _WIN32 - ASSERT(SetCurrentDirectory(L"E:\\"), - "failed to change directory to E:\\"); -#else - const char *tmpdir = getenv("TMPDIR"); - if (!tmpdir) - tmpdir = P_tmpdir; + const wchar_t *tmpdir = _wgetenv(T("TMPDIR")); + + ASSERT(tmpdir != NULL, "TMPDIR must be set"); + _wmkdir(tmpdir); + ASSERT(!_wchdir(tmpdir), + "failed to change to temporary directory '%ls'", tmpdir); +#else /* _WIN32 */ + const char *tmpdir = getenv("TMPDIR") ?: P_tmpdir; + struct statfs fs; + + mkdir(tmpdir, 0700); ASSERT(!chdir(tmpdir), - "failed to change to temporary directory \"%s\": %m", tmpdir); -#endif + "failed to change to temporary directory '%s': %m", tmpdir); + ASSERT(!statfs(".", &fs), "statfs of '%s' failed: %m", tmpdir); + filesystem_type = fs.f_type; +#endif /* !_WIN32 */ } static void __attribute__((unused)) @@ -334,20 +348,26 @@ delete_directory_tree(const tchar *name) #endif /* !_WIN32 */ -static uint32_t +static u64 random_state; + +static u32 rand32(void) { - static uint64_t state; - - /* A simple linear congruential generator */ - state = (state * 25214903917 + 11) & (((uint64_t)1 << 48) - 1); - return state >> 16; + /* A simple linear congruential generator */ + random_state = (random_state * 25214903917 + 11) % (1ULL << 48); + return random_state >> 16; } -static inline bool +static bool randbool(void) { - return rand32() & 1; + return rand32() % 2; +} + +static u64 +rand64(void) +{ + return ((u64)rand32() << 32) | rand32(); } static tchar wimfile[32]; @@ -498,7 +518,7 @@ get_random_write_flags(void) return write_flags; } -static uint32_t +static u32 get_random_chunk_size(int min_order, int max_order) { return 1 << (min_order + (rand32() % (max_order - min_order + 1))); @@ -511,8 +531,8 @@ op__create_new_wim(void) const tchar *wimfile; enum wimlib_compression_type ctype = WIMLIB_COMPRESSION_TYPE_NONE; - uint32_t chunk_size = 0; - uint32_t solid_chunk_size = 0; + u32 chunk_size = 0; + u32 solid_chunk_size = 0; int write_flags; WIMStruct *wim; @@ -786,6 +806,8 @@ op__apply_and_capture_test(void) extract_flags |= WIMLIB_EXTRACT_FLAG_UNIX_DATA; add_flags |= WIMLIB_ADD_FLAG_UNIX_DATA; cmp_flags |= WIMLIB_CMP_FLAG_UNIX_MODE; + if (filesystem_type == EXT4_SUPER_MAGIC) + cmp_flags |= WIMLIB_CMP_FLAG_EXT4; #endif /* !_WIN32 */ } add_flags |= WIMLIB_ADD_FLAG_NORPFIX; @@ -812,10 +834,15 @@ op__apply_and_capture_test(void) #ifdef _WIN32 -/* Enumerate and unregister all backing WIMs from the specified volume */ +/* + * Enumerate and unregister all backing WIMs from the volume containing the + * current directory. + */ static void -unregister_all_backing_wims(const tchar drive_letter) +unregister_all_backing_wims(void) { + wchar_t full_path[MAX_PATH]; + DWORD path_len; wchar_t volume[7]; HANDLE h; void *overlay_list; @@ -826,8 +853,12 @@ unregister_all_backing_wims(const tchar drive_letter) WIM_PROVIDER_REMOVE_OVERLAY_INPUT wim; } in; - wsprintf(volume, L"\\\\.\\%lc:", drive_letter); + path_len = GetFullPathName(L".", ARRAY_LEN(full_path), full_path, NULL); + ASSERT(path_len > 0, + "Failed to get full path of current directory; error=%u", + (unsigned)GetLastError()); + wsprintf(volume, L"\\\\.\\%lc:", full_path[0]); h = CreateFile(volume, GENERIC_READ | GENERIC_WRITE, FILE_SHARE_VALID_FLAGS, NULL, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL); @@ -859,11 +890,11 @@ unregister_all_backing_wims(const tchar drive_letter) ASSERT(DeviceIoControl(h, FSCTL_REMOVE_OVERLAY, &in, sizeof(in), NULL, 0, &bytes_returned, NULL), "FSCTL_REMOVE_OVERLAY failed; error=%u", - (unsigned )GetLastError()); + (unsigned)GetLastError()); if (entry->NextEntryOffset == 0) break; entry = (const WIM_PROVIDER_OVERLAY_ENTRY *) - ((const uint8_t *)entry + entry->NextEntryOffset); + ((const u8 *)entry + entry->NextEntryOffset); } free(overlay_list); CloseHandle(h); @@ -884,7 +915,7 @@ op__wimboot_test(void) index = select_random_wimfile_index(); - unregister_all_backing_wims(L'E'); + unregister_all_backing_wims(); copy_file(get_wimfile(index), L"wimboot.wim"); CHECK_RET(wimlib_open_wim(L"wimboot.wim", 0, &wim)); @@ -953,7 +984,7 @@ op__split_test(void) WIMStruct *wim; WIMStruct *swm; WIMStruct *joined_wim; - uint64_t part_size; + u64 part_size; int write_flags; const tchar *globs[] = { T("tmp*.swm") }; int image_count; @@ -1052,27 +1083,36 @@ int wmain(int argc, wchar_t **argv); int main(int argc, tchar **argv) { - unsigned long long num_iterations; - - if (argc < 2) { - num_iterations = ULLONG_MAX; - printf("Starting test runner\n"); - } else { - num_iterations = tstrtoull(argv[1], NULL, 10); - printf("Starting test runner with %llu iterations\n", - num_iterations); - } + unsigned long time_limit = 0; + time_t start_time; + u64 i; + + /* If you want to make the tests deterministic, delete this line. */ + random_state = ((u64)time(NULL) << 16) ^ getpid(); + + if (argc >= 2) + time_limit = tstrtoul(argv[1], NULL, 10); + + if (time_limit == 0) + printf("Starting wlfuzz with no time limit\n"); + else + printf("Starting wlfuzz with time limit of %lu seconds\n", + time_limit); - CHECK_RET(wimlib_global_init(0)); + CHECK_RET(wimlib_global_init(WIMLIB_INIT_FLAG_STRICT_APPLY_PRIVILEGES | + WIMLIB_INIT_FLAG_STRICT_CAPTURE_PRIVILEGES)); wimlib_set_print_errors(true); + wimlib_seed_random(rand64()); change_to_temporary_directory(); - for (int i = 0; i < MAX_NUM_WIMS; i++) + for (i = 0; i < MAX_NUM_WIMS; i++) ASSERT(!tunlink(get_wimfile(i)) || errno == ENOENT, "unlink: %m"); - for (unsigned long long i = 0; i < num_iterations; i++) { - printf("--> iteration %llu\n", i); + i = 0; + start_time = time(NULL); + while (time_limit == 0 || time(NULL) < start_time + time_limit) { + printf("--> iteration %"PRIu64"\n", ++i); (*operation_table[rand32() % ARRAY_LEN(operation_table)])(); } -- 2.43.0