X-Git-Url: https://wimlib.net/git/?a=blobdiff_plain;f=src%2Ftest_support.c;h=4df34d4cc8a8f06278313ddac92da82333121ecf;hb=681d8fcc1af3b0eb94031311e51cb0f422382ee6;hp=313ad0858632c20cf060cbc0d86ee501cd526c87;hpb=76689b1cac26c545260568997ae7fb949846f302;p=wimlib diff --git a/src/test_support.c b/src/test_support.c index 313ad085..4df34d4c 100644 --- a/src/test_support.c +++ b/src/test_support.c @@ -3,7 +3,7 @@ */ /* - * Copyright (C) 2015-2016 Eric Biggers + * Copyright (C) 2015-2017 Eric Biggers * * This file is free software; you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -35,6 +35,9 @@ #include #include +#include +#include +#include #include "wimlib.h" #include "wimlib/endianness.h" @@ -42,10 +45,13 @@ #include "wimlib/metadata.h" #include "wimlib/dentry.h" #include "wimlib/inode.h" +#include "wimlib/object_id.h" #include "wimlib/reparse.h" #include "wimlib/scan.h" #include "wimlib/security_descriptor.h" #include "wimlib/test_support.h" +#include "wimlib/unix_data.h" +#include "wimlib/xattr.h" /*----------------------------------------------------------------------------* * File tree generation * @@ -100,6 +106,134 @@ generate_random_timestamp(void) return (1 + rand64()) & ~(1ULL << 63); } +static inline bool +is_valid_windows_filename_char(utf16lechar c) +{ + return le16_to_cpu(c) > 31 && + c != cpu_to_le16('/') && + c != cpu_to_le16('<') && + c != cpu_to_le16('>') && + c != cpu_to_le16(':') && + c != cpu_to_le16('"') && + c != cpu_to_le16('/' ) && + c != cpu_to_le16('\\') && + c != cpu_to_le16('|') && + c != cpu_to_le16('?') && + c != cpu_to_le16('*'); +} + +/* Is the character valid in a filename on the current platform? */ +static inline bool +is_valid_filename_char(utf16lechar c) +{ +#ifdef __WIN32__ + return is_valid_windows_filename_char(c); +#else + return c != cpu_to_le16('\0') && c != cpu_to_le16('/'); +#endif +} + +/* Generate a random filename and return its length. */ +static int +generate_random_filename(utf16lechar name[], int max_len, + struct generation_context *ctx) +{ + int len; + + /* Choose the length of the name. */ + switch (rand32() % 8) { + default: + /* short name */ + len = 1 + (rand32() % 6); + break; + case 2: + case 3: + case 4: + /* medium-length name */ + len = 7 + (rand32() % 8); + break; + case 5: + case 6: + /* long name */ + len = 15 + (rand32() % 15); + break; + case 7: + /* very long name */ + len = 30 + (rand32() % 90); + break; + } + len = min(len, max_len); + +retry: + /* Generate the characters in the name. */ + for (int i = 0; i < len; i++) { + do { + name[i] = rand16(); + } while (!is_valid_filename_char(name[i])); + } + + /* Add a null terminator. */ + name[len] = cpu_to_le16('\0'); + + /* Don't generate . and .. */ + if (name[0] == cpu_to_le16('.') && + (len == 1 || (len == 2 && name[1] == cpu_to_le16('.')))) + goto retry; + + return len; +} + +/* The set of characters which are valid in short filenames. */ +static const char valid_short_name_chars[] = { + 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', + 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', + '!', '#', '$', '%', '&', '\'', '(', ')', '-', '@', '^', '_', '`', '{', + '}', '~', + /* Note: Windows does not allow space and 128-255 in short filenames + * (tested on both NTFS and FAT). */ +}; + +static int +generate_short_name_component(utf16lechar p[], int len) +{ + for (int i = 0; i < len; i++) { + char c = valid_short_name_chars[rand32() % + ARRAY_LEN(valid_short_name_chars)]; + p[i] = cpu_to_le16(c); + } + return len; +} + +/* Generate a random short (8.3) filename and return its length. + * The @name array must have length >= 13 (8 + 1 + 3 + 1). */ +static int +generate_random_short_name(utf16lechar name[], struct generation_context *ctx) +{ + /* + * Legal short names on Windows consist of 1 to 8 characters, optionally + * followed by a dot then 1 to 3 more characters. Only certain + * characters are allowed. + */ + int base_len = 1 + (rand32() % 8); + int ext_len = rand32() % 4; + int total_len; + + base_len = generate_short_name_component(name, base_len); + + if (ext_len) { + name[base_len] = cpu_to_le16('.'); + ext_len = generate_short_name_component(&name[base_len + 1], + ext_len); + total_len = base_len + 1 + ext_len; + } else { + total_len = base_len; + } + name[total_len] = cpu_to_le16('\0'); + return total_len; +} + + static const struct { u8 num_subauthorities; u64 identifier_authority; @@ -248,16 +382,172 @@ generate_random_security_descriptor(void *_desc, struct generation_context *ctx) return p - (u8 *)desc; } +static bool +am_root(void) +{ +#ifdef __WIN32__ + return false; +#else + return (getuid() == 0); +#endif +} + +static u32 +generate_uid(void) +{ +#ifdef __WIN32__ + return 0; +#else + if (am_root()) + return rand32(); + return getuid(); +#endif +} + +static u32 +generate_gid(void) +{ +#ifdef __WIN32__ + return 0; +#else + if (am_root()) + return rand32(); + return getgid(); +#endif +} + +#ifdef __WIN32__ +# ifndef S_IFLNK +# define S_IFLNK 0120000 +# endif +# ifndef S_IFSOCK +# define S_IFSOCK 0140000 +# endif +#endif + +static int +set_random_unix_metadata(struct wim_inode *inode) +{ + struct wimlib_unix_data dat; + + dat.uid = generate_uid(); + dat.gid = generate_gid(); + if (inode_is_symlink(inode)) + dat.mode = S_IFLNK | 0777; + else if (inode->i_attributes & FILE_ATTRIBUTE_DIRECTORY) + dat.mode = S_IFDIR | 0700 | (rand32() % 07777); + else if (is_zero_hash(inode_get_hash_of_unnamed_data_stream(inode)) && + randbool() && am_root()) + { + dat.mode = rand32() % 07777; + switch (rand32() % 4) { + case 0: + dat.mode |= S_IFIFO; + break; + case 1: + dat.mode |= S_IFCHR; + dat.rdev = 261; /* /dev/zero */ + break; + case 2: + dat.mode |= S_IFBLK; + dat.rdev = 261; /* /dev/zero */ + break; + default: + dat.mode |= S_IFSOCK; + break; + } + } else { + dat.mode = S_IFREG | 0400 | (rand32() % 07777); + } + dat.rdev = 0; + + if (!inode_set_unix_data(inode, &dat, UNIX_DATA_ALL)) + return WIMLIB_ERR_NOMEM; + + return 0; +} + +static noinline_for_stack int +set_random_xattrs(struct wim_inode *inode) +{ + int num_xattrs = 1 + rand32() % 16; + char entries[8192] _aligned_attribute(4); + struct wimlib_xattr_entry *entry = (void *)entries; + size_t entries_size; + struct wimlib_unix_data unix_data; + const char *prefix = "user."; + static const char capability_name[] = "security.capability"; + bool generated_capability_xattr = false; + + /* + * On Linux, xattrs in the "user" namespace are only permitted on + * regular files and directories. For other types of files we can use + * the "trusted" namespace, but this requires root. + */ + if (inode_is_symlink(inode) || + (inode_get_unix_data(inode, &unix_data) && + !S_ISREG(unix_data.mode) && !S_ISDIR(unix_data.mode))) + { + if (!am_root()) + return 0; + prefix = "trusted."; + } + + for (int i = 0; i < num_xattrs; i++) { + int value_len = rand32() % 64; + u8 *p; + + entry->reserved = 0; + entry->value_len = cpu_to_le32(value_len); + + if (rand32() % 16 == 0 && am_root() && + !generated_capability_xattr) { + int name_len = sizeof(capability_name) - 1; + entry->name_len = cpu_to_le16(name_len); + p = mempcpy(entry->name, capability_name, name_len); + generated_capability_xattr = true; + } else { + int name_len = 1 + rand32() % 64; + + entry->name_len = cpu_to_le16(strlen(prefix) + name_len); + p = mempcpy(entry->name, prefix, strlen(prefix)); + *p++ = 'a' + i; + for (int j = 1; j < name_len; j++) { + do { + *p = rand8(); + } while (*p == '\0'); + p++; + } + } + for (int j = 0; j < value_len; j++) + *p++ = rand8(); + + while ((uintptr_t)p % 4) + *p++ = 0; + + entry = (void *)p; + } + + entries_size = (char *)entry - entries; + wimlib_assert(entries_size > 0 && entries_size % 4 == 0 && + entries_size <= sizeof(entries)); + + if (!inode_set_linux_xattrs(inode, entries, entries_size)) + return WIMLIB_ERR_NOMEM; + + return 0; +} + static int set_random_metadata(struct wim_inode *inode, struct generation_context *ctx) { - u32 v = rand32(); - u32 attrib = (v & (FILE_ATTRIBUTE_READONLY | - FILE_ATTRIBUTE_HIDDEN | - FILE_ATTRIBUTE_SYSTEM | - FILE_ATTRIBUTE_ARCHIVE | - FILE_ATTRIBUTE_NOT_CONTENT_INDEXED | - FILE_ATTRIBUTE_COMPRESSED)); + u32 attrib = (rand32() & (FILE_ATTRIBUTE_READONLY | + FILE_ATTRIBUTE_HIDDEN | + FILE_ATTRIBUTE_SYSTEM | + FILE_ATTRIBUTE_ARCHIVE | + FILE_ATTRIBUTE_NOT_CONTENT_INDEXED | + FILE_ATTRIBUTE_COMPRESSED | + FILE_ATTRIBUTE_SPARSE_FILE)); /* File attributes */ inode->i_attributes |= attrib; @@ -282,6 +572,30 @@ set_random_metadata(struct wim_inode *inode, struct generation_context *ctx) return WIMLIB_ERR_NOMEM; } + /* Object ID */ + if (rand32() % 32 == 0) { + struct wimlib_object_id object_id; + + for (int i = 0; i < sizeof(object_id); i++) + *((u8 *)&object_id + i) = rand8(); + if (!inode_set_object_id(inode, &object_id, sizeof(object_id))) + return WIMLIB_ERR_NOMEM; + } + + /* Standard UNIX permissions and special files */ + if (rand32() % 16 == 0) { + int ret = set_random_unix_metadata(inode); + if (ret) + return ret; + } + + /* Extended attributes */ + if (rand32() % 32 == 0) { + int ret = set_random_xattrs(inode); + if (ret) + return ret; + } + return 0; } @@ -323,8 +637,13 @@ generate_data(u8 *buffer, size_t size, struct generation_context *ctx) size_t mask = -1; size_t num_byte_fills = rand32() % 256; + if (size == 0) + return; + + /* Start by initializing to a random byte */ memset(buffer, rand32() % 256, size); + /* Add some random bytes in some random places */ for (size_t i = 0; i < num_byte_fills; i++) { u8 b = rand8(); @@ -342,6 +661,7 @@ generate_data(u8 *buffer, size_t size, struct generation_context *ctx) mask = (size_t)-1 << rand32() % 4; } + /* Sometimes add a wave pattern */ if (rand32() % 8 == 0) { double magnitude = rand32() % 128; double scale = 1.0 / (1 + (rand32() % 256)); @@ -349,6 +669,17 @@ generate_data(u8 *buffer, size_t size, struct generation_context *ctx) for (size_t i = 0; i < size; i++) buffer[i] += (int)(magnitude * cos(i * scale)); } + + /* Sometimes add some zero regions (holes) */ + if (rand32() % 4 == 0) { + size_t num_holes = 1 + (rand32() % 16); + for (size_t i = 0; i < num_holes; i++) { + size_t hole_offset = rand32() % size; + size_t hole_len = min(size - hole_offset, + size / (1 + (rand32() % 16))); + memset(&buffer[hole_offset], 0, hole_len); + } + } } static int @@ -381,35 +712,59 @@ err_nomem: return WIMLIB_ERR_NOMEM; } -static int +static noinline_for_stack int set_random_reparse_point(struct wim_inode *inode, struct generation_context *ctx) { - void *buffer = NULL; - size_t rpdatalen = select_stream_size(ctx) % (REPARSE_DATA_MAX_SIZE + 1); - - if (rpdatalen) { - buffer = MALLOC(rpdatalen); - if (!buffer) - return WIMLIB_ERR_NOMEM; - generate_data(buffer, rpdatalen, ctx); - } + struct reparse_buffer_disk rpbuf; + size_t rpdatalen; inode->i_attributes |= FILE_ATTRIBUTE_REPARSE_POINT; - inode->i_rp_reserved = rand16(); - - if (rpdatalen >= GUID_SIZE && randbool()) { - /* Non-Microsoft reparse tag (16-byte GUID required) */ - u8 *guid = buffer; - guid[6] = (guid[6] & 0x0F) | 0x40; - guid[8] = (guid[8] & 0x3F) | 0x80; - inode->i_reparse_tag = 0x00000100; + + if (randbool()) { + /* Symlink */ + int target_nchars; + utf16lechar *targets = (utf16lechar *)rpbuf.link.symlink.data; + + inode->i_reparse_tag = WIM_IO_REPARSE_TAG_SYMLINK; + + target_nchars = generate_random_filename(targets, 255, ctx); + + rpbuf.link.substitute_name_offset = cpu_to_le16(0); + rpbuf.link.substitute_name_nbytes = cpu_to_le16(2*target_nchars); + rpbuf.link.print_name_offset = cpu_to_le16(2*(target_nchars + 1)); + rpbuf.link.print_name_nbytes = cpu_to_le16(2*target_nchars); + targets[target_nchars] = cpu_to_le16(0); + memcpy(&targets[target_nchars + 1], targets, 2*target_nchars); + targets[target_nchars + 1 + target_nchars] = cpu_to_le16(0); + + rpbuf.link.symlink.flags = cpu_to_le32(SYMBOLIC_LINK_RELATIVE); + rpdatalen = ((u8 *)targets - rpbuf.rpdata) + + 2*(target_nchars + 1 + target_nchars + 1); } else { - /* Microsoft reparse tag */ - inode->i_reparse_tag = 0x80000000; + rpdatalen = select_stream_size(ctx) % REPARSE_DATA_MAX_SIZE; + generate_data(rpbuf.rpdata, rpdatalen, ctx); + + if (rpdatalen >= GUID_SIZE && randbool()) { + /* Non-Microsoft reparse tag (16-byte GUID required) */ + u8 *guid = rpbuf.rpdata; + guid[6] = (guid[6] & 0x0F) | 0x40; + guid[8] = (guid[8] & 0x3F) | 0x80; + inode->i_reparse_tag = 0x00000100; + } else { + /* Microsoft reparse tag */ + inode->i_reparse_tag = 0x80000000; + } + inode->i_rp_reserved = rand16(); } - return add_stream(inode, ctx, STREAM_TYPE_REPARSE_POINT, NO_STREAM_NAME, - buffer, rpdatalen); + wimlib_assert(rpdatalen < REPARSE_DATA_MAX_SIZE); + + if (!inode_add_stream_with_data(inode, STREAM_TYPE_REPARSE_POINT, + NO_STREAM_NAME, rpbuf.rpdata, + rpdatalen, ctx->params->blob_table)) + return WIMLIB_ERR_NOMEM; + + return 0; } static int @@ -432,21 +787,21 @@ add_random_data_stream(struct wim_inode *inode, struct generation_context *ctx, } static int -set_random_streams(struct wim_inode *inode, struct generation_context *ctx, - bool reparse_ok) +set_random_streams(struct wim_inode *inode, struct generation_context *ctx) { int ret; u32 r; /* Reparse point (sometimes) */ - if (reparse_ok && rand32() % 8 == 0) { + if (inode->i_attributes & FILE_ATTRIBUTE_REPARSE_POINT) { ret = set_random_reparse_point(inode, ctx); if (ret) return ret; } - /* Unnamed data stream (nondirectories only) */ - if (!(inode->i_attributes & FILE_ATTRIBUTE_DIRECTORY)) { + /* Unnamed data stream (nondirectories and non-symlinks only) */ + if (!(inode->i_attributes & FILE_ATTRIBUTE_DIRECTORY) && + !inode_is_symlink(inode)) { ret = add_random_data_stream(inode, ctx, NO_STREAM_NAME); if (ret) return ret; @@ -468,144 +823,6 @@ set_random_streams(struct wim_inode *inode, struct generation_context *ctx, return 0; } -static inline bool -is_valid_windows_filename_char(utf16lechar c) -{ - return le16_to_cpu(c) > 31 && - c != cpu_to_le16('/') && - c != cpu_to_le16('<') && - c != cpu_to_le16('>') && - c != cpu_to_le16(':') && - c != cpu_to_le16('"') && - c != cpu_to_le16('/' ) && - c != cpu_to_le16('\\') && - c != cpu_to_le16('|') && - c != cpu_to_le16('?') && - c != cpu_to_le16('*'); -} - -/* Is the character valid in a filename on the current platform? */ -static inline bool -is_valid_filename_char(utf16lechar c) -{ -#ifdef __WIN32__ - return is_valid_windows_filename_char(c); -#else - return c != cpu_to_le16('\0') && c != cpu_to_le16('/'); -#endif -} - -/* Generate a random filename and return its length. */ -static int -generate_random_filename(utf16lechar name[], int max_len, - struct generation_context *ctx) -{ - int len; - - /* Choose the length of the name. */ - switch (rand32() % 8) { - default: - /* short name */ - len = 1 + (rand32() % 6); - break; - case 2: - case 3: - case 4: - /* medium-length name */ - len = 7 + (rand32() % 8); - break; - case 5: - case 6: - /* long name */ - len = 15 + (rand32() % 15); - break; - case 7: - /* very long name */ - len = 30 + (rand32() % 90); - break; - } - len = min(len, max_len); - - /* Generate the characters in the name. */ - for (int i = 0; i < len; i++) { - do { - name[i] = rand16(); - } while (!is_valid_filename_char(name[i])); - } - - /* Add a null terminator. */ - name[len] = cpu_to_le16('\0'); - - return len; -} - -/* The set of characters which are valid in short filenames. */ -static const char valid_short_name_chars[] = { - 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', - 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', - '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', - '!', '#', '$', '%', '&', '\'', '(', ')', '-', '@', '^', '_', '`', '{', - '}', '~', - /* TODO: why doesn't Windows accept these characters? */ - /* ' ', */ - /*128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141,*/ - /*142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155,*/ - /*156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169,*/ - /*170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183,*/ - /*184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197,*/ - /*198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211,*/ - /*212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225,*/ - /*226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239,*/ - /*240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253,*/ - /*254, 255*/ -}; - -static int -generate_short_name_component(utf16lechar p[], int len) -{ - for (int i = 0; i < len; i++) { - char c = valid_short_name_chars[rand32() % - ARRAY_LEN(valid_short_name_chars)]; - p[i] = cpu_to_le16(c); - } -#if 0 /* TODO: we aren't using space yet anyway */ - while (len > 1 && p[len - 1] == cpu_to_le16(' ')) - len--; - if (p[len - 1] == cpu_to_le16(' ')) - p[len - 1] = cpu_to_le16('A'); -#endif - return len; -} - -/* Generate a random short (8.3) filename and return its length. - * The @name array must have length >= 13 (8 + 1 + 3 + 1). */ -static int -generate_random_short_name(utf16lechar name[], struct generation_context *ctx) -{ - /* - * Legal short names on Windows consist of 1 to 8 characters, optionally - * followed by a dot then 1 to 3 more characters. Only certain - * characters are allowed. In addition, trailing spaces are not - * significant. - */ - int base_len = 1 + (rand32() % 8); - int ext_len = rand32() % 4; - int total_len; - - base_len = generate_short_name_component(name, base_len); - - if (ext_len) { - name[base_len] = cpu_to_le16('.'); - ext_len = generate_short_name_component(&name[base_len + 1], - ext_len); - total_len = base_len + 1 + ext_len; - } else { - total_len = base_len; - } - name[total_len] = cpu_to_le16('\0'); - return total_len; -} - static u64 select_inode_number(struct generation_context *ctx) { @@ -635,29 +852,12 @@ is_name_valid_in_win32_namespace(const utf16lechar *name) { const utf16lechar *p; - static const utf16lechar forbidden_names[][5] = { - { cpu_to_le16('C'), cpu_to_le16('O'), cpu_to_le16('N'), }, - { cpu_to_le16('P'), cpu_to_le16('R'), cpu_to_le16('N'), }, - { cpu_to_le16('A'), cpu_to_le16('U'), cpu_to_le16('X'), }, - { cpu_to_le16('N'), cpu_to_le16('U'), cpu_to_le16('L'), }, - { cpu_to_le16('C'), cpu_to_le16('O'), cpu_to_le16('M'), cpu_to_le16('1'), }, - { cpu_to_le16('C'), cpu_to_le16('O'), cpu_to_le16('M'), cpu_to_le16('2'), }, - { cpu_to_le16('C'), cpu_to_le16('O'), cpu_to_le16('M'), cpu_to_le16('3'), }, - { cpu_to_le16('C'), cpu_to_le16('O'), cpu_to_le16('M'), cpu_to_le16('4'), }, - { cpu_to_le16('C'), cpu_to_le16('O'), cpu_to_le16('M'), cpu_to_le16('5'), }, - { cpu_to_le16('C'), cpu_to_le16('O'), cpu_to_le16('M'), cpu_to_le16('6'), }, - { cpu_to_le16('C'), cpu_to_le16('O'), cpu_to_le16('M'), cpu_to_le16('7'), }, - { cpu_to_le16('C'), cpu_to_le16('O'), cpu_to_le16('M'), cpu_to_le16('8'), }, - { cpu_to_le16('C'), cpu_to_le16('O'), cpu_to_le16('M'), cpu_to_le16('9'), }, - { cpu_to_le16('L'), cpu_to_le16('P'), cpu_to_le16('T'), cpu_to_le16('1'), }, - { cpu_to_le16('L'), cpu_to_le16('P'), cpu_to_le16('T'), cpu_to_le16('2'), }, - { cpu_to_le16('L'), cpu_to_le16('P'), cpu_to_le16('T'), cpu_to_le16('3'), }, - { cpu_to_le16('L'), cpu_to_le16('P'), cpu_to_le16('T'), cpu_to_le16('4'), }, - { cpu_to_le16('L'), cpu_to_le16('P'), cpu_to_le16('T'), cpu_to_le16('5'), }, - { cpu_to_le16('L'), cpu_to_le16('P'), cpu_to_le16('T'), cpu_to_le16('6'), }, - { cpu_to_le16('L'), cpu_to_le16('P'), cpu_to_le16('T'), cpu_to_le16('7'), }, - { cpu_to_le16('L'), cpu_to_le16('P'), cpu_to_le16('T'), cpu_to_le16('8'), }, - { cpu_to_le16('L'), cpu_to_le16('P'), cpu_to_le16('T'), cpu_to_le16('9'), }, + static const char * const reserved_names[] = { + "CON", "PRN", "AUX", "NUL", + "COM1", "COM2", "COM3", "COM4", "COM5", + "COM6", "COM7", "COM8", "COM9", + "LPT1", "LPT2", "LPT3", "LPT4", "LPT5", + "LPT6", "LPT7", "LPT8", "LPT9", }; /* The name must be nonempty. */ @@ -669,14 +869,24 @@ is_name_valid_in_win32_namespace(const utf16lechar *name) if (!is_valid_windows_filename_char(*p)) return false; - /* There can't be a trailing dot or space. */ - if (p[-1] == cpu_to_le16('.') || p[-1] == cpu_to_le16(' ')) - return false; - - /* The name can't be one of the reserved names (case insensitively). */ - for (size_t i = 0; i < ARRAY_LEN(forbidden_names); i++) - if (!cmp_utf16le_strings_z(forbidden_names[i], name, true)) - return false; + /* Note: a trailing dot or space is permitted, even though on Windows + * such a file can only be accessed using a WinNT-style path. */ + + /* The name can't be one of the reserved names or be a reserved name + * with an extension. Case insensitive. */ + for (size_t i = 0; i < ARRAY_LEN(reserved_names); i++) { + for (size_t j = 0; ; j++) { + u16 c1 = le16_to_cpu(name[j]); + u16 c2 = reserved_names[i][j]; + if (c2 == '\0') { + if (c1 == '\0' || c1 == '.') + return false; + break; + } + if (upcase[c1] != upcase[c2]) + break; + } + } return true; } @@ -765,23 +975,27 @@ generate_dentry_tree_recursive(struct wim_dentry *dir, u32 depth, /* Generate the next child dentry. */ struct wim_inode *inode; u64 ino; - bool is_directory; + bool is_directory = (rand32() % 16 <= 6); + bool is_reparse = (rand32() % 8 == 0); utf16lechar name[63 + 1]; /* for UNIX extraction: 63 * 4 <= 255 */ int name_len; struct wim_dentry *duplicate; - /* Decide whether to create a directory or not. If not a - * directory, also decide on the inode number (i.e. we may - * generate a "hard link" to an existing file). */ - is_directory = ((rand32() % 16) <= 6); - if (is_directory) + /* + * Select an inode number for the new file. Sometimes choose an + * existing inode number (i.e. create a hard link). However, + * wimlib intentionally doesn't honor directory hard links, and + * reparse points cannot be represented in the WIM file format + * at all; so don't create hard links for such files. + */ + if (is_directory || is_reparse) ino = 0; else ino = select_inode_number(ctx); /* Create the dentry. */ ret = inode_table_new_dentry(ctx->params->inode_table, NULL, - ino, 0, is_directory, &child); + ino, 0, ino == 0, &child); if (ret) return ret; @@ -812,19 +1026,19 @@ generate_dentry_tree_recursive(struct wim_dentry *dir, u32 depth, if (is_directory) inode->i_attributes |= FILE_ATTRIBUTE_DIRECTORY; + if (is_reparse) + inode->i_attributes |= FILE_ATTRIBUTE_REPARSE_POINT; - ret = set_random_metadata(inode, ctx); + ret = set_random_streams(inode, ctx); if (ret) return ret; - ret = set_random_streams(inode, ctx, true); + ret = set_random_metadata(inode, ctx); if (ret) return ret; /* Recurse if it's a directory. */ - if (is_directory && - !(inode->i_attributes & FILE_ATTRIBUTE_REPARSE_POINT)) - { + if (is_directory && !is_reparse) { ret = generate_dentry_tree_recursive(child, depth + 1, ctx); if (ret) @@ -859,10 +1073,10 @@ generate_dentry_tree(struct wim_dentry **root_ret, const tchar *_ignored, ret = inode_table_new_dentry(params->inode_table, NULL, 0, 0, true, &root); if (!ret) { root->d_inode->i_attributes = FILE_ATTRIBUTE_DIRECTORY; - ret = set_random_metadata(root->d_inode, &ctx); + ret = set_random_streams(root->d_inode, &ctx); } if (!ret) - ret = set_random_streams(root->d_inode, &ctx, false); + ret = set_random_metadata(root->d_inode, &ctx); if (!ret) ret = generate_dentry_tree_recursive(root, 1, &ctx); if (!ret) @@ -920,7 +1134,7 @@ calc_corresponding_files_recursive(struct wim_dentry *d1, struct wim_dentry *d2, /* Compare short filenames, case insensitively. */ if (!(d2->d_short_name_nbytes == 0 && - (cmp_flags & WIMLIB_CMP_FLAG_SHORT_NAMES_NOT_PRESERVED)) && + (cmp_flags & WIMLIB_CMP_FLAG_UNIX_MODE)) && cmp_utf16le_strings(d1->d_short_name, d1->d_short_name_nbytes / 2, d2->d_short_name, d2->d_short_name_nbytes / 2, true)) @@ -998,60 +1212,350 @@ check_hard_link(struct wim_dentry *dentry, void *_ignore) return WIMLIB_ERR_IMAGES_ARE_DIFFERENT; } +static const struct { + u32 flag; + const char *name; +} file_attr_flags[] = { + {FILE_ATTRIBUTE_READONLY, "READONLY"}, + {FILE_ATTRIBUTE_HIDDEN, "HIDDEN"}, + {FILE_ATTRIBUTE_SYSTEM, "SYSTEM"}, + {FILE_ATTRIBUTE_DIRECTORY, "DIRECTORY"}, + {FILE_ATTRIBUTE_ARCHIVE, "ARCHIVE"}, + {FILE_ATTRIBUTE_DEVICE, "DEVICE"}, + {FILE_ATTRIBUTE_NORMAL, "NORMAL"}, + {FILE_ATTRIBUTE_TEMPORARY, "TEMPORARY"}, + {FILE_ATTRIBUTE_SPARSE_FILE, "SPARSE_FILE"}, + {FILE_ATTRIBUTE_REPARSE_POINT, "REPARSE_POINT"}, + {FILE_ATTRIBUTE_COMPRESSED, "COMPRESSED"}, + {FILE_ATTRIBUTE_OFFLINE, "OFFLINE"}, + {FILE_ATTRIBUTE_NOT_CONTENT_INDEXED, "NOT_CONTENT_INDEXED"}, + {FILE_ATTRIBUTE_ENCRYPTED, "ENCRYPTED"}, + {FILE_ATTRIBUTE_VIRTUAL, "VIRTUAL"}, +}; + static int -cmp_inodes(const struct wim_inode *inode1, const struct wim_inode *inode2, - const struct wim_image_metadata *imd1, - const struct wim_image_metadata *imd2, int cmp_flags) +cmp_attributes(const struct wim_inode *inode1, + const struct wim_inode *inode2, int cmp_flags) { - const u32 attrib_diff = inode1->i_attributes ^ inode2->i_attributes; - bool reparse_point_should_preserved = true; + const u32 changed = inode1->i_attributes ^ inode2->i_attributes; + const u32 set = inode2->i_attributes & ~inode1->i_attributes; + const u32 cleared = inode1->i_attributes & ~inode2->i_attributes; + + /* NORMAL may change, but it must never be set along with other + * attributes. */ + if ((inode2->i_attributes & FILE_ATTRIBUTE_NORMAL) && + (inode2->i_attributes & ~FILE_ATTRIBUTE_NORMAL)) + goto mismatch; + + /* DIRECTORY may change in UNIX mode for symlinks. */ + if (changed & FILE_ATTRIBUTE_DIRECTORY) { + if (!(inode_is_symlink(inode1) && + (cmp_flags & WIMLIB_CMP_FLAG_UNIX_MODE))) + goto mismatch; + } - /* Compare attributes */ - if (cmp_flags & WIMLIB_CMP_FLAG_ATTRIBUTES_NOT_PRESERVED) { + /* REPARSE_POINT may be cleared in UNIX mode if the inode is not a + * symlink. */ + if ((changed & FILE_ATTRIBUTE_REPARSE_POINT) && + !((cleared & FILE_ATTRIBUTE_REPARSE_POINT) && + (cmp_flags & WIMLIB_CMP_FLAG_UNIX_MODE) && + !inode_is_symlink(inode1))) + goto mismatch; + + /* SPARSE_FILE may be cleared in UNIX and NTFS-3G modes, or in Windows + * mode if the inode is a directory. */ + if ((changed & FILE_ATTRIBUTE_SPARSE_FILE) && + !((cleared & FILE_ATTRIBUTE_SPARSE_FILE) && + ((cmp_flags & (WIMLIB_CMP_FLAG_UNIX_MODE | + WIMLIB_CMP_FLAG_NTFS_3G_MODE)) || + ((cmp_flags & WIMLIB_CMP_FLAG_WINDOWS_MODE) && + (inode1->i_attributes & FILE_ATTRIBUTE_DIRECTORY))))) + goto mismatch; + + /* COMPRESSED may change in UNIX and NTFS-3G modes. (It *should* be + * preserved in NTFS-3G mode, but it's not implemented yet.) */ + if ((changed & FILE_ATTRIBUTE_COMPRESSED) && + !(cmp_flags & (WIMLIB_CMP_FLAG_UNIX_MODE | + WIMLIB_CMP_FLAG_NTFS_3G_MODE))) + goto mismatch; + + /* All other attributes can change in UNIX mode, but not in any other + * mode. */ + if ((changed & ~(FILE_ATTRIBUTE_NORMAL | + FILE_ATTRIBUTE_DIRECTORY | + FILE_ATTRIBUTE_REPARSE_POINT | + FILE_ATTRIBUTE_SPARSE_FILE | + FILE_ATTRIBUTE_COMPRESSED)) && + !(cmp_flags & WIMLIB_CMP_FLAG_UNIX_MODE)) + goto mismatch; - /* In this mode, we expect that most attributes are not - * preserved. However, FILE_ATTRIBUTE_DIRECTORY should always - * match. */ - if (attrib_diff & FILE_ATTRIBUTE_DIRECTORY) - goto attrib_mismatch; + return 0; - /* We may also expect FILE_ATTRIBUTE_REPARSE_POINT to be - * preserved for symlinks. It also shouldn't be set if it - * wasn't set before. */ +mismatch: + ERROR("Attribute mismatch for %"TS": 0x%08"PRIx32" vs. 0x%08"PRIx32":", + inode_any_full_path(inode1), inode1->i_attributes, + inode2->i_attributes); + for (size_t i = 0; i < ARRAY_LEN(file_attr_flags); i++) { + u32 flag = file_attr_flags[i].flag; + if (changed & flag) { + fprintf(stderr, "\tFILE_ATTRIBUTE_%s was %s\n", + file_attr_flags[i].name, + (set & flag) ? "set" : "cleared"); + } + } + return WIMLIB_ERR_IMAGES_ARE_DIFFERENT; +} - if ((cmp_flags & WIMLIB_CMP_FLAG_IMAGE2_SHOULD_HAVE_SYMLINKS) && - inode_is_symlink(inode1)) - reparse_point_should_preserved = true; - else - reparse_point_should_preserved = false; +static int +cmp_object_ids(const struct wim_inode *inode1, + const struct wim_inode *inode2, int cmp_flags) +{ + const void *objid1, *objid2; + u32 len1, len2; + + objid1 = inode_get_object_id(inode1, &len1); + objid2 = inode_get_object_id(inode2, &len2); + + if (!objid1 && !objid2) + return 0; + + if (objid1 && !objid2) { + if (cmp_flags & WIMLIB_CMP_FLAG_UNIX_MODE) + return 0; + ERROR("%"TS" unexpectedly lost its object ID", + inode_any_full_path(inode1)); + return WIMLIB_ERR_IMAGES_ARE_DIFFERENT; + } + + if (!objid1 && objid2) { + ERROR("%"TS" unexpectedly gained an object ID", + inode_any_full_path(inode1)); + return WIMLIB_ERR_IMAGES_ARE_DIFFERENT; + } + + if (len1 != len2 || memcmp(objid1, objid2, len1) != 0) { + ERROR("Object ID of %"TS" differs", + inode_any_full_path(inode1)); + fprintf(stderr, "objid1="); + print_byte_field(objid1, len1, stderr); + fprintf(stderr, "\nobjid2="); + print_byte_field(objid2, len2, stderr); + fprintf(stderr, "\n"); + return WIMLIB_ERR_IMAGES_ARE_DIFFERENT; + } + + return 0; +} + +static int +cmp_unix_metadata(const struct wim_inode *inode1, + const struct wim_inode *inode2, int cmp_flags) +{ + struct wimlib_unix_data dat1, dat2; + bool present1, present2; + + present1 = inode_get_unix_data(inode1, &dat1); + present2 = inode_get_unix_data(inode2, &dat2); + + if (!present1 && !present2) + return 0; + + if (present1 && !present2) { + if (cmp_flags & (WIMLIB_CMP_FLAG_NTFS_3G_MODE | + WIMLIB_CMP_FLAG_WINDOWS_MODE)) + return 0; + ERROR("%"TS" unexpectedly lost its UNIX metadata", + inode_any_full_path(inode1)); + return WIMLIB_ERR_IMAGES_ARE_DIFFERENT; + } + + if (!present1 && present2) { + if (cmp_flags & WIMLIB_CMP_FLAG_UNIX_MODE) + return 0; + ERROR("%"TS" unexpectedly gained UNIX metadata", + inode_any_full_path(inode1)); + return WIMLIB_ERR_IMAGES_ARE_DIFFERENT; + } + + if (memcmp(&dat1, &dat2, sizeof(dat1)) != 0) { + ERROR("UNIX metadata of %"TS" differs: " + "[uid=%u, gid=%u, mode=0%o, rdev=%u] vs. " + "[uid=%u, gid=%u, mode=0%o, rdev=%u]", + inode_any_full_path(inode1), + dat1.uid, dat1.gid, dat1.mode, dat1.rdev, + dat2.uid, dat2.gid, dat2.mode, dat2.rdev); + return WIMLIB_ERR_IMAGES_ARE_DIFFERENT; + } + + return 0; +} + +static int +cmp_xattr_names(const void *p1, const void *p2) +{ + const struct wimlib_xattr_entry *entry1 = *(const struct wimlib_xattr_entry **)p1; + const struct wimlib_xattr_entry *entry2 = *(const struct wimlib_xattr_entry **)p2; + u16 name_len1 = le16_to_cpu(entry1->name_len); + u16 name_len2 = le16_to_cpu(entry2->name_len); + int res; + + res = cmp_u32(name_len1, name_len2); + if (res) + return res; + + return memcmp(entry1->name, entry2->name, name_len1); +} + +/* Validate and sort by name a list of extended attributes */ +static int +parse_xattrs(const void *xattrs, u32 len, + const struct wimlib_xattr_entry *entries[], + u32 *num_entries_p) +{ + u32 limit = *num_entries_p; + u32 num_entries = 0; + const struct wimlib_xattr_entry *entry = xattrs; + + while ((void *)entry < xattrs + len) { + if (!valid_xattr_entry(entry, xattrs + len - (void *)entry)) { + ERROR("Invalid xattr entry"); + return WIMLIB_ERR_INVALID_XATTR; + } + if (num_entries >= limit) { + ERROR("Too many xattr entries"); + return WIMLIB_ERR_INVALID_XATTR; + } + entries[num_entries++] = entry; + entry = xattr_entry_next(entry); + } + + if (num_entries == 0) { + ERROR("No xattr entries"); + return WIMLIB_ERR_INVALID_XATTR; + } + + qsort(entries, num_entries, sizeof(entries[0]), cmp_xattr_names); + + for (u32 i = 1; i < num_entries; i++) { + if (cmp_xattr_names(&entries[i - 1], &entries[i]) == 0) { + ERROR("Duplicate xattr names"); + return WIMLIB_ERR_INVALID_XATTR; + } + } + + *num_entries_p = num_entries; + return 0; +} - if ((attrib_diff & FILE_ATTRIBUTE_REPARSE_POINT) && - (reparse_point_should_preserved || - (inode2->i_attributes & FILE_ATTRIBUTE_REPARSE_POINT))) - goto attrib_mismatch; +static int +cmp_linux_xattrs(const struct wim_inode *inode1, + const struct wim_inode *inode2, int cmp_flags) +{ + const void *xattrs1, *xattrs2; + u32 len1, len2; + + xattrs1 = inode_get_linux_xattrs(inode1, &len1); + xattrs2 = inode_get_linux_xattrs(inode2, &len2); + + if (!xattrs1 && !xattrs2) { + return 0; + } else if (xattrs1 && !xattrs2) { + if (cmp_flags & (WIMLIB_CMP_FLAG_NTFS_3G_MODE | + WIMLIB_CMP_FLAG_WINDOWS_MODE)) + return 0; + ERROR("%"TS" unexpectedly lost its xattrs", + inode_any_full_path(inode1)); + return WIMLIB_ERR_IMAGES_ARE_DIFFERENT; + } else if (!xattrs1 && xattrs2) { + ERROR("%"TS" unexpectedly gained xattrs", + inode_any_full_path(inode1)); + return WIMLIB_ERR_IMAGES_ARE_DIFFERENT; } else { + const int max_entries = 64; + const struct wimlib_xattr_entry *entries1[max_entries]; + const struct wimlib_xattr_entry *entries2[max_entries]; + u32 xattr_count1 = max_entries; + u32 xattr_count2 = max_entries; + int ret; + + ret = parse_xattrs(xattrs1, len1, entries1, &xattr_count1); + if (ret) { + ERROR("%"TS": invalid xattrs", + inode_any_full_path(inode1)); + return ret; + } + ret = parse_xattrs(xattrs2, len2, entries2, &xattr_count2); + if (ret) { + ERROR("%"TS": invalid xattrs", + inode_any_full_path(inode2)); + return ret; + } + if (xattr_count1 != xattr_count2) { + ERROR("%"TS": number of xattrs changed. had %u " + "before, now has %u", inode_any_full_path(inode1), + xattr_count1, xattr_count2); + } + for (u32 i = 0; i < xattr_count1; i++) { + const struct wimlib_xattr_entry *entry1 = entries1[i]; + const struct wimlib_xattr_entry *entry2 = entries2[i]; + + if (entry1->name_len != entry2->name_len || + entry1->value_len != entry2->value_len || + entry1->reserved != entry2->reserved || + memcmp(entry1->name, entry2->name, + le16_to_cpu(entry1->name_len)) || + memcmp(entry1->name + le16_to_cpu(entry1->name_len), + entry2->name + le16_to_cpu(entry1->name_len), + le32_to_cpu(entry1->value_len))) + { + ERROR("xattr %.*s of %"TS" differs", + le16_to_cpu(entry1->name_len), + entry1->name, inode_any_full_path(inode1)); + return WIMLIB_ERR_IMAGES_ARE_DIFFERENT; + } + } + return 0; + } +} - /* Most attributes should be preserved. */ +static int +cmp_timestamps(const struct wim_inode *inode1, const struct wim_inode *inode2, + int cmp_flags) +{ + if (inode1->i_creation_time != inode2->i_creation_time && + !(cmp_flags & WIMLIB_CMP_FLAG_UNIX_MODE)) { + ERROR("Creation time of %"TS" differs", + inode_any_full_path(inode1)); + return WIMLIB_ERR_IMAGES_ARE_DIFFERENT; + } - /* Nothing other than COMPRESSED and NORMAL should have changed. - */ - if (attrib_diff & ~(FILE_ATTRIBUTE_COMPRESSED | - FILE_ATTRIBUTE_NORMAL)) - goto attrib_mismatch; - - /* COMPRESSED shouldn't have changed unless specifically - * excluded. */ - if ((attrib_diff & FILE_ATTRIBUTE_COMPRESSED) && - !(cmp_flags & WIMLIB_CMP_FLAG_COMPRESSION_NOT_PRESERVED)) - goto attrib_mismatch; - - /* We allow NORMAL to change, but not if the file ended up with - * other attributes set as well. */ - if ((attrib_diff & FILE_ATTRIBUTE_NORMAL) && - (inode2->i_attributes & ~FILE_ATTRIBUTE_NORMAL)) - goto attrib_mismatch; + if (inode1->i_last_write_time != inode2->i_last_write_time) { + ERROR("Last write time of %"TS" differs", + inode_any_full_path(inode1)); + return WIMLIB_ERR_IMAGES_ARE_DIFFERENT; } + if (inode1->i_last_access_time != inode2->i_last_access_time) { + ERROR("Last access time of %"TS" differs", + inode_any_full_path(inode1)); + return WIMLIB_ERR_IMAGES_ARE_DIFFERENT; + } + + return 0; +} + +static int +cmp_inodes(const struct wim_inode *inode1, const struct wim_inode *inode2, + const struct wim_image_metadata *imd1, + const struct wim_image_metadata *imd2, int cmp_flags) +{ + int ret; + + /* Compare attributes */ + ret = cmp_attributes(inode1, inode2, cmp_flags); + if (ret) + return ret; + /* Compare security descriptors */ if (inode_has_security_descriptor(inode1)) { if (inode_has_security_descriptor(inode2)) { @@ -1065,7 +1569,7 @@ cmp_inodes(const struct wim_inode *inode1, const struct wim_inode *inode2, inode_any_full_path(inode1)); return WIMLIB_ERR_IMAGES_ARE_DIFFERENT; } - } else if (!(cmp_flags & WIMLIB_CMP_FLAG_SECURITY_NOT_PRESERVED)) { + } else if (!(cmp_flags & WIMLIB_CMP_FLAG_UNIX_MODE)) { ERROR("%"TS" has a security descriptor in the first image but " "not in the second image!", inode_any_full_path(inode1)); return WIMLIB_ERR_IMAGES_ARE_DIFFERENT; @@ -1084,7 +1588,8 @@ cmp_inodes(const struct wim_inode *inode1, const struct wim_inode *inode2, const struct wim_inode_stream *strm2; if (strm1->stream_type == STREAM_TYPE_REPARSE_POINT && - !reparse_point_should_preserved) + (cmp_flags & WIMLIB_CMP_FLAG_UNIX_MODE && + !inode_is_symlink(inode1))) continue; if (strm1->stream_type == STREAM_TYPE_UNKNOWN) @@ -1096,7 +1601,7 @@ cmp_inodes(const struct wim_inode *inode1, const struct wim_inode *inode2, if (!strm2) { /* Corresponding stream not found */ if (stream_is_named(strm1) && - (cmp_flags & WIMLIB_CMP_FLAG_ADS_NOT_PRESERVED)) + (cmp_flags & WIMLIB_CMP_FLAG_UNIX_MODE)) continue; ERROR("Stream of %"TS" is missing in second image; " "type %d, named=%d, empty=%d", @@ -1114,14 +1619,27 @@ cmp_inodes(const struct wim_inode *inode1, const struct wim_inode *inode2, } } - return 0; + /* Compare object IDs */ + ret = cmp_object_ids(inode1, inode2, cmp_flags); + if (ret) + return ret; -attrib_mismatch: - ERROR("Attribute mismatch; %"TS" has attributes 0x%08"PRIx32" " - "in first image but attributes 0x%08"PRIx32" in second image", - inode_any_full_path(inode1), inode1->i_attributes, - inode2->i_attributes); - return WIMLIB_ERR_IMAGES_ARE_DIFFERENT; + /* Compare timestamps */ + ret = cmp_timestamps(inode1, inode2, cmp_flags); + if (ret) + return ret; + + /* Compare standard UNIX metadata */ + ret = cmp_unix_metadata(inode1, inode2, cmp_flags); + if (ret) + return ret; + + /* Compare Linux-style xattrs */ + ret = cmp_linux_xattrs(inode1, inode2, cmp_flags); + if (ret) + return ret; + + return 0; } static int