From: Eric Biggers Date: Mon, 23 Dec 2013 19:31:42 +0000 (-0600) Subject: Cleanup X-Git-Tag: v1.6.0~128 X-Git-Url: https://wimlib.net/git/?p=wimlib;a=commitdiff_plain;h=e176e9731e696562bab8de7b9bd34c019deef3e8 Cleanup --- diff --git a/include/wimlib.h b/include/wimlib.h index efc770cf..cf47751a 100644 --- a/include/wimlib.h +++ b/include/wimlib.h @@ -426,7 +426,11 @@ enum wimlib_compression_type { /** Compressed resources in the WIM use XPRESS compression. */ WIMLIB_COMPRESSION_TYPE_XPRESS = 2, - /** TODO */ + /** Compressed resources in the WIM use LZMS compression. Currently, + * wimlib has a decompressor for this format but not a compressor. LZMS + * compression is only compatible with wimlib v1.6.0 and later and with + * WIMGAPI Windows 8 and later (and some restrictions apply on the + * latter). */ WIMLIB_COMPRESSION_TYPE_LZMS = 3, }; @@ -947,55 +951,64 @@ struct wimlib_wim_info { uint32_t reserved[9]; }; -/** Information about a unique resource in the WIM file. - */ +/** Information about a unique stream in the WIM file. (A stream is the same + * thing as a "resource", except in the case of packed resources.) */ struct wimlib_resource_entry { - /** Uncompressed size of the resource in bytes. */ + /** Uncompressed size of the stream in bytes. */ uint64_t uncompressed_size; - /** Compressed size of the resource in bytes. This will be the same as - * @p uncompressed_size if the resource is uncompressed. */ + /** Compressed size of the stream in bytes. This will be the same as @p + * uncompressed_size if the stream is uncompressed. Or, if @p + * is_packed_streams is 1, this will be 0. */ uint64_t compressed_size; - /** Offset, in bytes, of this resource from the start of the WIM file. + /** Offset, in bytes, of this stream from the start of the WIM file. Or + * if @p packed is 1, then this is actually the offset at which this + * stream begins in the uncompressed contents of the packed resource. */ uint64_t offset; - /** SHA1 message digest of the resource's uncompressed contents. */ + /** SHA1 message digest of the stream's uncompressed contents. */ uint8_t sha1_hash[20]; - /** Which part number of the split WIM this resource is in. This should + /** Which part number of the split WIM this stream is in. This should * be the same as the part number provided by wimlib_get_wim_info(). */ uint32_t part_number; - /** Number of times this resource is referenced over all WIM images. */ + /** Number of times this stream is referenced over all WIM images. */ uint32_t reference_count; - /** 1 if this resource is compressed. */ + /** 1 if this stream is compressed. */ uint32_t is_compressed : 1; - /** 1 if this resource is a metadata resource rather than a file - * resource. */ + /** 1 if this stream is a metadata resource rather than a file resource. + * */ uint32_t is_metadata : 1; uint32_t is_free : 1; uint32_t is_spanned : 1; - /** 1 if this resource was not found in the lookup table of the + /** 1 if this stream was not found in the lookup table of the * ::WIMStruct. This normally implies a missing call to * wimlib_reference_resource_files() or wimlib_reference_resources(). - */ + * */ uint32_t is_missing : 1; - uint32_t is_packed_streams : 1; + /** 1 if this stream is located in a packed resource which may contain + * other streams (all compressed together) as well. */ + uint32_t packed : 1; uint32_t reserved_flags : 26; + /** If @p packed is 1, then this will specify the offset of the packed + * resource in the WIM. */ uint64_t raw_resource_offset_in_wim; - uint64_t raw_resource_uncompressed_size; + + /** If @p is_packed_streams is 1, then this will specify the compressed + * size of the packed resource in the WIM. */ uint64_t raw_resource_compressed_size; - uint64_t reserved[1]; + uint64_t reserved[2]; }; /** A stream of a file in the WIM. */ @@ -3045,7 +3058,8 @@ wimlib_mount_image(WIMStruct *wim, * chunk of the WIM does not match the corresponding message digest given * in the integrity table. * @retval ::WIMLIB_ERR_INVALID_CHUNK_SIZE - * Resources in @p wim_file are compressed, but the chunk size is not 32768. + * Resources in @p wim_file are compressed, but the chunk size was invalid + * for the WIM's compression format. * @retval ::WIMLIB_ERR_INVALID_COMPRESSION_TYPE * The header of @p wim_file says that resources in the WIM are compressed, * but the header flag indicating LZX or XPRESS compression is not set. @@ -3441,7 +3455,8 @@ wimlib_set_image_descripton(WIMStruct *wim, int image, * * WARNING: Changing the compression chunk size to any value other than the * default of 32768 bytes eliminates compatibility with Microsoft's software, - * except when increasing the XPRESS chunk size before Windows 8. + * except when increasing the XPRESS chunk size before Windows 8. Chunk sizes + * other than 32768 are also incompatible with wimlib v1.5.3 and earlier. * * @param wim * ::WIMStruct for a WIM. @@ -3982,7 +3997,7 @@ wimlib_write_to_fd(WIMStruct *wim, * library clients looking to make use of wimlib's compression code for another * purpose. * - * As of wimlib v1.5.4, this function can be used with @p chunk_size greater + * As of wimlib v1.6.0, this function can be used with @p chunk_size greater * than 32768 bytes and is only limited by available memory. However, the * XPRESS format itself still caps match offsets to 65535, so if a larger chunk * size is chosen, then the matching will effectively occur in a sliding window diff --git a/include/wimlib/error.h b/include/wimlib/error.h index 8dd48c18..8d02120b 100644 --- a/include/wimlib/error.h +++ b/include/wimlib/error.h @@ -42,7 +42,7 @@ extern void wimlib_warning_with_errno(const tchar *format, ...) _format_attribute(printf, 1, 2) _cold_attribute; # define ERROR(format, ...) wimlib_error(T(format), ## __VA_ARGS__) -# define ERROR_WITH_ERRNO(format, ...) wimlib_error_with_errno(T(format), ## __VA_ARGS__) +# define ERROR_WITH_ERRNO(format, ...) wimlib_error_with_errno(T(format), ## __VA_ARGS__) # define WARNING(format, ...) wimlib_warning(T(format), ## __VA_ARGS__) # define WARNING_WITH_ERRNO(format, ...) wimlib_warning_with_errno(T(format), ## __VA_ARGS__) extern bool wimlib_print_errors; @@ -68,7 +68,7 @@ extern void wimlib_debug(const tchar *file, int line, const char *func, const tchar *format, ...); # define DEBUG(format, ...) \ - wimlib_debug(T(__FILE__), __LINE__, __func__, T(format), ## __VA_ARGS__) + wimlib_debug(T(__FILE__), __LINE__, __func__, T(format), ## __VA_ARGS__) #else # define DEBUG(format, ...) dummy_tprintf(T(format), ## __VA_ARGS__) diff --git a/include/wimlib/header.h b/include/wimlib/header.h index d67811c1..6f701891 100644 --- a/include/wimlib/header.h +++ b/include/wimlib/header.h @@ -16,55 +16,55 @@ /* Version number used for WIMs that allow multiple streams packed into one * resource (WIM_RESHDR_FLAG_PACKED_STREAMS). New as of Windows 8 WIMGAPI; used - * for the Windows 8 web downloader, but yet properly documented by Microsoft. - */ + * for the Windows 8 web downloader, but yet not yet properly documented by + * Microsoft. */ #define WIM_VERSION_PACKED_STREAMS 0xe00 /* Note: there is another WIM version from Vista pre-releases, but it is not * supported by wimlib. */ -/* WIM magic characters, translated to a single 64-bit little endian number. */ -#define WIM_MAGIC \ - cpu_to_le64(((u64)'M' << 0) | \ - ((u64)'S' << 8) | \ - ((u64)'W' << 16) | \ - ((u64)'I' << 24) | \ - ((u64)'M' << 32) | \ - ((u64)'\0' << 40) | \ - ((u64)'\0' << 48) | \ - ((u64)'\0' << 54)) - -/* wimlib pipable WIM magic characters, translated to a single 64-bit little - * endian number. */ -#define PWM_MAGIC \ - cpu_to_le64(((u64)'W' << 0) | \ - ((u64)'L' << 8) | \ - ((u64)'P' << 16) | \ - ((u64)'W' << 24) | \ - ((u64)'M' << 32) | \ - ((u64)'\0' << 40) | \ - ((u64)'\0' << 48) | \ - ((u64)'\0' << 54)) +/* WIM magic characters, translated to a single 64-bit number. */ +#define WIM_MAGIC \ + (((u64)'M' << 0) | \ + ((u64)'S' << 8) | \ + ((u64)'W' << 16) | \ + ((u64)'I' << 24) | \ + ((u64)'M' << 32) | \ + ((u64)'\0' << 40) | \ + ((u64)'\0' << 48) | \ + ((u64)'\0' << 54)) + +/* wimlib pipable WIM magic characters, translated to a single 64-bit number. + * */ +#define PWM_MAGIC \ + (((u64)'W' << 0) | \ + ((u64)'L' << 8) | \ + ((u64)'P' << 16) | \ + ((u64)'W' << 24) | \ + ((u64)'M' << 32) | \ + ((u64)'\0' << 40) | \ + ((u64)'\0' << 48) | \ + ((u64)'\0' << 54)) /* On-disk format of the WIM header. */ struct wim_header_disk { - /* +0x00: Magic characters "MSWIM\0\0\0" */ + /* +0x00: Magic characters "MSWIM\0\0\0". */ le64 magic; /* +0x08: Size of the WIM header, in bytes; WIM_HEADER_DISK_SIZE - * expected (currently the only supported value). */ + * expected (currently the only supported value). */ u32 hdr_size; - /* +0x0c: Version of the WIM file - * TODO */ + /* +0x0c: Version of the WIM file. Recognized values are the + * WIM_VERSION_* constants from above. */ u32 wim_version; - /* +0x10: Flags for the WIM file (WIM_HDR_FLAG_*) */ + /* +0x10: Flags for the WIM file (WIM_HDR_FLAG_*). */ u32 wim_flags; - /* +0x14: Chunk size for compressed resources in the WIM, or 0 if the - * WIM is uncompressed. */ + /* +0x14: Uncompressed chunk size for compressed resources in the WIM + * other than packed resources, or 0 if the WIM is uncompressed. */ u32 chunk_size; /* +0x18: Globally unique identifier for the WIM file. Basically a @@ -79,84 +79,53 @@ struct wim_header_disk { * is not split. */ u16 total_parts; - /* +0x2c: Number of images in the WIM. */ + /* +0x2c: Number of images in the WIM. */ u32 image_count; - /* +0x30: Location and size of the WIM's lookup table. */ + /* +0x30: Location and size of the WIM's lookup table. */ struct wim_reshdr_disk lookup_table_reshdr; - /* +0x48: Location and size of the WIM's XML data. */ + /* +0x48: Location and size of the WIM's XML data. */ struct wim_reshdr_disk xml_data_reshdr; /* +0x60: Location and size of metadata resource for the bootable image - * of the WIM, or all zeroes if no image is bootable. */ + * of the WIM, or all zeroes if no image is bootable. */ struct wim_reshdr_disk boot_metadata_reshdr; /* +0x78: 1-based index of the bootable image of the WIM, or 0 if no - * image is bootable. */ + * image is bootable. */ u32 boot_idx; /* +0x7c: Location and size of the WIM's integrity table, or all zeroes * if the WIM has no integrity table. * - * Note the integrity_table_reshdr here is 4-byte aligned even though - * it would ordinarily be 8-byte aligned--- hence, the _packed_attribute - * on the `struct wim_header_disk' is essential. */ + * Note the integrity_table_reshdr here is 4-byte aligned even though it + * would ordinarily be 8-byte aligned--- hence, the _packed_attribute on + * this structure is essential. */ struct wim_reshdr_disk integrity_table_reshdr; - /* +0x94: Unused bytes. */ + /* +0x94: Unused bytes. */ u8 unused[60]; /* +0xd0 (208) */ } _packed_attribute; -/* Header at the very beginning of the WIM file. This is the in-memory - * representation and does not include all fields; see `struct wim_header_disk' - * for the on-disk structure. */ +/* In-memory representation of a WIM header. See `struct wim_header_disk' for + * field descriptions. */ struct wim_header { - - /* Magic characters: either WIM_MAGIC or PWM_MAGIC. */ - le64 magic; - - /* Version of the WIM file */ + u64 magic; u32 wim_version; - - /* Bitwise OR of one or more of the WIM_HDR_FLAG_* defined below. */ u32 flags; - - /* Compressed resource chunk size */ u32 chunk_size; - - /* A unique identifier for the WIM file. */ u8 guid[WIM_GID_LEN]; - - /* Part number of the WIM file in a spanned set. */ u16 part_number; - - /* Total number of parts in a spanned set. */ u16 total_parts; - - /* Number of images in the WIM file. */ u32 image_count; - - /* Location, size, and flags of the lookup table of the WIM. */ struct wim_reshdr lookup_table_reshdr; - - /* Location, size, and flags for the XML data of the WIM. */ struct wim_reshdr xml_data_reshdr; - - /* Location, size, and flags for the boot metadata. This means the - * metadata resource for the image specified by boot_idx below. Should - * be zeroed out if boot_idx is 0. */ struct wim_reshdr boot_metadata_reshdr; - - /* The index of the bootable image in the WIM file. If 0, there are no - * bootable images available. */ u32 boot_idx; - - /* The location of the optional integrity table used to verify the - * integrity WIM. Zeroed out if there is no integrity table.*/ struct wim_reshdr integrity_table_reshdr; }; diff --git a/include/wimlib/list.h b/include/wimlib/list.h index 4ac2d0b4..e2731d1a 100644 --- a/include/wimlib/list.h +++ b/include/wimlib/list.h @@ -275,7 +275,7 @@ static inline void list_splice_tail(struct list_head *list, */ #define list_for_each_entry(pos, head, member) \ for (pos = list_entry((head)->next, typeof(*pos), member); \ - &pos->member != (head); \ + &pos->member != (head); \ pos = list_entry(pos->member.next, typeof(*pos), member)) /** @@ -288,7 +288,7 @@ static inline void list_splice_tail(struct list_head *list, #define list_for_each_entry_safe(pos, n, head, member) \ for (pos = list_entry((head)->next, typeof(*pos), member), \ n = list_entry(pos->member.next, typeof(*pos), member); \ - &pos->member != (head); \ + &pos->member != (head); \ pos = n, n = list_entry(n->member.next, typeof(*n), member)) /* @@ -383,9 +383,9 @@ static inline void hlist_add_head(struct hlist_node *n, struct hlist_head *h) * @head: the head for your list. * @member: the name of the hlist_node within the struct. */ -#define hlist_for_each_entry_safe(tpos, pos, n, head, member) \ +#define hlist_for_each_entry_safe(tpos, pos, n, head, member) \ for (pos = (head)->first; \ - pos && ({ n = pos->next; 1; }) && \ + pos && ({ n = pos->next; 1; }) && \ ({ tpos = hlist_entry(pos, typeof(*tpos), member); 1;}); \ pos = n) diff --git a/include/wimlib/lookup_table.h b/include/wimlib/lookup_table.h index 6a934e49..f99f38d4 100644 --- a/include/wimlib/lookup_table.h +++ b/include/wimlib/lookup_table.h @@ -62,12 +62,13 @@ enum resource_location { * `struct wim_resource_spec' pointed to by @rspec. @offset_in_res * identifies the offset at which this particular stream begins in the * uncompressed data of the resource; this is normally 0, but in general - * a WIM resource may contain multiple streams. */ + * a WIM resource may be "packed" and potentially contain multiple + * streams. */ RESOURCE_IN_WIM, /* The stream is located in the external file named by @file_on_disk. - * On Windows, @file_on_disk may actually specify a named data stream. - */ + * On Windows, @file_on_disk may actually specify a named data stream + * (file path, then colon, then name of the stream). */ RESOURCE_IN_FILE_ON_DISK, /* The stream is directly attached in the in-memory buffer pointed to by @@ -98,43 +99,42 @@ enum resource_location { }; -/* - * An entry in the lookup table in the WIM file. - * - * It is used to find data streams for files in the WIM. +/* Specification for a stream, which may be the contents of a file (unnamed data + * stream), a named data stream, reparse point data, or a WIM metadata resource. * - * Metadata resources and reparse point data buffers will also have lookup table - * entries associated with the data. - * - * The lookup_table_entry for a given dentry or alternate stream entry in the - * WIM is found using the SHA1 message digest field. - */ + * One instance of this structure is created for each entry in the WIM's lookup + * table, hence the name of the struct. Each of these entries contains the SHA1 + * message digest of a stream and the location of the stream data in the WIM + * file (size, location, flags). The in-memory lookup table is a map from SHA1 + * message digests to stream locations. */ struct wim_lookup_table_entry { - /* List of lookup table entries in this hash bucket */ + /* List node for a hash bucket of the lookup table. */ struct hlist_node hash_list; - /* Uncompressed size of the stream. */ + /* Uncompressed size of this stream. */ u64 size; /* Stream flags (WIM_RESHDR_FLAG_*). */ u32 flags : 8; - /* One of the `enum resource_location' values documented above. */ - u32 resource_location : 5; + /* One of the `enum resource_location' values documented above. */ + u32 resource_location : 4; /* 1 if this stream has not had a SHA1 message digest calculated for it - * yet */ + * yet. */ u32 unhashed : 1; - /* Temoorary files used for writing; set as documented for + /* Temoorary fields used when writing streams; set as documented for * prepare_stream_list_for_write(). */ u32 unique_size : 1; u32 will_be_in_output_wim : 1; /* Set to 1 when a metadata entry has its checksum changed; in such - * cases the hash is no longer valid to verify the data if the metadata - * resource is read again. */ + * cases the hash cannot be used to verify the data if the metadata + * resource is read again. (This could be avoided if we used separate + * fields for input/output checksum, but most stream entries wouldn't + * need this.) */ u32 dont_check_metadata_hash : 1; union { @@ -162,27 +162,31 @@ struct wim_lookup_table_entry { }; /* Number of times this lookup table entry is referenced by dentries in - * the WIM. */ + * the WIM. When a WIM's lookup table is read, this field is + * initialized from a corresponding entry; while it should be correct, + * in general it may not be. wim_recalculate_refcnts() recalculates the + * reference counts for all streams and is run before doing any + * deletions. */ u32 refcnt; - /* Actual reference count to this stream (only used while verifying an - * image). */ - u32 real_refcnt; - - /* When a WIM file is written, out_refcnt starts at 0 and is incremented - * whenever the stream pointed to by this lookup table entry needs to be - * written. The stream only need to be written when out_refcnt is - * nonzero, since otherwise it is not referenced by any dentries. */ + /* When a WIM file is written, this is set to the number of references + * (by dentries) to this stream in the output WIM file. + * + * During extraction, this is set to the number of times the stream must + * be extracted. + * + * During image export, this is set to the number of references of this + * stream that originated from the source WIM. */ u32 out_refcnt; #ifdef WITH_FUSE - /* Number of times this stream has been opened (used only during - * mounting) */ + /* Number of times this stream has been opened; used only during + * mounting. */ u16 num_opened_fds; #endif - /* Pointers to somewhere where the stream is actually located. See the - * comments for the @resource_location field above. */ + /* Specification of where this stream is actually located. Which member + * is valid is determined by the @resource_location field. */ union { struct { struct wim_resource_spec *rspec; @@ -199,45 +203,59 @@ struct wim_lookup_table_entry { }; /* Links together streams that share the same underlying WIM resource. - * The head is wim_resource_spec.stream_list. */ + * The head is the `stream_list' member of `struct wim_resource_spec'. + */ struct list_head rspec_node; - /* This field is used for the special hardlink or symlink image - * extraction mode. In these mode, all identical files are linked - * together, and @extracted_file will be set to the filename of the - * first extracted file containing this stream. */ + /* This field is used during the hardlink and symlink image extraction + * modes. In these modes, all identical files are linked together, and + * @extracted_file will be set to the filename of the first extracted + * file containing this stream. */ tchar *extracted_file; /* Temporary fields */ union { - /* Used temporarily during WIM file writing */ + /* Fields used temporarily during WIM file writing. */ struct { union { + /* List node used for stream size table. */ struct hlist_node hash_list_2; + + /* Metadata for the underlying packed resource + * in the WIM being written (only valid if + * WIM_RESHDR_FLAG_PACKED_STREAMS set in + * out_reshdr.flags). */ struct { u64 out_res_offset_in_wim; u64 out_res_size_in_wim; - u64 out_res_uncompressed_size; }; }; + /* Links streams being written to the WIM. */ struct list_head write_streams_list; + /* Metadata for this stream in the WIM being written. + */ struct wim_reshdr out_reshdr; }; /* Used temporarily during extraction */ union { - /* out_refcnt tracks number of slots filled */ - struct wim_dentry *inline_lte_dentries[8]; + /* Dentries to extract that reference this stream. + * out_refcnt tracks the number of slots filled. */ + struct wim_dentry *inline_lte_dentries[7]; struct { struct wim_dentry **lte_dentries; - unsigned long alloc_lte_dentries; + size_t alloc_lte_dentries; }; }; + + /* Actual reference count to this stream (only used while + * verifying an image). */ + u32 real_refcnt; }; - /* Temporary list fields */ + /* Temporary list fields. */ union { /* Links streams for writing lookup table. */ struct list_head lookup_table_list; @@ -294,7 +312,7 @@ free_lookup_table(struct wim_lookup_table *table); extern void lookup_table_insert(struct wim_lookup_table *table, struct wim_lookup_table_entry *lte); -/* Unlinks a lookup table entry from the table; does not free it. */ +/* Unlinks a lookup table entry from the table; does not free it. */ static inline void lookup_table_unlink(struct wim_lookup_table *table, struct wim_lookup_table_entry *lte) { @@ -381,7 +399,6 @@ static inline void lte_unbind_wim_resource_spec(struct wim_lookup_table_entry *lte) { list_del(<e->rspec_node); - lte->rspec = NULL; lte->resource_location = RESOURCE_NONEXISTENT; } diff --git a/include/wimlib/lzms.h b/include/wimlib/lzms.h index 15128cb8..de965fe7 100644 --- a/include/wimlib/lzms.h +++ b/include/wimlib/lzms.h @@ -31,7 +31,7 @@ #define LZMS_NUM_LITERAL_SYMS 256 #define LZMS_NUM_LEN_SYMS 54 #define LZMS_NUM_DELTA_POWER_SYMS 8 -#define LZMS_MAX_NUM_OFFSET_SYMS 799 +#define LZMS_MAX_NUM_OFFSET_SYMS 799 #define LZMS_MAX_NUM_SYMS 799 #define LZMS_MAX_CODEWORD_LEN 15 diff --git a/include/wimlib/lzx.h b/include/wimlib/lzx.h index f72bfb1b..1332645f 100644 --- a/include/wimlib/lzx.h +++ b/include/wimlib/lzx.h @@ -71,7 +71,7 @@ #define LZX_ALIGNEDCODE_NUM_SYMBOLS 8 #define LZX_ALIGNEDCODE_TABLEBITS 7 -#define LZX_ALIGNEDCODE_ELEMENT_SIZE 3 +#define LZX_ALIGNEDCODE_ELEMENT_SIZE 3 /* Maximum allowed length of Huffman codewords. */ #define LZX_MAX_MAIN_CODEWORD_LEN 16 diff --git a/include/wimlib/resource.h b/include/wimlib/resource.h index 3d58bd24..2c1cd9b4 100644 --- a/include/wimlib/resource.h +++ b/include/wimlib/resource.h @@ -31,7 +31,8 @@ struct wim_resource_spec { u64 offset_in_wim; /* The size of this resource in the WIM file. For compressed resources - * this is the compressed size. */ + * this is the compressed size, including overhead such as the chunk + * table. */ u64 size_in_wim; /* The number of bytes of uncompressed data this resource decompresses @@ -41,12 +42,12 @@ struct wim_resource_spec { /* The list of streams this resource contains. */ struct list_head stream_list; - /* Flags for this resource (WIM_RESHDR_FLAG_*) */ + /* Flags for this resource (WIM_RESHDR_FLAG_*). */ u32 flags : 8; - /* This flag will be set if the WIM is pipable. In such cases, the - * resource will be in a slightly different format if it is compressed. - * This is a wimlib extension. */ + /* [wimlib extension] This flag will be set if the WIM is pipable. In + * such cases, the resource will be in a slightly different format if it + * is compressed. */ u32 is_pipable : 1; /* Temporary flag. */ @@ -70,7 +71,7 @@ struct wim_reshdr_disk { le64 uncompressed_size; } _packed_attribute; -/* In-memory version of a WIM resource header. */ +/* In-memory version of a WIM resource header (`struct wim_reshdr_disk'). */ struct wim_reshdr { u64 size_in_wim : 56; u64 flags : 8; @@ -78,7 +79,8 @@ struct wim_reshdr { u64 uncompressed_size; }; -/* Flags for the `flags' field of WIM resource headers. */ +/* Flags for the `flags' field of WIM resource headers (`struct wim_reshdr'). + */ /* Unknown meaning; may be intended to indicate spaces in the WIM that are free * to overwrite. Currently ignored by wimlib. */ @@ -151,18 +153,21 @@ struct alt_chunk_table_header_disk { le64 res_usize; /* Number of bytes each compressed chunk decompresses into, except - * possibly the last which decompresses into the remainder. */ + * possibly the last which decompresses into the remainder. This + * overrides the chunk size specified by the WIM header. */ le32 chunk_size; /* Compression format used for compressed chunks: * 0 = None * 1 = LZX * 2 = XPRESS - * 3 = LZMS */ + * 3 = LZMS + * + * This overrides the compression type specified by the WIM header. */ le32 compression_format; /* This header is directly followed by a table of compressed sizes of - * the chunks. */ + * the chunks (4 bytes per entry). */ } _packed_attribute; /* wimlib internal flags used when writing resources. */ @@ -190,22 +195,49 @@ wim_reshdr_to_data(const struct wim_reshdr *reshdr, extern int skip_wim_stream(struct wim_lookup_table_entry *lte); -extern int -read_stream_prefix(const struct wim_lookup_table_entry *lte, - u64 size, consume_data_callback_t cb, - void *cb_ctx); - +/* + * Type of callback function for beginning to read a stream. + * + * @lte: + * Stream that is about to be read. + * + * @is_partial_res: + * Set to true if the stream is just one of several being read from a + * single pack and therefore would be extra expensive to read + * independently. + * + * @ctx: + * User-provided context. + * + * Must return 0 on success, a positive error code on failure, or the special + * value BEGIN_STREAM_STATUS_SKIP_STREAM to indicate that the stream should not + * be read, and read_stream_list() should continue on to the next stream + * (without calling @consume_chunk or @end_stream). + */ typedef int (*read_stream_list_begin_stream_t)(struct wim_lookup_table_entry *lte, bool is_partial_res, void *ctx); + +#define BEGIN_STREAM_STATUS_SKIP_STREAM -1 + +/* + * Type of callback function for finishing reading a stream. + * + * @lte: + * Stream that has been fully read, or stream that started being read but + * could not be fully read due to a read error. + * + * @status: + * 0 if reading the stream was successful; otherwise a nonzero error code + * that specifies the return status. + * + * @ctx: + * User-provided context. + */ typedef int (*read_stream_list_end_stream_t)(struct wim_lookup_table_entry *lte, int status, void *ctx); -#define VERIFY_STREAM_HASHES 0x1 -#define COMPUTE_MISSING_STREAM_HASHES 0x2 -#define STREAM_LIST_ALREADY_SORTED 0x4 -#define BEGIN_STREAM_STATUS_SKIP_STREAM -1 /* Callback functions and contexts for read_stream_list(). */ struct read_stream_list_callbacks { @@ -216,7 +248,8 @@ struct read_stream_list_callbacks { /* Called when a chunk of data has been read. */ consume_data_callback_t consume_chunk; - /* Called when a stream has been fully read. */ + /* Called when a stream has been fully read. A successful call to + * @begin_stream will always be matched by a call to @end_stream. */ read_stream_list_end_stream_t end_stream; /* Parameter passed to @begin_stream. */ @@ -229,6 +262,11 @@ struct read_stream_list_callbacks { void *end_stream_ctx; }; +/* Flags for read_stream_list() */ +#define VERIFY_STREAM_HASHES 0x1 +#define COMPUTE_MISSING_STREAM_HASHES 0x2 +#define STREAM_LIST_ALREADY_SORTED 0x4 + extern int read_stream_list(struct list_head *stream_list, size_t list_head_offset, diff --git a/include/wimlib/win32.h b/include/wimlib/win32.h index f4220127..6bb5b34b 100644 --- a/include/wimlib/win32.h +++ b/include/wimlib/win32.h @@ -15,17 +15,13 @@ extern int read_win32_file_prefix(const struct wim_lookup_table_entry *lte, u64 size, consume_data_callback_t cb, - u32 in_chunk_size, - void *ctx_or_buf, - int _ignored_flags); + void *cb_ctx); extern int read_win32_encrypted_file_prefix(const struct wim_lookup_table_entry *lte, u64 size, consume_data_callback_t cb, - u32 in_chunk_size, - void *ctx_or_buf, - int _ignored_flags); + void *cb_ctx); extern int @@ -46,6 +42,9 @@ fsync(int fd); extern unsigned win32_get_number_of_processors(void); +extern u64 +win32_get_avail_memory(void); + extern tchar * realpath(const tchar *path, tchar *resolved_path); diff --git a/include/wimlib/xpress.h b/include/wimlib/xpress.h index 2b57bae7..37f2fdfa 100644 --- a/include/wimlib/xpress.h +++ b/include/wimlib/xpress.h @@ -7,7 +7,7 @@ //#define ENABLE_XPRESS_DEBUG #ifdef ENABLE_XPRESS_DEBUG # define XPRESS_DEBUG DEBUG -# define XPRESS_ASSERT wimlib_assert +# define XPRESS_ASSERT wimlib_assert #else # define XPRESS_DEBUG(format, ...) # define XPRESS_ASSERT(...) diff --git a/programs/imagex.c b/programs/imagex.c index b983c7f3..c280bf33 100644 --- a/programs/imagex.c +++ b/programs/imagex.c @@ -2785,10 +2785,7 @@ print_resource(const struct wimlib_resource_entry *resource, { tprintf(T("Uncompressed size = %"PRIu64" bytes\n"), resource->uncompressed_size); - if (resource->is_packed_streams) { - tprintf(T("Raw uncompressed size = %"PRIu64" bytes\n"), - resource->raw_resource_uncompressed_size); - + if (resource->packed) { tprintf(T("Raw compressed size = %"PRIu64" bytes\n"), resource->raw_resource_compressed_size); @@ -2805,7 +2802,6 @@ print_resource(const struct wimlib_resource_entry *resource, resource->offset); } - tprintf(T("Part Number = %u\n"), resource->part_number); tprintf(T("Reference Count = %u\n"), resource->reference_count); @@ -2822,7 +2818,7 @@ print_resource(const struct wimlib_resource_entry *resource, tprintf(T("WIM_RESHDR_FLAG_FREE ")); if (resource->is_spanned) tprintf(T("WIM_RESHDR_FLAG_SPANNED ")); - if (resource->is_packed_streams) + if (resource->packed) tprintf(T("WIM_RESHDR_FLAG_PACKED_STREAMS ")); tputchar(T('\n')); tputchar(T('\n')); diff --git a/src/capture_common.c b/src/capture_common.c index 967d58eb..28a830fa 100644 --- a/src/capture_common.c +++ b/src/capture_common.c @@ -167,7 +167,7 @@ match_pattern(const tchar *path, * replacement function in win32.c. */ if (fnmatch(pat, string, FNM_PATHNAME | FNM_NOESCAPE #ifdef FNM_CASEFOLD - | FNM_CASEFOLD + | FNM_CASEFOLD #endif ) == 0) { diff --git a/src/compress.c b/src/compress.c index 7d5e2b6e..79fe4ddf 100644 --- a/src/compress.c +++ b/src/compress.c @@ -232,21 +232,21 @@ huffman_tree_compute_path_lengths(HuffmanNode *base_node, u16 cur_len) * @num_syms: The number of symbols in the alphabet. * * @max_codeword_len: The maximum allowed length of a codeword in the code. - * Note that if the code being created runs up against - * this restriction, the code ultimately created will be - * suboptimal, although there are some advantages for - * limiting the length of the codewords. + * Note that if the code being created runs up against + * this restriction, the code ultimately created will be + * suboptimal, although there are some advantages for + * limiting the length of the codewords. * * @freq_tab: An array of length @num_syms that contains the frequencies - * of each symbol in the uncompressed data. + * of each symbol in the uncompressed data. * * @lens: An array of length @num_syms into which the lengths of the - * codewords for each symbol will be written. + * codewords for each symbol will be written. * * @codewords: An array of @num_syms short integers into which the - * codewords for each symbol will be written. The first - * lens[i] bits of codewords[i] will contain the codeword - * for symbol i. + * codewords for each symbol will be written. The first + * lens[i] bits of codewords[i] will contain the codeword + * for symbol i. */ void make_canonical_huffman_code(unsigned num_syms, diff --git a/src/decompress.c b/src/decompress.c index 502ca47d..c94696dd 100644 --- a/src/decompress.c +++ b/src/decompress.c @@ -52,29 +52,29 @@ * though). * * @decode_table: The array in which to create the fast huffman decoding - * table. It must have a length of at least - * (2**table_bits) + 2 * num_syms to guarantee - * that there is enough space. Also must be 16-byte - * aligned (at least when USE_SSE2_FILL gets defined). + * table. It must have a length of at least + * (2**table_bits) + 2 * num_syms to guarantee + * that there is enough space. Also must be 16-byte + * aligned (at least when USE_SSE2_FILL gets defined). * - * @num_syms: Number of symbols in the alphabet, including symbols + * @num_syms: Number of symbols in the alphabet, including symbols * that do not appear in this particular input chunk. * * @table_bits: Any symbols with a code length of table_bits or less can - * be decoded in one lookup of the table. 2**table_bits - * must be greater than or equal to @num_syms if there are - * any Huffman codes longer than @table_bits. + * be decoded in one lookup of the table. 2**table_bits + * must be greater than or equal to @num_syms if there are + * any Huffman codes longer than @table_bits. * * @lens: An array of length @num_syms, indexable by symbol, that - * gives the length of the Huffman codeword for that - * symbol. Because the Huffman tree is in canonical form, - * it can be reconstructed by only knowing the length of - * the codeword for each symbol. It is assumed, but not - * checked, that every length is less than - * @max_codeword_len. + * gives the length of the Huffman codeword for that + * symbol. Because the Huffman tree is in canonical form, + * it can be reconstructed by only knowing the length of + * the codeword for each symbol. It is assumed, but not + * checked, that every length is less than + * @max_codeword_len. * * @max_codeword_len: The longest codeword length allowed in the compression - * format. + * format. * * Returns 0 on success; returns -1 if the length values do not correspond to a * valid Huffman tree. diff --git a/src/extract.c b/src/extract.c index a7e62697..1c7c2b07 100644 --- a/src/extract.c +++ b/src/extract.c @@ -1435,7 +1435,7 @@ read_pwm_stream_header(WIMStruct *pwm, struct wim_lookup_table_entry *lte, return 0; } - if (buf.stream_hdr.magic != PWM_STREAM_MAGIC) { + if (le64_to_cpu(buf.stream_hdr.magic) != PWM_STREAM_MAGIC) { ERROR("Data read on pipe is invalid (expected stream header)."); return WIMLIB_ERR_INVALID_PIPABLE_WIM; } @@ -1508,7 +1508,7 @@ extract_streams_from_pipe(struct apply_ctx *ctx) && (needed_lte = lookup_resource(lookup_table, found_lte->hash)) && (needed_lte->out_refcnt)) { - char *tmpfile_name = NULL; + tchar *tmpfile_name = NULL; struct wim_lookup_table_entry *lte_override; struct wim_lookup_table_entry tmpfile_lte; diff --git a/src/file_io.c b/src/file_io.c index 94f1ced5..7deaf5e3 100644 --- a/src/file_io.c +++ b/src/file_io.c @@ -126,7 +126,7 @@ full_pread(struct filedes *fd, void *buf, size_t count, off_t offset) for (bytes_remaining = count; bytes_remaining != 0; bytes_remaining -= bytes_read, buf += bytes_read, - offset += bytes_read) + offset += bytes_read) { bytes_read = pread(fd->fd, buf, bytes_remaining, offset); if (unlikely(bytes_read <= 0)) { @@ -194,7 +194,7 @@ full_pwrite(struct filedes *fd, const void *buf, size_t count, off_t offset) for (bytes_remaining = count; bytes_remaining != 0; bytes_remaining -= bytes_written, buf += bytes_written, - offset += bytes_written) + offset += bytes_written) { bytes_written = pwrite(fd->fd, buf, bytes_remaining, offset); if (unlikely(bytes_written < 0)) { @@ -206,6 +206,7 @@ full_pwrite(struct filedes *fd, const void *buf, size_t count, off_t offset) return 0; } +#if 0 /* Wrapper around writev() that checks for errors and keep retrying until all * requested bytes have been written. * @@ -242,6 +243,7 @@ full_writev(struct filedes *fd, struct iovec *iov, int iovcnt) fd->offset += total_bytes_written; return 0; } +#endif ssize_t raw_pread(struct filedes *fd, void *buf, size_t count, off_t offset) diff --git a/src/header.c b/src/header.c index 893874b5..90d40ded 100644 --- a/src/header.c +++ b/src/header.c @@ -89,8 +89,10 @@ read_wim_header(WIMStruct *wim, struct wim_header *hdr) if (ret) goto read_error; - if (disk_hdr.magic != WIM_MAGIC) { - if (disk_hdr.magic == PWM_MAGIC) { + hdr->magic = le64_to_cpu(disk_hdr.magic); + + if (hdr->magic != WIM_MAGIC) { + if (hdr->magic == PWM_MAGIC) { /* Pipable WIM: Use header at end instead, unless * actually reading from a pipe. */ if (!in_fd->is_pipe) { @@ -104,7 +106,6 @@ read_wim_header(WIMStruct *wim, struct wim_header *hdr) return WIMLIB_ERR_NOT_A_WIM_FILE; } } - hdr->magic = disk_hdr.magic; if (le32_to_cpu(disk_hdr.hdr_size) != sizeof(struct wim_header_disk)) { ERROR("\"%"TS"\": Header size is invalid (%u bytes)", @@ -171,7 +172,7 @@ write_wim_header_at_offset(const struct wim_header *hdr, struct filedes *out_fd, ((hdr->magic == PWM_MAGIC) ? "pipable " : ""), offset); - disk_hdr.magic = hdr->magic; + disk_hdr.magic = cpu_to_le64(hdr->magic); disk_hdr.hdr_size = cpu_to_le32(sizeof(struct wim_header_disk)); disk_hdr.wim_version = cpu_to_le32(hdr->wim_version); disk_hdr.wim_flags = cpu_to_le32(hdr->flags); @@ -273,7 +274,7 @@ struct hdr_flag { const char *name; }; struct hdr_flag hdr_flags[] = { - {WIM_HDR_FLAG_RESERVED, "RESERVED"}, + {WIM_HDR_FLAG_RESERVED, "RESERVED"}, {WIM_HDR_FLAG_COMPRESSION, "COMPRESSION"}, {WIM_HDR_FLAG_READONLY, "READONLY"}, {WIM_HDR_FLAG_SPANNED, "SPANNED"}, diff --git a/src/join.c b/src/join.c index 19049353..622919e9 100644 --- a/src/join.c +++ b/src/join.c @@ -38,17 +38,17 @@ * correspond to a spanned set. * * @wim: - * Part 1 of the set. + * Part 1 of the set. * * @additional_swms: - * All parts of the set other than part 1. + * All parts of the set other than part 1. * * @num_additional_swms: - * Number of WIMStructs in @additional_swms. Or, the total number of parts - * in the set minus 1. + * Number of WIMStructs in @additional_swms. Or, the total number of parts + * in the set minus 1. * * @return: - * 0 on success; WIMLIB_ERR_SPLIT_INVALID if the set is not valid. + * 0 on success; WIMLIB_ERR_SPLIT_INVALID if the set is not valid. */ static int verify_swm_set(WIMStruct *wim, WIMStruct **additional_swms, diff --git a/src/lookup_table.c b/src/lookup_table.c index a899bf95..0538dd05 100644 --- a/src/lookup_table.c +++ b/src/lookup_table.c @@ -337,8 +337,11 @@ cmp_streams_by_sequential_order(const void *p1, const void *p2) if (v) return v; - return cmp_u64(lte1->rspec->offset_in_wim + lte1->offset_in_res, - lte2->rspec->offset_in_wim + lte2->offset_in_res); + if (lte1->rspec->offset_in_wim != lte2->rspec->offset_in_wim) + return cmp_u64(lte1->rspec->offset_in_wim, + lte2->rspec->offset_in_wim); + + return cmp_u64(lte1->offset_in_res, lte2->offset_in_res); case RESOURCE_IN_FILE_ON_DISK: #ifdef WITH_FUSE @@ -486,7 +489,8 @@ validate_resource(const struct wim_resource_spec *rspec) goto invalid; /* Verify that each stream in the resource has a valid offset and size, - * and that no streams overlap. */ + * and that no streams overlap, and that the streams were added in order + * of increasing offset. */ cur_offset = 0; list_for_each_entry(lte, &rspec->stream_list, rspec_node) { if (lte->offset_in_res + lte->size < lte->size || @@ -645,7 +649,7 @@ read_wim_lookup_table(WIMStruct *wim) if (reshdr.flags & WIM_RESHDR_FLAG_PACKED_STREAMS) { cur_rspec->size_in_wim = 0; cur_rspec->uncompressed_size = 0; - cur_rspec->flags = WIM_RESHDR_FLAG_PACKED_STREAMS; + cur_rspec->offset_in_wim = 0; } if (prev_entry) { @@ -1037,7 +1041,7 @@ lte_to_wimlib_resource_entry(const struct wim_lookup_table_entry *lte, wentry->offset = lte->rspec->offset_in_wim; } wentry->raw_resource_offset_in_wim = lte->rspec->offset_in_wim; - wentry->raw_resource_uncompressed_size = lte->rspec->uncompressed_size; + /*wentry->raw_resource_uncompressed_size = lte->rspec->uncompressed_size;*/ wentry->raw_resource_compressed_size = lte->rspec->size_in_wim; } copy_hash(wentry->sha1_hash, lte->hash); @@ -1046,7 +1050,7 @@ lte_to_wimlib_resource_entry(const struct wim_lookup_table_entry *lte, wentry->is_metadata = (lte->flags & WIM_RESHDR_FLAG_METADATA) != 0; wentry->is_free = (lte->flags & WIM_RESHDR_FLAG_FREE) != 0; wentry->is_spanned = (lte->flags & WIM_RESHDR_FLAG_SPANNED) != 0; - wentry->is_packed_streams = (lte->flags & WIM_RESHDR_FLAG_PACKED_STREAMS) != 0; + wentry->packed = (lte->flags & WIM_RESHDR_FLAG_PACKED_STREAMS) != 0; } struct iterate_lte_context { diff --git a/src/lz77.c b/src/lz77.c index 3b66dee8..5729bd4c 100644 --- a/src/lz77.c +++ b/src/lz77.c @@ -86,18 +86,18 @@ insert_string(input_idx_t hash_tab[], input_idx_t prev_tab[], * @window: The window of uncompressed data. * @bytes_remaining: The number of bytes remaining in the window. * @strstart: The index of the start of the string in the window that - * we are trying to find a match for. + * we are trying to find a match for. * @prev_tab: The array of prev pointers for the hash table. * @cur_match: The index of the head of the hash chain for matches - * having the hash value of the string beginning - * at index @strstart. + * having the hash value of the string beginning + * at index @strstart. * @prev_len: The length of the match that was found for the string - * beginning at (@strstart - 1). + * beginning at (@strstart - 1). * @match_start_ret: A location into which the index of the start of the - * match will be returned. + * match will be returned. * @params: Parameters that affect how long the search will proceed - * before going with the best that has been found - * so far. + * before going with the best that has been found + * so far. * @min_start_pos: If the chain reaches a match starting before this * position (including the end-of-chain 0), the search will * be terminated. @@ -193,8 +193,8 @@ longest_match(const u8 window[], unsigned bytes_remaining, * @record_literal: Consumer for literals. * @record_ctx: Context passed to @record_match and @record_literal. * @params: Structure that contains parameters that affect how the - * analysis proceeds (mainly how good the matches - * have to be). + * analysis proceeds (mainly how good the matches + * have to be). * @prev_tab: Temporary space containing least @window_size elements. */ void diff --git a/src/lzx-decompress.c b/src/lzx-decompress.c index 5a3c78a7..b7872520 100644 --- a/src/lzx-decompress.c +++ b/src/lzx-decompress.c @@ -190,10 +190,10 @@ read_huffsym_using_alignedtree(struct input_bitstream *istream, * code length values from the input. * * @istream: The bit stream for the input. It is positioned on the beginning - * of the pretree for the code length values. + * of the pretree for the code length values. * @lens: An array that contains the length values from the previous time - * the code lengths for this Huffman tree were read, or all - * 0's if this is the first time. + * the code lengths for this Huffman tree were read, or all + * 0's if this is the first time. * @num_lens: Number of length values to decode and return. * */ @@ -310,15 +310,15 @@ lzx_read_code_lens(struct input_bitstream *istream, u8 lens[], * * @istream: The input bitstream. * @block_size_ret: A pointer to an int into which the size of the block, - * in bytes, will be returned. + * in bytes, will be returned. * @block_type_ret: A pointer to an int into which the type of the block - * (LZX_BLOCKTYPE_*) will be returned. + * (LZX_BLOCKTYPE_*) will be returned. * @tables: A pointer to a lzx_tables structure in which the - * main tree, the length tree, and possibly the - * aligned offset tree will be constructed. + * main tree, the length tree, and possibly the + * aligned offset tree will be constructed. * @queue: A pointer to the least-recently-used queue into which - * R0, R1, and R2 will be written (only for uncompressed - * blocks, which contain this information in the header) + * R0, R1, and R2 will be written (only for uncompressed + * blocks, which contain this information in the header) */ static int lzx_read_block_header(struct input_bitstream *istream, @@ -518,14 +518,14 @@ lzx_read_block_header(struct input_bitstream *istream, * tree. * * @block_type: The type of the block (LZX_BLOCKTYPE_ALIGNED or - * LZX_BLOCKTYPE_VERBATIM) + * LZX_BLOCKTYPE_VERBATIM) * * @bytes_remaining: The amount of uncompressed data remaining to be - * uncompressed in this block. It is an error if the match - * is longer than this number. + * uncompressed in this block. It is an error if the match + * is longer than this number. * * @window: A pointer to the window into which the uncompressed - * data is being written. + * data is being written. * * @window_pos: The current byte offset in the window. * @@ -539,9 +539,9 @@ lzx_read_block_header(struct input_bitstream *istream, * * Returns the length of the match, or a negative number on error. The possible * error cases are: - * - Match would exceed the amount of data remaining to be uncompressed. - * - Match refers to data before the window. - * - The input bitstream ended unexpectedly. + * - Match would exceed the amount of data remaining to be uncompressed. + * - Match refers to data before the window. + * - The input bitstream ended unexpectedly. */ static int lzx_decode_match(unsigned main_element, int block_type, @@ -764,14 +764,14 @@ undo_call_insn_preprocessing(u8 uncompressed_data[], int uncompressed_data_len) * been read. * * @block_type: The type of the block (LZX_BLOCKTYPE_VERBATIM or - * LZX_BLOCKTYPE_ALIGNED) + * LZX_BLOCKTYPE_ALIGNED) * @block_size: The size of the block, in bytes. * @num_main_syms: Number of symbols in the main alphabet. * @window: Pointer to the decompression window. * @window_pos: The current position in the window. Will be 0 for the first - * block. + * block. * @tables: The Huffman decoding tables for the block (main, length, and - * aligned offset, the latter only for LZX_BLOCKTYPE_ALIGNED) + * aligned offset, the latter only for LZX_BLOCKTYPE_ALIGNED) * @queue: The least-recently-used queue for match offsets. * @istream: The input bitstream for the compressed literals. */ diff --git a/src/mount_image.c b/src/mount_image.c index 7543b6e4..9a0c8a72 100644 --- a/src/mount_image.c +++ b/src/mount_image.c @@ -1943,7 +1943,7 @@ wimfs_read(const char *path, char *buf, size_t size, case RESOURCE_IN_WIM: if (read_partial_wim_stream_into_buf(fd->f_lte, size, offset, buf)) - ret = -errno; + ret = errno ? -errno : -EIO; else ret = size; break; diff --git a/src/ntfs-3g_capture.c b/src/ntfs-3g_capture.c index 17fa9f06..b6d92262 100644 --- a/src/ntfs-3g_capture.c +++ b/src/ntfs-3g_capture.c @@ -87,7 +87,7 @@ read_ntfs_file_prefix(const struct wim_lookup_table_entry *lte, u64 size, int ret; u8 buf[BUFFER_SIZE]; - ni = ntfs_pathname_to_inode(vol, NULL, loc->path); + ni = ntfs_pathname_to_inode(vol, NULL, loc->path); if (!ni) { ERROR_WITH_ERRNO("Can't find NTFS inode for \"%"TS"\"", loc->path); ret = WIMLIB_ERR_NTFS_3G; diff --git a/src/resource.c b/src/resource.c index 373279c1..6bc6fbf7 100644 --- a/src/resource.c +++ b/src/resource.c @@ -582,8 +582,8 @@ read_error: /* Read raw data from a file descriptor at the specified offset, feeding the * data it in chunks into the specified callback function. */ static int -read_raw_file_data(struct filedes *in_fd, u64 size, - consume_data_callback_t cb, void *cb_ctx, u64 offset) +read_raw_file_data(struct filedes *in_fd, u64 offset, u64 size, + consume_data_callback_t cb, void *cb_ctx) { u8 buf[BUFFER_SIZE]; size_t bytes_to_read; @@ -672,10 +672,10 @@ read_partial_wim_resource(const struct wim_resource_spec *rspec, cb, cb_ctx); } else { return read_raw_file_data(&rspec->wim->in_fd, + rspec->offset_in_wim + offset, size, cb, - cb_ctx, - rspec->offset_in_wim + offset); + cb_ctx); } } @@ -754,7 +754,7 @@ read_file_on_disk_prefix(const struct wim_lookup_table_entry *lte, u64 size, return WIMLIB_ERR_OPEN; } filedes_init(&fd, raw_fd); - ret = read_raw_file_data(&fd, size, cb, cb_ctx, 0); + ret = read_raw_file_data(&fd, 0, size, cb, cb_ctx); filedes_close(&fd); return ret; } @@ -790,7 +790,7 @@ typedef int (*read_stream_prefix_handler_t)(const struct wim_lookup_table_entry * reasons, depending on the stream location), or if @cb returned nonzero in * which case that error code will be returned. */ -int +static int read_stream_prefix(const struct wim_lookup_table_entry *lte, u64 size, consume_data_callback_t cb, void *cb_ctx) { @@ -1109,7 +1109,21 @@ read_full_stream_with_sha1(struct wim_lookup_table_entry *lte, * @cbs * Callback functions to accept the stream data. * @flags + * Bitwise OR of zero or more of the following flags: * + * VERIFY_STREAM_HASHES: + * For all streams being read that have already had SHA1 message + * digests computed, calculate the SHA1 message digest of the read + * data and compare it with the previously computed value. If they + * do not match, return WIMLIB_ERR_INVALID_RESOURCE_HASH. + * + * COMPUTE_MISSING_STREAM_HASHES + * For all streams being read that have not yet had their SHA1 + * message digests computed, calculate and save their SHA1 message + * digests. + * + * STREAM_LIST_ALREADY_SORTED + * @stream_list is already sorted in sequential order for reading. * * Returns 0 on success; a nonzero error code on failure. Failure can occur due * to an error reading the data or due to an error status being returned by any @@ -1261,6 +1275,7 @@ int extract_stream(struct wim_lookup_table_entry *lte, u64 size, consume_data_callback_t extract_chunk, void *extract_chunk_arg) { + wimlib_assert(size <= lte->size); if (size == lte->size) { /* Do SHA1. */ struct read_stream_list_callbacks cbs = { @@ -1313,15 +1328,15 @@ sha1_stream(struct wim_lookup_table_entry *lte) * specification. */ void wim_res_hdr_to_spec(const struct wim_reshdr *reshdr, WIMStruct *wim, - struct wim_resource_spec *spec) + struct wim_resource_spec *rspec) { - spec->wim = wim; - spec->offset_in_wim = reshdr->offset_in_wim; - spec->size_in_wim = reshdr->size_in_wim; - spec->uncompressed_size = reshdr->uncompressed_size; - INIT_LIST_HEAD(&spec->stream_list); - spec->flags = reshdr->flags; - spec->is_pipable = wim_is_pipable(wim); + rspec->wim = wim; + rspec->offset_in_wim = reshdr->offset_in_wim; + rspec->size_in_wim = reshdr->size_in_wim; + rspec->uncompressed_size = reshdr->uncompressed_size; + INIT_LIST_HEAD(&rspec->stream_list); + rspec->flags = reshdr->flags; + rspec->is_pipable = wim_is_pipable(wim); } /* Convert a stand-alone resource specification to a WIM resource header. */ diff --git a/src/security.c b/src/security.c index aab9808e..27c68a4f 100644 --- a/src/security.c +++ b/src/security.c @@ -51,20 +51,20 @@ new_wim_security_data(void) * Reads the security data from the metadata resource of a WIM image. * * @metadata_resource: An array that contains the uncompressed metadata - * resource for the WIM image. + * resource for the WIM image. * @metadata_resource_len: The length of @metadata_resource. It must be at * least 8 bytes. * @sd_ret: A pointer to a pointer to a wim_security_data structure that - * will be filled in with a pointer to a new wim_security_data - * structure containing the security data on success. + * will be filled in with a pointer to a new wim_security_data + * structure containing the security data on success. * * Note: There is no `offset' argument because the security data is located at * the beginning of the metadata resource. * * Return values: * WIMLIB_ERR_SUCCESS (0) - * WIMLIB_ERR_INVALID_METADATA_RESOURCE - * WIMLIB_ERR_NOMEM + * WIMLIB_ERR_INVALID_METADATA_RESOURCE + * WIMLIB_ERR_NOMEM */ int read_wim_security_data(const u8 metadata_resource[], size_t metadata_resource_len, diff --git a/src/unix_capture.c b/src/unix_capture.c index 1d116d4b..d16eae6d 100644 --- a/src/unix_capture.c +++ b/src/unix_capture.c @@ -296,8 +296,8 @@ out: /* * unix_build_dentry_tree(): - * Builds a tree of WIM dentries from an on-disk directory tree (UNIX - * version; no NTFS-specific data is captured). + * Builds a tree of WIM dentries from an on-disk directory tree (UNIX + * version; no NTFS-specific data is captured). * * @root_ret: Place to return a pointer to the root of the dentry tree. Only * modified if successful. Set to NULL if the file or directory was @@ -348,7 +348,7 @@ unix_build_dentry_tree(struct wim_dentry **root_ret, if (path_len >= path_bufsz) return WIMLIB_ERR_INVALID_PARAM; - path_buf = MALLOC(path_bufsz); + path_buf = MALLOC(path_bufsz); if (!path_buf) return WIMLIB_ERR_NOMEM; memcpy(path_buf, root_disk_path, path_len + 1); diff --git a/src/update_image.c b/src/update_image.c index 8f981ec6..eb848e36 100644 --- a/src/update_image.c +++ b/src/update_image.c @@ -88,13 +88,13 @@ do_overlay(struct wim_dentry *target, struct wim_dentry *branch) /* Attach or overlay a branch onto the WIM image. * * @root_p: - * Pointer to the root of the WIM image, or pointer to NULL if it has not - * been created yet. + * Pointer to the root of the WIM image, or pointer to NULL if it has not + * been created yet. * @branch - * Branch to add. + * Branch to add. * @target_path: - * Path in the WIM image to add the branch, with leading and trailing - * slashes stripped. + * Path in the WIM image to add the branch, with leading and trailing + * slashes stripped. */ static int attach_branch(struct wim_dentry **root_p, struct wim_dentry *branch, diff --git a/src/wim.c b/src/wim.c index 3d173f9a..97e156dd 100644 --- a/src/wim.c +++ b/src/wim.c @@ -151,7 +151,12 @@ wim_chunk_size_valid(u32 chunk_size, int ctype) static u32 wim_default_chunk_size(int ctype) { - return 32768; + switch (ctype) { + case WIMLIB_COMPRESSION_TYPE_LZMS: + return 131072; + default: + return 32768; + } } /* @@ -481,12 +486,15 @@ wimlib_set_output_chunk_size(WIMStruct *wim, uint32_t chunk_size) "32768, 65536, 131072, ..., 2097152."); break; case WIMLIB_COMPRESSION_TYPE_LZMS: - ERROR("Valid chunk sizes for LZMS are 131072."); + ERROR("Valid chunk sizes for LZMS are " + "32768, 65536, 131072, ..., 67108864."); break; } return WIMLIB_ERR_INVALID_CHUNK_SIZE; } - if (chunk_size != 32768) { + if (chunk_size != 32768 && + wim->out_compression_type != WIMLIB_COMPRESSION_TYPE_LZMS) + { WARNING ("Changing the compression chunk size to any value other than\n" " the default of 32768 bytes eliminates compatibility with\n" " Microsoft's software!"); diff --git a/src/win32_capture.c b/src/win32_capture.c index eb8bd7f9..6e3552f5 100644 --- a/src/win32_capture.c +++ b/src/win32_capture.c @@ -58,14 +58,11 @@ int read_win32_file_prefix(const struct wim_lookup_table_entry *lte, u64 size, consume_data_callback_t cb, - u32 in_chunk_size, - void *ctx_or_buf, - int _ignored_flags) + void *cb_ctx) { int ret = 0; - void *out_buf; - bool out_buf_malloced; u64 bytes_remaining; + u8 buf[BUFFER_SIZE]; HANDLE hFile = win32_open_existing_file(lte->file_on_disk, FILE_READ_DATA); @@ -75,28 +72,12 @@ read_win32_file_prefix(const struct wim_lookup_table_entry *lte, return WIMLIB_ERR_OPEN; } - out_buf_malloced = false; - if (cb) { - if (in_chunk_size <= STACK_MAX) { - out_buf = alloca(in_chunk_size); - } else { - out_buf = MALLOC(in_chunk_size); - if (out_buf == NULL) { - ret = WIMLIB_ERR_NOMEM; - goto out_close_handle; - } - out_buf_malloced = true; - } - } else { - out_buf = ctx_or_buf; - } - bytes_remaining = size; while (bytes_remaining) { DWORD bytesToRead, bytesRead; - bytesToRead = min(in_chunk_size, bytes_remaining); - if (!ReadFile(hFile, out_buf, bytesToRead, &bytesRead, NULL) || + bytesToRead = min(sizeof(buf), bytes_remaining); + if (!ReadFile(hFile, buf, bytesToRead, &bytesRead, NULL) || bytesRead != bytesToRead) { set_errno_from_GetLastError(); @@ -106,16 +87,10 @@ read_win32_file_prefix(const struct wim_lookup_table_entry *lte, break; } bytes_remaining -= bytesRead; - if (cb) { - ret = (*cb)(out_buf, bytesRead, ctx_or_buf); - if (ret) - break; - } else { - out_buf += bytesRead; - } + ret = (*cb)(buf, bytesRead, cb_ctx); + if (ret) + break; } - if (out_buf_malloced) - FREE(out_buf); out_close_handle: CloseHandle(hFile); return ret; @@ -123,62 +98,26 @@ out_close_handle: struct win32_encrypted_read_ctx { consume_data_callback_t read_prefix_cb; - void *read_prefix_ctx_or_buf; + void *read_prefix_ctx; int wimlib_err_code; - void *buf; - size_t buf_filled; u64 bytes_remaining; - u32 in_chunk_size; }; static DWORD WINAPI -win32_encrypted_export_cb(unsigned char *_data, void *_ctx, unsigned long len) +win32_encrypted_export_cb(unsigned char *data, void *_ctx, unsigned long len) { - const void *data = _data; struct win32_encrypted_read_ctx *ctx = _ctx; - u32 in_chunk_size = ctx->in_chunk_size; int ret; + size_t bytes_to_consume = min(len, ctx->bytes_remaining); - DEBUG("len = %lu", len); - if (ctx->read_prefix_cb) { - /* The length of the buffer passed to the ReadEncryptedFileRaw() - * export callback is undocumented, so we assume it may be of - * arbitrary size. */ - size_t bytes_to_buffer = min(ctx->bytes_remaining - ctx->buf_filled, - len); - while (bytes_to_buffer) { - size_t bytes_to_copy_to_buf = - min(bytes_to_buffer, in_chunk_size - ctx->buf_filled); - - memcpy(ctx->buf + ctx->buf_filled, data, - bytes_to_copy_to_buf); - ctx->buf_filled += bytes_to_copy_to_buf; - data += bytes_to_copy_to_buf; - bytes_to_buffer -= bytes_to_copy_to_buf; - - if (ctx->buf_filled == in_chunk_size || - ctx->buf_filled == ctx->bytes_remaining) - { - ret = (*ctx->read_prefix_cb)(ctx->buf, - ctx->buf_filled, - ctx->read_prefix_ctx_or_buf); - if (ret) { - ctx->wimlib_err_code = ret; - /* Shouldn't matter what error code is returned - * here, as long as it isn't ERROR_SUCCESS. */ - return ERROR_READ_FAULT; - } - ctx->bytes_remaining -= ctx->buf_filled; - ctx->buf_filled = 0; - } - } - } else { - size_t len_to_copy = min(len, ctx->bytes_remaining); - ctx->read_prefix_ctx_or_buf = mempcpy(ctx->read_prefix_ctx_or_buf, - data, - len_to_copy); - ctx->bytes_remaining -= len_to_copy; + ret = (*ctx->read_prefix_cb)(data, bytes_to_consume, ctx->read_prefix_ctx); + if (ret) { + ctx->wimlib_err_code = ret; + /* Shouldn't matter what error code is returned here, as long as + * it isn't ERROR_SUCCESS. */ + return ERROR_READ_FAULT; } + ctx->bytes_remaining -= bytes_to_consume; return ERROR_SUCCESS; } @@ -186,9 +125,7 @@ int read_win32_encrypted_file_prefix(const struct wim_lookup_table_entry *lte, u64 size, consume_data_callback_t cb, - u32 in_chunk_size, - void *ctx_or_buf, - int _ignored_flags) + void *cb_ctx) { struct win32_encrypted_read_ctx export_ctx; DWORD err; @@ -199,16 +136,8 @@ read_win32_encrypted_file_prefix(const struct wim_lookup_table_entry *lte, size, lte->file_on_disk); export_ctx.read_prefix_cb = cb; - export_ctx.read_prefix_ctx_or_buf = ctx_or_buf; + export_ctx.read_prefix_ctx = cb_ctx; export_ctx.wimlib_err_code = 0; - if (cb) { - export_ctx.buf = MALLOC(in_chunk_size); - if (!export_ctx.buf) - return WIMLIB_ERR_NOMEM; - } else { - export_ctx.buf = NULL; - } - export_ctx.buf_filled = 0; export_ctx.bytes_remaining = size; err = OpenEncryptedFileRaw(lte->file_on_disk, 0, &file_ctx); @@ -216,8 +145,7 @@ read_win32_encrypted_file_prefix(const struct wim_lookup_table_entry *lte, set_errno_from_win32_error(err); ERROR_WITH_ERRNO("Failed to open encrypted file \"%ls\" " "for raw read", lte->file_on_disk); - ret = WIMLIB_ERR_OPEN; - goto out_free_buf; + return WIMLIB_ERR_OPEN; } err = ReadEncryptedFileRaw(win32_encrypted_export_cb, &export_ctx, file_ctx); @@ -238,8 +166,6 @@ read_win32_encrypted_file_prefix(const struct wim_lookup_table_entry *lte, ret = 0; } CloseEncryptedFileRaw(file_ctx); -out_free_buf: - FREE(export_ctx.buf); return ret; } diff --git a/src/win32_replacements.c b/src/win32_replacements.c index 41706382..610afbdf 100644 --- a/src/win32_replacements.c +++ b/src/win32_replacements.c @@ -57,7 +57,7 @@ err: return -1; } -/* Use the Win32 API to get the number of processors */ +/* Use the Win32 API to get the number of processors. */ unsigned win32_get_number_of_processors(void) { @@ -66,6 +66,17 @@ win32_get_number_of_processors(void) return sysinfo.dwNumberOfProcessors; } +/* Use the Win32 API to get the amount of available memory. */ +u64 +win32_get_avail_memory(void) +{ + MEMORYSTATUSEX status = { + .dwLength = sizeof(status), + }; + GlobalMemoryStatusEx(&status); + return status.ullTotalPhys; +} + /* Replacement for POSIX-2008 realpath(). Warning: partial functionality only * (resolved_path must be NULL). Also I highly doubt that GetFullPathName * really does the right thing under all circumstances. */ @@ -299,6 +310,7 @@ pwrite(int fd, const void *buf, size_t count, off_t offset) return do_pread_or_pwrite(fd, (void*)buf, count, offset, true); } +#if 0 /* Dumb Windows implementation of writev(). It writes the vectors one at a * time. */ ssize_t writev(int fd, const struct iovec *iov, int iovcnt) @@ -323,6 +335,7 @@ ssize_t writev(int fd, const struct iovec *iov, int iovcnt) } return total_bytes_written; } +#endif int win32_get_file_and_vol_ids(const wchar_t *path, u64 *ino_ret, u64 *dev_ret) diff --git a/src/write.c b/src/write.c index 95365744..8ade0da9 100644 --- a/src/write.c +++ b/src/write.c @@ -43,6 +43,7 @@ #include "wimlib/lookup_table.h" #include "wimlib/metadata.h" #include "wimlib/resource.h" +#include "wimlib/win32.h" #include "wimlib/write.h" #include "wimlib/xml.h" @@ -220,7 +221,7 @@ stream_set_out_reshdr_for_reuse(struct wim_lookup_table_entry *lte) lte->out_res_offset_in_wim = rspec->offset_in_wim; lte->out_res_size_in_wim = rspec->size_in_wim; - lte->out_res_uncompressed_size = rspec->uncompressed_size; + /*lte->out_res_uncompressed_size = rspec->uncompressed_size;*/ } else { wimlib_assert(!(lte->flags & WIM_RESHDR_FLAG_PACKED_STREAMS)); @@ -242,7 +243,7 @@ write_pwm_stream_header(const struct wim_lookup_table_entry *lte, u32 reshdr_flags; int ret; - stream_hdr.magic = PWM_STREAM_MAGIC; + stream_hdr.magic = cpu_to_le64(PWM_STREAM_MAGIC); stream_hdr.uncompressed_size = cpu_to_le64(lte->size); if (additional_reshdr_flags & PWM_RESHDR_FLAG_UNHASHED) { zero_out_hash(stream_hdr.hash); @@ -402,9 +403,10 @@ struct write_streams_ctx { }; static u64 -get_chunk_entry_size(u64 res_size) +get_chunk_entry_size(u64 res_size, int write_resource_flags) { - if (res_size <= UINT32_MAX) + if (res_size <= UINT32_MAX || + (write_resource_flags & WIM_RESHDR_FLAG_PACKED_STREAMS)) return 4; else return 8; @@ -459,7 +461,9 @@ begin_chunk_table(struct write_streams_ctx *ctx, u64 res_expected_size) * prove to be needed. At this point, we just use @chunk_csizes * for a buffer of 0's because the actual compressed chunk sizes * are unknown. */ - reserve_size = expected_num_chunk_entries * get_chunk_entry_size(res_expected_size); + reserve_size = expected_num_chunk_entries * + get_chunk_entry_size(res_expected_size, + ctx->write_resource_flags); if (ctx->write_resource_flags & WIMLIB_WRITE_RESOURCE_FLAG_PACK_STREAMS) reserve_size += sizeof(struct alt_chunk_table_header_disk); memset(ctx->chunk_csizes, 0, reserve_size); @@ -505,7 +509,8 @@ end_chunk_table(struct write_streams_ctx *ctx, u64 res_actual_size, if (!(ctx->write_resource_flags & WIMLIB_WRITE_RESOURCE_FLAG_PACK_STREAMS)) actual_num_chunk_entries--; - chunk_entry_size = get_chunk_entry_size(res_actual_size); + chunk_entry_size = get_chunk_entry_size(res_actual_size, + ctx->write_resource_flags); typedef le64 __attribute__((may_alias)) aliased_le64_t; typedef le32 __attribute__((may_alias)) aliased_le32_t; @@ -1296,7 +1301,7 @@ write_stream_list(struct list_head *stream_list, lte->out_reshdr.offset_in_wim = offset_in_res; lte->out_res_offset_in_wim = reshdr.offset_in_wim; lte->out_res_size_in_wim = reshdr.size_in_wim; - lte->out_res_uncompressed_size = reshdr.uncompressed_size; + /*lte->out_res_uncompressed_size = reshdr.uncompressed_size;*/ offset_in_res += lte->size; } wimlib_assert(offset_in_res == reshdr.uncompressed_size); @@ -1901,7 +1906,7 @@ cmp_streams_by_out_rspec(const void *p1, const void *p2) if (lte2->out_reshdr.flags & WIM_RESHDR_FLAG_PACKED_STREAMS) return -1; } - return cmp_u64(lte1->out_reshdr.offset_in_wim, + return cmp_u64(lte1->out_reshdr.offset_in_wim, lte2->out_reshdr.offset_in_wim); } @@ -2929,6 +2934,9 @@ can_overwrite_wim_inplace(const WIMStruct *wim, int write_flags) return false; if (wim->chunk_size != wim->out_chunk_size) return false; + } else { + if (write_flags & WIMLIB_WRITE_FLAG_NO_PACK_STREAMS) + return false; } return true; diff --git a/src/xml.c b/src/xml.c index 920eeeb5..8179ebce 100644 --- a/src/xml.c +++ b/src/xml.c @@ -690,7 +690,7 @@ xml_write_strings_from_specs(xmlTextWriter *writer, for (size_t i = 0; i < num_specs; i++) { int rc = xml_write_string(writer, specs[i].name, *(const tchar * const *) - (struct_with_strings + specs[i].offset)); + (struct_with_strings + specs[i].offset)); if (rc) return rc; } diff --git a/src/xpress-decompress.c b/src/xpress-decompress.c index b71f6be2..f08cccbb 100644 --- a/src/xpress-decompress.c +++ b/src/xpress-decompress.c @@ -55,10 +55,10 @@ * * Also, a caveat--- according to Microsoft's documentation for XPRESS, * - * "Some implementation of the decompression algorithm expect an extra - * symbol to mark the end of the data. Specifically, some implementations - * fail during decompression if the Huffman symbol 256 is not found after - * the actual data." + * "Some implementation of the decompression algorithm expect an extra + * symbol to mark the end of the data. Specifically, some implementations + * fail during decompression if the Huffman symbol 256 is not found after + * the actual data." * * This is the case for the implementation in WIMGAPI. However, wimlib's * decompressor in this file currently does not care if this extra symbol is