From: Eric Biggers Date: Fri, 20 Dec 2013 22:59:15 +0000 (-0600) Subject: Cleanup and add more comments X-Git-Tag: v1.6.0~136 X-Git-Url: https://wimlib.net/git/?p=wimlib;a=commitdiff_plain;h=7cfb9777313e1a2f60a49d6b9ef87910f27f1a51 Cleanup and add more comments --- diff --git a/include/wimlib/callback.h b/include/wimlib/callback.h index a805f551..e83ff68f 100644 --- a/include/wimlib/callback.h +++ b/include/wimlib/callback.h @@ -3,6 +3,8 @@ #include -typedef int (*consume_data_callback_t)(const void *buf, size_t len, void *ctx); +/* Callback for processing a chunk of data. Returns 0 on success, or an error + * code on failure. */ +typedef int (*consume_data_callback_t)(const void *chunk, size_t size, void *ctx); #endif diff --git a/include/wimlib/header.h b/include/wimlib/header.h index 2fe76d4c..5f26f7c6 100644 --- a/include/wimlib/header.h +++ b/include/wimlib/header.h @@ -11,19 +11,17 @@ /* Length of the WIM header on disk. */ #define WIM_HEADER_DISK_SIZE 208 -/* Version of the WIM file. There is an older version (used for prerelease - * versions of Windows Vista), but wimlib doesn't support it. The differences - * between the versions are undocumented. */ +/* Default WIM version number. Streams are always compressed independently. */ #define WIM_VERSION_DEFAULT 0x10d00 -/* Version number used for a different WIM format, which as of Windows 8 can be - * created by passing 0x20000000 in dwFlagsAndAttributes to WIMGAPI's - * WIMCreateFile() and specifying either NONE, XPRESS, or LZMS compression. - * This format is currently undocumented by Microsoft and is seemingly - * incompatible with their own ImageX and Dism programs; however, it seems to be - * used for Windows 8 updates. The format appears to feature a new flag (0x10) - * in resource entries, which I've named WIM_RESHDR_FLAG_CONCAT. */ -#define WIM_VERSION_STREAM_CONCAT 0xe00 +/* Version number used for WIMs that allow multiple streams packed into one + * resource (WIM_RESHDR_FLAG_PACKED_STREAMS). New as of Windows 8 WIMGAPI; used + * for the Windows 8 web downloader, but yet properly documented by Microsoft. + */ +#define WIM_VERSION_PACKED_STREAMS 0xe00 + +/* Note: there is another WIM version from Vista pre-releases, but it is not + * supported by wimlib. */ /* WIM magic characters, translated to a single 64-bit little endian number. */ #define WIM_MAGIC \ diff --git a/include/wimlib/lookup_table.h b/include/wimlib/lookup_table.h index bc765ca8..f0cfb768 100644 --- a/include/wimlib/lookup_table.h +++ b/include/wimlib/lookup_table.h @@ -267,25 +267,6 @@ struct wim_lookup_table_entry { struct list_head wim_resource_list; }; -static inline int -lte_ctype(const struct wim_lookup_table_entry *lte) -{ - if (lte->resource_location == RESOURCE_IN_WIM) - return lte->rspec->ctype; - else - return WIMLIB_COMPRESSION_TYPE_NONE; -} - -static inline u32 -lte_cchunk_size(const struct wim_lookup_table_entry * lte) -{ - if (lte->resource_location == RESOURCE_IN_WIM && - lte->rspec->ctype != WIMLIB_COMPRESSION_TYPE_NONE) - return lte->rspec->cchunk_size; - else - return 32768; -} - static inline bool lte_is_partial(const struct wim_lookup_table_entry * lte) { @@ -398,7 +379,7 @@ lte_bind_wim_resource_spec(struct wim_lookup_table_entry *lte, { lte->resource_location = RESOURCE_IN_WIM; lte->rspec = rspec; - list_add_tail(<e->wim_resource_list, &rspec->lte_list); + list_add_tail(<e->wim_resource_list, &rspec->stream_list); } static inline void diff --git a/include/wimlib/resource.h b/include/wimlib/resource.h index 1d68649d..16e28dfc 100644 --- a/include/wimlib/resource.h +++ b/include/wimlib/resource.h @@ -1,12 +1,11 @@ #ifndef _WIMLIB_RESOURCE_H #define _WIMLIB_RESOURCE_H -#include "wimlib/types.h" -#include "wimlib/endianness.h" #include "wimlib/callback.h" #include "wimlib/file_io.h" #include "wimlib/list.h" #include "wimlib/sha1.h" +#include "wimlib/types.h" struct wim_lookup_table_entry; struct wim_image_metadata; @@ -18,65 +17,57 @@ struct wim_image_metadata; * an instance of this structure. * * Normally, there is a one-to-one correspondence between WIM lookup table - * entries ("streams") and WIM resources. However, the flag - * WIM_RESHDR_FLAG_CONCAT can be used to specify that two streams be combined - * into the same resource and compressed together. Caveats about this flag are - * noted in the comment above the definition of WIM_VERSION_STREAM_CONCAT. */ + * entries ("streams", each of which may be the contents of a file, for example) + * and WIM resources. However, WIM resources with the + * WIM_RESHDR_FLAG_PACKED_STREAMS flag set may actually contain multiple streams + * compressed together. */ struct wim_resource_spec { - /* The WIM file containing this resource. */ + /* The WIM containing this resource. @wim->in_fd is expected to be a + * file descriptor to the underlying WIM file, opened for reading. */ WIMStruct *wim; - /* Offset, in bytes, from the start of WIM file at which this resource - * starts. */ + /* The offset, in bytes, from the start of WIM file at which this + * resource starts. */ u64 offset_in_wim; /* The size of this resource in the WIM file. For compressed resources * this is the compressed size. */ u64 size_in_wim; - /* Number of bytes of uncompressed data this resource uncompresses to. - */ + /* The number of bytes of uncompressed data this resource decompresses + * to. */ u64 uncompressed_size; - /* List of streams this resource contains. */ - struct list_head lte_list; + /* The list of streams this resource contains. */ + struct list_head stream_list; - /* Resource flags. */ + /* Flags for this resource (WIM_RESHDR_FLAG_*) */ u32 flags : 8; - /* This flag will be set if the WIM is pipable and therefore the - * resource will be in a slightly different format if it is compressed - * (wimlib extension). */ + /* This flag will be set if the WIM is pipable. In such cases, the + * resource will be in a slightly different format if it is compressed. + * This is a wimlib extension. */ u32 is_pipable : 1; - - /* Compression type of this resource as one of WIMLIB_COMPRESSION_TYPE_* - * constants. */ - u32 ctype : 3; - - /* Compression chunk size. */ - u32 cchunk_size; }; - -/* On-disk version of a WIM resource header: This structure specifies the - * location, size, and flags (e.g. compressed or not compressed) for a resource - * in the WIM file. */ +/* On-disk version of a WIM resource header. */ struct wim_reshdr_disk { /* Size of the resource as it appears in the WIM file (possibly * compressed). */ u8 size_in_wim[7]; - /* WIM_RESHDR_FLAG_* flags. */ + /* Zero or more of the WIM_RESHDR_FLAG_* flags. These indicate, for + * example, whether the resource is compressed or not. */ u8 flags; - /* Offset of the resource from the start of the WIM file. */ + /* Offset of the resource from the start of the WIM file, in bytes. */ le64 offset_in_wim; - /* Uncompressed size of the resource. */ + /* Uncompressed size of the resource, in bytes. */ le64 uncompressed_size; } _packed_attribute; -/* In-memory version of wim_reshdr_disk. */ +/* In-memory version of a WIM resource header. */ struct wim_reshdr { u64 size_in_wim : 56; u64 flags : 8; @@ -84,31 +75,39 @@ struct wim_reshdr { u64 uncompressed_size; }; -/* Flags for the `flags' field of the struct resource_entry structure. */ +/* Flags for the `flags' field of WIM resource headers. */ -/* I haven't seen this flag used in any of the WIMs I have examined. I assume - * it means that there are no references to the stream, so the space is free. - * However, even after deleting files from a WIM mounted with `imagex.exe - * /mountrw', I could not see this flag being used. Either way, wimlib doesn't - * actually use this flag for anything. */ +/* Unknown meaning; may be intended to indicate spaces in the WIM that are free + * to overwrite. Currently ignored by wimlib. */ #define WIM_RESHDR_FLAG_FREE 0x01 -/* Indicates that the stream is a metadata resource for a WIM image. This flag - * is also set in the resource entry for the lookup table in the WIM header. */ +/* The resource is a metadata resource for a WIM image, or is the lookup table + * or XML data for the WIM. */ #define WIM_RESHDR_FLAG_METADATA 0x02 -/* Indicates that the stream is compressed (using the WIM's set compression - * type). */ +/* The resource is compressed using the WIM's default compression type and uses + * the regular chunk table format. */ #define WIM_RESHDR_FLAG_COMPRESSED 0x04 -/* I haven't seen this flag used in any of the WIMs I have examined. Perhaps it - * means that a stream could possibly be split among multiple split WIM parts. - * However, `imagex.exe /split' does not seem to create any WIMs like this. - * Either way, wimlib doesn't actually use this flag for anything. */ +/* Unknown meaning; may be intended to indicate a partial stream. Currently + * ignored by wimlib. */ #define WIM_RESHDR_FLAG_SPANNED 0x08 -/* TODO */ -#define WIM_RESHDR_FLAG_CONCAT 0x10 +/* The resource is packed in a special format that may contain multiple + * underlying streams, or this resource entry represents a stream packed into + * one such resource. When resources have this flag set, the WIM version number + * should be WIM_VERSION_PACKED_STREAMS. */ +#define WIM_RESHDR_FLAG_PACKED_STREAMS 0x10 + +/* Returns true if the specified WIM resource is compressed, using either the + * original chunk table layout or the alternate layout for resources that may + * contain multiple packed streams. */ +static inline bool +resource_is_compressed(const struct wim_resource_spec *rspec) +{ + return (rspec->flags & (WIM_RESHDR_FLAG_COMPRESSED | + WIM_RESHDR_FLAG_PACKED_STREAMS)); +} static inline void copy_reshdr(struct wim_reshdr *dest, const struct wim_reshdr *src) @@ -122,7 +121,6 @@ zero_reshdr(struct wim_reshdr *reshdr) memset(reshdr, 0, sizeof(struct wim_reshdr)); } - extern void wim_res_hdr_to_spec(const struct wim_reshdr *reshdr, WIMStruct *wim, struct wim_resource_spec *rspec); @@ -157,9 +155,6 @@ extern int read_partial_wim_stream_into_buf(const struct wim_lookup_table_entry *lte, size_t size, u64 offset, void *buf); -extern int -skip_wim_stream(struct wim_lookup_table_entry *lte); - extern int read_full_stream_into_buf(const struct wim_lookup_table_entry *lte, void *buf); @@ -171,6 +166,9 @@ extern int wim_reshdr_to_data(const struct wim_reshdr *reshdr, WIMStruct *wim, void **buf_ret); +extern int +skip_wim_stream(struct wim_lookup_table_entry *lte); + extern int read_stream_prefix(const struct wim_lookup_table_entry *lte, u64 size, consume_data_callback_t cb, @@ -183,7 +181,7 @@ typedef int (*read_stream_list_end_stream_t)(struct wim_lookup_table_entry *lte, int status, void *ctx); -/* Callbacks for read_stream_list(). */ +/* Callback functions and contexts for read_stream_list(). */ struct read_stream_list_callbacks { /* Called when a stream is about to be read. */ @@ -211,7 +209,7 @@ read_stream_list(struct list_head *stream_list, u32 cb_chunk_size, const struct read_stream_list_callbacks *cbs); -/* Functions for stream extraction. */ +/* Functions to extract streams. */ extern int extract_stream(struct wim_lookup_table_entry *lte, @@ -266,5 +264,4 @@ struct pwm_chunk_hdr { le32 compressed_size; } _packed_attribute; - #endif /* _WIMLIB_RESOURCE_H */ diff --git a/src/header.c b/src/header.c index feea7c68..893874b5 100644 --- a/src/header.c +++ b/src/header.c @@ -114,7 +114,7 @@ read_wim_header(WIMStruct *wim, struct wim_header *hdr) hdr->wim_version = le32_to_cpu(disk_hdr.wim_version); if (hdr->wim_version != WIM_VERSION_DEFAULT && - hdr->wim_version != WIM_VERSION_STREAM_CONCAT) + hdr->wim_version != WIM_VERSION_PACKED_STREAMS) { ERROR("\"%"TS"\": Unknown WIM version: %u", hdr->wim_version); return WIMLIB_ERR_UNKNOWN_VERSION; diff --git a/src/lookup_table.c b/src/lookup_table.c index fe0872cc..99e25ffb 100644 --- a/src/lookup_table.c +++ b/src/lookup_table.c @@ -97,7 +97,7 @@ clone_lookup_table_entry(const struct wim_lookup_table_entry *old) new->extracted_file = NULL; switch (new->resource_location) { case RESOURCE_IN_WIM: - list_add(&new->wim_resource_list, &new->rspec->lte_list); + list_add(&new->wim_resource_list, &new->rspec->stream_list); break; case RESOURCE_IN_FILE_ON_DISK: @@ -156,7 +156,7 @@ free_lookup_table_entry(struct wim_lookup_table_entry *lte) switch (lte->resource_location) { case RESOURCE_IN_WIM: list_del(<e->wim_resource_list); - if (list_empty(<e->rspec->lte_list)) + if (list_empty(<e->rspec->stream_list)) FREE(lte->rspec); break; case RESOURCE_IN_FILE_ON_DISK: @@ -471,8 +471,8 @@ static int validate_resource(const struct wim_resource_spec *rspec) { struct wim_lookup_table_entry *lte; - if (!list_is_singular(&rspec->lte_list)) { - list_for_each_entry(lte, &rspec->lte_list, wim_resource_list) { + if (!list_is_singular(&rspec->stream_list)) { + list_for_each_entry(lte, &rspec->stream_list, wim_resource_list) { if (rspec->flags & WIM_RESHDR_FLAG_COMPRESSED) lte->flags |= WIM_RESHDR_FLAG_COMPRESSED; else @@ -576,7 +576,7 @@ read_wim_lookup_table(WIMStruct *wim) } if (cur_rspec == NULL || - !(reshdr.flags & WIM_RESHDR_FLAG_CONCAT)) + !(reshdr.flags & WIM_RESHDR_FLAG_PACKED_STREAMS)) { /* Starting new run of stream entries that all share the * same WIM resource (streams concatenated together); or @@ -595,7 +595,7 @@ read_wim_lookup_table(WIMStruct *wim) goto out_free_cur_entry; } wim_res_hdr_to_spec(&reshdr, wim, cur_rspec); - if (reshdr.flags & WIM_RESHDR_FLAG_CONCAT) { + if (reshdr.flags & WIM_RESHDR_FLAG_PACKED_STREAMS) { cur_rspec->size_in_wim = 0; cur_rspec->uncompressed_size = 0; } @@ -613,7 +613,7 @@ read_wim_lookup_table(WIMStruct *wim) continue; } - if (reshdr.flags & WIM_RESHDR_FLAG_CONCAT) { + if (reshdr.flags & WIM_RESHDR_FLAG_PACKED_STREAMS) { /* Continuing the run with another stream. */ DEBUG("Continuing concat run with stream: " "%"PRIu64" uncompressed bytes @ resource offset %"PRIu64")", @@ -622,7 +622,7 @@ read_wim_lookup_table(WIMStruct *wim) } lte_bind_wim_resource_spec(cur_entry, cur_rspec); - if (reshdr.flags & WIM_RESHDR_FLAG_CONCAT) { + if (reshdr.flags & WIM_RESHDR_FLAG_PACKED_STREAMS) { /* In concatenation runs, the offset field is used for * in-resource offset, not the in-WIM offset, and the * size field is used for the uncompressed size, not the @@ -967,8 +967,8 @@ print_lookup_table_entry(const struct wim_lookup_table_entry *lte, FILE *out) tfputs(T("WIM_RESHDR_FLAG_METADATA, "), out); if (flags & WIM_RESHDR_FLAG_SPANNED) tfputs(T("WIM_RESHDR_FLAG_SPANNED, "), out); - if (flags & WIM_RESHDR_FLAG_CONCAT) - tfputs(T("WIM_RESHDR_FLAG_CONCAT, "), out); + if (flags & WIM_RESHDR_FLAG_PACKED_STREAMS) + tfputs(T("WIM_RESHDR_FLAG_PACKED_STREAMS, "), out); tputc(T('\n'), out); switch (lte->resource_location) { case RESOURCE_IN_WIM: diff --git a/src/mount_image.c b/src/mount_image.c index 99db4ab3..7543b6e4 100644 --- a/src/mount_image.c +++ b/src/mount_image.c @@ -2414,11 +2414,6 @@ wimlib_mount_image(WIMStruct *wim, int image, const char *dir, return ret; } - if (wim->hdr.wim_version == WIM_VERSION_STREAM_CONCAT) { - WARNING("WIM contains streams not compressed independently; " - "access may be slow."); - } - ret = select_wim_image(wim, image); if (ret) return ret; diff --git a/src/resource.c b/src/resource.c index 90197d79..78d07a7f 100644 --- a/src/resource.c +++ b/src/resource.c @@ -1,7 +1,7 @@ /* * resource.c * - * Read uncompressed and compressed metadata and file resources from a WIM file. + * Code for reading streams and resources, including compressed WIM resources. */ /* @@ -108,7 +108,7 @@ decompress(const void *cchunk, unsigned clen, void *uchunk, unsigned ulen, case WIMLIB_COMPRESSION_TYPE_LZMS: return wimlib_lzms_decompress(cchunk, clen, uchunk, ulen); default: - wimlib_assert(0); + ERROR("Invalid compression format (%d)", ctype); return -1; } } @@ -118,18 +118,22 @@ struct data_range { u64 size; }; -/* Alternate chunk table format for resources with WIM_RESHDR_FLAG_CONCAT set. - */ +/* Alternate chunk table format for resources with + * WIM_RESHDR_FLAG_PACKED_STREAMS set. */ struct alt_chunk_table_header_disk { - /* Uncompressed size of the resource. */ + /* Uncompressed size of the resource in bytes. */ le64 res_usize; /* Number of bytes each compressed chunk decompresses into, except * possibly the last which decompresses into the remainder. */ le32 chunk_size; - /* ??? */ - le32 unknown; + /* Compression format used for compressed chunks: + * 0 = None + * 1 = LZX + * 2 = XPRESS + * 3 = LZMS */ + le32 compression_format; /* This header is directly followed by a table of compressed sizes of * the chunks. */ @@ -190,8 +194,7 @@ read_compressed_wim_resource(const struct wim_resource_spec * const rspec, /* Sanity checks */ wimlib_assert(rspec != NULL); - wimlib_assert(rspec->ctype != WIMLIB_COMPRESSION_TYPE_NONE); - wimlib_assert(is_power_of_2(rspec->cchunk_size)); + wimlib_assert(resource_is_compressed(rspec)); wimlib_assert(cb != NULL); wimlib_assert(num_ranges != 0); for (size_t i = 0; i < num_ranges; i++) { @@ -222,14 +225,17 @@ read_compressed_wim_resource(const struct wim_resource_spec * const rspec, const bool is_pipe_read = !filedes_is_seekable(in_fd); /* Determine if the chunk table is in an altenate format. */ - const bool alt_chunk_table = (rspec->flags & WIM_RESHDR_FLAG_CONCAT) && !is_pipe_read; + const bool alt_chunk_table = (rspec->flags & WIM_RESHDR_FLAG_PACKED_STREAMS) + && !is_pipe_read; /* Get the maximum size of uncompressed chunks in this resource, which * we require be a power of 2. */ u32 chunk_size; u64 cur_read_offset = rspec->offset_in_wim; + int ctype; if (alt_chunk_table) { - /* Alternate chunk table format. */ + /* Alternate chunk table format. Its header specifies the chunk + * size and compression format. */ struct alt_chunk_table_header_disk hdr; ret = full_pread(in_fd, &hdr, sizeof(hdr), cur_read_offset); @@ -238,16 +244,27 @@ read_compressed_wim_resource(const struct wim_resource_spec * const rspec, cur_read_offset += sizeof(hdr); chunk_size = le32_to_cpu(hdr.chunk_size); - - if (!is_power_of_2(chunk_size)) { - ERROR("Invalid compressed resource: " - "expected power-of-2 chunk size (got %u)", chunk_size); - ret = WIMLIB_ERR_INVALID_CHUNK_SIZE; - goto out_free_memory; - } + ctype = le32_to_cpu(hdr.compression_format); + + /* Format numbers must be the same as in WIMGAPI to be + * compatible. */ + BUILD_BUG_ON(WIMLIB_COMPRESSION_TYPE_NONE != 0); + BUILD_BUG_ON(WIMLIB_COMPRESSION_TYPE_LZX != 1); + BUILD_BUG_ON(WIMLIB_COMPRESSION_TYPE_XPRESS != 2); + BUILD_BUG_ON(WIMLIB_COMPRESSION_TYPE_LZMS != 3); } else { - chunk_size = rspec->cchunk_size; + /* "Normal" format: the maximum uncompressed chunk size and the + * compression format default to those of the WIM itself. */ + chunk_size = rspec->wim->chunk_size; + ctype = rspec->wim->compression_type; + } + if (!is_power_of_2(chunk_size)) { + ERROR("Invalid compressed resource: " + "expected power-of-2 chunk size (got %u)", chunk_size); + ret = WIMLIB_ERR_INVALID_CHUNK_SIZE; + goto out_free_memory; } + const u32 chunk_order = bsr32(chunk_size); /* Calculate the total number of chunks the resource is divided into. */ @@ -511,7 +528,7 @@ read_compressed_wim_resource(const struct wim_resource_spec * const rspec, chunk_csize, ubuf, chunk_usize, - rspec->ctype, + ctype, chunk_size); if (ret) { ERROR("Failed to decompress data!"); @@ -626,6 +643,8 @@ read_raw_file_data(struct filedes *in_fd, u64 size, return 0; } +/* A consume_data_callback_t implementation that simply concatenates all chunks + * into a buffer. */ static int bufferer_cb(const void *chunk, size_t size, void *_ctx) { @@ -640,7 +659,8 @@ bufferer_cb(const void *chunk, size_t size, void *_ctx) * * Read a range of data from an uncompressed or compressed resource in a WIM * file. Data is fed chunk-by-chunk into the callback function @cb, passing it - * the argument @cb_ctx. + * the argument @cb_ctx. The chunks are of unspecified size unless the + * RAW_CHUNKS mode is requested. * * By default, this function provides the uncompressed data of the resource, and * @offset and @size and interpreted relative to the uncompressed contents of @@ -694,7 +714,7 @@ read_partial_wim_resource(const struct wim_resource_spec *rspec, return 0; if ((flags & WIMLIB_READ_RESOURCE_FLAG_RAW_FULL) || - rspec->ctype == WIMLIB_COMPRESSION_TYPE_NONE) + !resource_is_compressed(rspec)) { return read_raw_file_data(&rspec->wim->in_fd, size, @@ -702,7 +722,7 @@ read_partial_wim_resource(const struct wim_resource_spec *rspec, cb_ctx, rspec->offset_in_wim + offset); } else { - bool raw_chunks = !!(flags & WIMLIB_READ_RESOURCE_FLAG_RAW_CHUNKS); + bool raw_chunks = (flags & WIMLIB_READ_RESOURCE_FLAG_RAW_CHUNKS); struct data_range range = { .offset = offset, .size = size, @@ -712,6 +732,8 @@ read_partial_wim_resource(const struct wim_resource_spec *rspec, } } +/* Read the specified range of uncompressed data from the specified stream, + * which must be located into a WIM file, into the specified buffer. */ int read_partial_wim_stream_into_buf(const struct wim_lookup_table_entry *lte, size_t size, u64 offset, void *_buf) @@ -728,6 +750,8 @@ read_partial_wim_stream_into_buf(const struct wim_lookup_table_entry *lte, 0); } +/* A consume_data_callback_t implementation that simply ignores the data + * received. */ static int skip_chunk_cb(const void *chunk, size_t size, void *_ctx) { @@ -818,7 +842,8 @@ typedef int (*read_stream_prefix_handler_t)(const struct wim_lookup_table_entry * any one of several locations, such as in a WIM file (compressed or * uncompressed), in an external file, or directly in an in-memory buffer. * - * This function feeds the data to a callback function @cb. + * This function feeds the data to a callback function @cb in chunks of + * unspecified size. * * If the stream is located in a WIM file, @flags can be set as documented in * read_partial_wim_resource(). Otherwise @flags are ignored. @@ -864,8 +889,8 @@ read_full_stream_into_buf(const struct wim_lookup_table_entry *lte, void *_buf) return read_stream_prefix(lte, lte->size, bufferer_cb, &buf, 0); } -/* Read the full uncompressed data of the specified stream. A buffer sufficient - * to hold the data is allocated and returned in @buf_ret. */ +/* Retrieve the full uncompressed data of the specified stream. A buffer large + * enough hold the data is allocated and returned in @buf_ret. */ int read_full_stream_into_alloc_buf(const struct wim_lookup_table_entry *lte, void **buf_ret) @@ -893,7 +918,8 @@ read_full_stream_into_alloc_buf(const struct wim_lookup_table_entry *lte, return 0; } -/* Retrieve the full uncompressed data of the specified WIM resource. */ +/* Retrieve the full uncompressed data of the specified WIM resource. A buffer + * large enough hold the data is allocated and returned in @buf_ret. */ static int wim_resource_spec_to_data(struct wim_resource_spec *rspec, void **buf_ret) { @@ -917,13 +943,16 @@ wim_resource_spec_to_data(struct wim_resource_spec *rspec, void **buf_ret) return ret; } -/* Retrieve the full uncompressed data of the specified WIM resource. */ +/* Retrieve the full uncompressed data of a WIM resource specified as a raw + * `wim_reshdr' and the corresponding WIM file. A large enough hold the data is + * allocated and returned in @buf_ret. */ int wim_reshdr_to_data(const struct wim_reshdr *reshdr, WIMStruct *wim, void **buf_ret) { DEBUG("offset_in_wim=%"PRIu64", size_in_wim=%"PRIu64", " "uncompressed_size=%"PRIu64, - reshdr->offset_in_wim, reshdr->size_in_wim, reshdr->uncompressed_size); + reshdr->offset_in_wim, reshdr->size_in_wim, + reshdr->uncompressed_size); struct wim_resource_spec rspec; wim_res_hdr_to_spec(reshdr, wim, &rspec); @@ -938,7 +967,9 @@ struct streamifier_context { size_t list_head_offset; }; -/* Callback for translating raw resource data into streams. */ +/* A consume_data_callback_t implementation that translates raw resource data + * into streams, calling the begin_stream, consume_chunk, and end_stream + * callback functions as appropriate. */ static int streamifier_cb(const void *chunk, size_t size, void *_ctx) { @@ -948,39 +979,47 @@ streamifier_cb(const void *chunk, size_t size, void *_ctx) DEBUG("%zu bytes passed to streamifier", size); wimlib_assert(ctx->cur_stream != NULL); + wimlib_assert(size <= ctx->cur_stream->size - ctx->cur_stream_offset); if (ctx->cur_stream_offset == 0) { /* Starting a new stream. */ - DEBUG("Begin new stream (size=%"PRIu64").", ctx->cur_stream->size); - ret = (*ctx->cbs.begin_stream)(ctx->cur_stream, true, ctx->cbs.begin_stream_ctx); + DEBUG("Begin new stream (size=%"PRIu64").", + ctx->cur_stream->size); + ret = (*ctx->cbs.begin_stream)(ctx->cur_stream, true, + ctx->cbs.begin_stream_ctx); if (ret) return ret; } /* Consume the chunk. */ - ret = (*ctx->cbs.consume_chunk)(chunk, size, ctx->cbs.consume_chunk_ctx); + ret = (*ctx->cbs.consume_chunk)(chunk, size, + ctx->cbs.consume_chunk_ctx); if (ret) return ret; ctx->cur_stream_offset += size; if (ctx->cur_stream_offset == ctx->cur_stream->size) { - /* Finished reading all the data for a stream; advance to the - * next one. */ + /* Finished reading all the data for a stream. */ DEBUG("End stream (size=%"PRIu64").", ctx->cur_stream->size); - ret = (*ctx->cbs.end_stream)(ctx->cur_stream, 0, ctx->cbs.end_stream_ctx); + ret = (*ctx->cbs.end_stream)(ctx->cur_stream, 0, + ctx->cbs.end_stream_ctx); if (ret) return ret; if (ctx->cur_stream != ctx->final_stream) { - struct list_head *cur = (struct list_head *) - ((u8*)ctx->cur_stream + ctx->list_head_offset); - struct list_head *next = cur->next; + /* Advance to next stream. */ + struct list_head *cur, *next; + + cur = (struct list_head *) + ((u8*)ctx->cur_stream + ctx->list_head_offset); + next = cur->next; ctx->cur_stream = (struct wim_lookup_table_entry *) ((u8*)next - ctx->list_head_offset); ctx->cur_stream_offset = 0; } else { + /* No more streams. */ ctx->cur_stream = NULL; } } @@ -1009,8 +1048,10 @@ hasher_begin_stream(struct wim_lookup_table_entry *lte, bool is_partial_res, ctx->cbs.begin_stream_ctx); } -/* Callback for continuing to read a stream while calculating its SHA1 message - * digest. */ +/* A consume_data_callback_t implementation that continues calculating the SHA1 + * message digest of the stream being read, then optionally passes the data on + * to another consume_data_callback_t implementation. This allows checking the + * SHA1 message digest of a stream being extracted, for example. */ static int hasher_consume_chunk(const void *chunk, size_t size, void *_ctx) { @@ -1033,20 +1074,22 @@ hasher_end_stream(struct wim_lookup_table_entry *lte, int status, void *_ctx) int ret; if (status) { + /* Error occurred; the full stream may not have been read. */ ret = status; goto out_next_cb; } + /* Retrieve the final SHA1 message digest. */ sha1_final(hash, &ctx->sha_ctx); if (lte->unhashed) { - /* No SHA1 message digest was present before; fill it in with - * the calculated value. */ + /* No SHA1 message digest was previously present for the stream. + * Set it to the one just calculated. */ DEBUG("Set SHA1 message digest for stream (size=%"PRIu64").", lte->size); copy_hash(lte->hash, hash); } else { - /* A SHA1 message digest was present before. Verify that it is - * the same as the calculated value. */ + /* The stream already had a SHA1 message digest present. Verify + * that it is the same as the calculated value. */ if (!hashes_equal(hash, lte->hash)) { if (wimlib_print_errors) { ERROR("Invalid SHA1 message digest " @@ -1069,9 +1112,9 @@ out_next_cb: return (*ctx->cbs.end_stream)(lte, ret, ctx->cbs.end_stream_ctx); } -/* Read the full data of the stream @lte, passing the data into the specified - * callbacks (all of which are optional) and either checking or computing the - * SHA1 message digest of the stream. */ +/* Read the full data of the specified stream, passing the data into the + * specified callbacks (all of which are optional) and either checking or + * computing the SHA1 message digest of the stream. */ static int read_full_stream_with_sha1(struct wim_lookup_table_entry *lte, const struct read_stream_list_callbacks *cbs) @@ -1355,11 +1398,13 @@ read_stream_list(struct list_head *stream_list, return 0; } -/* Extracts the first @size bytes of a stream to somewhere. In the process, the - * SHA1 message digest of the uncompressed stream is checked if the full stream - * is being extracted. +/* Extract the first @size bytes of the specified stream. + * + * If @size specifies the full uncompressed size of the stream, then the SHA1 + * message digest of the uncompressed stream is checked while being extracted. * - * @extract_chunk is the callback to extract each chunk of the stream. */ + * The uncompressed data of the resource is passed in chunks of unspecified size + * to the @extract_chunk function, passing it @extract_chunk_arg. */ int extract_stream(struct wim_lookup_table_entry *lte, u64 size, consume_data_callback_t extract_chunk, void *extract_chunk_arg) @@ -1378,8 +1423,8 @@ extract_stream(struct wim_lookup_table_entry *lte, u64 size, } } -/* Write a chunk of data to a file descriptor. This function can be passed as a - * consume_data_callback_t. */ +/* A consume_data_callback_t implementation that writes the chunk of data to a + * file descriptor. */ int extract_chunk_to_fd(const void *chunk, size_t size, void *_fd_p) { @@ -1390,13 +1435,11 @@ extract_chunk_to_fd(const void *chunk, size_t size, void *_fd_p) ERROR_WITH_ERRNO("Error writing to file descriptor"); return ret; } - return 0; } /* Extract the first @size bytes of the specified stream to the specified file - * descriptor. If @size is the full size of the stream, its SHA1 message digest - * is also checked. */ + * descriptor. */ int extract_stream_to_fd(struct wim_lookup_table_entry *lte, struct filedes *fd, u64 size) @@ -1404,16 +1447,18 @@ extract_stream_to_fd(struct wim_lookup_table_entry *lte, return extract_stream(lte, size, extract_chunk_to_fd, fd); } -/* Calculate the SHA1 message digest of a stream, storing it in @lte->hash. */ +/* Calculate the SHA1 message digest of a stream and store it in @lte->hash. */ int sha1_stream(struct wim_lookup_table_entry *lte) { + wimlib_assert(lte->unhashed); struct read_stream_list_callbacks cbs = { }; return read_full_stream_with_sha1(lte, &cbs); } -/* Convert a WIM resource header to a stand-alone resource specification. */ +/* Convert a short WIM resource header to a stand-alone WIM resource + * specification. */ void wim_res_hdr_to_spec(const struct wim_reshdr *reshdr, WIMStruct *wim, struct wim_resource_spec *spec) @@ -1422,16 +1467,9 @@ wim_res_hdr_to_spec(const struct wim_reshdr *reshdr, WIMStruct *wim, spec->offset_in_wim = reshdr->offset_in_wim; spec->size_in_wim = reshdr->size_in_wim; spec->uncompressed_size = reshdr->uncompressed_size; - INIT_LIST_HEAD(&spec->lte_list); + INIT_LIST_HEAD(&spec->stream_list); spec->flags = reshdr->flags; spec->is_pipable = wim_is_pipable(wim); - if (spec->flags & (WIM_RESHDR_FLAG_COMPRESSED | WIM_RESHDR_FLAG_CONCAT)) { - spec->ctype = wim->compression_type; - spec->cchunk_size = wim->chunk_size; - } else { - spec->ctype = WIMLIB_COMPRESSION_TYPE_NONE; - spec->cchunk_size = 0; - } } /* Convert a stand-alone resource specification to a WIM resource header. */ @@ -1453,12 +1491,12 @@ get_wim_reshdr(const struct wim_reshdr_disk *disk_reshdr, { reshdr->offset_in_wim = le64_to_cpu(disk_reshdr->offset_in_wim); reshdr->size_in_wim = (((u64)disk_reshdr->size_in_wim[0] << 0) | - ((u64)disk_reshdr->size_in_wim[1] << 8) | - ((u64)disk_reshdr->size_in_wim[2] << 16) | - ((u64)disk_reshdr->size_in_wim[3] << 24) | - ((u64)disk_reshdr->size_in_wim[4] << 32) | - ((u64)disk_reshdr->size_in_wim[5] << 40) | - ((u64)disk_reshdr->size_in_wim[6] << 48)); + ((u64)disk_reshdr->size_in_wim[1] << 8) | + ((u64)disk_reshdr->size_in_wim[2] << 16) | + ((u64)disk_reshdr->size_in_wim[3] << 24) | + ((u64)disk_reshdr->size_in_wim[4] << 32) | + ((u64)disk_reshdr->size_in_wim[5] << 40) | + ((u64)disk_reshdr->size_in_wim[6] << 48)); reshdr->uncompressed_size = le64_to_cpu(disk_reshdr->uncompressed_size); reshdr->flags = disk_reshdr->flags; diff --git a/src/write.c b/src/write.c index a3f4213f..4ae01c6e 100644 --- a/src/write.c +++ b/src/write.c @@ -76,15 +76,13 @@ can_raw_copy(const struct wim_lookup_table_entry *lte, { if (lte->resource_location != RESOURCE_IN_WIM) return false; - if (out_ctype == WIMLIB_COMPRESSION_TYPE_NONE) - return false; - if (lte->rspec->ctype != out_ctype) + if (lte->rspec->flags & WIM_RESHDR_FLAG_PACKED_STREAMS) return false; - if (out_chunk_size != lte->rspec->cchunk_size) + if (out_ctype == WIMLIB_COMPRESSION_TYPE_NONE) return false; - if (lte->offset_in_res != 0) + if (lte->rspec->wim->compression_type != out_ctype) return false; - if (lte->size != lte->rspec->uncompressed_size) + if (lte->rspec->wim->chunk_size != out_chunk_size) return false; return true; } @@ -268,7 +266,7 @@ write_pwm_stream_header(const struct wim_lookup_table_entry *lte, copy_hash(stream_hdr.hash, lte->hash); } - reshdr_flags = lte->flags & ~(WIM_RESHDR_FLAG_COMPRESSED | WIM_RESHDR_FLAG_CONCAT); + reshdr_flags = lte->flags & ~(WIM_RESHDR_FLAG_COMPRESSED | WIM_RESHDR_FLAG_PACKED_STREAMS); reshdr_flags |= additional_reshdr_flags; stream_hdr.flags = cpu_to_le32(reshdr_flags); ret = full_write(out_fd, &stream_hdr, sizeof(stream_hdr)); @@ -528,7 +526,7 @@ write_wim_resource(struct wim_lookup_table_entry *lte, write_ctx.resource_flags = resource_flags; try_write_again: if (write_ctx.out_ctype == WIMLIB_COMPRESSION_TYPE_NONE) - in_chunk_size = lte_cchunk_size(lte); + in_chunk_size = 0; else in_chunk_size = out_chunk_size; ret = read_stream_prefix(lte, read_size, write_resource_cb, @@ -557,7 +555,7 @@ try_write_again: /* Fill in out_reshdr with information about the newly written * resource. */ out_reshdr->size_in_wim = out_fd->offset - res_start_offset; - out_reshdr->flags = lte->flags & ~WIM_RESHDR_FLAG_CONCAT; + out_reshdr->flags = lte->flags & ~WIM_RESHDR_FLAG_PACKED_STREAMS; if (out_ctype == WIMLIB_COMPRESSION_TYPE_NONE) out_reshdr->flags &= ~WIM_RESHDR_FLAG_COMPRESSED; else