#ifndef _WIMLIB_RESOURCE_H
#define _WIMLIB_RESOURCE_H
-#include "wimlib/types.h"
-#include "wimlib/endianness.h"
#include "wimlib/callback.h"
#include "wimlib/file_io.h"
#include "wimlib/list.h"
#include "wimlib/sha1.h"
+#include "wimlib/types.h"
struct wim_lookup_table_entry;
struct wim_image_metadata;
* an instance of this structure.
*
* Normally, there is a one-to-one correspondence between WIM lookup table
- * entries ("streams") and WIM resources. However, the flag
- * WIM_RESHDR_FLAG_CONCAT can be used to specify that two streams be combined
- * into the same resource and compressed together. Caveats about this flag are
- * noted in the comment above the definition of WIM_VERSION_STREAM_CONCAT. */
+ * entries ("streams", each of which may be the contents of a file, for example)
+ * and WIM resources. However, WIM resources with the
+ * WIM_RESHDR_FLAG_PACKED_STREAMS flag set may actually contain multiple streams
+ * compressed together. */
struct wim_resource_spec {
- /* The WIM file containing this resource. */
+ /* The WIM containing this resource. @wim->in_fd is expected to be a
+ * file descriptor to the underlying WIM file, opened for reading. */
WIMStruct *wim;
- /* Offset, in bytes, from the start of WIM file at which this resource
- * starts. */
+ /* The offset, in bytes, from the start of WIM file at which this
+ * resource starts. */
u64 offset_in_wim;
/* The size of this resource in the WIM file. For compressed resources
* this is the compressed size. */
u64 size_in_wim;
- /* Number of bytes of uncompressed data this resource uncompresses to.
- */
+ /* The number of bytes of uncompressed data this resource decompresses
+ * to. */
u64 uncompressed_size;
- /* List of streams this resource contains. */
- struct list_head lte_list;
+ /* The list of streams this resource contains. */
+ struct list_head stream_list;
- /* Resource flags. */
+ /* Flags for this resource (WIM_RESHDR_FLAG_*) */
u32 flags : 8;
- /* This flag will be set if the WIM is pipable and therefore the
- * resource will be in a slightly different format if it is compressed
- * (wimlib extension). */
+ /* This flag will be set if the WIM is pipable. In such cases, the
+ * resource will be in a slightly different format if it is compressed.
+ * This is a wimlib extension. */
u32 is_pipable : 1;
-
- /* Compression type of this resource as one of WIMLIB_COMPRESSION_TYPE_*
- * constants. */
- u32 ctype : 3;
-
- /* Compression chunk size. */
- u32 cchunk_size;
};
-
-/* On-disk version of a WIM resource header: This structure specifies the
- * location, size, and flags (e.g. compressed or not compressed) for a resource
- * in the WIM file. */
+/* On-disk version of a WIM resource header. */
struct wim_reshdr_disk {
/* Size of the resource as it appears in the WIM file (possibly
* compressed). */
u8 size_in_wim[7];
- /* WIM_RESHDR_FLAG_* flags. */
+ /* Zero or more of the WIM_RESHDR_FLAG_* flags. These indicate, for
+ * example, whether the resource is compressed or not. */
u8 flags;
- /* Offset of the resource from the start of the WIM file. */
+ /* Offset of the resource from the start of the WIM file, in bytes. */
le64 offset_in_wim;
- /* Uncompressed size of the resource. */
+ /* Uncompressed size of the resource, in bytes. */
le64 uncompressed_size;
} _packed_attribute;
-/* In-memory version of wim_reshdr_disk. */
+/* In-memory version of a WIM resource header. */
struct wim_reshdr {
u64 size_in_wim : 56;
u64 flags : 8;
u64 uncompressed_size;
};
-/* Flags for the `flags' field of the struct resource_entry structure. */
+/* Flags for the `flags' field of WIM resource headers. */
-/* I haven't seen this flag used in any of the WIMs I have examined. I assume
- * it means that there are no references to the stream, so the space is free.
- * However, even after deleting files from a WIM mounted with `imagex.exe
- * /mountrw', I could not see this flag being used. Either way, wimlib doesn't
- * actually use this flag for anything. */
+/* Unknown meaning; may be intended to indicate spaces in the WIM that are free
+ * to overwrite. Currently ignored by wimlib. */
#define WIM_RESHDR_FLAG_FREE 0x01
-/* Indicates that the stream is a metadata resource for a WIM image. This flag
- * is also set in the resource entry for the lookup table in the WIM header. */
+/* The resource is a metadata resource for a WIM image, or is the lookup table
+ * or XML data for the WIM. */
#define WIM_RESHDR_FLAG_METADATA 0x02
-/* Indicates that the stream is compressed (using the WIM's set compression
- * type). */
+/* The resource is compressed using the WIM's default compression type and uses
+ * the regular chunk table format. */
#define WIM_RESHDR_FLAG_COMPRESSED 0x04
-/* I haven't seen this flag used in any of the WIMs I have examined. Perhaps it
- * means that a stream could possibly be split among multiple split WIM parts.
- * However, `imagex.exe /split' does not seem to create any WIMs like this.
- * Either way, wimlib doesn't actually use this flag for anything. */
+/* Unknown meaning; may be intended to indicate a partial stream. Currently
+ * ignored by wimlib. */
#define WIM_RESHDR_FLAG_SPANNED 0x08
-/* TODO */
-#define WIM_RESHDR_FLAG_CONCAT 0x10
+/* The resource is packed in a special format that may contain multiple
+ * underlying streams, or this resource entry represents a stream packed into
+ * one such resource. When resources have this flag set, the WIM version number
+ * should be WIM_VERSION_PACKED_STREAMS. */
+#define WIM_RESHDR_FLAG_PACKED_STREAMS 0x10
+
+/* Returns true if the specified WIM resource is compressed, using either the
+ * original chunk table layout or the alternate layout for resources that may
+ * contain multiple packed streams. */
+static inline bool
+resource_is_compressed(const struct wim_resource_spec *rspec)
+{
+ return (rspec->flags & (WIM_RESHDR_FLAG_COMPRESSED |
+ WIM_RESHDR_FLAG_PACKED_STREAMS));
+}
static inline void
copy_reshdr(struct wim_reshdr *dest, const struct wim_reshdr *src)
memset(reshdr, 0, sizeof(struct wim_reshdr));
}
-
extern void
wim_res_hdr_to_spec(const struct wim_reshdr *reshdr, WIMStruct *wim,
struct wim_resource_spec *rspec);
read_partial_wim_stream_into_buf(const struct wim_lookup_table_entry *lte,
size_t size, u64 offset, void *buf);
-extern int
-skip_wim_stream(struct wim_lookup_table_entry *lte);
-
extern int
read_full_stream_into_buf(const struct wim_lookup_table_entry *lte, void *buf);
wim_reshdr_to_data(const struct wim_reshdr *reshdr,
WIMStruct *wim, void **buf_ret);
+extern int
+skip_wim_stream(struct wim_lookup_table_entry *lte);
+
extern int
read_stream_prefix(const struct wim_lookup_table_entry *lte,
u64 size, consume_data_callback_t cb,
int status,
void *ctx);
-/* Callbacks for read_stream_list(). */
+/* Callback functions and contexts for read_stream_list(). */
struct read_stream_list_callbacks {
/* Called when a stream is about to be read. */
u32 cb_chunk_size,
const struct read_stream_list_callbacks *cbs);
-/* Functions for stream extraction. */
+/* Functions to extract streams. */
extern int
extract_stream(struct wim_lookup_table_entry *lte,
le32 compressed_size;
} _packed_attribute;
-
#endif /* _WIMLIB_RESOURCE_H */
/*
* resource.c
*
- * Read uncompressed and compressed metadata and file resources from a WIM file.
+ * Code for reading streams and resources, including compressed WIM resources.
*/
/*
case WIMLIB_COMPRESSION_TYPE_LZMS:
return wimlib_lzms_decompress(cchunk, clen, uchunk, ulen);
default:
- wimlib_assert(0);
+ ERROR("Invalid compression format (%d)", ctype);
return -1;
}
}
u64 size;
};
-/* Alternate chunk table format for resources with WIM_RESHDR_FLAG_CONCAT set.
- */
+/* Alternate chunk table format for resources with
+ * WIM_RESHDR_FLAG_PACKED_STREAMS set. */
struct alt_chunk_table_header_disk {
- /* Uncompressed size of the resource. */
+ /* Uncompressed size of the resource in bytes. */
le64 res_usize;
/* Number of bytes each compressed chunk decompresses into, except
* possibly the last which decompresses into the remainder. */
le32 chunk_size;
- /* ??? */
- le32 unknown;
+ /* Compression format used for compressed chunks:
+ * 0 = None
+ * 1 = LZX
+ * 2 = XPRESS
+ * 3 = LZMS */
+ le32 compression_format;
/* This header is directly followed by a table of compressed sizes of
* the chunks. */
/* Sanity checks */
wimlib_assert(rspec != NULL);
- wimlib_assert(rspec->ctype != WIMLIB_COMPRESSION_TYPE_NONE);
- wimlib_assert(is_power_of_2(rspec->cchunk_size));
+ wimlib_assert(resource_is_compressed(rspec));
wimlib_assert(cb != NULL);
wimlib_assert(num_ranges != 0);
for (size_t i = 0; i < num_ranges; i++) {
const bool is_pipe_read = !filedes_is_seekable(in_fd);
/* Determine if the chunk table is in an altenate format. */
- const bool alt_chunk_table = (rspec->flags & WIM_RESHDR_FLAG_CONCAT) && !is_pipe_read;
+ const bool alt_chunk_table = (rspec->flags & WIM_RESHDR_FLAG_PACKED_STREAMS)
+ && !is_pipe_read;
/* Get the maximum size of uncompressed chunks in this resource, which
* we require be a power of 2. */
u32 chunk_size;
u64 cur_read_offset = rspec->offset_in_wim;
+ int ctype;
if (alt_chunk_table) {
- /* Alternate chunk table format. */
+ /* Alternate chunk table format. Its header specifies the chunk
+ * size and compression format. */
struct alt_chunk_table_header_disk hdr;
ret = full_pread(in_fd, &hdr, sizeof(hdr), cur_read_offset);
cur_read_offset += sizeof(hdr);
chunk_size = le32_to_cpu(hdr.chunk_size);
-
- if (!is_power_of_2(chunk_size)) {
- ERROR("Invalid compressed resource: "
- "expected power-of-2 chunk size (got %u)", chunk_size);
- ret = WIMLIB_ERR_INVALID_CHUNK_SIZE;
- goto out_free_memory;
- }
+ ctype = le32_to_cpu(hdr.compression_format);
+
+ /* Format numbers must be the same as in WIMGAPI to be
+ * compatible. */
+ BUILD_BUG_ON(WIMLIB_COMPRESSION_TYPE_NONE != 0);
+ BUILD_BUG_ON(WIMLIB_COMPRESSION_TYPE_LZX != 1);
+ BUILD_BUG_ON(WIMLIB_COMPRESSION_TYPE_XPRESS != 2);
+ BUILD_BUG_ON(WIMLIB_COMPRESSION_TYPE_LZMS != 3);
} else {
- chunk_size = rspec->cchunk_size;
+ /* "Normal" format: the maximum uncompressed chunk size and the
+ * compression format default to those of the WIM itself. */
+ chunk_size = rspec->wim->chunk_size;
+ ctype = rspec->wim->compression_type;
+ }
+ if (!is_power_of_2(chunk_size)) {
+ ERROR("Invalid compressed resource: "
+ "expected power-of-2 chunk size (got %u)", chunk_size);
+ ret = WIMLIB_ERR_INVALID_CHUNK_SIZE;
+ goto out_free_memory;
}
+
const u32 chunk_order = bsr32(chunk_size);
/* Calculate the total number of chunks the resource is divided into. */
chunk_csize,
ubuf,
chunk_usize,
- rspec->ctype,
+ ctype,
chunk_size);
if (ret) {
ERROR("Failed to decompress data!");
return 0;
}
+/* A consume_data_callback_t implementation that simply concatenates all chunks
+ * into a buffer. */
static int
bufferer_cb(const void *chunk, size_t size, void *_ctx)
{
*
* Read a range of data from an uncompressed or compressed resource in a WIM
* file. Data is fed chunk-by-chunk into the callback function @cb, passing it
- * the argument @cb_ctx.
+ * the argument @cb_ctx. The chunks are of unspecified size unless the
+ * RAW_CHUNKS mode is requested.
*
* By default, this function provides the uncompressed data of the resource, and
* @offset and @size and interpreted relative to the uncompressed contents of
return 0;
if ((flags & WIMLIB_READ_RESOURCE_FLAG_RAW_FULL) ||
- rspec->ctype == WIMLIB_COMPRESSION_TYPE_NONE)
+ !resource_is_compressed(rspec))
{
return read_raw_file_data(&rspec->wim->in_fd,
size,
cb_ctx,
rspec->offset_in_wim + offset);
} else {
- bool raw_chunks = !!(flags & WIMLIB_READ_RESOURCE_FLAG_RAW_CHUNKS);
+ bool raw_chunks = (flags & WIMLIB_READ_RESOURCE_FLAG_RAW_CHUNKS);
struct data_range range = {
.offset = offset,
.size = size,
}
}
+/* Read the specified range of uncompressed data from the specified stream,
+ * which must be located into a WIM file, into the specified buffer. */
int
read_partial_wim_stream_into_buf(const struct wim_lookup_table_entry *lte,
size_t size, u64 offset, void *_buf)
0);
}
+/* A consume_data_callback_t implementation that simply ignores the data
+ * received. */
static int
skip_chunk_cb(const void *chunk, size_t size, void *_ctx)
{
* any one of several locations, such as in a WIM file (compressed or
* uncompressed), in an external file, or directly in an in-memory buffer.
*
- * This function feeds the data to a callback function @cb.
+ * This function feeds the data to a callback function @cb in chunks of
+ * unspecified size.
*
* If the stream is located in a WIM file, @flags can be set as documented in
* read_partial_wim_resource(). Otherwise @flags are ignored.
return read_stream_prefix(lte, lte->size, bufferer_cb, &buf, 0);
}
-/* Read the full uncompressed data of the specified stream. A buffer sufficient
- * to hold the data is allocated and returned in @buf_ret. */
+/* Retrieve the full uncompressed data of the specified stream. A buffer large
+ * enough hold the data is allocated and returned in @buf_ret. */
int
read_full_stream_into_alloc_buf(const struct wim_lookup_table_entry *lte,
void **buf_ret)
return 0;
}
-/* Retrieve the full uncompressed data of the specified WIM resource. */
+/* Retrieve the full uncompressed data of the specified WIM resource. A buffer
+ * large enough hold the data is allocated and returned in @buf_ret. */
static int
wim_resource_spec_to_data(struct wim_resource_spec *rspec, void **buf_ret)
{
return ret;
}
-/* Retrieve the full uncompressed data of the specified WIM resource. */
+/* Retrieve the full uncompressed data of a WIM resource specified as a raw
+ * `wim_reshdr' and the corresponding WIM file. A large enough hold the data is
+ * allocated and returned in @buf_ret. */
int
wim_reshdr_to_data(const struct wim_reshdr *reshdr, WIMStruct *wim, void **buf_ret)
{
DEBUG("offset_in_wim=%"PRIu64", size_in_wim=%"PRIu64", "
"uncompressed_size=%"PRIu64,
- reshdr->offset_in_wim, reshdr->size_in_wim, reshdr->uncompressed_size);
+ reshdr->offset_in_wim, reshdr->size_in_wim,
+ reshdr->uncompressed_size);
struct wim_resource_spec rspec;
wim_res_hdr_to_spec(reshdr, wim, &rspec);
size_t list_head_offset;
};
-/* Callback for translating raw resource data into streams. */
+/* A consume_data_callback_t implementation that translates raw resource data
+ * into streams, calling the begin_stream, consume_chunk, and end_stream
+ * callback functions as appropriate. */
static int
streamifier_cb(const void *chunk, size_t size, void *_ctx)
{
DEBUG("%zu bytes passed to streamifier", size);
wimlib_assert(ctx->cur_stream != NULL);
+ wimlib_assert(size <= ctx->cur_stream->size - ctx->cur_stream_offset);
if (ctx->cur_stream_offset == 0) {
/* Starting a new stream. */
- DEBUG("Begin new stream (size=%"PRIu64").", ctx->cur_stream->size);
- ret = (*ctx->cbs.begin_stream)(ctx->cur_stream, true, ctx->cbs.begin_stream_ctx);
+ DEBUG("Begin new stream (size=%"PRIu64").",
+ ctx->cur_stream->size);
+ ret = (*ctx->cbs.begin_stream)(ctx->cur_stream, true,
+ ctx->cbs.begin_stream_ctx);
if (ret)
return ret;
}
/* Consume the chunk. */
- ret = (*ctx->cbs.consume_chunk)(chunk, size, ctx->cbs.consume_chunk_ctx);
+ ret = (*ctx->cbs.consume_chunk)(chunk, size,
+ ctx->cbs.consume_chunk_ctx);
if (ret)
return ret;
ctx->cur_stream_offset += size;
if (ctx->cur_stream_offset == ctx->cur_stream->size) {
- /* Finished reading all the data for a stream; advance to the
- * next one. */
+ /* Finished reading all the data for a stream. */
DEBUG("End stream (size=%"PRIu64").", ctx->cur_stream->size);
- ret = (*ctx->cbs.end_stream)(ctx->cur_stream, 0, ctx->cbs.end_stream_ctx);
+ ret = (*ctx->cbs.end_stream)(ctx->cur_stream, 0,
+ ctx->cbs.end_stream_ctx);
if (ret)
return ret;
if (ctx->cur_stream != ctx->final_stream) {
- struct list_head *cur = (struct list_head *)
- ((u8*)ctx->cur_stream + ctx->list_head_offset);
- struct list_head *next = cur->next;
+ /* Advance to next stream. */
+ struct list_head *cur, *next;
+
+ cur = (struct list_head *)
+ ((u8*)ctx->cur_stream + ctx->list_head_offset);
+ next = cur->next;
ctx->cur_stream = (struct wim_lookup_table_entry *)
((u8*)next - ctx->list_head_offset);
ctx->cur_stream_offset = 0;
} else {
+ /* No more streams. */
ctx->cur_stream = NULL;
}
}
ctx->cbs.begin_stream_ctx);
}
-/* Callback for continuing to read a stream while calculating its SHA1 message
- * digest. */
+/* A consume_data_callback_t implementation that continues calculating the SHA1
+ * message digest of the stream being read, then optionally passes the data on
+ * to another consume_data_callback_t implementation. This allows checking the
+ * SHA1 message digest of a stream being extracted, for example. */
static int
hasher_consume_chunk(const void *chunk, size_t size, void *_ctx)
{
int ret;
if (status) {
+ /* Error occurred; the full stream may not have been read. */
ret = status;
goto out_next_cb;
}
+ /* Retrieve the final SHA1 message digest. */
sha1_final(hash, &ctx->sha_ctx);
if (lte->unhashed) {
- /* No SHA1 message digest was present before; fill it in with
- * the calculated value. */
+ /* No SHA1 message digest was previously present for the stream.
+ * Set it to the one just calculated. */
DEBUG("Set SHA1 message digest for stream (size=%"PRIu64").", lte->size);
copy_hash(lte->hash, hash);
} else {
- /* A SHA1 message digest was present before. Verify that it is
- * the same as the calculated value. */
+ /* The stream already had a SHA1 message digest present. Verify
+ * that it is the same as the calculated value. */
if (!hashes_equal(hash, lte->hash)) {
if (wimlib_print_errors) {
ERROR("Invalid SHA1 message digest "
return (*ctx->cbs.end_stream)(lte, ret, ctx->cbs.end_stream_ctx);
}
-/* Read the full data of the stream @lte, passing the data into the specified
- * callbacks (all of which are optional) and either checking or computing the
- * SHA1 message digest of the stream. */
+/* Read the full data of the specified stream, passing the data into the
+ * specified callbacks (all of which are optional) and either checking or
+ * computing the SHA1 message digest of the stream. */
static int
read_full_stream_with_sha1(struct wim_lookup_table_entry *lte,
const struct read_stream_list_callbacks *cbs)
return 0;
}
-/* Extracts the first @size bytes of a stream to somewhere. In the process, the
- * SHA1 message digest of the uncompressed stream is checked if the full stream
- * is being extracted.
+/* Extract the first @size bytes of the specified stream.
+ *
+ * If @size specifies the full uncompressed size of the stream, then the SHA1
+ * message digest of the uncompressed stream is checked while being extracted.
*
- * @extract_chunk is the callback to extract each chunk of the stream. */
+ * The uncompressed data of the resource is passed in chunks of unspecified size
+ * to the @extract_chunk function, passing it @extract_chunk_arg. */
int
extract_stream(struct wim_lookup_table_entry *lte, u64 size,
consume_data_callback_t extract_chunk, void *extract_chunk_arg)
}
}
-/* Write a chunk of data to a file descriptor. This function can be passed as a
- * consume_data_callback_t. */
+/* A consume_data_callback_t implementation that writes the chunk of data to a
+ * file descriptor. */
int
extract_chunk_to_fd(const void *chunk, size_t size, void *_fd_p)
{
ERROR_WITH_ERRNO("Error writing to file descriptor");
return ret;
}
-
return 0;
}
/* Extract the first @size bytes of the specified stream to the specified file
- * descriptor. If @size is the full size of the stream, its SHA1 message digest
- * is also checked. */
+ * descriptor. */
int
extract_stream_to_fd(struct wim_lookup_table_entry *lte,
struct filedes *fd, u64 size)
return extract_stream(lte, size, extract_chunk_to_fd, fd);
}
-/* Calculate the SHA1 message digest of a stream, storing it in @lte->hash. */
+/* Calculate the SHA1 message digest of a stream and store it in @lte->hash. */
int
sha1_stream(struct wim_lookup_table_entry *lte)
{
+ wimlib_assert(lte->unhashed);
struct read_stream_list_callbacks cbs = {
};
return read_full_stream_with_sha1(lte, &cbs);
}
-/* Convert a WIM resource header to a stand-alone resource specification. */
+/* Convert a short WIM resource header to a stand-alone WIM resource
+ * specification. */
void
wim_res_hdr_to_spec(const struct wim_reshdr *reshdr, WIMStruct *wim,
struct wim_resource_spec *spec)
spec->offset_in_wim = reshdr->offset_in_wim;
spec->size_in_wim = reshdr->size_in_wim;
spec->uncompressed_size = reshdr->uncompressed_size;
- INIT_LIST_HEAD(&spec->lte_list);
+ INIT_LIST_HEAD(&spec->stream_list);
spec->flags = reshdr->flags;
spec->is_pipable = wim_is_pipable(wim);
- if (spec->flags & (WIM_RESHDR_FLAG_COMPRESSED | WIM_RESHDR_FLAG_CONCAT)) {
- spec->ctype = wim->compression_type;
- spec->cchunk_size = wim->chunk_size;
- } else {
- spec->ctype = WIMLIB_COMPRESSION_TYPE_NONE;
- spec->cchunk_size = 0;
- }
}
/* Convert a stand-alone resource specification to a WIM resource header. */
{
reshdr->offset_in_wim = le64_to_cpu(disk_reshdr->offset_in_wim);
reshdr->size_in_wim = (((u64)disk_reshdr->size_in_wim[0] << 0) |
- ((u64)disk_reshdr->size_in_wim[1] << 8) |
- ((u64)disk_reshdr->size_in_wim[2] << 16) |
- ((u64)disk_reshdr->size_in_wim[3] << 24) |
- ((u64)disk_reshdr->size_in_wim[4] << 32) |
- ((u64)disk_reshdr->size_in_wim[5] << 40) |
- ((u64)disk_reshdr->size_in_wim[6] << 48));
+ ((u64)disk_reshdr->size_in_wim[1] << 8) |
+ ((u64)disk_reshdr->size_in_wim[2] << 16) |
+ ((u64)disk_reshdr->size_in_wim[3] << 24) |
+ ((u64)disk_reshdr->size_in_wim[4] << 32) |
+ ((u64)disk_reshdr->size_in_wim[5] << 40) |
+ ((u64)disk_reshdr->size_in_wim[6] << 48));
reshdr->uncompressed_size = le64_to_cpu(disk_reshdr->uncompressed_size);
reshdr->flags = disk_reshdr->flags;