/*
* resource.c
*
- * Read uncompressed and compressed metadata and file resources from a WIM file.
+ * Code for reading streams and resources, including compressed WIM resources.
*/
/*
case WIMLIB_COMPRESSION_TYPE_LZMS:
return wimlib_lzms_decompress(cchunk, clen, uchunk, ulen);
default:
- wimlib_assert(0);
+ ERROR("Invalid compression format (%d)", ctype);
return -1;
}
}
u64 size;
};
-/* Alternate chunk table format for resources with WIM_RESHDR_FLAG_CONCAT set.
- */
+/* Alternate chunk table format for resources with
+ * WIM_RESHDR_FLAG_PACKED_STREAMS set. */
struct alt_chunk_table_header_disk {
- /* Uncompressed size of the resource. */
+ /* Uncompressed size of the resource in bytes. */
le64 res_usize;
/* Number of bytes each compressed chunk decompresses into, except
* possibly the last which decompresses into the remainder. */
le32 chunk_size;
- /* ??? */
- le32 unknown;
+ /* Compression format used for compressed chunks:
+ * 0 = None
+ * 1 = LZX
+ * 2 = XPRESS
+ * 3 = LZMS */
+ le32 compression_format;
/* This header is directly followed by a table of compressed sizes of
* the chunks. */
/* Sanity checks */
wimlib_assert(rspec != NULL);
- wimlib_assert(rspec->ctype != WIMLIB_COMPRESSION_TYPE_NONE);
- wimlib_assert(is_power_of_2(rspec->cchunk_size));
+ wimlib_assert(resource_is_compressed(rspec));
wimlib_assert(cb != NULL);
wimlib_assert(num_ranges != 0);
for (size_t i = 0; i < num_ranges; i++) {
const bool is_pipe_read = !filedes_is_seekable(in_fd);
/* Determine if the chunk table is in an altenate format. */
- const bool alt_chunk_table = (rspec->flags & WIM_RESHDR_FLAG_CONCAT) && !is_pipe_read;
+ const bool alt_chunk_table = (rspec->flags & WIM_RESHDR_FLAG_PACKED_STREAMS)
+ && !is_pipe_read;
/* Get the maximum size of uncompressed chunks in this resource, which
* we require be a power of 2. */
u32 chunk_size;
u64 cur_read_offset = rspec->offset_in_wim;
+ int ctype;
if (alt_chunk_table) {
- /* Alternate chunk table format. */
+ /* Alternate chunk table format. Its header specifies the chunk
+ * size and compression format. */
struct alt_chunk_table_header_disk hdr;
ret = full_pread(in_fd, &hdr, sizeof(hdr), cur_read_offset);
cur_read_offset += sizeof(hdr);
chunk_size = le32_to_cpu(hdr.chunk_size);
-
- if (!is_power_of_2(chunk_size)) {
- ERROR("Invalid compressed resource: "
- "expected power-of-2 chunk size (got %u)", chunk_size);
- ret = WIMLIB_ERR_INVALID_CHUNK_SIZE;
- goto out_free_memory;
- }
+ ctype = le32_to_cpu(hdr.compression_format);
+
+ /* Format numbers must be the same as in WIMGAPI to be
+ * compatible. */
+ BUILD_BUG_ON(WIMLIB_COMPRESSION_TYPE_NONE != 0);
+ BUILD_BUG_ON(WIMLIB_COMPRESSION_TYPE_LZX != 1);
+ BUILD_BUG_ON(WIMLIB_COMPRESSION_TYPE_XPRESS != 2);
+ BUILD_BUG_ON(WIMLIB_COMPRESSION_TYPE_LZMS != 3);
} else {
- chunk_size = rspec->cchunk_size;
+ /* "Normal" format: the maximum uncompressed chunk size and the
+ * compression format default to those of the WIM itself. */
+ chunk_size = rspec->wim->chunk_size;
+ ctype = rspec->wim->compression_type;
+ }
+ if (!is_power_of_2(chunk_size)) {
+ ERROR("Invalid compressed resource: "
+ "expected power-of-2 chunk size (got %u)", chunk_size);
+ ret = WIMLIB_ERR_INVALID_CHUNK_SIZE;
+ goto out_free_memory;
}
+
const u32 chunk_order = bsr32(chunk_size);
/* Calculate the total number of chunks the resource is divided into. */
/* Read the chunk and feed data to the callback
* function. */
- u8 *cb_buf;
+ u8 *read_buf;
+
+ if (chunk_csize == chunk_usize || raw_chunks_mode)
+ read_buf = ubuf;
+ else
+ read_buf = cbuf;
ret = full_pread(in_fd,
- cbuf,
+ read_buf,
chunk_csize,
cur_read_offset);
if (ret)
goto read_error;
- if (chunk_csize != chunk_usize && !raw_chunks_mode) {
+ if (read_buf == cbuf) {
DEBUG("Decompressing chunk %"PRIu64" "
"(csize=%"PRIu64" usize=%"PRIu64"",
i, chunk_csize, chunk_usize);
chunk_csize,
ubuf,
chunk_usize,
- rspec->ctype,
+ ctype,
chunk_size);
if (ret) {
ERROR("Failed to decompress data!");
errno = EINVAL;
goto out_free_memory;
}
- cb_buf = ubuf;
- } else {
- /* Raw chunks requested, or data stored
- * uncompressed. */
- cb_buf = cbuf;
}
cur_read_offset += chunk_csize;
size = end - start;
if (raw_chunks_mode)
- ret = (*cb)(&cb_buf[0], chunk_csize, cb_ctx);
+ ret = (*cb)(&ubuf[0], chunk_csize, cb_ctx);
else
- ret = (*cb)(&cb_buf[start], size, cb_ctx);
+ ret = (*cb)(&ubuf[start], size, cb_ctx);
if (ret)
goto out_free_memory;
return 0;
}
+/* A consume_data_callback_t implementation that simply concatenates all chunks
+ * into a buffer. */
static int
bufferer_cb(const void *chunk, size_t size, void *_ctx)
{
*
* Read a range of data from an uncompressed or compressed resource in a WIM
* file. Data is fed chunk-by-chunk into the callback function @cb, passing it
- * the argument @cb_ctx.
+ * the argument @cb_ctx. The chunks are of unspecified size unless the
+ * RAW_CHUNKS mode is requested.
*
* By default, this function provides the uncompressed data of the resource, and
* @offset and @size and interpreted relative to the uncompressed contents of
wimlib_assert(offset + size <= rspec->size_in_wim);
} else if (flags & WIMLIB_READ_RESOURCE_FLAG_RAW_CHUNKS) {
wimlib_assert(offset == 0);
- wimlib_assert(offset == rspec->uncompressed_size);
+ wimlib_assert(size == rspec->uncompressed_size);
} else {
wimlib_assert(offset + size >= offset);
wimlib_assert(offset + size <= rspec->uncompressed_size);
return 0;
if ((flags & WIMLIB_READ_RESOURCE_FLAG_RAW_FULL) ||
- rspec->ctype == WIMLIB_COMPRESSION_TYPE_NONE)
+ !resource_is_compressed(rspec))
{
return read_raw_file_data(&rspec->wim->in_fd,
size,
cb_ctx,
rspec->offset_in_wim + offset);
} else {
- bool raw_chunks = !!(flags & WIMLIB_READ_RESOURCE_FLAG_RAW_CHUNKS);
+ bool raw_chunks = (flags & WIMLIB_READ_RESOURCE_FLAG_RAW_CHUNKS);
struct data_range range = {
.offset = offset,
.size = size,
}
}
+/* Read the specified range of uncompressed data from the specified stream,
+ * which must be located into a WIM file, into the specified buffer. */
int
read_partial_wim_stream_into_buf(const struct wim_lookup_table_entry *lte,
size_t size, u64 offset, void *_buf)
0);
}
+/* A consume_data_callback_t implementation that simply ignores the data
+ * received. */
static int
skip_chunk_cb(const void *chunk, size_t size, void *_ctx)
{
* any one of several locations, such as in a WIM file (compressed or
* uncompressed), in an external file, or directly in an in-memory buffer.
*
- * This function feeds the data to a callback function @cb.
+ * This function feeds the data to a callback function @cb in chunks of
+ * unspecified size.
*
* If the stream is located in a WIM file, @flags can be set as documented in
* read_partial_wim_resource(). Otherwise @flags are ignored.
return read_stream_prefix(lte, lte->size, bufferer_cb, &buf, 0);
}
-/* Read the full uncompressed data of the specified stream. A buffer sufficient
- * to hold the data is allocated and returned in @buf_ret. */
+/* Retrieve the full uncompressed data of the specified stream. A buffer large
+ * enough hold the data is allocated and returned in @buf_ret. */
int
read_full_stream_into_alloc_buf(const struct wim_lookup_table_entry *lte,
void **buf_ret)
return 0;
}
-/* Retrieve the full uncompressed data of the specified WIM resource. */
+/* Retrieve the full uncompressed data of the specified WIM resource. A buffer
+ * large enough hold the data is allocated and returned in @buf_ret. */
static int
wim_resource_spec_to_data(struct wim_resource_spec *rspec, void **buf_ret)
{
return ret;
}
-/* Retrieve the full uncompressed data of the specified WIM resource. */
+/* Retrieve the full uncompressed data of a WIM resource specified as a raw
+ * `wim_reshdr' and the corresponding WIM file. A large enough hold the data is
+ * allocated and returned in @buf_ret. */
int
wim_reshdr_to_data(const struct wim_reshdr *reshdr, WIMStruct *wim, void **buf_ret)
{
DEBUG("offset_in_wim=%"PRIu64", size_in_wim=%"PRIu64", "
"uncompressed_size=%"PRIu64,
- reshdr->offset_in_wim, reshdr->size_in_wim, reshdr->uncompressed_size);
+ reshdr->offset_in_wim, reshdr->size_in_wim,
+ reshdr->uncompressed_size);
struct wim_resource_spec rspec;
wim_res_hdr_to_spec(reshdr, wim, &rspec);
size_t list_head_offset;
};
-/* Callback for translating raw resource data into streams. */
+/* A consume_data_callback_t implementation that translates raw resource data
+ * into streams, calling the begin_stream, consume_chunk, and end_stream
+ * callback functions as appropriate. */
static int
streamifier_cb(const void *chunk, size_t size, void *_ctx)
{
DEBUG("%zu bytes passed to streamifier", size);
wimlib_assert(ctx->cur_stream != NULL);
+ wimlib_assert(size <= ctx->cur_stream->size - ctx->cur_stream_offset);
if (ctx->cur_stream_offset == 0) {
/* Starting a new stream. */
- DEBUG("Begin new stream (size=%"PRIu64").", ctx->cur_stream->size);
- ret = (*ctx->cbs.begin_stream)(ctx->cur_stream, true, ctx->cbs.begin_stream_ctx);
+ DEBUG("Begin new stream (size=%"PRIu64").",
+ ctx->cur_stream->size);
+ ret = (*ctx->cbs.begin_stream)(ctx->cur_stream, true,
+ ctx->cbs.begin_stream_ctx);
if (ret)
return ret;
}
/* Consume the chunk. */
- ret = (*ctx->cbs.consume_chunk)(chunk, size, ctx->cbs.consume_chunk_ctx);
+ ret = (*ctx->cbs.consume_chunk)(chunk, size,
+ ctx->cbs.consume_chunk_ctx);
if (ret)
return ret;
ctx->cur_stream_offset += size;
if (ctx->cur_stream_offset == ctx->cur_stream->size) {
- /* Finished reading all the data for a stream; advance to the
- * next one. */
+ /* Finished reading all the data for a stream. */
DEBUG("End stream (size=%"PRIu64").", ctx->cur_stream->size);
- ret = (*ctx->cbs.end_stream)(ctx->cur_stream, 0, ctx->cbs.end_stream_ctx);
+ ret = (*ctx->cbs.end_stream)(ctx->cur_stream, 0,
+ ctx->cbs.end_stream_ctx);
if (ret)
return ret;
if (ctx->cur_stream != ctx->final_stream) {
- struct list_head *cur = (struct list_head *)
- ((u8*)ctx->cur_stream + ctx->list_head_offset);
- struct list_head *next = cur->next;
+ /* Advance to next stream. */
+ struct list_head *cur, *next;
+
+ cur = (struct list_head *)
+ ((u8*)ctx->cur_stream + ctx->list_head_offset);
+ next = cur->next;
ctx->cur_stream = (struct wim_lookup_table_entry *)
((u8*)next - ctx->list_head_offset);
ctx->cur_stream_offset = 0;
} else {
+ /* No more streams. */
ctx->cur_stream = NULL;
}
}
ctx->cbs.begin_stream_ctx);
}
-/* Callback for continuing to read a stream while calculating its SHA1 message
- * digest. */
+/* A consume_data_callback_t implementation that continues calculating the SHA1
+ * message digest of the stream being read, then optionally passes the data on
+ * to another consume_data_callback_t implementation. This allows checking the
+ * SHA1 message digest of a stream being extracted, for example. */
static int
hasher_consume_chunk(const void *chunk, size_t size, void *_ctx)
{
int ret;
if (status) {
+ /* Error occurred; the full stream may not have been read. */
ret = status;
goto out_next_cb;
}
+ /* Retrieve the final SHA1 message digest. */
sha1_final(hash, &ctx->sha_ctx);
if (lte->unhashed) {
- /* No SHA1 message digest was present before; fill it in with
- * the calculated value. */
+ /* No SHA1 message digest was previously present for the stream.
+ * Set it to the one just calculated. */
DEBUG("Set SHA1 message digest for stream (size=%"PRIu64").", lte->size);
copy_hash(lte->hash, hash);
} else {
- /* A SHA1 message digest was present before. Verify that it is
- * the same as the calculated value. */
+ /* The stream already had a SHA1 message digest present. Verify
+ * that it is the same as the calculated value. */
if (!hashes_equal(hash, lte->hash)) {
if (wimlib_print_errors) {
ERROR("Invalid SHA1 message digest "
return (*ctx->cbs.end_stream)(lte, ret, ctx->cbs.end_stream_ctx);
}
-/* Read the full data of the stream @lte, passing the data into the specified
- * callbacks (all of which are optional) and either checking or computing the
- * SHA1 message digest of the stream. */
+/* Read the full data of the specified stream, passing the data into the
+ * specified callbacks (all of which are optional) and either checking or
+ * computing the SHA1 message digest of the stream. */
static int
read_full_stream_with_sha1(struct wim_lookup_table_entry *lte,
const struct read_stream_list_callbacks *cbs)
return 0;
}
-/* Extracts the first @size bytes of a stream to somewhere. In the process, the
- * SHA1 message digest of the uncompressed stream is checked if the full stream
- * is being extracted.
+/* Extract the first @size bytes of the specified stream.
+ *
+ * If @size specifies the full uncompressed size of the stream, then the SHA1
+ * message digest of the uncompressed stream is checked while being extracted.
*
- * @extract_chunk is the callback to extract each chunk of the stream. */
+ * The uncompressed data of the resource is passed in chunks of unspecified size
+ * to the @extract_chunk function, passing it @extract_chunk_arg. */
int
extract_stream(struct wim_lookup_table_entry *lte, u64 size,
consume_data_callback_t extract_chunk, void *extract_chunk_arg)
}
}
-/* Write a chunk of data to a file descriptor. This function can be passed as a
- * consume_data_callback_t. */
+/* A consume_data_callback_t implementation that writes the chunk of data to a
+ * file descriptor. */
int
extract_chunk_to_fd(const void *chunk, size_t size, void *_fd_p)
{
ERROR_WITH_ERRNO("Error writing to file descriptor");
return ret;
}
-
return 0;
}
/* Extract the first @size bytes of the specified stream to the specified file
- * descriptor. If @size is the full size of the stream, its SHA1 message digest
- * is also checked. */
+ * descriptor. */
int
extract_stream_to_fd(struct wim_lookup_table_entry *lte,
struct filedes *fd, u64 size)
return extract_stream(lte, size, extract_chunk_to_fd, fd);
}
-/* Calculate the SHA1 message digest of a stream, storing it in @lte->hash. */
+/* Calculate the SHA1 message digest of a stream and store it in @lte->hash. */
int
sha1_stream(struct wim_lookup_table_entry *lte)
{
+ wimlib_assert(lte->unhashed);
struct read_stream_list_callbacks cbs = {
};
return read_full_stream_with_sha1(lte, &cbs);
}
-/* Convert a WIM resource header to a stand-alone resource specification. */
+/* Convert a short WIM resource header to a stand-alone WIM resource
+ * specification. */
void
wim_res_hdr_to_spec(const struct wim_reshdr *reshdr, WIMStruct *wim,
struct wim_resource_spec *spec)
spec->offset_in_wim = reshdr->offset_in_wim;
spec->size_in_wim = reshdr->size_in_wim;
spec->uncompressed_size = reshdr->uncompressed_size;
- INIT_LIST_HEAD(&spec->lte_list);
+ INIT_LIST_HEAD(&spec->stream_list);
spec->flags = reshdr->flags;
spec->is_pipable = wim_is_pipable(wim);
- if (spec->flags & (WIM_RESHDR_FLAG_COMPRESSED | WIM_RESHDR_FLAG_CONCAT)) {
- spec->ctype = wim->compression_type;
- spec->cchunk_size = wim->chunk_size;
- } else {
- spec->ctype = WIMLIB_COMPRESSION_TYPE_NONE;
- spec->cchunk_size = 0;
- }
}
/* Convert a stand-alone resource specification to a WIM resource header. */
/* Translates a WIM resource header from the on-disk format into an in-memory
* format. */
-int
+void
get_wim_reshdr(const struct wim_reshdr_disk *disk_reshdr,
struct wim_reshdr *reshdr)
{
reshdr->offset_in_wim = le64_to_cpu(disk_reshdr->offset_in_wim);
reshdr->size_in_wim = (((u64)disk_reshdr->size_in_wim[0] << 0) |
- ((u64)disk_reshdr->size_in_wim[1] << 8) |
- ((u64)disk_reshdr->size_in_wim[2] << 16) |
- ((u64)disk_reshdr->size_in_wim[3] << 24) |
- ((u64)disk_reshdr->size_in_wim[4] << 32) |
- ((u64)disk_reshdr->size_in_wim[5] << 40) |
- ((u64)disk_reshdr->size_in_wim[6] << 48));
+ ((u64)disk_reshdr->size_in_wim[1] << 8) |
+ ((u64)disk_reshdr->size_in_wim[2] << 16) |
+ ((u64)disk_reshdr->size_in_wim[3] << 24) |
+ ((u64)disk_reshdr->size_in_wim[4] << 32) |
+ ((u64)disk_reshdr->size_in_wim[5] << 40) |
+ ((u64)disk_reshdr->size_in_wim[6] << 48));
reshdr->uncompressed_size = le64_to_cpu(disk_reshdr->uncompressed_size);
reshdr->flags = disk_reshdr->flags;
-
- /* Avoid possible overflows. */
- if (reshdr->offset_in_wim & 0xc000000000000000ULL)
- return WIMLIB_ERR_INVALID_LOOKUP_TABLE_ENTRY;
-
- if (reshdr->uncompressed_size & 0xc000000000000000ULL)
- return WIMLIB_ERR_INVALID_LOOKUP_TABLE_ENTRY;
-
return 0;
}
git://wimlib.net / wimlib / blobdiff