X-Git-Url: https://wimlib.net/git/?p=wimlib;a=blobdiff_plain;f=src%2Fresource.c;h=d5c26b40c00ace2b4392e8821879b22276fb7284;hp=90197d79eacfa1b59c44c82f1ca2e0d3485322e4;hb=f18b7fc3361c4daac0ddd104af65a8eff8466fec;hpb=f7e62e27ceae4cf2ad2cc74b1a97ebf3015b95eb diff --git a/src/resource.c b/src/resource.c index 90197d79..d5c26b40 100644 --- a/src/resource.c +++ b/src/resource.c @@ -1,41 +1,47 @@ /* * resource.c * - * Read uncompressed and compressed metadata and file resources from a WIM file. + * Code for reading streams and resources, including compressed WIM resources. */ /* * Copyright (C) 2012, 2013 Eric Biggers * - * This file is part of wimlib, a library for working with WIM files. + * This file is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation; either version 3 of the License, or (at your option) any + * later version. * - * wimlib is free software; you can redistribute it and/or modify it under the - * terms of the GNU General Public License as published by the Free Software - * Foundation; either version 3 of the License, or (at your option) any later - * version. + * This file is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. * - * wimlib is distributed in the hope that it will be useful, but WITHOUT ANY - * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR - * A PARTICULAR PURPOSE. See the GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along with - * wimlib; if not, see http://www.gnu.org/licenses/. + * You should have received a copy of the GNU Lesser General Public License + * along with this file; if not, see http://www.gnu.org/licenses/. */ #ifdef HAVE_CONFIG_H # include "config.h" #endif -#include "wimlib.h" +#include +#include +#include + +#include "wimlib/alloca.h" +#include "wimlib/assert.h" +#include "wimlib/bitops.h" #include "wimlib/endianness.h" #include "wimlib/error.h" #include "wimlib/file_io.h" #include "wimlib/lookup_table.h" #include "wimlib/resource.h" #include "wimlib/sha1.h" +#include "wimlib/wim.h" #ifdef __WIN32__ -/* for read_win32_file_prefix(), read_win32_encrypted_file_prefix() */ +/* for read_winnt_file_prefix(), read_win32_encrypted_file_prefix() */ # include "wimlib/win32.h" #endif @@ -44,13 +50,6 @@ # include "wimlib/ntfs_3g.h" #endif -#ifdef HAVE_ALLOCA_H -# include -#endif -#include -#include -#include -#include /* * Compressed WIM resources @@ -90,51 +89,11 @@ */ -/* Decompress the specified chunk that uses the specified compression type - * @ctype, part of a WIM with default chunk size @wim_chunk_size. For LZX the - * separate @wim_chunk_size is needed because it determines the window size used - * for LZX compression. */ -static int -decompress(const void *cchunk, unsigned clen, void *uchunk, unsigned ulen, - int ctype, u32 wim_chunk_size) -{ - switch (ctype) { - case WIMLIB_COMPRESSION_TYPE_LZX: - return wimlib_lzx_decompress2(cchunk, clen, - uchunk, ulen, wim_chunk_size); - case WIMLIB_COMPRESSION_TYPE_XPRESS: - return wimlib_xpress_decompress(cchunk, clen, - uchunk, ulen); - case WIMLIB_COMPRESSION_TYPE_LZMS: - return wimlib_lzms_decompress(cchunk, clen, uchunk, ulen); - default: - wimlib_assert(0); - return -1; - } -} - struct data_range { u64 offset; u64 size; }; -/* Alternate chunk table format for resources with WIM_RESHDR_FLAG_CONCAT set. - */ -struct alt_chunk_table_header_disk { - /* Uncompressed size of the resource. */ - le64 res_usize; - - /* Number of bytes each compressed chunk decompresses into, except - * possibly the last which decompresses into the remainder. */ - le32 chunk_size; - - /* ??? */ - le32 unknown; - - /* This header is directly followed by a table of compressed sizes of - * the chunks. */ -} _packed_attribute; - /* * read_compressed_wim_resource() - * @@ -149,16 +108,11 @@ struct alt_chunk_table_header_disk { * Number of ranges in @ranges; must be at least 1. * @cb * Callback function to feed the data being read. Each call provides the - * next chunk of the requested data. Each chunk will be of nonzero size - * and will not cross range boundaries, but otherwise is of unspecified - * size. + * next chunk of the requested data, uncompressed. Each chunk will be of + * nonzero size and will not cross range boundaries, but otherwise will be + * of unspecified size. * @cb_ctx * Parameter to pass to @cb_ctx. - * @raw_chunks_mode - * If %true, this function will provide the raw compressed chunks of the - * resource rather than the uncompressed data. In this mode, only a single - * data range can be requested, and it must cover the entire uncompressed - * resource. * * Possible return values: * @@ -175,8 +129,7 @@ read_compressed_wim_resource(const struct wim_resource_spec * const rspec, const struct data_range * const ranges, const size_t num_ranges, const consume_data_callback_t cb, - void * const cb_ctx, - const bool raw_chunks_mode) + void * const cb_ctx) { int ret; int errno_save; @@ -187,11 +140,11 @@ read_compressed_wim_resource(const struct wim_resource_spec * const rspec, bool chunk_offsets_malloced = false; bool ubuf_malloced = false; bool cbuf_malloced = false; + struct wimlib_decompressor *decompressor = NULL; /* Sanity checks */ wimlib_assert(rspec != NULL); - wimlib_assert(rspec->ctype != WIMLIB_COMPRESSION_TYPE_NONE); - wimlib_assert(is_power_of_2(rspec->cchunk_size)); + wimlib_assert(resource_is_compressed(rspec)); wimlib_assert(cb != NULL); wimlib_assert(num_ranges != 0); for (size_t i = 0; i < num_ranges; i++) { @@ -205,12 +158,6 @@ read_compressed_wim_resource(const struct wim_resource_spec * const rspec, for (size_t i = 0; i < num_ranges - 1; i++) wimlib_assert(ranges[i].offset + ranges[i].size <= ranges[i + 1].offset); - if (raw_chunks_mode) { - wimlib_assert(num_ranges == 1); - wimlib_assert(ranges[0].offset == 0); - wimlib_assert(ranges[0].size == rspec->uncompressed_size); - } - /* Get the offsets of the first and last bytes of the read. */ const u64 first_offset = ranges[0].offset; const u64 last_offset = ranges[num_ranges - 1].offset + ranges[num_ranges - 1].size - 1; @@ -219,36 +166,53 @@ read_compressed_wim_resource(const struct wim_resource_spec * const rspec, struct filedes * const in_fd = &rspec->wim->in_fd; /* Determine if we're reading a pipable resource from a pipe or not. */ - const bool is_pipe_read = !filedes_is_seekable(in_fd); + const bool is_pipe_read = (rspec->is_pipable && !filedes_is_seekable(in_fd)); - /* Determine if the chunk table is in an altenate format. */ - const bool alt_chunk_table = (rspec->flags & WIM_RESHDR_FLAG_CONCAT) && !is_pipe_read; + /* Determine if the chunk table is in an alternate format. */ + const bool alt_chunk_table = (rspec->flags & WIM_RESHDR_FLAG_PACKED_STREAMS) + && !is_pipe_read; /* Get the maximum size of uncompressed chunks in this resource, which * we require be a power of 2. */ - u32 chunk_size; u64 cur_read_offset = rspec->offset_in_wim; + int ctype = rspec->compression_type; + u32 chunk_size = rspec->chunk_size; if (alt_chunk_table) { - /* Alternate chunk table format. */ - struct alt_chunk_table_header_disk hdr; - - ret = full_pread(in_fd, &hdr, sizeof(hdr), cur_read_offset); - if (ret) - goto read_error; - cur_read_offset += sizeof(hdr); + /* Alternate chunk table format. Its header specifies the chunk + * size and compression format. Note: it could be read here; + * however, the relevant data was already loaded into @rspec by + * read_wim_lookup_table(). */ + cur_read_offset += sizeof(struct alt_chunk_table_header_disk); + } - chunk_size = le32_to_cpu(hdr.chunk_size); + if (!is_power_of_2(chunk_size)) { + ERROR("Invalid compressed resource: " + "expected power-of-2 chunk size (got %"PRIu32")", + chunk_size); + ret = WIMLIB_ERR_INVALID_CHUNK_SIZE; + errno = EINVAL; + goto out_free_memory; + } - if (!is_power_of_2(chunk_size)) { - ERROR("Invalid compressed resource: " - "expected power-of-2 chunk size (got %u)", chunk_size); - ret = WIMLIB_ERR_INVALID_CHUNK_SIZE; + /* Get valid decompressor. */ + if (ctype == rspec->wim->decompressor_ctype && + chunk_size == rspec->wim->decompressor_max_block_size) + { + /* Cached decompressor. */ + decompressor = rspec->wim->decompressor; + rspec->wim->decompressor_ctype = WIMLIB_COMPRESSION_TYPE_NONE; + rspec->wim->decompressor = NULL; + } else { + ret = wimlib_create_decompressor(ctype, chunk_size, + &decompressor); + if (ret) { + if (ret != WIMLIB_ERR_NOMEM) + errno = EINVAL; goto out_free_memory; } - } else { - chunk_size = rspec->cchunk_size; } - const u32 chunk_order = bsr32(chunk_size); + + const u32 chunk_order = fls32(chunk_size); /* Calculate the total number of chunks the resource is divided into. */ const u64 num_chunks = (rspec->uncompressed_size + chunk_size - 1) >> chunk_order; @@ -276,11 +240,9 @@ read_compressed_wim_resource(const struct wim_resource_spec * const rspec, const u64 num_chunk_entries = (alt_chunk_table ? num_chunks : num_chunks - 1); /* Set the size of each chunk table entry based on the resource's - * uncompressed size. XXX: Does the alternate chunk table really - * always have 4-byte entries? */ - const u64 chunk_entry_size = - (rspec->uncompressed_size > (1ULL << 32) && !alt_chunk_table) - ? 8 : 4; + * uncompressed size. */ + const u64 chunk_entry_size = get_chunk_entry_size(rspec->uncompressed_size, + alt_chunk_table); /* Calculate the size of the chunk table in bytes. */ const u64 chunk_table_size = num_chunk_entries * chunk_entry_size; @@ -354,7 +316,7 @@ read_compressed_wim_resource(const struct wim_resource_spec * const rspec, chunk_offsets_alloc_size - chunk_table_size_to_read; - ret = full_pread(in_fd, chunk_table_data, chunk_table_size, + ret = full_pread(in_fd, chunk_table_data, chunk_table_size_to_read, file_offset_of_needed_chunk_entries); if (ret) goto read_error; @@ -362,8 +324,8 @@ read_compressed_wim_resource(const struct wim_resource_spec * const rspec, /* Now fill in chunk_offsets from the entries we have read in * chunk_tab_data. We break aliasing rules here to avoid having * to allocate yet another array. */ - typedef le64 __attribute__((may_alias)) aliased_le64_t; - typedef le32 __attribute__((may_alias)) aliased_le32_t; + typedef le64 _may_alias_attribute aliased_le64_t; + typedef le32 _may_alias_attribute aliased_le32_t; u64 * chunk_offsets_p = chunk_offsets; if (alt_chunk_table) { @@ -411,19 +373,17 @@ read_compressed_wim_resource(const struct wim_resource_spec * const rspec, ubuf_malloced = true; } - /* Unless the raw compressed data was requested, allocate a temporary - * buffer for reading compressed chunks, each of which can be at most - * @chunk_size - 1 bytes. This excludes compressed chunks that are a - * full @chunk_size bytes, which are actually stored uncompressed. */ - if (!raw_chunks_mode) { - if (chunk_size - 1 <= STACK_MAX) { - cbuf = alloca(chunk_size - 1); - } else { - cbuf = MALLOC(chunk_size - 1); - if (cbuf == NULL) - goto oom; - cbuf_malloced = true; - } + /* Allocate a temporary buffer for reading compressed chunks, each of + * which can be at most @chunk_size - 1 bytes. This excludes compressed + * chunks that are a full @chunk_size bytes, which are actually stored + * uncompressed. */ + if (chunk_size - 1 <= STACK_MAX) { + cbuf = alloca(chunk_size - 1); + } else { + cbuf = MALLOC(chunk_size - 1); + if (cbuf == NULL) + goto oom; + cbuf_malloced = true; } /* Set current data range. */ @@ -494,36 +454,35 @@ read_compressed_wim_resource(const struct wim_resource_spec * const rspec, /* Read the chunk and feed data to the callback * function. */ - u8 *cb_buf; + u8 *read_buf; + + if (chunk_csize == chunk_usize) + read_buf = ubuf; + else + read_buf = cbuf; ret = full_pread(in_fd, - cbuf, + read_buf, chunk_csize, cur_read_offset); if (ret) goto read_error; - if (chunk_csize != chunk_usize && !raw_chunks_mode) { + if (read_buf == cbuf) { DEBUG("Decompressing chunk %"PRIu64" " - "(csize=%"PRIu64" usize=%"PRIu64"", + "(csize=%"PRIu32" usize=%"PRIu32")", i, chunk_csize, chunk_usize); - ret = decompress(cbuf, - chunk_csize, - ubuf, - chunk_usize, - rspec->ctype, - chunk_size); + ret = wimlib_decompress(cbuf, + chunk_csize, + ubuf, + chunk_usize, + decompressor); if (ret) { ERROR("Failed to decompress data!"); ret = WIMLIB_ERR_DECOMPRESSION; errno = EINVAL; goto out_free_memory; } - cb_buf = ubuf; - } else { - /* Raw chunks requested, or data stored - * uncompressed. */ - cb_buf = cbuf; } cur_read_offset += chunk_csize; @@ -540,10 +499,7 @@ read_compressed_wim_resource(const struct wim_resource_spec * const rspec, end = min(cur_range_end, chunk_end_offset) - chunk_start_offset; size = end - start; - if (raw_chunks_mode) - ret = (*cb)(&cb_buf[0], chunk_csize, cb_ctx); - else - ret = (*cb)(&cb_buf[start], size, cb_ctx); + ret = (*cb)(&ubuf[start], size, cb_ctx); if (ret) goto out_free_memory; @@ -577,8 +533,15 @@ read_compressed_wim_resource(const struct wim_resource_spec * const rspec, goto read_error; } ret = 0; + out_free_memory: errno_save = errno; + if (decompressor) { + wimlib_free_decompressor(rspec->wim->decompressor); + rspec->wim->decompressor = decompressor; + rspec->wim->decompressor_ctype = ctype; + rspec->wim->decompressor_max_block_size = chunk_size; + } if (chunk_offsets_malloced) FREE(chunk_offsets); if (ubuf_malloced) @@ -600,11 +563,33 @@ read_error: goto out_free_memory; } +static int +fill_zeroes(u64 size, consume_data_callback_t cb, void *cb_ctx) +{ + if (unlikely(size)) { + u8 buf[min(size, BUFFER_SIZE)]; + + memset(buf, 0, sizeof(buf)); + + do { + size_t len; + int ret; + + len = min(size, BUFFER_SIZE); + ret = cb(buf, len, cb_ctx); + if (ret) + return ret; + size -= len; + } while (size); + } + return 0; +} + /* Read raw data from a file descriptor at the specified offset, feeding the * data it in chunks into the specified callback function. */ static int -read_raw_file_data(struct filedes *in_fd, u64 size, - consume_data_callback_t cb, void *cb_ctx, u64 offset) +read_raw_file_data(struct filedes *in_fd, u64 offset, u64 size, + consume_data_callback_t cb, void *cb_ctx) { u8 buf[BUFFER_SIZE]; size_t bytes_to_read; @@ -626,6 +611,8 @@ read_raw_file_data(struct filedes *in_fd, u64 size, return 0; } +/* A consume_data_callback_t implementation that simply concatenates all chunks + * into a buffer. */ static int bufferer_cb(const void *chunk, size_t size, void *_ctx) { @@ -639,23 +626,21 @@ bufferer_cb(const void *chunk, size_t size, void *_ctx) * read_partial_wim_resource()- * * Read a range of data from an uncompressed or compressed resource in a WIM - * file. Data is fed chunk-by-chunk into the callback function @cb, passing it - * the argument @cb_ctx. - * - * By default, this function provides the uncompressed data of the resource, and - * @offset and @size and interpreted relative to the uncompressed contents of - * the resource. This behavior can be modified by either of the following - * flags: - * - * WIMLIB_READ_RESOURCE_FLAG_RAW_FULL: - * Read @size bytes at @offset of the raw contents of the compressed - * resource. In the case of pipable resources, this excludes the stream - * header. Exclusive with WIMLIB_READ_RESOURCE_FLAG_RAW_CHUNKS. + * file. * - * WIMLIB_READ_RESOURCE_FLAG_RAW_CHUNKS: - * Read the raw compressed chunks of the compressed resource. @size must - * be the full uncompressed size, @offset must be 0, and @cb_chunk_size - * must be the resource chunk size. + * @rspec + * Specification of the WIM resource to read from. + * @offset + * Offset within the uncompressed resource at which to start reading. + * @size + * Number of bytes to read. + * @cb + * Callback function to feed the data being read. Each call provides the + * next chunk of the requested data, uncompressed. Each chunk will be of + * nonzero size and will not cross range boundaries, but otherwise will be + * of unspecified size. + * @cb_ctx + * Parameter to pass to @cb_ctx. * * Return values: * WIMLIB_ERR_SUCCESS (0) @@ -668,50 +653,66 @@ bufferer_cb(const void *chunk, size_t size, void *_ctx) */ static int read_partial_wim_resource(const struct wim_resource_spec *rspec, - u64 offset, u64 size, consume_data_callback_t cb, - void *cb_ctx, int flags) + u64 offset, u64 size, + consume_data_callback_t cb, void *cb_ctx) { /* Sanity checks. */ - if (flags & WIMLIB_READ_RESOURCE_FLAG_RAW_FULL) { - wimlib_assert(!(flags & WIMLIB_READ_RESOURCE_FLAG_RAW_CHUNKS)); - wimlib_assert(offset + size >= offset); - wimlib_assert(offset + size <= rspec->size_in_wim); - } else if (flags & WIMLIB_READ_RESOURCE_FLAG_RAW_CHUNKS) { - wimlib_assert(offset == 0); - wimlib_assert(offset == rspec->uncompressed_size); - } else { - wimlib_assert(offset + size >= offset); - wimlib_assert(offset + size <= rspec->uncompressed_size); - } + wimlib_assert(offset + size >= offset); + wimlib_assert(offset + size <= rspec->uncompressed_size); DEBUG("Reading %"PRIu64" @ %"PRIu64" from WIM resource " - "%"PRIu64" => %"PRIu64" @ %"PRIu64" (flags 0x%08x)", + "%"PRIu64" => %"PRIu64" @ %"PRIu64, size, offset, rspec->uncompressed_size, - rspec->size_in_wim, rspec->offset_in_wim, flags); + rspec->size_in_wim, rspec->offset_in_wim); /* Trivial case. */ if (size == 0) return 0; - if ((flags & WIMLIB_READ_RESOURCE_FLAG_RAW_FULL) || - rspec->ctype == WIMLIB_COMPRESSION_TYPE_NONE) - { - return read_raw_file_data(&rspec->wim->in_fd, - size, - cb, - cb_ctx, - rspec->offset_in_wim + offset); - } else { - bool raw_chunks = !!(flags & WIMLIB_READ_RESOURCE_FLAG_RAW_CHUNKS); + if (resource_is_compressed(rspec)) { struct data_range range = { .offset = offset, .size = size, }; return read_compressed_wim_resource(rspec, &range, 1, - cb, cb_ctx, raw_chunks); + cb, cb_ctx); + } else { + /* Reading uncompressed resource. For completeness, handle the + * weird case where size_in_wim < uncompressed_size. */ + + u64 read_size; + u64 zeroes_size; + int ret; + + if (likely(offset + size <= rspec->size_in_wim) || + rspec->is_pipable) + { + read_size = size; + zeroes_size = 0; + } else { + if (offset >= rspec->size_in_wim) { + read_size = 0; + zeroes_size = size; + } else { + read_size = rspec->size_in_wim - offset; + zeroes_size = offset + size - rspec->size_in_wim; + } + } + + ret = read_raw_file_data(&rspec->wim->in_fd, + rspec->offset_in_wim + offset, + read_size, + cb, + cb_ctx); + if (ret) + return ret; + + return fill_zeroes(zeroes_size, cb, cb_ctx); } } +/* Read the specified range of uncompressed data from the specified stream, + * which must be located into a WIM file, into the specified buffer. */ int read_partial_wim_stream_into_buf(const struct wim_lookup_table_entry *lte, size_t size, u64 offset, void *_buf) @@ -724,10 +725,11 @@ read_partial_wim_stream_into_buf(const struct wim_lookup_table_entry *lte, lte->offset_in_res + offset, size, bufferer_cb, - &buf, - 0); + &buf); } +/* A consume_data_callback_t implementation that simply ignores the data + * received. */ static int skip_chunk_cb(const void *chunk, size_t size, void *_ctx) { @@ -740,28 +742,23 @@ int skip_wim_stream(struct wim_lookup_table_entry *lte) { wimlib_assert(lte->resource_location == RESOURCE_IN_WIM); - wimlib_assert(!lte_is_partial(lte)); + wimlib_assert(!(lte->flags & WIM_RESHDR_FLAG_PACKED_STREAMS)); + DEBUG("Skipping stream (size=%"PRIu64")", lte->size); return read_partial_wim_resource(lte->rspec, 0, lte->rspec->uncompressed_size, skip_chunk_cb, - NULL, - WIMLIB_READ_RESOURCE_FLAG_RAW_CHUNKS); + NULL); } static int read_wim_stream_prefix(const struct wim_lookup_table_entry *lte, u64 size, - consume_data_callback_t cb, void *cb_ctx, int flags) + consume_data_callback_t cb, void *cb_ctx) { - return read_partial_wim_resource(lte->rspec, - lte->offset_in_res, - size, - cb, - cb_ctx, - flags); + return read_partial_wim_resource(lte->rspec, lte->offset_in_res, size, + cb, cb_ctx); } -#ifndef __WIN32__ /* This function handles reading stream data that is located in an external * file, such as a file that has been added to the WIM image through execution * of a wimlib_add_command. @@ -772,8 +769,7 @@ read_wim_stream_prefix(const struct wim_lookup_table_entry *lte, u64 size, * encrypted), so Windows uses its own code for its equivalent case. */ static int read_file_on_disk_prefix(const struct wim_lookup_table_entry *lte, u64 size, - consume_data_callback_t cb, void *cb_ctx, - int _ignored_flags) + consume_data_callback_t cb, void *cb_ctx) { int ret; int raw_fd; @@ -783,24 +779,50 @@ read_file_on_disk_prefix(const struct wim_lookup_table_entry *lte, u64 size, DEBUG("Reading %"PRIu64" bytes from \"%"TS"\"", size, lte->file_on_disk); - raw_fd = open(lte->file_on_disk, O_BINARY | O_RDONLY); + raw_fd = topen(lte->file_on_disk, O_BINARY | O_RDONLY); if (raw_fd < 0) { ERROR_WITH_ERRNO("Can't open \"%"TS"\"", lte->file_on_disk); return WIMLIB_ERR_OPEN; } filedes_init(&fd, raw_fd); - ret = read_raw_file_data(&fd, size, cb, cb_ctx, 0); + ret = read_raw_file_data(&fd, 0, size, cb, cb_ctx); + filedes_close(&fd); + return ret; +} + +#ifdef WITH_FUSE +static int +read_staging_file_prefix(const struct wim_lookup_table_entry *lte, u64 size, + consume_data_callback_t cb, void *cb_ctx) +{ + int raw_fd; + struct filedes fd; + int ret; + + wimlib_assert(size <= lte->size); + + DEBUG("Reading %"PRIu64" bytes from staging file \"%s\"", + size, lte->staging_file_name); + + raw_fd = openat(lte->staging_dir_fd, lte->staging_file_name, + O_RDONLY | O_NOFOLLOW); + if (raw_fd < 0) { + ERROR_WITH_ERRNO("Can't open staging file \"%s\"", + lte->staging_file_name); + return WIMLIB_ERR_OPEN; + } + filedes_init(&fd, raw_fd); + ret = read_raw_file_data(&fd, 0, size, cb, cb_ctx); filedes_close(&fd); return ret; } -#endif /* !__WIN32__ */ +#endif /* This function handles the trivial case of reading stream data that is, in * fact, already located in an in-memory buffer. */ static int read_buffer_prefix(const struct wim_lookup_table_entry *lte, - u64 size, consume_data_callback_t cb, - void *cb_ctx, int _ignored_flags) + u64 size, consume_data_callback_t cb, void *cb_ctx) { wimlib_assert(size <= lte->size); return (*cb)(lte->attached_buffer, size, cb_ctx); @@ -809,7 +831,7 @@ read_buffer_prefix(const struct wim_lookup_table_entry *lte, typedef int (*read_stream_prefix_handler_t)(const struct wim_lookup_table_entry *lte, u64 size, consume_data_callback_t cb, - void *cb_ctx, int flags); + void *cb_ctx); /* * read_stream_prefix()- @@ -818,41 +840,36 @@ typedef int (*read_stream_prefix_handler_t)(const struct wim_lookup_table_entry * any one of several locations, such as in a WIM file (compressed or * uncompressed), in an external file, or directly in an in-memory buffer. * - * This function feeds the data to a callback function @cb. - * - * If the stream is located in a WIM file, @flags can be set as documented in - * read_partial_wim_resource(). Otherwise @flags are ignored. + * This function feeds the data to a callback function @cb in chunks of + * unspecified size. * * Returns 0 on success; nonzero on error. A nonzero value will be returned if * the stream data cannot be successfully read (for a number of different * reasons, depending on the stream location), or if @cb returned nonzero in * which case that error code will be returned. */ -int +static int read_stream_prefix(const struct wim_lookup_table_entry *lte, u64 size, - consume_data_callback_t cb, void *cb_ctx, int flags) + consume_data_callback_t cb, void *cb_ctx) { static const read_stream_prefix_handler_t handlers[] = { [RESOURCE_IN_WIM] = read_wim_stream_prefix, - #ifdef __WIN32__ - [RESOURCE_IN_FILE_ON_DISK] = read_win32_file_prefix, - #else [RESOURCE_IN_FILE_ON_DISK] = read_file_on_disk_prefix, - #endif [RESOURCE_IN_ATTACHED_BUFFER] = read_buffer_prefix, #ifdef WITH_FUSE - [RESOURCE_IN_STAGING_FILE] = read_file_on_disk_prefix, + [RESOURCE_IN_STAGING_FILE] = read_staging_file_prefix, #endif #ifdef WITH_NTFS_3G [RESOURCE_IN_NTFS_VOLUME] = read_ntfs_file_prefix, #endif #ifdef __WIN32__ + [RESOURCE_IN_WINNT_FILE_ON_DISK] = read_winnt_file_prefix, [RESOURCE_WIN32_ENCRYPTED] = read_win32_encrypted_file_prefix, #endif }; wimlib_assert(lte->resource_location < ARRAY_LEN(handlers) && handlers[lte->resource_location] != NULL); - return handlers[lte->resource_location](lte, size, cb, cb_ctx, flags); + return handlers[lte->resource_location](lte, size, cb, cb_ctx); } /* Read the full uncompressed data of the specified stream into the specified @@ -861,11 +878,11 @@ int read_full_stream_into_buf(const struct wim_lookup_table_entry *lte, void *_buf) { u8 *buf = _buf; - return read_stream_prefix(lte, lte->size, bufferer_cb, &buf, 0); + return read_stream_prefix(lte, lte->size, bufferer_cb, &buf); } -/* Read the full uncompressed data of the specified stream. A buffer sufficient - * to hold the data is allocated and returned in @buf_ret. */ +/* Retrieve the full uncompressed data of the specified stream. A buffer large + * enough hold the data is allocated and returned in @buf_ret. */ int read_full_stream_into_alloc_buf(const struct wim_lookup_table_entry *lte, void **buf_ret) @@ -893,7 +910,8 @@ read_full_stream_into_alloc_buf(const struct wim_lookup_table_entry *lte, return 0; } -/* Retrieve the full uncompressed data of the specified WIM resource. */ +/* Retrieve the full uncompressed data of the specified WIM resource. A buffer + * large enough hold the data is allocated and returned in @buf_ret. */ static int wim_resource_spec_to_data(struct wim_resource_spec *rspec, void **buf_ret) { @@ -904,7 +922,6 @@ wim_resource_spec_to_data(struct wim_resource_spec *rspec, void **buf_ret) if (lte == NULL) return WIMLIB_ERR_NOMEM; - lte->unhashed = 1; lte_bind_wim_resource_spec(lte, rspec); lte->flags = rspec->flags; lte->size = rspec->uncompressed_size; @@ -917,28 +934,72 @@ wim_resource_spec_to_data(struct wim_resource_spec *rspec, void **buf_ret) return ret; } -/* Retrieve the full uncompressed data of the specified WIM resource. */ +/* Retrieve the full uncompressed data of a WIM resource specified as a raw + * `wim_reshdr' and the corresponding WIM file. A large enough hold the data is + * allocated and returned in @buf_ret. */ int wim_reshdr_to_data(const struct wim_reshdr *reshdr, WIMStruct *wim, void **buf_ret) { DEBUG("offset_in_wim=%"PRIu64", size_in_wim=%"PRIu64", " "uncompressed_size=%"PRIu64, - reshdr->offset_in_wim, reshdr->size_in_wim, reshdr->uncompressed_size); + reshdr->offset_in_wim, reshdr->size_in_wim, + reshdr->uncompressed_size); struct wim_resource_spec rspec; wim_res_hdr_to_spec(reshdr, wim, &rspec); return wim_resource_spec_to_data(&rspec, buf_ret); } +int +wim_reshdr_to_hash(const struct wim_reshdr *reshdr, WIMStruct *wim, + u8 hash[SHA1_HASH_SIZE]) +{ + struct wim_resource_spec rspec; + int ret; + struct wim_lookup_table_entry *lte; + + wim_res_hdr_to_spec(reshdr, wim, &rspec); + + lte = new_lookup_table_entry(); + if (lte == NULL) + return WIMLIB_ERR_NOMEM; + + lte_bind_wim_resource_spec(lte, &rspec); + lte->flags = rspec.flags; + lte->size = rspec.uncompressed_size; + lte->offset_in_res = 0; + lte->unhashed = 1; + + ret = sha1_stream(lte); + + lte_unbind_wim_resource_spec(lte); + copy_hash(hash, lte->hash); + free_lookup_table_entry(lte); + return ret; +} + struct streamifier_context { struct read_stream_list_callbacks cbs; struct wim_lookup_table_entry *cur_stream; + struct wim_lookup_table_entry *next_stream; u64 cur_stream_offset; struct wim_lookup_table_entry *final_stream; size_t list_head_offset; }; -/* Callback for translating raw resource data into streams. */ +static struct wim_lookup_table_entry * +next_stream(struct wim_lookup_table_entry *lte, size_t list_head_offset) +{ + struct list_head *cur; + + cur = (struct list_head*)((u8*)lte + list_head_offset); + + return (struct wim_lookup_table_entry*)((u8*)cur->next - list_head_offset); +} + +/* A consume_data_callback_t implementation that translates raw resource data + * into streams, calling the begin_stream, consume_chunk, and end_stream + * callback functions as appropriate. */ static int streamifier_cb(const void *chunk, size_t size, void *_ctx) { @@ -948,40 +1009,46 @@ streamifier_cb(const void *chunk, size_t size, void *_ctx) DEBUG("%zu bytes passed to streamifier", size); wimlib_assert(ctx->cur_stream != NULL); + wimlib_assert(size <= ctx->cur_stream->size - ctx->cur_stream_offset); if (ctx->cur_stream_offset == 0) { + /* Starting a new stream. */ - DEBUG("Begin new stream (size=%"PRIu64").", ctx->cur_stream->size); - ret = (*ctx->cbs.begin_stream)(ctx->cur_stream, true, ctx->cbs.begin_stream_ctx); + DEBUG("Begin new stream (size=%"PRIu64").", + ctx->cur_stream->size); + + ret = (*ctx->cbs.begin_stream)(ctx->cur_stream, + ctx->cbs.begin_stream_ctx); if (ret) return ret; } /* Consume the chunk. */ - ret = (*ctx->cbs.consume_chunk)(chunk, size, ctx->cbs.consume_chunk_ctx); + ret = (*ctx->cbs.consume_chunk)(chunk, size, + ctx->cbs.consume_chunk_ctx); + ctx->cur_stream_offset += size; if (ret) return ret; - ctx->cur_stream_offset += size; if (ctx->cur_stream_offset == ctx->cur_stream->size) { - /* Finished reading all the data for a stream; advance to the - * next one. */ + /* Finished reading all the data for a stream. */ + + ctx->cur_stream_offset = 0; + DEBUG("End stream (size=%"PRIu64").", ctx->cur_stream->size); - ret = (*ctx->cbs.end_stream)(ctx->cur_stream, 0, ctx->cbs.end_stream_ctx); + ret = (*ctx->cbs.end_stream)(ctx->cur_stream, 0, + ctx->cbs.end_stream_ctx); if (ret) return ret; - if (ctx->cur_stream != ctx->final_stream) { - struct list_head *cur = (struct list_head *) - ((u8*)ctx->cur_stream + ctx->list_head_offset); - struct list_head *next = cur->next; - - ctx->cur_stream = (struct wim_lookup_table_entry *) - ((u8*)next - ctx->list_head_offset); - - ctx->cur_stream_offset = 0; - } else { - ctx->cur_stream = NULL; + /* Advance to next stream. */ + ctx->cur_stream = ctx->next_stream; + if (ctx->cur_stream != NULL) { + if (ctx->cur_stream != ctx->final_stream) + ctx->next_stream = next_stream(ctx->cur_stream, + ctx->list_head_offset); + else + ctx->next_stream = NULL; } } return 0; @@ -989,14 +1056,14 @@ streamifier_cb(const void *chunk, size_t size, void *_ctx) struct hasher_context { SHA_CTX sha_ctx; + int flags; struct read_stream_list_callbacks cbs; }; /* Callback for starting to read a stream while calculating its SHA1 message * digest. */ static int -hasher_begin_stream(struct wim_lookup_table_entry *lte, bool is_partial_res, - void *_ctx) +hasher_begin_stream(struct wim_lookup_table_entry *lte, void *_ctx) { struct hasher_context *ctx = _ctx; @@ -1005,12 +1072,13 @@ hasher_begin_stream(struct wim_lookup_table_entry *lte, bool is_partial_res, if (ctx->cbs.begin_stream == NULL) return 0; else - return (*ctx->cbs.begin_stream)(lte, is_partial_res, - ctx->cbs.begin_stream_ctx); + return (*ctx->cbs.begin_stream)(lte, ctx->cbs.begin_stream_ctx); } -/* Callback for continuing to read a stream while calculating its SHA1 message - * digest. */ +/* A consume_data_callback_t implementation that continues calculating the SHA1 + * message digest of the stream being read, then optionally passes the data on + * to another consume_data_callback_t implementation. This allows checking the + * SHA1 message digest of a stream being extracted, for example. */ static int hasher_consume_chunk(const void *chunk, size_t size, void *_ctx) { @@ -1033,33 +1101,44 @@ hasher_end_stream(struct wim_lookup_table_entry *lte, int status, void *_ctx) int ret; if (status) { + /* Error occurred; the full stream may not have been read. */ ret = status; goto out_next_cb; } + /* Retrieve the final SHA1 message digest. */ sha1_final(hash, &ctx->sha_ctx); if (lte->unhashed) { - /* No SHA1 message digest was present before; fill it in with - * the calculated value. */ - DEBUG("Set SHA1 message digest for stream (size=%"PRIu64").", lte->size); - copy_hash(lte->hash, hash); + if (ctx->flags & COMPUTE_MISSING_STREAM_HASHES) { + /* No SHA1 message digest was previously present for the + * stream. Set it to the one just calculated. */ + DEBUG("Set SHA1 message digest for stream " + "(size=%"PRIu64").", lte->size); + copy_hash(lte->hash, hash); + } } else { - /* A SHA1 message digest was present before. Verify that it is - * the same as the calculated value. */ - if (!hashes_equal(hash, lte->hash)) { - if (wimlib_print_errors) { - ERROR("Invalid SHA1 message digest " - "on the following WIM stream:"); - print_lookup_table_entry(lte, stderr); - if (lte->resource_location == RESOURCE_IN_WIM) - ERROR("The WIM file appears to be corrupt!"); + if (ctx->flags & VERIFY_STREAM_HASHES) { + /* The stream already had a SHA1 message digest present. Verify + * that it is the same as the calculated value. */ + if (!hashes_equal(hash, lte->hash)) { + if (wimlib_print_errors) { + tchar expected_hashstr[SHA1_HASH_SIZE * 2 + 1]; + tchar actual_hashstr[SHA1_HASH_SIZE * 2 + 1]; + sprint_hash(lte->hash, expected_hashstr); + sprint_hash(hash, actual_hashstr); + ERROR("The stream is corrupted!\n" + " (Expected SHA1=%"TS",\n" + " got SHA1=%"TS")", + expected_hashstr, actual_hashstr); + } + ret = WIMLIB_ERR_INVALID_RESOURCE_HASH; + errno = EINVAL; + goto out_next_cb; } - ret = WIMLIB_ERR_INVALID_RESOURCE_HASH; - errno = EINVAL; - goto out_next_cb; + DEBUG("SHA1 message digest okay for " + "stream (size=%"PRIu64").", lte->size); } - DEBUG("SHA1 message digest okay for stream (size=%"PRIu64").", lte->size); } ret = 0; out_next_cb: @@ -1069,89 +1148,118 @@ out_next_cb: return (*ctx->cbs.end_stream)(lte, ret, ctx->cbs.end_stream_ctx); } -/* Read the full data of the stream @lte, passing the data into the specified - * callbacks (all of which are optional) and either checking or computing the - * SHA1 message digest of the stream. */ static int -read_full_stream_with_sha1(struct wim_lookup_table_entry *lte, - const struct read_stream_list_callbacks *cbs) +read_full_stream_with_cbs(struct wim_lookup_table_entry *lte, + const struct read_stream_list_callbacks *cbs) { int ret; - struct hasher_context hasher_ctx = { - .cbs = *cbs, - }; - - ret = hasher_begin_stream(lte, false, &hasher_ctx); + ret = (*cbs->begin_stream)(lte, cbs->begin_stream_ctx); if (ret) return ret; - ret = read_stream_prefix(lte, lte->size, hasher_consume_chunk, - &hasher_ctx, 0); + ret = read_stream_prefix(lte, lte->size, cbs->consume_chunk, + cbs->consume_chunk_ctx); - return hasher_end_stream(lte, ret, &hasher_ctx); + return (*cbs->end_stream)(lte, ret, cbs->end_stream_ctx); } -struct rechunkifier_context { - u8 *buffer; - u32 buffer_filled; - u32 cb_chunk_size; - - const struct data_range *ranges; - size_t num_ranges; - size_t cur_range; - u64 range_bytes_remaining; - - consume_data_callback_t cb; - void *cb_ctx; -}; +/* Read the full data of the specified stream, passing the data into the + * specified callbacks (all of which are optional) and either checking or + * computing the SHA1 message digest of the stream. */ +static int +read_full_stream_with_sha1(struct wim_lookup_table_entry *lte, + const struct read_stream_list_callbacks *cbs) +{ + struct hasher_context hasher_ctx = { + .flags = VERIFY_STREAM_HASHES | COMPUTE_MISSING_STREAM_HASHES, + .cbs = *cbs, + }; + struct read_stream_list_callbacks hasher_cbs = { + .begin_stream = hasher_begin_stream, + .begin_stream_ctx = &hasher_ctx, + .consume_chunk = hasher_consume_chunk, + .consume_chunk_ctx = &hasher_ctx, + .end_stream = hasher_end_stream, + .end_stream_ctx = &hasher_ctx, + }; + return read_full_stream_with_cbs(lte, &hasher_cbs); +} -/* Wrapper callback for adjusting the data chunk size. */ static int -rechunkifier_cb(const void *chunk, size_t size, void *_ctx) +read_packed_streams(struct wim_lookup_table_entry *first_stream, + struct wim_lookup_table_entry *last_stream, + u64 stream_count, + size_t list_head_offset, + const struct read_stream_list_callbacks *sink_cbs) { - struct rechunkifier_context *ctx = _ctx; - const u8 *chunkptr = chunk; - size_t bytes_to_copy; + struct data_range *ranges; + bool ranges_malloced; + struct wim_lookup_table_entry *cur_stream; + size_t i; int ret; + u64 ranges_alloc_size; - wimlib_assert(ctx->cur_range != ctx->num_ranges); + DEBUG("Reading %"PRIu64" streams combined in same WIM resource", + stream_count); - while (size) { + /* Setup data ranges array (one range per stream to read); this way + * read_compressed_wim_resource() does not need to be aware of streams. + */ - /* Append more data to the buffer. */ - bytes_to_copy = size; + ranges_alloc_size = stream_count * sizeof(ranges[0]); - if (bytes_to_copy > ctx->cb_chunk_size - ctx->buffer_filled) - bytes_to_copy = ctx->cb_chunk_size - ctx->buffer_filled; + if (unlikely((size_t)ranges_alloc_size != ranges_alloc_size)) { + ERROR("Too many streams in one resource!"); + return WIMLIB_ERR_NOMEM; + } + if (likely(ranges_alloc_size <= STACK_MAX)) { + ranges = alloca(ranges_alloc_size); + ranges_malloced = false; + } else { + ranges = MALLOC(ranges_alloc_size); + if (ranges == NULL) { + ERROR("Too many streams in one resource!"); + return WIMLIB_ERR_NOMEM; + } + ranges_malloced = true; + } - if (bytes_to_copy > ctx->range_bytes_remaining - ctx->buffer_filled) - bytes_to_copy = ctx->range_bytes_remaining - ctx->buffer_filled; + for (i = 0, cur_stream = first_stream; + i < stream_count; + i++, cur_stream = next_stream(cur_stream, list_head_offset)) + { + ranges[i].offset = cur_stream->offset_in_res; + ranges[i].size = cur_stream->size; + } - memcpy(&ctx->buffer[ctx->buffer_filled], chunkptr, bytes_to_copy); + struct streamifier_context streamifier_ctx = { + .cbs = *sink_cbs, + .cur_stream = first_stream, + .next_stream = next_stream(first_stream, list_head_offset), + .cur_stream_offset = 0, + .final_stream = last_stream, + .list_head_offset = list_head_offset, + }; - ctx->buffer_filled += bytes_to_copy; - chunkptr += bytes_to_copy; - size -= bytes_to_copy; - ctx->range_bytes_remaining -= bytes_to_copy; + ret = read_compressed_wim_resource(first_stream->rspec, + ranges, + stream_count, + streamifier_cb, + &streamifier_ctx); - if (ctx->buffer_filled == ctx->cb_chunk_size || - ctx->range_bytes_remaining == 0) - { - /* Maximum chunk size reached, or current range ended. - * Call the next consume_data_callback_t and empty the - * buffer */ - ret = (*ctx->cb)(ctx->buffer, ctx->buffer_filled, ctx->cb_ctx); - if (ret) - return ret; - ctx->buffer_filled = 0; + if (ranges_malloced) + FREE(ranges); - if (ctx->range_bytes_remaining == 0 && - ++ctx->cur_range != ctx->num_ranges) - ctx->range_bytes_remaining = ctx->ranges[ctx->cur_range].size; + if (ret) { + if (streamifier_ctx.cur_stream_offset != 0) { + ret = (*streamifier_ctx.cbs.end_stream) + (streamifier_ctx.cur_stream, + ret, + streamifier_ctx.cbs.end_stream_ctx); } } - return 0; + return ret; } /* @@ -1167,11 +1275,27 @@ rechunkifier_cb(const void *chunk, size_t size, void *_ctx) * @list_head_offset * Offset of the `struct list_head' within each `struct * wim_lookup_table_entry' that makes up the @stream_list. - * @cb_chunk_size - * Size of chunks to provide to @consume_chunk. For a given stream, all - * the chunks will be this size, except possibly the last which will be the - * remainder. If @ - * @cbs TODO + * @cbs + * Callback functions to accept the stream data. + * @flags + * Bitwise OR of zero or more of the following flags: + * + * VERIFY_STREAM_HASHES: + * For all streams being read that have already had SHA1 message + * digests computed, calculate the SHA1 message digest of the read + * data and compare it with the previously computed value. If they + * do not match, return WIMLIB_ERR_INVALID_RESOURCE_HASH. + * + * COMPUTE_MISSING_STREAM_HASHES + * For all streams being read that have not yet had their SHA1 + * message digests computed, calculate and save their SHA1 message + * digests. + * + * STREAM_LIST_ALREADY_SORTED + * @stream_list is already sorted in sequential order for reading. + * + * The callback functions are allowed to delete the current stream from the list + * if necessary. * * Returns 0 on success; a nonzero error code on failure. Failure can occur due * to an error reading the data or due to an error status being returned by any @@ -1180,16 +1304,39 @@ rechunkifier_cb(const void *chunk, size_t size, void *_ctx) int read_stream_list(struct list_head *stream_list, size_t list_head_offset, - u32 cb_chunk_size, - const struct read_stream_list_callbacks *cbs) + const struct read_stream_list_callbacks *cbs, + int flags) { int ret; struct list_head *cur, *next; struct wim_lookup_table_entry *lte; + struct hasher_context *hasher_ctx; + struct read_stream_list_callbacks *sink_cbs; - ret = sort_stream_list_by_sequential_order(stream_list, list_head_offset); - if (ret) - return ret; + if (!(flags & STREAM_LIST_ALREADY_SORTED)) { + ret = sort_stream_list_by_sequential_order(stream_list, list_head_offset); + if (ret) + return ret; + } + + if (flags & (VERIFY_STREAM_HASHES | COMPUTE_MISSING_STREAM_HASHES)) { + hasher_ctx = alloca(sizeof(*hasher_ctx)); + *hasher_ctx = (struct hasher_context) { + .flags = flags, + .cbs = *cbs, + }; + sink_cbs = alloca(sizeof(*sink_cbs)); + *sink_cbs = (struct read_stream_list_callbacks) { + .begin_stream = hasher_begin_stream, + .begin_stream_ctx = hasher_ctx, + .consume_chunk = hasher_consume_chunk, + .consume_chunk_ctx = hasher_ctx, + .end_stream = hasher_end_stream, + .end_stream_ctx = hasher_ctx, + }; + } else { + sink_cbs = (struct read_stream_list_callbacks*)cbs; + } for (cur = stream_list->next, next = cur->next; cur != stream_list; @@ -1197,11 +1344,13 @@ read_stream_list(struct list_head *stream_list, { lte = (struct wim_lookup_table_entry*)((u8*)cur - list_head_offset); - if (lte_is_partial(lte)) { + if (lte->flags & WIM_RESHDR_FLAG_PACKED_STREAMS && + lte->size != lte->rspec->uncompressed_size) + { struct wim_lookup_table_entry *lte_next, *lte_last; struct list_head *next2; - size_t stream_count; + u64 stream_count; /* The next stream is a proper sub-sequence of a WIM * resource. See if there are other streams in the same @@ -1230,140 +1379,36 @@ read_stream_list(struct list_head *stream_list, * first stream in the resource that needs to be * read and @lte_last specifies the last stream * in the resource that needs to be read. */ - - DEBUG("Reading %zu streams combined in same " - "WIM resource", stream_count); - next = next2; - - struct data_range ranges[stream_count]; - - { - struct list_head *next3; - size_t i; - struct wim_lookup_table_entry *lte_cur; - - next3 = cur; - for (i = 0; i < stream_count; i++) { - lte_cur = (struct wim_lookup_table_entry*) - ((u8*)next3 - list_head_offset); - ranges[i].offset = lte_cur->offset_in_res; - ranges[i].size = lte_cur->size; - next3 = next3->next; - } - } - - /* Set up a chain of callbacks. - * - * The first level is the - * streamifier_cb, - * which takes in chunks of data and divides - * them into the constituent streams. - * - * The second level are the SHA1 message digest - * callbacks, which checksum each stream. - * - * rechunkifier_cb handles dividing the read - * data into chunks of maximum size - * @cb_chunk_size. If @cb_chunk_size is 0, then - * this callback is not needed. - * - * Finally, the last level of callbacks are - * @cbs, passed as arguments to this function. - */ - - struct rechunkifier_context *rechunkifier_ctx = NULL; - consume_data_callback_t last_cb; - void *last_cb_ctx; - - if (cb_chunk_size != 0) { - rechunkifier_ctx = alloca(sizeof(*rechunkifier_ctx)); - *rechunkifier_ctx = (struct rechunkifier_context) { - .buffer = MALLOC(cb_chunk_size), - .buffer_filled = 0, - .cb_chunk_size = cb_chunk_size, - .ranges = ranges, - .num_ranges = stream_count, - .cur_range = 0, - .range_bytes_remaining = ranges[0].size, - .cb = cbs->consume_chunk, - .cb_ctx = cbs->consume_chunk_ctx, - }; - - if (rechunkifier_ctx->buffer == NULL) - return WIMLIB_ERR_NOMEM; - last_cb = rechunkifier_cb; - last_cb_ctx = rechunkifier_ctx; - } else { - rechunkifier_ctx = NULL; - last_cb = cbs->consume_chunk; - last_cb_ctx = cbs->consume_chunk_ctx; - } - - struct hasher_context hasher_ctx = { - .cbs = { - .begin_stream = cbs->begin_stream, - .begin_stream_ctx = cbs->begin_stream_ctx, - .consume_chunk = last_cb, - .consume_chunk_ctx = last_cb_ctx, - .end_stream = cbs->end_stream, - .end_stream_ctx = cbs->end_stream_ctx, - }, - }; - - struct streamifier_context streamifier_ctx = { - .cbs = { - .begin_stream = hasher_begin_stream, - .begin_stream_ctx = &hasher_ctx, - .consume_chunk = hasher_consume_chunk, - .consume_chunk_ctx = &hasher_ctx, - .end_stream = hasher_end_stream, - .end_stream_ctx = &hasher_ctx, - }, - .cur_stream = lte, - .cur_stream_offset = 0, - .final_stream = lte_last, - .list_head_offset = list_head_offset, - }; - - ret = read_compressed_wim_resource(lte->rspec, - ranges, - stream_count, - streamifier_cb, - &streamifier_ctx, - false); - if (rechunkifier_ctx != NULL) - FREE(rechunkifier_ctx->buffer); - - if (ret) { - if (streamifier_ctx.cur_stream_offset != 0) { - ret = (*streamifier_ctx.cbs.end_stream) - (streamifier_ctx.cur_stream, - ret, - streamifier_ctx.cbs.end_stream_ctx); - } + ret = read_packed_streams(lte, lte_last, + stream_count, + list_head_offset, + sink_cbs); + if (ret) return ret; - } continue; } } - ret = read_full_stream_with_sha1(lte, cbs); - if (ret > 0) + ret = read_full_stream_with_cbs(lte, sink_cbs); + if (ret && ret != BEGIN_STREAM_STATUS_SKIP_STREAM) return ret; } return 0; } -/* Extracts the first @size bytes of a stream to somewhere. In the process, the - * SHA1 message digest of the uncompressed stream is checked if the full stream - * is being extracted. +/* Extract the first @size bytes of the specified stream. * - * @extract_chunk is the callback to extract each chunk of the stream. */ + * If @size specifies the full uncompressed size of the stream, then the SHA1 + * message digest of the uncompressed stream is checked while being extracted. + * + * The uncompressed data of the resource is passed in chunks of unspecified size + * to the @extract_chunk function, passing it @extract_chunk_arg. */ int extract_stream(struct wim_lookup_table_entry *lte, u64 size, consume_data_callback_t extract_chunk, void *extract_chunk_arg) { + wimlib_assert(size <= lte->size); if (size == lte->size) { /* Do SHA1. */ struct read_stream_list_callbacks cbs = { @@ -1374,13 +1419,13 @@ extract_stream(struct wim_lookup_table_entry *lte, u64 size, } else { /* Don't do SHA1. */ return read_stream_prefix(lte, size, extract_chunk, - extract_chunk_arg, 0); + extract_chunk_arg); } } -/* Write a chunk of data to a file descriptor. This function can be passed as a - * consume_data_callback_t. */ -int +/* A consume_data_callback_t implementation that writes the chunk of data to a + * file descriptor. */ +static int extract_chunk_to_fd(const void *chunk, size_t size, void *_fd_p) { struct filedes *fd = _fd_p; @@ -1390,13 +1435,11 @@ extract_chunk_to_fd(const void *chunk, size_t size, void *_fd_p) ERROR_WITH_ERRNO("Error writing to file descriptor"); return ret; } - return 0; } /* Extract the first @size bytes of the specified stream to the specified file - * descriptor. If @size is the full size of the stream, its SHA1 message digest - * is also checked. */ + * descriptor. */ int extract_stream_to_fd(struct wim_lookup_table_entry *lte, struct filedes *fd, u64 size) @@ -1404,33 +1447,47 @@ extract_stream_to_fd(struct wim_lookup_table_entry *lte, return extract_stream(lte, size, extract_chunk_to_fd, fd); } -/* Calculate the SHA1 message digest of a stream, storing it in @lte->hash. */ +/* Extract the full uncompressed contents of the specified stream to the + * specified file descriptor. */ +int +extract_full_stream_to_fd(struct wim_lookup_table_entry *lte, + struct filedes *fd) +{ + return extract_stream_to_fd(lte, fd, lte->size); +} + +/* Calculate the SHA1 message digest of a stream and store it in @lte->hash. */ int sha1_stream(struct wim_lookup_table_entry *lte) { + wimlib_assert(lte->unhashed); struct read_stream_list_callbacks cbs = { }; return read_full_stream_with_sha1(lte, &cbs); } -/* Convert a WIM resource header to a stand-alone resource specification. */ +/* Convert a short WIM resource header to a stand-alone WIM resource + * specification. + * + * Note: for packed resources some fields still need to be overridden. + */ void wim_res_hdr_to_spec(const struct wim_reshdr *reshdr, WIMStruct *wim, - struct wim_resource_spec *spec) + struct wim_resource_spec *rspec) { - spec->wim = wim; - spec->offset_in_wim = reshdr->offset_in_wim; - spec->size_in_wim = reshdr->size_in_wim; - spec->uncompressed_size = reshdr->uncompressed_size; - INIT_LIST_HEAD(&spec->lte_list); - spec->flags = reshdr->flags; - spec->is_pipable = wim_is_pipable(wim); - if (spec->flags & (WIM_RESHDR_FLAG_COMPRESSED | WIM_RESHDR_FLAG_CONCAT)) { - spec->ctype = wim->compression_type; - spec->cchunk_size = wim->chunk_size; + rspec->wim = wim; + rspec->offset_in_wim = reshdr->offset_in_wim; + rspec->size_in_wim = reshdr->size_in_wim; + rspec->uncompressed_size = reshdr->uncompressed_size; + INIT_LIST_HEAD(&rspec->stream_list); + rspec->flags = reshdr->flags; + rspec->is_pipable = wim_is_pipable(wim); + if (rspec->flags & WIM_RESHDR_FLAG_COMPRESSED) { + rspec->compression_type = wim->compression_type; + rspec->chunk_size = wim->chunk_size; } else { - spec->ctype = WIMLIB_COMPRESSION_TYPE_NONE; - spec->cchunk_size = 0; + rspec->compression_type = WIMLIB_COMPRESSION_TYPE_NONE; + rspec->chunk_size = 0; } } @@ -1447,29 +1504,20 @@ wim_res_spec_to_hdr(const struct wim_resource_spec *rspec, /* Translates a WIM resource header from the on-disk format into an in-memory * format. */ -int +void get_wim_reshdr(const struct wim_reshdr_disk *disk_reshdr, struct wim_reshdr *reshdr) { reshdr->offset_in_wim = le64_to_cpu(disk_reshdr->offset_in_wim); reshdr->size_in_wim = (((u64)disk_reshdr->size_in_wim[0] << 0) | - ((u64)disk_reshdr->size_in_wim[1] << 8) | - ((u64)disk_reshdr->size_in_wim[2] << 16) | - ((u64)disk_reshdr->size_in_wim[3] << 24) | - ((u64)disk_reshdr->size_in_wim[4] << 32) | - ((u64)disk_reshdr->size_in_wim[5] << 40) | - ((u64)disk_reshdr->size_in_wim[6] << 48)); + ((u64)disk_reshdr->size_in_wim[1] << 8) | + ((u64)disk_reshdr->size_in_wim[2] << 16) | + ((u64)disk_reshdr->size_in_wim[3] << 24) | + ((u64)disk_reshdr->size_in_wim[4] << 32) | + ((u64)disk_reshdr->size_in_wim[5] << 40) | + ((u64)disk_reshdr->size_in_wim[6] << 48)); reshdr->uncompressed_size = le64_to_cpu(disk_reshdr->uncompressed_size); reshdr->flags = disk_reshdr->flags; - - /* Avoid possible overflows. */ - if (reshdr->offset_in_wim & 0xc000000000000000ULL) - return WIMLIB_ERR_INVALID_LOOKUP_TABLE_ENTRY; - - if (reshdr->uncompressed_size & 0xc000000000000000ULL) - return WIMLIB_ERR_INVALID_LOOKUP_TABLE_ENTRY; - - return 0; } /* Translates a WIM resource header from an in-memory format into the on-disk