X-Git-Url: https://wimlib.net/git/?p=wimlib;a=blobdiff_plain;f=src%2Fresource.c;h=300aec8f1cbec29e0b919bba8243984e7e518271;hp=192ffb2e988470252f92fda501a226664ee3f04c;hb=9e5404f2c773173ebaae519d7ebacb2d802b263f;hpb=4f433755e8f9ef79dbb4699430d047f74e338e82 diff --git a/src/resource.c b/src/resource.c index 192ffb2e..300aec8f 100644 --- a/src/resource.c +++ b/src/resource.c @@ -27,15 +27,17 @@ #endif #include "wimlib.h" +#include "wimlib/assert.h" #include "wimlib/endianness.h" #include "wimlib/error.h" #include "wimlib/file_io.h" #include "wimlib/lookup_table.h" #include "wimlib/resource.h" #include "wimlib/sha1.h" +#include "wimlib/wim.h" #ifdef __WIN32__ -/* for read_win32_file_prefix(), read_win32_encrypted_file_prefix() */ +/* for read_winnt_file_prefix(), read_win32_encrypted_file_prefix() */ # include "wimlib/win32.h" #endif @@ -136,6 +138,7 @@ read_compressed_wim_resource(const struct wim_resource_spec * const rspec, int errno_save; u64 *chunk_offsets = NULL; + u8 *_ubuf = NULL; u8 *ubuf = NULL; void *cbuf = NULL; bool chunk_offsets_malloced = false; @@ -167,46 +170,31 @@ read_compressed_wim_resource(const struct wim_resource_spec * const rspec, struct filedes * const in_fd = &rspec->wim->in_fd; /* Determine if we're reading a pipable resource from a pipe or not. */ - const bool is_pipe_read = !filedes_is_seekable(in_fd); + const bool is_pipe_read = (rspec->is_pipable && !filedes_is_seekable(in_fd)); - /* Determine if the chunk table is in an altenate format. */ + /* Determine if the chunk table is in an alternate format. */ const bool alt_chunk_table = (rspec->flags & WIM_RESHDR_FLAG_PACKED_STREAMS) && !is_pipe_read; /* Get the maximum size of uncompressed chunks in this resource, which * we require be a power of 2. */ - u32 chunk_size; u64 cur_read_offset = rspec->offset_in_wim; - int ctype; + int ctype = rspec->compression_type; + u32 chunk_size = rspec->chunk_size; if (alt_chunk_table) { /* Alternate chunk table format. Its header specifies the chunk - * size and compression format. */ - struct alt_chunk_table_header_disk hdr; - - ret = full_pread(in_fd, &hdr, sizeof(hdr), cur_read_offset); - if (ret) - goto read_error; - cur_read_offset += sizeof(hdr); - - chunk_size = le32_to_cpu(hdr.chunk_size); - ctype = le32_to_cpu(hdr.compression_format); - - /* Format numbers must be the same as in WIMGAPI to be - * compatible. */ - BUILD_BUG_ON(WIMLIB_COMPRESSION_TYPE_NONE != 0); - BUILD_BUG_ON(WIMLIB_COMPRESSION_TYPE_LZX != 1); - BUILD_BUG_ON(WIMLIB_COMPRESSION_TYPE_XPRESS != 2); - BUILD_BUG_ON(WIMLIB_COMPRESSION_TYPE_LZMS != 3); - } else { - /* "Normal" format: the maximum uncompressed chunk size and the - * compression format default to those of the WIM itself. */ - chunk_size = rspec->wim->chunk_size; - ctype = rspec->wim->compression_type; + * size and compression format. Note: it could be read here; + * however, the relevant data was already loaded into @rspec by + * read_wim_lookup_table(). */ + cur_read_offset += sizeof(struct alt_chunk_table_header_disk); } + if (!is_power_of_2(chunk_size)) { ERROR("Invalid compressed resource: " - "expected power-of-2 chunk size (got %u)", chunk_size); + "expected power-of-2 chunk size (got %"PRIu32")", + chunk_size); ret = WIMLIB_ERR_INVALID_CHUNK_SIZE; + errno = EINVAL; goto out_free_memory; } @@ -221,8 +209,11 @@ read_compressed_wim_resource(const struct wim_resource_spec * const rspec, } else { ret = wimlib_create_decompressor(ctype, chunk_size, NULL, &decompressor); - if (ret) + if (ret) { + if (ret != WIMLIB_ERR_NOMEM) + errno = EINVAL; goto out_free_memory; + } } const u32 chunk_order = bsr32(chunk_size); @@ -253,11 +244,9 @@ read_compressed_wim_resource(const struct wim_resource_spec * const rspec, const u64 num_chunk_entries = (alt_chunk_table ? num_chunks : num_chunks - 1); /* Set the size of each chunk table entry based on the resource's - * uncompressed size. XXX: Does the alternate chunk table really - * always have 4-byte entries? */ - const u64 chunk_entry_size = - (rspec->uncompressed_size > (1ULL << 32) && !alt_chunk_table) - ? 8 : 4; + * uncompressed size. */ + const u64 chunk_entry_size = get_chunk_entry_size(rspec->uncompressed_size, + alt_chunk_table); /* Calculate the size of the chunk table in bytes. */ const u64 chunk_table_size = num_chunk_entries * chunk_entry_size; @@ -380,13 +369,14 @@ read_compressed_wim_resource(const struct wim_resource_spec * const rspec, /* Allocate buffer for holding the uncompressed data of each chunk. */ if (chunk_size <= STACK_MAX) { - ubuf = alloca(chunk_size); + _ubuf = alloca(chunk_size + 15); } else { - ubuf = MALLOC(chunk_size); - if (ubuf == NULL) + _ubuf = MALLOC(chunk_size + 15); + if (_ubuf == NULL) goto oom; ubuf_malloced = true; } + ubuf = (u8 *)(((uintptr_t)_ubuf + 15) & ~15); /* Allocate a temporary buffer for reading compressed chunks, each of * which can be at most @chunk_size - 1 bytes. This excludes compressed @@ -560,7 +550,7 @@ out_free_memory: if (chunk_offsets_malloced) FREE(chunk_offsets); if (ubuf_malloced) - FREE(ubuf); + FREE(_ubuf); if (cbuf_malloced) FREE(cbuf); errno = errno_save; @@ -578,6 +568,28 @@ read_error: goto out_free_memory; } +static int +fill_zeroes(u64 size, consume_data_callback_t cb, void *cb_ctx) +{ + if (unlikely(size)) { + u8 buf[min(size, BUFFER_SIZE)]; + + memset(buf, 0, sizeof(buf)); + + do { + size_t len; + int ret; + + len = min(size, BUFFER_SIZE); + ret = cb(buf, len, cb_ctx); + if (ret) + return ret; + size -= len; + } while (size); + } + return 0; +} + /* Read raw data from a file descriptor at the specified offset, feeding the * data it in chunks into the specified callback function. */ static int @@ -670,11 +682,37 @@ read_partial_wim_resource(const struct wim_resource_spec *rspec, return read_compressed_wim_resource(rspec, &range, 1, cb, cb_ctx); } else { - return read_raw_file_data(&rspec->wim->in_fd, - rspec->offset_in_wim + offset, - size, - cb, - cb_ctx); + /* Reading uncompressed resource. For completeness, handle the + * weird case where size_in_wim < uncompressed_size. */ + + u64 read_size; + u64 zeroes_size; + int ret; + + if (likely(offset + size <= rspec->size_in_wim) || + rspec->is_pipable) + { + read_size = size; + zeroes_size = 0; + } else { + if (offset >= rspec->size_in_wim) { + read_size = 0; + zeroes_size = size; + } else { + read_size = rspec->size_in_wim - offset; + zeroes_size = offset + size - rspec->size_in_wim; + } + } + + ret = read_raw_file_data(&rspec->wim->in_fd, + rspec->offset_in_wim + offset, + read_size, + cb, + cb_ctx); + if (ret) + return ret; + + return fill_zeroes(zeroes_size, cb, cb_ctx); } } @@ -726,7 +764,6 @@ read_wim_stream_prefix(const struct wim_lookup_table_entry *lte, u64 size, cb, cb_ctx); } -#ifndef __WIN32__ /* This function handles reading stream data that is located in an external * file, such as a file that has been added to the WIM image through execution * of a wimlib_add_command. @@ -747,7 +784,7 @@ read_file_on_disk_prefix(const struct wim_lookup_table_entry *lte, u64 size, DEBUG("Reading %"PRIu64" bytes from \"%"TS"\"", size, lte->file_on_disk); - raw_fd = open(lte->file_on_disk, O_BINARY | O_RDONLY); + raw_fd = topen(lte->file_on_disk, O_BINARY | O_RDONLY); if (raw_fd < 0) { ERROR_WITH_ERRNO("Can't open \"%"TS"\"", lte->file_on_disk); return WIMLIB_ERR_OPEN; @@ -757,7 +794,34 @@ read_file_on_disk_prefix(const struct wim_lookup_table_entry *lte, u64 size, filedes_close(&fd); return ret; } -#endif /* !__WIN32__ */ + +#ifdef WITH_FUSE +static int +read_staging_file_prefix(const struct wim_lookup_table_entry *lte, u64 size, + consume_data_callback_t cb, void *cb_ctx) +{ + int raw_fd; + struct filedes fd; + int ret; + + wimlib_assert(size <= lte->size); + + DEBUG("Reading %"PRIu64" bytes from staging file \"%s\"", + size, lte->staging_file_name); + + raw_fd = openat(lte->staging_dir_fd, lte->staging_file_name, + O_RDONLY | O_NOFOLLOW); + if (raw_fd < 0) { + ERROR_WITH_ERRNO("Can't open staging file \"%s\"", + lte->staging_file_name); + return WIMLIB_ERR_OPEN; + } + filedes_init(&fd, raw_fd); + ret = read_raw_file_data(&fd, 0, size, cb, cb_ctx); + filedes_close(&fd); + return ret; +} +#endif /* This function handles the trivial case of reading stream data that is, in * fact, already located in an in-memory buffer. */ @@ -795,19 +859,16 @@ read_stream_prefix(const struct wim_lookup_table_entry *lte, u64 size, { static const read_stream_prefix_handler_t handlers[] = { [RESOURCE_IN_WIM] = read_wim_stream_prefix, - #ifdef __WIN32__ - [RESOURCE_IN_FILE_ON_DISK] = read_win32_file_prefix, - #else [RESOURCE_IN_FILE_ON_DISK] = read_file_on_disk_prefix, - #endif [RESOURCE_IN_ATTACHED_BUFFER] = read_buffer_prefix, #ifdef WITH_FUSE - [RESOURCE_IN_STAGING_FILE] = read_file_on_disk_prefix, + [RESOURCE_IN_STAGING_FILE] = read_staging_file_prefix, #endif #ifdef WITH_NTFS_3G [RESOURCE_IN_NTFS_VOLUME] = read_ntfs_file_prefix, #endif #ifdef __WIN32__ + [RESOURCE_IN_WINNT_FILE_ON_DISK] = read_winnt_file_prefix, [RESOURCE_WIN32_ENCRYPTED] = read_win32_encrypted_file_prefix, #endif }; @@ -894,6 +955,34 @@ wim_reshdr_to_data(const struct wim_reshdr *reshdr, WIMStruct *wim, void **buf_r return wim_resource_spec_to_data(&rspec, buf_ret); } +int +wim_reshdr_to_hash(const struct wim_reshdr *reshdr, WIMStruct *wim, + u8 hash[SHA1_HASH_SIZE]) +{ + struct wim_resource_spec rspec; + int ret; + struct wim_lookup_table_entry *lte; + + wim_res_hdr_to_spec(reshdr, wim, &rspec); + + lte = new_lookup_table_entry(); + if (lte == NULL) + return WIMLIB_ERR_NOMEM; + + lte_bind_wim_resource_spec(lte, &rspec); + lte->flags = rspec.flags; + lte->size = rspec.uncompressed_size; + lte->offset_in_res = 0; + lte->unhashed = 1; + + ret = sha1_stream(lte); + + lte_unbind_wim_resource_spec(lte); + copy_hash(hash, lte->hash); + free_lookup_table_entry(lte); + return ret; +} + struct streamifier_context { struct read_stream_list_callbacks cbs; struct wim_lookup_table_entry *cur_stream; @@ -928,10 +1017,17 @@ streamifier_cb(const void *chunk, size_t size, void *_ctx) wimlib_assert(size <= ctx->cur_stream->size - ctx->cur_stream_offset); if (ctx->cur_stream_offset == 0) { + u32 flags; + /* Starting a new stream. */ DEBUG("Begin new stream (size=%"PRIu64").", ctx->cur_stream->size); - ret = (*ctx->cbs.begin_stream)(ctx->cur_stream, true, + + flags = BEGIN_STREAM_FLAG_PARTIAL_RESOURCE; + if (size == ctx->cur_stream->size) + flags |= BEGIN_STREAM_FLAG_WHOLE_STREAM; + ret = (*ctx->cbs.begin_stream)(ctx->cur_stream, + flags, ctx->cbs.begin_stream_ctx); if (ret) return ret; @@ -940,9 +1036,9 @@ streamifier_cb(const void *chunk, size_t size, void *_ctx) /* Consume the chunk. */ ret = (*ctx->cbs.consume_chunk)(chunk, size, ctx->cbs.consume_chunk_ctx); + ctx->cur_stream_offset += size; if (ret) return ret; - ctx->cur_stream_offset += size; if (ctx->cur_stream_offset == ctx->cur_stream->size) { /* Finished reading all the data for a stream. */ @@ -977,7 +1073,7 @@ struct hasher_context { /* Callback for starting to read a stream while calculating its SHA1 message * digest. */ static int -hasher_begin_stream(struct wim_lookup_table_entry *lte, bool is_partial_res, +hasher_begin_stream(struct wim_lookup_table_entry *lte, u32 flags, void *_ctx) { struct hasher_context *ctx = _ctx; @@ -987,7 +1083,7 @@ hasher_begin_stream(struct wim_lookup_table_entry *lte, bool is_partial_res, if (ctx->cbs.begin_stream == NULL) return 0; else - return (*ctx->cbs.begin_stream)(lte, is_partial_res, + return (*ctx->cbs.begin_stream)(lte, flags, ctx->cbs.begin_stream_ctx); } @@ -1007,6 +1103,13 @@ hasher_consume_chunk(const void *chunk, size_t size, void *_ctx) return (*ctx->cbs.consume_chunk)(chunk, size, ctx->cbs.consume_chunk_ctx); } +static void +get_sha1_string(const u8 md[SHA1_HASH_SIZE], tchar *str) +{ + for (size_t i = 0; i < SHA1_HASH_SIZE; i++) + str += tsprintf(str, T("%02x"), md[i]); +} + /* Callback for finishing reading a stream while calculating its SHA1 message * digest. */ static int @@ -1039,9 +1142,14 @@ hasher_end_stream(struct wim_lookup_table_entry *lte, int status, void *_ctx) * that it is the same as the calculated value. */ if (!hashes_equal(hash, lte->hash)) { if (wimlib_print_errors) { - ERROR("Invalid SHA1 message digest " - "on the following WIM stream:"); - print_lookup_table_entry(lte, stderr); + tchar expected_hashstr[SHA1_HASH_SIZE * 2 + 1]; + tchar actual_hashstr[SHA1_HASH_SIZE * 2 + 1]; + get_sha1_string(lte->hash, expected_hashstr); + get_sha1_string(hash, actual_hashstr); + ERROR("The stream is corrupted!\n" + " (Expected SHA1=%"TS",\n" + " got SHA1=%"TS")", + expected_hashstr, actual_hashstr); } ret = WIMLIB_ERR_INVALID_RESOURCE_HASH; errno = EINVAL; @@ -1065,7 +1173,7 @@ read_full_stream_with_cbs(struct wim_lookup_table_entry *lte, { int ret; - ret = (*cbs->begin_stream)(lte, false, cbs->begin_stream_ctx); + ret = (*cbs->begin_stream)(lte, 0, cbs->begin_stream_ctx); if (ret) return ret; @@ -1093,11 +1201,86 @@ read_full_stream_with_sha1(struct wim_lookup_table_entry *lte, .consume_chunk_ctx = &hasher_ctx, .end_stream = hasher_end_stream, .end_stream_ctx = &hasher_ctx, - }; return read_full_stream_with_cbs(lte, &hasher_cbs); } +static int +read_packed_streams(struct wim_lookup_table_entry *first_stream, + struct wim_lookup_table_entry *last_stream, + u64 stream_count, + size_t list_head_offset, + const struct read_stream_list_callbacks *sink_cbs) +{ + struct data_range *ranges; + bool ranges_malloced; + struct wim_lookup_table_entry *cur_stream; + size_t i; + int ret; + u64 ranges_alloc_size; + + DEBUG("Reading %"PRIu64" streams combined in same WIM resource", + stream_count); + + /* Setup data ranges array (one range per stream to read); this way + * read_compressed_wim_resource() does not need to be aware of streams. + */ + + ranges_alloc_size = stream_count * sizeof(ranges[0]); + + if (unlikely((size_t)ranges_alloc_size != ranges_alloc_size)) { + ERROR("Too many streams in one resource!"); + return WIMLIB_ERR_NOMEM; + } + if (likely(ranges_alloc_size <= STACK_MAX)) { + ranges = alloca(ranges_alloc_size); + ranges_malloced = false; + } else { + ranges = MALLOC(ranges_alloc_size); + if (ranges == NULL) { + ERROR("Too many streams in one resource!"); + return WIMLIB_ERR_NOMEM; + } + ranges_malloced = true; + } + + for (i = 0, cur_stream = first_stream; + i < stream_count; + i++, cur_stream = next_stream(cur_stream, list_head_offset)) + { + ranges[i].offset = cur_stream->offset_in_res; + ranges[i].size = cur_stream->size; + } + + struct streamifier_context streamifier_ctx = { + .cbs = *sink_cbs, + .cur_stream = first_stream, + .next_stream = next_stream(first_stream, list_head_offset), + .cur_stream_offset = 0, + .final_stream = last_stream, + .list_head_offset = list_head_offset, + }; + + ret = read_compressed_wim_resource(first_stream->rspec, + ranges, + stream_count, + streamifier_cb, + &streamifier_ctx); + + if (ranges_malloced) + FREE(ranges); + + if (ret) { + if (streamifier_ctx.cur_stream_offset != 0) { + ret = (*streamifier_ctx.cbs.end_stream) + (streamifier_ctx.cur_stream, + ret, + streamifier_ctx.cbs.end_stream_ctx); + } + } + return ret; +} + /* * Read a list of streams, each of which may be in any supported location (e.g. * in a WIM or in an external file). Unlike read_stream_prefix() or the @@ -1186,7 +1369,7 @@ read_stream_list(struct list_head *stream_list, struct wim_lookup_table_entry *lte_next, *lte_last; struct list_head *next2; - size_t stream_count; + u64 stream_count; /* The next stream is a proper sub-sequence of a WIM * resource. See if there are other streams in the same @@ -1215,53 +1398,13 @@ read_stream_list(struct list_head *stream_list, * first stream in the resource that needs to be * read and @lte_last specifies the last stream * in the resource that needs to be read. */ - - DEBUG("Reading %zu streams combined in same " - "WIM resource", stream_count); - next = next2; - - struct data_range ranges[stream_count]; - - { - struct list_head *next3; - size_t i; - struct wim_lookup_table_entry *lte_cur; - - next3 = cur; - for (i = 0; i < stream_count; i++) { - lte_cur = (struct wim_lookup_table_entry*) - ((u8*)next3 - list_head_offset); - ranges[i].offset = lte_cur->offset_in_res; - ranges[i].size = lte_cur->size; - next3 = next3->next; - } - } - - struct streamifier_context streamifier_ctx = { - .cbs = *sink_cbs, - .cur_stream = lte, - .next_stream = next_stream(lte, list_head_offset), - .cur_stream_offset = 0, - .final_stream = lte_last, - .list_head_offset = list_head_offset, - }; - - ret = read_compressed_wim_resource(lte->rspec, - ranges, - stream_count, - streamifier_cb, - &streamifier_ctx); - - if (ret) { - if (streamifier_ctx.cur_stream_offset != 0) { - ret = (*streamifier_ctx.cbs.end_stream) - (streamifier_ctx.cur_stream, - ret, - streamifier_ctx.cbs.end_stream_ctx); - } + ret = read_packed_streams(lte, lte_last, + stream_count, + list_head_offset, + sink_cbs); + if (ret) return ret; - } continue; } } @@ -1323,6 +1466,15 @@ extract_stream_to_fd(struct wim_lookup_table_entry *lte, return extract_stream(lte, size, extract_chunk_to_fd, fd); } +/* Extract the full uncompressed contents of the specified stream to the + * specified file descriptor. */ +int +extract_full_stream_to_fd(struct wim_lookup_table_entry *lte, + struct filedes *fd) +{ + return extract_stream_to_fd(lte, fd, lte->size); +} + /* Calculate the SHA1 message digest of a stream and store it in @lte->hash. */ int sha1_stream(struct wim_lookup_table_entry *lte) @@ -1334,7 +1486,10 @@ sha1_stream(struct wim_lookup_table_entry *lte) } /* Convert a short WIM resource header to a stand-alone WIM resource - * specification. */ + * specification. + * + * Note: for packed resources some fields still need to be overridden. + */ void wim_res_hdr_to_spec(const struct wim_reshdr *reshdr, WIMStruct *wim, struct wim_resource_spec *rspec) @@ -1346,6 +1501,13 @@ wim_res_hdr_to_spec(const struct wim_reshdr *reshdr, WIMStruct *wim, INIT_LIST_HEAD(&rspec->stream_list); rspec->flags = reshdr->flags; rspec->is_pipable = wim_is_pipable(wim); + if (rspec->flags & WIM_RESHDR_FLAG_COMPRESSED) { + rspec->compression_type = wim->compression_type; + rspec->chunk_size = wim->chunk_size; + } else { + rspec->compression_type = WIMLIB_COMPRESSION_TYPE_NONE; + rspec->chunk_size = 0; + } } /* Convert a stand-alone resource specification to a WIM resource header. */