X-Git-Url: https://wimlib.net/git/?a=blobdiff_plain;f=src%2Fresource.c;h=4b91be6b010104fc2c26cf65f5a16b77606f357a;hb=83d1242cf793d718c666d6d50faa397ce5234ed2;hp=90e45ca1092db0eb3f0a65617734613beceab1c0;hpb=9e2571b03cd9c71d11b3dad9ea5dcfa43f50deb4;p=wimlib diff --git a/src/resource.c b/src/resource.c index 90e45ca1..4b91be6b 100644 --- a/src/resource.c +++ b/src/resource.c @@ -38,51 +38,9 @@ #include #include -/* Write @n bytes from @buf to the file descriptor @fd, retrying on internupt - * and on short writes. - * - * Returns short count and set errno on failure. */ -static ssize_t -full_write(int fd, const void *buf, size_t n) -{ - const void *p = buf; - ssize_t ret; - ssize_t total = 0; - - while (total != n) { - ret = write(fd, p, n); - if (ret < 0) { - if (errno == EINTR) - continue; - else - break; - } - total += ret; - p += ret; - } - return total; -} - -/* Read @n bytes from the file descriptor @fd to the buffer @buf, retrying on - * internupt and on short reads. - * - * Returns short count and set errno on failure. */ -static size_t -full_read(int fd, void *buf, size_t n) -{ - size_t bytes_remaining = n; - while (bytes_remaining) { - ssize_t bytes_read = read(fd, buf, bytes_remaining); - if (bytes_read < 0) { - if (errno == EINTR) - continue; - break; - } - bytes_remaining -= bytes_read; - buf += bytes_read; - } - return n - bytes_remaining; -} +#ifdef HAVE_ALLOCA_H +# include +#endif /* * Reads all or part of a compressed WIM resource. @@ -90,10 +48,13 @@ full_read(int fd, void *buf, size_t n) * Returns zero on success, nonzero on failure. */ static int -read_compressed_resource(FILE *fp, u64 resource_compressed_size, +read_compressed_resource(int in_fd, + u64 resource_compressed_size, u64 resource_uncompressed_size, - u64 resource_offset, int resource_ctype, - u64 len, u64 offset, + u64 resource_offset, + int resource_ctype, + u64 len, + u64 offset, consume_data_callback_t cb, void *ctx_or_buf) { @@ -122,7 +83,8 @@ read_compressed_resource(FILE *fp, u64 resource_compressed_size, * follows the chunk table and therefore must have an offset of 0. */ - /* Calculate how many chunks the resource conists of in its entirety. */ + /* Calculate how many chunks the resource consists of in its entirety. + * */ u64 num_chunks = (resource_uncompressed_size + WIM_CHUNK_SIZE - 1) / WIM_CHUNK_SIZE; /* As mentioned, the first chunk has no entry in the chunk table. */ @@ -149,9 +111,22 @@ read_compressed_resource(FILE *fp, u64 resource_compressed_size, if (end_chunk != num_chunks - 1) num_needed_chunks++; - /* Declare the chunk table. It will only contain offsets for the chunks - * that are actually needed for this read. */ - u64 chunk_offsets[num_needed_chunks]; + /* Allocate the chunk table. It will only contain offsets for the + * chunks that are actually needed for this read. */ + u64 *chunk_offsets; + bool chunk_offsets_malloced; + if (num_needed_chunks < 1000) { + chunk_offsets = alloca(num_needed_chunks * sizeof(u64)); + chunk_offsets_malloced = false; + } else { + chunk_offsets = malloc(num_needed_chunks * sizeof(u64)); + if (!chunk_offsets) { + ERROR("Failed to allocate chunk table " + "with %"PRIu64" entries", num_needed_chunks); + return WIMLIB_ERR_NOMEM; + } + chunk_offsets_malloced = true; + } /* Set the implicit offset of the first chunk if it is included in the * needed chunks. @@ -187,49 +162,46 @@ read_compressed_resource(FILE *fp, u64 resource_compressed_size, /* Skip over unneeded chunk table entries. */ u64 file_offset_of_needed_chunk_entries = resource_offset + start_table_idx * chunk_entry_size; - if (fseeko(fp, file_offset_of_needed_chunk_entries, SEEK_SET)) - goto read_error; /* Number of bytes we need to read from the chunk table. */ size_t size = num_needed_chunk_entries * chunk_entry_size; - { - u8 chunk_tab_buf[size]; + /* Read the raw data into the end of the chunk_offsets array to + * avoid allocating another array. */ + void *chunk_tab_buf = (void*)&chunk_offsets[num_needed_chunks] - size; - if (fread(chunk_tab_buf, 1, size, fp) != size) - goto read_error; + if (full_pread(in_fd, chunk_tab_buf, size, + file_offset_of_needed_chunk_entries) != size) + goto read_error; - /* Now fill in chunk_offsets from the entries we have read in - * chunk_tab_buf. */ + /* Now fill in chunk_offsets from the entries we have read in + * chunk_tab_buf. */ - u64 *chunk_tab_p = chunk_offsets; - if (start_chunk == 0) - chunk_tab_p++; + u64 *chunk_tab_p = chunk_offsets; + if (start_chunk == 0) + chunk_tab_p++; - if (chunk_entry_size == 4) { - u32 *entries = (u32*)chunk_tab_buf; - while (num_needed_chunk_entries--) - *chunk_tab_p++ = le32_to_cpu(*entries++); - } else { - u64 *entries = (u64*)chunk_tab_buf; - while (num_needed_chunk_entries--) - *chunk_tab_p++ = le64_to_cpu(*entries++); - } + if (chunk_entry_size == 4) { + u32 *entries = (u32*)chunk_tab_buf; + while (num_needed_chunk_entries--) + *chunk_tab_p++ = le32_to_cpu(*entries++); + } else { + u64 *entries = (u64*)chunk_tab_buf; + while (num_needed_chunk_entries--) + *chunk_tab_p++ = le64_to_cpu(*entries++); } /* Done with the chunk table now. We must now seek to the first chunk * that is needed for the read. */ - u64 file_offset_of_first_needed_chunk = resource_offset + - chunk_table_size + chunk_offsets[0]; - if (fseeko(fp, file_offset_of_first_needed_chunk, SEEK_SET)) - goto read_error; + u64 cur_read_offset = resource_offset + chunk_table_size + chunk_offsets[0]; /* Pointer to current position in the output buffer for uncompressed - * data. */ + * data. Alternatively, if using a callback function, we repeatedly + * fill a temporary buffer to feed data into the callback function. */ u8 *out_p; if (cb) - out_p = alloca(32768); + out_p = alloca(WIM_CHUNK_SIZE); else out_p = ctx_or_buf; @@ -298,19 +270,24 @@ read_compressed_resource(FILE *fp, u64 resource_compressed_size, * is equal to the uncompressed chunk size. */ if (compressed_chunk_size == uncompressed_chunk_size) { /* Uncompressed chunk */ - - if (start_offset != 0) - if (fseeko(fp, start_offset, SEEK_CUR)) - goto read_error; - if (fread(out_p, 1, partial_chunk_size, fp) != partial_chunk_size) + if (full_pread(in_fd, + cb ? out_p + start_offset : out_p, + partial_chunk_size, + cur_read_offset + start_offset) != partial_chunk_size) + { goto read_error; + } } else { /* Compressed chunk */ /* Read the compressed data into compressed_buf. */ - if (fread(compressed_buf, 1, compressed_chunk_size, - fp) != compressed_chunk_size) + if (full_pread(in_fd, + compressed_buf, + compressed_chunk_size, + cur_read_offset) != compressed_chunk_size) + { goto read_error; + } /* For partial chunks and when writing directly to a * buffer, we must buffer the uncompressed data because @@ -341,7 +318,8 @@ read_compressed_resource(FILE *fp, u64 resource_compressed_size, } if (cb) { /* Feed the data to the callback function */ - ret = cb(out_p, partial_chunk_size, ctx_or_buf); + ret = cb(out_p + start_offset, + partial_chunk_size, ctx_or_buf); if (ret) goto out; } else { @@ -351,46 +329,21 @@ read_compressed_resource(FILE *fp, u64 resource_compressed_size, * written. */ out_p += partial_chunk_size; } + cur_read_offset += compressed_chunk_size; } ret = 0; out: + if (chunk_offsets_malloced) + FREE(chunk_offsets); return ret; read_error: - if (feof(fp)) - ERROR("Unexpected EOF in compressed file resource"); - else - ERROR_WITH_ERRNO("Error reading compressed file resource"); + ERROR_WITH_ERRNO("Error reading compressed file resource"); ret = WIMLIB_ERR_READ; goto out; } -/* - * Reads uncompressed data from an open file stream. - */ -int -read_uncompressed_resource(FILE *fp, u64 offset, u64 len, void *contents_ret) -{ - if (fseeko(fp, offset, SEEK_SET) != 0) { - ERROR("Failed to seek to byte %"PRIu64" of input file " - "to read uncompressed resource (len = %"PRIu64")", - offset, len); - return WIMLIB_ERR_READ; - } - if (fread(contents_ret, 1, len, fp) != len) { - if (feof(fp)) { - ERROR("Unexpected EOF in uncompressed file resource"); - } else { - ERROR("Failed to read %"PRIu64" bytes from " - "uncompressed resource at offset %"PRIu64, - len, offset); - } - return WIMLIB_ERR_READ; - } - return 0; -} - /* Reads the contents of a struct resource_entry, as represented in the on-disk * format, from the memory pointed to by @p, and fills in the fields of @entry. * A pointer to the byte after the memory read at @p is returned. */ @@ -435,66 +388,6 @@ put_resource_entry(void *p, const struct resource_entry *entry) return p; } -static FILE * -wim_get_fp(WIMStruct *w) -{ -#ifdef WITH_FUSE - pthread_mutex_lock(&w->fp_tab_mutex); - FILE *fp; - - wimlib_assert(w->filename != NULL); - - for (size_t i = 0; i < w->num_allocated_fps; i++) { - if (w->fp_tab[i]) { - fp = w->fp_tab[i]; - w->fp_tab[i] = NULL; - goto out_unlock; - } - } - DEBUG("Opening extra file descriptor to `%"TS"'", w->filename); - fp = tfopen(w->filename, T("rb")); - if (!fp) - ERROR_WITH_ERRNO("Failed to open `%"TS"'", w->filename); -out_unlock: - pthread_mutex_unlock(&w->fp_tab_mutex); -#else /* WITH_FUSE */ - fp = w->fp; -#endif /* !WITH_FUSE */ - return fp; -} - -static int -wim_release_fp(WIMStruct *w, FILE *fp) -{ - int ret = 0; -#ifdef WITH_FUSE - FILE **fp_tab; - - pthread_mutex_lock(&w->fp_tab_mutex); - - for (size_t i = 0; i < w->num_allocated_fps; i++) { - if (w->fp_tab[i] == NULL) { - w->fp_tab[i] = fp; - goto out_unlock; - } - } - - fp_tab = REALLOC(w->fp_tab, sizeof(FILE*) * (w->num_allocated_fps + 4)); - if (!fp_tab) { - ret = WIMLIB_ERR_NOMEM; - fclose(fp); - goto out_unlock; - } - w->fp_tab = fp_tab; - memset(&w->fp_tab[w->num_allocated_fps], 0, 4 * sizeof(FILE*)); - w->fp_tab[w->num_allocated_fps] = fp; - w->num_allocated_fps += 4; -out_unlock: - pthread_mutex_unlock(&w->fp_tab_mutex); -#endif /* WITH_FUSE */ - return ret; -} - static int read_partial_wim_resource(const struct wim_lookup_table_entry *lte, u64 size, @@ -503,31 +396,19 @@ read_partial_wim_resource(const struct wim_lookup_table_entry *lte, int flags, u64 offset) { - FILE *wim_fp; WIMStruct *wim; + int in_fd; int ret; wimlib_assert(lte->resource_location == RESOURCE_IN_WIM); - wimlib_assert(offset + size <= lte->resource_entry.original_size); wim = lte->wim; - - if (flags & WIMLIB_RESOURCE_FLAG_MULTITHREADED) { - wim_fp = wim_get_fp(wim); - if (!wim_fp) { - ret = -1; - goto out; - } - } else { - wim_fp = lte->wim->fp; - } - - wimlib_assert(wim_fp != NULL); + in_fd = wim->in_fd; if (lte->resource_entry.flags & WIM_RESHDR_FLAG_COMPRESSED && !(flags & WIMLIB_RESOURCE_FLAG_RAW)) { - ret = read_compressed_resource(wim_fp, + ret = read_compressed_resource(in_fd, lte->resource_entry.size, lte->resource_entry.original_size, lte->resource_entry.offset, @@ -537,42 +418,37 @@ read_partial_wim_resource(const struct wim_lookup_table_entry *lte, cb, ctx_or_buf); } else { - if (fseeko(wim_fp, offset, SEEK_SET)) { - ERROR_WITH_ERRNO("Failed to seek to offset %"PRIu64 - " in WIM", offset); - ret = WIMLIB_ERR_READ; - goto out_release_fp; - } + offset += lte->resource_entry.offset; if (cb) { - char buf[min(32768, size)]; + /* Send data to callback function */ + u8 buf[min(WIM_CHUNK_SIZE, size)]; while (size) { - size_t bytes_to_read = min(32768, size); - size_t bytes_read = fread(buf, 1, bytes_to_read, wim_fp); - + size_t bytes_to_read = min(WIM_CHUNK_SIZE, size); + size_t bytes_read = full_pread(in_fd, buf, + bytes_to_read, offset); if (bytes_read != bytes_to_read) goto read_error; ret = cb(buf, bytes_read, ctx_or_buf); if (ret) - goto out_release_fp; + goto out; + size -= bytes_read; + offset += bytes_read; } } else { - if (fread(ctx_or_buf, 1, size, wim_fp) != size) + /* Send data directly to a buffer */ + if (full_pread(in_fd, ctx_or_buf, size, offset) != size) goto read_error; } ret = 0; } - goto out_release_fp; + goto out; read_error: ERROR_WITH_ERRNO("Error reading data from WIM"); ret = WIMLIB_ERR_READ; -out_release_fp: - if (flags & WIMLIB_RESOURCE_FLAG_MULTITHREADED) - ret |= wim_release_fp(wim, wim_fp); out: if (ret) { if (errno == 0) errno = EIO; - ret = -1; } return ret; } @@ -580,12 +456,9 @@ out: int read_partial_wim_resource_into_buf(const struct wim_lookup_table_entry *lte, - size_t size, u64 offset, void *buf, - bool threadsafe) + size_t size, u64 offset, void *buf) { - return read_partial_wim_resource(lte, size, NULL, buf, - threadsafe ? WIMLIB_RESOURCE_FLAG_MULTITHREADED : 0, - offset); + return read_partial_wim_resource(lte, size, NULL, buf, 0, offset); } static int @@ -599,6 +472,7 @@ read_wim_resource_prefix(const struct wim_lookup_table_entry *lte, } +#ifndef __WIN32__ static int read_file_on_disk_prefix(const struct wim_lookup_table_entry *lte, u64 size, @@ -618,16 +492,17 @@ read_file_on_disk_prefix(const struct wim_lookup_table_entry *lte, } if (cb) { /* Send data to callback function */ - char buf[min(32768, size)]; + u8 buf[min(WIM_CHUNK_SIZE, size)]; size_t bytes_to_read; while (size) { - bytes_to_read = min(32768, size); + bytes_to_read = min(WIM_CHUNK_SIZE, size); bytes_read = full_read(fd, buf, bytes_to_read); if (bytes_read != bytes_to_read) goto read_error; ret = cb(buf, bytes_read, ctx_or_buf); if (ret) goto out_close; + size -= bytes_read; } } else { /* Send data directly to a buffer */ @@ -644,6 +519,7 @@ out_close: close(fd); return ret; } +#endif /* !__WIN32__ */ static int read_buffer_prefix(const struct wim_lookup_table_entry *lte, @@ -651,12 +527,21 @@ read_buffer_prefix(const struct wim_lookup_table_entry *lte, void *ctx_or_buf, int _ignored_flags) { const void *inbuf = lte->attached_buffer; + int ret; + if (cb) { - return cb(inbuf, size, ctx_or_buf); + while (size) { + size_t chunk_size = min(WIM_CHUNK_SIZE, size); + ret = cb(inbuf, chunk_size, ctx_or_buf); + if (ret) + return ret; + size -= chunk_size; + inbuf += chunk_size; + } } else { memcpy(ctx_or_buf, inbuf, size); - return 0; } + return 0; } typedef int (*read_resource_prefix_handler_t)(const struct wim_lookup_table_entry *lte, @@ -665,6 +550,23 @@ typedef int (*read_resource_prefix_handler_t)(const struct wim_lookup_table_entr void *ctx_or_buf, int flags); +/* + * Read the first @size bytes from a generic "resource", which may be located in + * the WIM (compressed or uncompressed), in an external file, or directly in an + * in-memory buffer. + * + * Feed the data either to a callback function (cb != NULL, passing it + * ctx_or_buf), or write it directly into a buffer (cb == NULL, ctx_or_buf + * specifies the buffer, which must have room for @size bytes). + * + * When using a callback function, it is called with chunks up to 32768 bytes in + * size until the resource is exhausted. + * + * If the resource is located in a WIM file, @flags can be: + * * WIMLIB_RESOURCE_FLAG_RAW if the raw compressed data is to be supplied + * instead of the uncompressed data. + * Otherwise, the @flags are ignored. + */ int read_resource_prefix(const struct wim_lookup_table_entry *lte, u64 size, consume_data_callback_t cb, void *ctx_or_buf, @@ -672,7 +574,9 @@ read_resource_prefix(const struct wim_lookup_table_entry *lte, { static const read_resource_prefix_handler_t handlers[] = { [RESOURCE_IN_WIM] = read_wim_resource_prefix, + #ifndef __WIN32__ [RESOURCE_IN_FILE_ON_DISK] = read_file_on_disk_prefix, + #endif [RESOURCE_IN_ATTACHED_BUFFER] = read_buffer_prefix, #ifdef WITH_FUSE [RESOURCE_IN_STAGING_FILE] = read_file_on_disk_prefix, @@ -692,12 +596,25 @@ read_resource_prefix(const struct wim_lookup_table_entry *lte, int read_full_resource_into_buf(const struct wim_lookup_table_entry *lte, - void *buf, bool thread_safe) + void *buf) { - return read_resource_prefix(lte, - wim_resource_size(lte), - NULL, buf, - thread_safe ? WIMLIB_RESOURCE_FLAG_MULTITHREADED : 0); + return read_resource_prefix(lte, wim_resource_size(lte), NULL, buf, 0); +} + +struct extract_ctx { + SHA_CTX sha_ctx; + consume_data_callback_t extract_chunk; + void *extract_chunk_arg; +}; + +static int +extract_chunk_sha1_wrapper(const void *chunk, size_t chunk_size, + void *_ctx) +{ + struct extract_ctx *ctx = _ctx; + + sha1_update(&ctx->sha_ctx, chunk, chunk_size); + return ctx->extract_chunk(chunk, chunk_size, ctx->extract_chunk_arg); } /* Extracts the first @size bytes of a WIM resource to somewhere. In the @@ -712,8 +629,36 @@ extract_wim_resource(const struct wim_lookup_table_entry *lte, consume_data_callback_t extract_chunk, void *extract_chunk_arg) { - return read_resource_prefix(lte, size, extract_chunk, - extract_chunk_arg, 0); + int ret; + if (size == wim_resource_size(lte)) { + /* Do SHA1 */ + struct extract_ctx ctx; + ctx.extract_chunk = extract_chunk; + ctx.extract_chunk_arg = extract_chunk_arg; + sha1_init(&ctx.sha_ctx); + ret = read_resource_prefix(lte, size, + extract_chunk_sha1_wrapper, + &ctx, 0); + if (ret == 0) { + u8 hash[SHA1_HASH_SIZE]; + sha1_final(hash, &ctx.sha_ctx); + if (!hashes_equal(hash, lte->hash)) { + #ifdef ENABLE_ERROR_MESSAGES + ERROR("Invalid SHA1 message digest " + "on the following WIM resource:"); + print_lookup_table_entry(lte, stderr); + if (lte->resource_location == RESOURCE_IN_WIM) + ERROR("The WIM file appears to be corrupt!"); + #endif + ret = WIMLIB_ERR_INVALID_RESOURCE_HASH; + } + } + } else { + /* Don't do SHA1 */ + ret = read_resource_prefix(lte, size, extract_chunk, + extract_chunk_arg, 0); + } + return ret; } static int @@ -736,6 +681,29 @@ extract_wim_resource_to_fd(const struct wim_lookup_table_entry *lte, return extract_wim_resource(lte, size, extract_wim_chunk_to_fd, &fd); } + +static int +sha1_chunk(const void *buf, size_t len, void *ctx) +{ + sha1_update(ctx, buf, len); + return 0; +} + +/* Calculate the SHA1 message digest of a stream. */ +int +sha1_resource(struct wim_lookup_table_entry *lte) +{ + int ret; + SHA_CTX sha_ctx; + + sha1_init(&sha_ctx); + ret = read_resource_prefix(lte, wim_resource_size(lte), + sha1_chunk, &sha_ctx, 0); + if (ret == 0) + sha1_final(lte->hash, &sha_ctx); + return ret; +} + /* * Copies the file resource specified by the lookup table entry @lte from the * input WIM to the output WIM that has its FILE * given by @@ -752,7 +720,7 @@ copy_resource(struct wim_lookup_table_entry *lte, void *wim) WIMStruct *w = wim; int ret; - ret = write_wim_resource(lte, w->out_fp, + ret = write_wim_resource(lte, w->out_fd, wim_resource_compression_type(lte), <e->output_resource_entry, 0); if (ret == 0) {