From 0541069de2f74283d44ebb02372b60fb608795f2 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 19 Nov 2012 22:24:30 -0600 Subject: [PATCH] Rewrite integrity calculation and checks - Read and write 4096 bytes at a time - Improve progress messages - Allow copying some, but possibly not all, entries from the previous integrity table. --- src/dentry.c | 7 +- src/integrity.c | 643 +++++++++++++++++++++++++++--------------- src/lookup_table.h | 8 - src/modify.c | 11 +- src/ntfs-capture.c | 2 +- src/resource.c | 16 +- src/security.c | 24 +- src/util.h | 6 + src/wim.c | 23 +- src/wimlib_internal.h | 20 +- src/write.c | 105 ++++--- 11 files changed, 548 insertions(+), 317 deletions(-) diff --git a/src/dentry.c b/src/dentry.c index d135f578..0e8d81f4 100644 --- a/src/dentry.c +++ b/src/dentry.c @@ -1037,9 +1037,8 @@ static int do_free_dentry(struct dentry *dentry, void *__lookup_table) */ void free_dentry_tree(struct dentry *root, struct lookup_table *lookup_table) { - if (!root || !root->parent) - return; - for_dentry_in_tree_depth(root, do_free_dentry, lookup_table); + if (root) + for_dentry_in_tree_depth(root, do_free_dentry, lookup_table); } int increment_dentry_refcnt(struct dentry *dentry, void *ignore) @@ -1819,9 +1818,7 @@ int read_dentry_tree(const u8 metadata_resource[], u64 metadata_resource_len, break; } memcpy(child, &cur_child, sizeof(struct dentry)); - dentry_add_child(dentry, child); - inode_add_dentry(child, child->d_inode); /* If there are children of this child, call this procedure diff --git a/src/integrity.c b/src/integrity.c index 5ad55a5b..bf02a6b1 100644 --- a/src/integrity.c +++ b/src/integrity.c @@ -33,323 +33,514 @@ * information. */ #define INTEGRITY_CHUNK_SIZE 10485760 -/* - * Verifies the integrity of a WIM. - * - * @fp: FILE* of the WIM, currently positioned at the end of the header. - * @num_bytes: Number of bytes to verify the integrity of. - * @chunk_size: Chunk size per SHA1 message digest. - * @sha1sums: Array of SHA1 message digests; 20 bytes each, one per chunk. - * @show_progress: Nonzero if the percent complete is to be printed after every - * chunk. - */ -static int verify_integrity(FILE *fp, u64 num_bytes, u32 chunk_size, - const u8 *sha1sums, int show_progress) +/* Only use a different chunk size for compatiblity with an existing integrity + * table if the chunk size is between these two numbers. */ +#define INTEGRITY_MIN_CHUNK_SIZE 4096 +#define INTEGRITY_MAX_CHUNK_SIZE 134217728 + +struct integrity_table { + u32 size; + u32 num_entries; + u32 chunk_size; + u8 sha1sums[0][20]; +}; + +static int calculate_chunk_sha1(FILE *fp, size_t this_chunk_size, + off_t offset, u8 sha1_md[]) { - u8 *chunk_buf; - u8 resblock[SHA1_HASH_SIZE]; - u64 bytes_remaining; + int ret; + u8 buf[BUFFER_SIZE]; + SHA_CTX ctx; + size_t bytes_remaining; size_t bytes_to_read; - uint percent_done; - int ret; + size_t bytes_read; - chunk_buf = MALLOC(chunk_size); - if (!chunk_buf) { - ERROR("Failed to allocate %u byte buffer for integrity chunks", - chunk_size); - return WIMLIB_ERR_NOMEM; + ret = fseeko(fp, offset, SEEK_SET); + if (ret != 0) { + ERROR_WITH_ERRNO("Can't seek to offset " + "%"PRIu64" in WIM", offset); + return WIMLIB_ERR_READ; } - bytes_remaining = num_bytes; - while (bytes_remaining != 0) { - if (show_progress) { - percent_done = (num_bytes - bytes_remaining) * 100 / - num_bytes; - printf("Verifying integrity of WIM (%"PRIu64" bytes " - "remaining, %u%% done) \r", - bytes_remaining, percent_done); - fflush(stdout); - } - bytes_to_read = min(chunk_size, bytes_remaining); - if (fread(chunk_buf, 1, bytes_to_read, fp) != bytes_to_read) { + bytes_remaining = this_chunk_size; + sha1_init(&ctx); + do { + bytes_to_read = min(bytes_remaining, sizeof(buf)); + bytes_read = fread(buf, 1, bytes_to_read, fp); + if (bytes_read != bytes_to_read) { if (feof(fp)) { - ERROR("Unexpected EOF while verifying " - "integrity of WIM"); + ERROR("Unexpected EOF while calculating " + "integrity checksums"); } else { ERROR_WITH_ERRNO("File stream error while " - "verifying integrity of WIM"); + "calculating integrity " + "checksums"); } - ret = WIMLIB_ERR_READ; - goto out; + return WIMLIB_ERR_READ; } - sha1_buffer(chunk_buf, bytes_to_read, resblock); - if (!hashes_equal(resblock, sha1sums)) { - ret = WIM_INTEGRITY_NOT_OK; - goto out; - } - sha1sums += SHA1_HASH_SIZE; - bytes_remaining -= bytes_to_read; - } - ret = WIM_INTEGRITY_OK; -out: - FREE(chunk_buf); - if (show_progress) - putchar('\n'); - return ret; + sha1_update(&ctx, buf, bytes_read); + bytes_remaining -= bytes_read; + } while (bytes_remaining); + sha1_final(sha1_md, &ctx); + return 0; } + /* - * Verifies the integrity of the WIM. + * Reads the integrity table from a WIM file. + * + * @res_entry: + * The resource entry that specifies the location of the integrity table. + * The integrity table must exist (i.e. res_entry->offset must not be 0). + * + * @fp: + * FILE * to the WIM file, opened for reading. + * + * @num_checked_bytes: + * Number of bytes of data that should be checked by the integrity table. + * + * @table ret: + * On success, a pointer to an in-memory structure containing the integrity + * information is written to this location. + * + * Returns 0 on success; nonzero on failure. The possible error codes are: + * + * * WIMLIB_ERR_INVALID_INTEGRITY_TABLE: The integrity table is invalid. + * * WIMLIB_ERR_NOMEM: Could not allocate memory to store the integrity + * data. + * * WIMLIB_ERR_READ: Could not read the integrity data from the WIM file. */ -int check_wim_integrity(WIMStruct *w, int show_progress) +static int read_integrity_table(const struct resource_entry *res_entry, + FILE *fp, + u64 num_checked_bytes, + struct integrity_table **table_ret) { - - struct resource_entry *res_entry; - u8 *buf = NULL; - int ret; - u32 integrity_table_size; - u32 num_entries; - u32 chunk_size; - const u8 *p; + struct integrity_table *table = NULL; + int ret = 0; u64 expected_size; - u64 end_lookup_table_offset; - u64 bytes_to_check; u64 expected_num_entries; - res_entry = &w->hdr.integrity; - if (res_entry->size == 0) { - DEBUG("No integrity information."); - return WIM_INTEGRITY_NONEXISTENT; - } if (res_entry->original_size < 12) { - ERROR("Integrity table is too short"); - return WIMLIB_ERR_INVALID_INTEGRITY_TABLE; + ERROR("Integrity table is too short (expected at least 12 bytes)"); + ret = WIMLIB_ERR_INVALID_INTEGRITY_TABLE; + goto out; } + if (res_entry->flags & WIM_RESHDR_FLAG_COMPRESSED) { ERROR("Didn't expect a compressed integrity table"); - return WIMLIB_ERR_INVALID_INTEGRITY_TABLE; + ret = WIMLIB_ERR_INVALID_INTEGRITY_TABLE; + goto out; } /* Read the integrity table into memory. */ if ((sizeof(size_t) < sizeof(u64) - && res_entry->original_size > ~(size_t)0) - || ((buf = MALLOC(res_entry->original_size)) == NULL)) + && res_entry->size > ~(size_t)0) + || ((table = MALLOC(res_entry->size)) == NULL)) { ERROR("Out of memory (needed %zu bytes for integrity table)", - (size_t)res_entry->original_size); + (size_t)res_entry->size); ret = WIMLIB_ERR_NOMEM; goto out; } - ret = read_uncompressed_resource(w->fp, res_entry->offset, - res_entry->original_size, buf); + + ret = read_uncompressed_resource(fp, res_entry->offset, + res_entry->size, (void*)table); + if (ret != 0) { ERROR("Failed to read integrity table (size = %"PRIu64", " - "original_size = %"PRIu64", offset = " - "%"PRIu64")", - (u64)res_entry->size, res_entry->original_size, - res_entry->offset); + " offset = %"PRIu64")", + (u64)res_entry->size, res_entry->offset); goto out; } - p = get_u32(buf, &integrity_table_size); - p = get_u32(p, &num_entries); - p = get_u32(p, &chunk_size); - - /* p now points to the array of SHA1 message digests for the WIM. */ + table->size = le32_to_cpu(table->size); + table->num_entries = le32_to_cpu(table->num_entries); + table->chunk_size = le32_to_cpu(table->chunk_size); - /* Make sure the integrity table is the right size. */ - if (integrity_table_size != res_entry->original_size) { - ERROR("Inconsistent integrity table sizes: header says %u " - "bytes but resource entry says " - "%"PRIu64" bytes", - integrity_table_size, res_entry->original_size); + if (table->size != res_entry->size) { + ERROR("Inconsistent integrity table sizes: Table header says " + "%u bytes but resource entry says %"PRIu64" bytes", + table->size, (u64)res_entry->size); ret = WIMLIB_ERR_INVALID_INTEGRITY_TABLE; goto out; } - DEBUG("integrity_table_size = %u, num_entries = %u, chunk_size = %u", - integrity_table_size, num_entries, chunk_size); + DEBUG("table->size = %u, table->num_entries = %u, " + "table->chunk_size = %u", + table->size, table->num_entries, table->chunk_size); + expected_size = (u64)table->num_entries * SHA1_HASH_SIZE + 12; - expected_size = num_entries * SHA1_HASH_SIZE + 12; - - if (integrity_table_size != expected_size) { + if (table->size != expected_size) { ERROR("Integrity table is %u bytes, but expected %"PRIu64" " "bytes to hold %u entries", - integrity_table_size, expected_size, num_entries); + table->size, expected_size, table->num_entries); ret = WIMLIB_ERR_INVALID_INTEGRITY_TABLE; goto out; } - if (chunk_size == 0) { + if (table->chunk_size == 0) { ERROR("Cannot use integrity chunk size of 0"); ret = WIMLIB_ERR_INVALID_INTEGRITY_TABLE; goto out; } - end_lookup_table_offset = w->hdr.lookup_table_res_entry.offset + - w->hdr.lookup_table_res_entry.size; - - if (end_lookup_table_offset < WIM_HEADER_DISK_SIZE) { - ERROR("WIM lookup table ends before WIM header ends???"); - ret = WIMLIB_ERR_INVALID_INTEGRITY_TABLE; - goto out; - } - - bytes_to_check = end_lookup_table_offset - WIM_HEADER_DISK_SIZE; - - expected_num_entries = (bytes_to_check + chunk_size - 1) / chunk_size; + expected_num_entries = DIV_ROUND_UP(num_checked_bytes, table->chunk_size); - if (num_entries != expected_num_entries) { + if (table->num_entries != expected_num_entries) { ERROR("%"PRIu64" entries would be required to checksum " "the %"PRIu64" bytes from the end of the header to the", - expected_num_entries, bytes_to_check); + expected_num_entries, num_checked_bytes); ERROR("end of the lookup table with a chunk size of %u, but " - "there were only %u entries", chunk_size, num_entries); + "there were only %u entries", + table->chunk_size, table->num_entries); ret = WIMLIB_ERR_INVALID_INTEGRITY_TABLE; - goto out; - } - - /* The integrity checking starts after the header, so seek to the offset - * in the WIM after the header. */ - - if (fseeko(w->fp, WIM_HEADER_DISK_SIZE, SEEK_SET) != 0) { - ERROR_WITH_ERRNO("Failed to seek to byte %u of WIM to check " - "integrity", WIM_HEADER_DISK_SIZE); - ret = WIMLIB_ERR_READ; - goto out; } - /* call verify_integrity(), which does the actual checking of the SHA1 - * message digests. */ - ret = verify_integrity(w->fp, bytes_to_check, chunk_size, p, - show_progress); out: - FREE(buf); + if (ret == 0) + *table_ret = table; + else + FREE(table); return ret; } /* - * Writes integrity information to the output stream for a WIM file being - * written. + * Calculates an integrity table for the data in a file beginning at offset 208 + * (WIM_HEADER_DISK_SIZE). + * + * @fp: + * FILE * for the file to be checked, opened for reading. Does not need to + * be at any specific location in the file. + * + * @new_check_end: + * Offset of byte after the last byte to be checked. + * + * @old_table: + * If non-NULL, a pointer to the table containing previously contained + * integrity data for a prefix of this file. * - * @end_header_offset is the offset of the byte after the header, which is the - * beginning of the region that is checksummed. + * @old_check_end: + * If @old_table is non-NULL, the byte after the last byte that was checked + * in the old table. Must be less than or equal to new_check_end. * - * @end_lookup_table_offset is the offset of the byte after the lookup table, - * which is the end of the region that is checksummed. + * @show_progress: + * True if progress information is to be shown while calculating the + * integrity data. + * + * @integrity_table_ret: + * On success, a pointer to the calculated integrity table is written into + * this location. + * + * Returns 0 on success; nonzero on failure. */ -int write_integrity_table(FILE *out, u64 end_header_offset, - u64 end_lookup_table_offset, int show_progress, - struct resource_entry *out_res_entry) +static int calculate_integrity_table(FILE *fp, + off_t new_check_end, + const struct integrity_table *old_table, + off_t old_check_end, + bool show_progress, + struct integrity_table **integrity_table_ret) { - u64 bytes_to_check; - u64 bytes_remaining; - u8 *buf; - u8 *p; - u8 *chunk_buf; - u32 num_entries; - u32 integrity_table_size; - int ret; - off_t start_offset; - - start_offset = ftello(out); - if (start_offset == -1) - return WIMLIB_ERR_WRITE; - - DEBUG("Calculating integrity table"); - if (fseeko(out, end_header_offset, SEEK_SET) != 0) { - ERROR_WITH_ERRNO("Failed to seek to byte %"PRIu64" of WIM to " - "calculate integrity data", end_header_offset); - return WIMLIB_ERR_WRITE; + int ret = 0; + size_t chunk_size = INTEGRITY_CHUNK_SIZE; + + /* If an old table is provided, set the chunk size to be compatible with + * the old chunk size, unless the old chunk size was weird. */ + if (old_table != NULL) { + if (old_table->chunk_size < INTEGRITY_MIN_CHUNK_SIZE || + old_table->chunk_size > INTEGRITY_MAX_CHUNK_SIZE) + old_table = NULL; + else + chunk_size = old_table->chunk_size; } - bytes_to_check = end_lookup_table_offset - end_header_offset; - num_entries = (bytes_to_check + INTEGRITY_CHUNK_SIZE - 1) / - INTEGRITY_CHUNK_SIZE; - integrity_table_size = num_entries * SHA1_HASH_SIZE + 3 * sizeof(u32); - DEBUG("integrity_table_size = %u", integrity_table_size); + u64 old_check_bytes = old_check_end - WIM_HEADER_DISK_SIZE; + u64 new_check_bytes = new_check_end - WIM_HEADER_DISK_SIZE; + + u32 old_num_chunks = DIV_ROUND_UP(old_check_bytes, chunk_size); + u32 new_num_chunks = DIV_ROUND_UP(new_check_bytes, chunk_size); + + size_t old_last_chunk_size = MODULO_NONZERO(old_check_bytes, chunk_size); + size_t new_last_chunk_size = MODULO_NONZERO(new_check_bytes, chunk_size); + + size_t new_table_size = 12 + new_num_chunks * SHA1_HASH_SIZE; - buf = MALLOC(integrity_table_size); - if (!buf) { - ERROR("Failed to allocate %u bytes for integrity table", - integrity_table_size); + struct integrity_table *new_table = MALLOC(new_table_size); + if (!new_table) return WIMLIB_ERR_NOMEM; + new_table->num_entries = new_num_chunks; + new_table->size = new_table_size; + new_table->chunk_size = chunk_size; + + u64 offset = WIM_HEADER_DISK_SIZE; + + for (u32 i = 0; i < new_num_chunks; i++) { + size_t this_chunk_size; + if (i == new_num_chunks - 1) + this_chunk_size = new_last_chunk_size; + else + this_chunk_size = chunk_size; + if (show_progress) { + unsigned percent_done; + u64 checked_bytes = offset - WIM_HEADER_DISK_SIZE; + percent_done = checked_bytes * 100 / new_check_bytes; + printf("\rCalculating integrity checksums for WIM: " + "%"PRIu64" MiB of %"PRIu64" MiB (%u%%) done", + checked_bytes >> 20, + new_check_bytes >> 20, + percent_done); + fflush(stdout); + } + + if (old_table && + ((this_chunk_size == chunk_size && i < old_num_chunks - 1) || + (i == old_num_chunks - 1 && this_chunk_size == old_last_chunk_size))) + { + /* Can use SHA1 message digest from old integrity table + * */ + copy_hash(new_table->sha1sums[i], old_table->sha1sums[i]); + } else { + /* Calculate the SHA1 message digest of this chunk */ + ret = calculate_chunk_sha1(fp, this_chunk_size, + offset, new_table->sha1sums[i]); + if (ret != 0) + break; + } + offset += this_chunk_size; } + if (ret != 0) { + FREE(new_table); + } else { + printf("\rCalculating integrity checksums for WIM: " + "%"PRIu64" MiB of %"PRIu64" MiB (100%%) done\n", + new_check_bytes >> 20, + new_check_bytes >> 20); + fflush(stdout); + *integrity_table_ret = new_table; + } + return ret; +} + +/* + * Writes a WIM integrity table (a list of SHA1 message digests of raw 10 MiB + * chunks of the file). + * + * This function can optionally re-use entries from an older integrity table. + * To do this, make @integrity_res_entry point to the resource entry for the + * older table (note: this is an input-output parameter), and set + * @old_lookup_table_end to the offset of the byte directly following the last + * byte checked by the old table. If the old integrity table is invalid or + * cannot be read, a warning is printed and the integrity information is + * re-calculated. + * + * @fp: + * FILE * to the WIM file, opened read-write, positioned at the location at + * which the integrity table is to be written. + * + * @integrity_res_entry: + * Resource entry which will be set to point to the integrity table on + * success. In addition, if @old_lookup_table_end != 0, this initially + * must point to the resource entry for the old integrity table for the + * WIM. + * + * @new_lookup_table_end: + * The offset of the byte directly following the lookup table in the WIM + * being written. + * + * @old_lookup_table_end: + * If nonzero, the offset of the byte directly following the old lookup + * table in the WIM. + * + * @show_progress: + * True if progress information is to be shown while writing the integrity + * table. + * + * Returns: + * 0 on success, nonzero on failure. The possible error codes are: + * * WIMLIB_ERR_WRITE: Could not write the integrity table. + * * WIMLIB_ERR_READ: Could not read a chunk of data that needed + * to be checked. + */ +int write_integrity_table(FILE *fp, + struct resource_entry *integrity_res_entry, + off_t new_lookup_table_end, + off_t old_lookup_table_end, + bool show_progress) +{ + struct integrity_table *old_table; + struct integrity_table *new_table; + int ret; + off_t cur_offset; + u32 new_table_size; - p = put_u32(buf, integrity_table_size); - p = put_u32(p, num_entries); - p = put_u32(p, INTEGRITY_CHUNK_SIZE); + cur_offset = ftello(fp); + if (cur_offset == -1) { + ERROR_WITH_ERRNO("Failed to get offset in WIM"); + return WIMLIB_ERR_WRITE; + } - chunk_buf = MALLOC(INTEGRITY_CHUNK_SIZE); - if (!chunk_buf) { - ERROR("Failed to allocate %u bytes for integrity chunk buffer", - INTEGRITY_CHUNK_SIZE); - ret = WIMLIB_ERR_NOMEM; - goto out_free_buf; + if (integrity_res_entry->offset == 0 || old_lookup_table_end == 0) { + old_table = NULL; + } else { + ret = read_integrity_table(integrity_res_entry, fp, + old_lookup_table_end - WIM_HEADER_DISK_SIZE, + &old_table); + if (ret == WIMLIB_ERR_INVALID_INTEGRITY_TABLE) { + WARNING("Old integrity table is invalid! " + "Ignoring it"); + } else if (ret != 0) { + WARNING("Can't read old integrity table! " + "Ignoring it"); + } } - bytes_remaining = bytes_to_check; + ret = calculate_integrity_table(fp, new_lookup_table_end, + old_table, old_lookup_table_end, + show_progress, &new_table); + if (ret != 0) + goto out_free_old_table; + + new_table_size = new_table->size; - DEBUG("Bytes to check = %"PRIu64, bytes_to_check); + new_table->size = cpu_to_le32(new_table->size); + new_table->num_entries = cpu_to_le32(new_table->num_entries); + new_table->chunk_size = cpu_to_le32(new_table->chunk_size); - while (bytes_remaining != 0) { + if (fseeko(fp, cur_offset, SEEK_SET) != 0) { + ERROR_WITH_ERRNO("Failed to seek to byte %"PRIu64" of WIM to " + "write integrity table", cur_offset); + ret = WIMLIB_ERR_WRITE; + goto out_free_new_table; + } - uint percent_done = (bytes_to_check - bytes_remaining) * - 100 / bytes_to_check; + if (fwrite(new_table, 1, new_table_size, fp) != new_table_size) { + ERROR_WITH_ERRNO("Failed to write WIM integrity table"); + ret = WIMLIB_ERR_WRITE; + } else { + integrity_res_entry->offset = cur_offset; + integrity_res_entry->size = new_table_size; + integrity_res_entry->original_size = new_table_size; + integrity_res_entry->flags = 0; + ret = 0; + } +out_free_new_table: + FREE(new_table); +out_free_old_table: + FREE(old_table); + return ret; +} + +/* + * Checks a WIM for consistency with the integrity table. + * + * @fp: + * FILE * to the WIM file, opened for reading. + * + * @table: + * The integrity table for the WIM, read into memory. + * + * @bytes_to_check: + * Number of bytes in the WIM that need to be checked (offset of end of the + * lookup table minus offset of end of the header). + * + * @show_progress: + * True if progress information is to be shown while checking the + * integrity. + * + * Returns: + * > 0 (WIMLIB_ERR_*) on error + * 0 (WIM_INTEGRITY_OK) if the integrity was checked successfully and there + * were no inconsistencies. + * -1 (WIM_INTEGRITY_NOT_OK) if the WIM failed the integrity check. + */ +static int verify_integrity(FILE *fp, const struct integrity_table *table, + u64 bytes_to_check, bool show_progress) +{ + int ret; + u64 offset = WIM_HEADER_DISK_SIZE; + u8 sha1_md[SHA1_HASH_SIZE]; + for (u32 i = 0; i < table->num_entries; i++) { + size_t this_chunk_size; + if (i == table->num_entries - 1) + this_chunk_size = MODULO_NONZERO(bytes_to_check, + table->chunk_size); + else + this_chunk_size = table->chunk_size; + + ret = calculate_chunk_sha1(fp, this_chunk_size, offset, sha1_md); + if (ret != 0) + return ret; + + if (!hashes_equal(sha1_md, table->sha1sums[i])) + return WIM_INTEGRITY_NOT_OK; if (show_progress) { - printf("Calculating integrity checksums for WIM " - "(%"PRIu64" bytes remaining, %u%% " - "done) \r", - bytes_remaining, percent_done); + u64 checked_bytes = offset - WIM_HEADER_DISK_SIZE; + unsigned percent_done = checked_bytes * 100 / bytes_to_check; + printf("\rVerifying integrity of WIM: " + "%"PRIu64" MiB of %"PRIu64" MiB (%u%%) done", + checked_bytes >> 20, + bytes_to_check >> 20, + percent_done); fflush(stdout); } + offset += this_chunk_size; + } + printf("\rVerifying integrity of WIM: " + "%"PRIu64" MiB of %"PRIu64" MiB (100%%) done\n", + bytes_to_check >> 20, + bytes_to_check >> 20); + fflush(stdout); + return WIM_INTEGRITY_OK; +} - size_t bytes_to_read = min(INTEGRITY_CHUNK_SIZE, bytes_remaining); - size_t bytes_read = fread(chunk_buf, 1, bytes_to_read, out); - if (bytes_read != bytes_to_read) { - if (feof(out)) { - ERROR("Unexpected EOF while calculating " - "integrity checksums"); - } else { - ERROR_WITH_ERRNO("File stream error while " - "calculating integrity " - "checksums"); - } - ret = WIMLIB_ERR_READ; - goto out_free_chunk_buf; - } - sha1_buffer(chunk_buf, bytes_read, p); - p += SHA1_HASH_SIZE; - bytes_remaining -= bytes_read; - } - if (show_progress) - puts("Calculating integrity checksums for WIM " - "(0 bytes remaining, 100% done)" - " "); +/* + * Verifies the integrity of the WIM by making sure the SHA1 message digests of + * ~10 MiB chunks of the WIM match up with the values given in the integrity + * tabel. + * + * @w: + * The WIM, opened for reading, and with the header already read. + * + * @show_progress: + * True if progress information is to be shown while checking the + * integrity. + * + * Returns: + * > 0 (WIMLIB_ERR_*) on error + * 0 (WIM_INTEGRITY_OK) if the integrity was checked successfully and there + * were no inconsistencies. + * -1 (WIM_INTEGRITY_NOT_OK) if the WIM failed the integrity check. + * -2 (WIM_INTEGRITY_NONEXISTENT) if the WIM contains no integrity + * information. + */ +int check_wim_integrity(WIMStruct *w, bool show_progress) +{ + int ret; + u64 bytes_to_check; + struct integrity_table *table; + u64 end_lookup_table_offset; - if (fseeko(out, start_offset, SEEK_SET) != 0) { - ERROR_WITH_ERRNO("Failed to seek to end of WIM"); - ret = WIMLIB_ERR_WRITE; - goto out_free_chunk_buf; + if (w->hdr.integrity.offset == 0) { + DEBUG("No integrity information."); + return WIM_INTEGRITY_NONEXISTENT; } - if (fwrite(buf, 1, integrity_table_size, out) != integrity_table_size) { - ERROR_WITH_ERRNO("Failed to write integrity table to end of " - "WIM"); - ret = WIMLIB_ERR_WRITE; - goto out_free_chunk_buf; + end_lookup_table_offset = w->hdr.lookup_table_res_entry.offset + + w->hdr.lookup_table_res_entry.size; + + if (end_lookup_table_offset < WIM_HEADER_DISK_SIZE) { + ERROR("WIM lookup table ends before WIM header ends!"); + return WIMLIB_ERR_INVALID_INTEGRITY_TABLE; } - out_res_entry->offset = start_offset; - out_res_entry->size = integrity_table_size; - out_res_entry->original_size = integrity_table_size; - out_res_entry->flags = 0; - ret = 0; -out_free_chunk_buf: - FREE(chunk_buf); -out_free_buf: - FREE(buf); + bytes_to_check = end_lookup_table_offset - WIM_HEADER_DISK_SIZE; + + ret = read_integrity_table(&w->hdr.integrity, w->fp, + bytes_to_check, &table); + if (ret != 0) + return ret; + ret = verify_integrity(w->fp, table, bytes_to_check, show_progress); + FREE(table); return ret; } diff --git a/src/lookup_table.h b/src/lookup_table.h index 2665c39c..2043cd10 100644 --- a/src/lookup_table.h +++ b/src/lookup_table.h @@ -288,14 +288,6 @@ extern int dentry_unresolve_ltes(struct dentry *dentry, void *ignore); int write_lookup_table(struct lookup_table *table, FILE *out, struct resource_entry *out_res_entry); -/* Unlinks and frees an entry from a lookup table. */ -static inline void lookup_table_remove(struct lookup_table *table, - struct lookup_table_entry *lte) -{ - lookup_table_unlink(table, lte); - free_lookup_table_entry(lte); -} - static inline struct resource_entry* wim_metadata_resource_entry(WIMStruct *w) { return &w->image_metadata[ diff --git a/src/modify.c b/src/modify.c index 455a7cec..3180c507 100644 --- a/src/modify.c +++ b/src/modify.c @@ -45,15 +45,18 @@ * the WIM image. */ #define WIMLIB_ADD_IMAGE_FLAG_ROOT 0x80000000 -void destroy_image_metadata(struct image_metadata *imd, struct lookup_table *lt) +void destroy_image_metadata(struct image_metadata *imd, + struct lookup_table *table) { - free_dentry_tree(imd->root_dentry, lt); + free_dentry_tree(imd->root_dentry, table); free_security_data(imd->security_data); /* Get rid of the lookup table entry for this image's metadata resource * */ - if (lt) - lookup_table_remove(lt, imd->metadata_lte); + if (table) { + lookup_table_unlink(table, imd->metadata_lte); + free_lookup_table_entry(imd->metadata_lte); + } } /* diff --git a/src/ntfs-capture.c b/src/ntfs-capture.c index d23776b8..fb670dc0 100644 --- a/src/ntfs-capture.c +++ b/src/ntfs-capture.c @@ -209,7 +209,7 @@ static int ntfs_attr_sha1sum(ntfs_inode *ni, ATTR_RECORD *ar, { s64 pos = 0; s64 bytes_remaining; - char buf[4096]; + char buf[BUFFER_SIZE]; ntfs_attr *na; SHA_CTX ctx; diff --git a/src/resource.c b/src/resource.c index 6c38a594..c29ed4fb 100644 --- a/src/resource.c +++ b/src/resource.c @@ -823,17 +823,18 @@ int read_metadata_resource(WIMStruct *w, struct image_metadata *imd) ret = read_dentry(buf, metadata_len, dentry_offset, dentry); + /* This is the root dentry, so set its parent to itself. */ + dentry->parent = dentry; - if (dentry->length == 0) { + if (ret == 0 && dentry->length == 0) { ERROR("Metadata resource cannot begin with end-of-directory entry!"); ret = WIMLIB_ERR_INVALID_DENTRY; } - if (ret != 0) - goto out_free_dentry_tree; - - /* This is the root dentry, so set its parent to itself. */ - dentry->parent = dentry; + if (ret != 0) { + FREE(dentry); + goto out_free_security_data; + } inode_add_dentry(dentry, dentry->d_inode); @@ -896,7 +897,8 @@ int write_metadata_resource(WIMStruct *w) u64 metadata_original_size; const struct wim_security_data *sd; - DEBUG("Writing metadata resource for image %d", w->current_image); + DEBUG("Writing metadata resource for image %d (offset = %"PRIu64")", + w->current_image, ftello(w->out_fp)); root = wim_root_dentry(w); sd = wim_security_data(w); diff --git a/src/security.c b/src/security.c index c9550500..c9158545 100644 --- a/src/security.c +++ b/src/security.c @@ -328,17 +328,17 @@ void print_security_data(const struct wim_security_data *sd) void free_security_data(struct wim_security_data *sd) { - if (!sd) - return; - wimlib_assert(sd->refcnt != 0); - if (--sd->refcnt == 0) { - u8 **descriptors = sd->descriptors; - u32 num_entries = sd->num_entries; - if (descriptors) - while (num_entries--) - FREE(*descriptors++); - FREE(sd->sizes); - FREE(sd->descriptors); - FREE(sd); + if (sd) { + wimlib_assert(sd->refcnt != 0); + if (--sd->refcnt == 0) { + u8 **descriptors = sd->descriptors; + u32 num_entries = sd->num_entries; + if (descriptors) + while (num_entries--) + FREE(*descriptors++); + FREE(sd->sizes); + FREE(sd->descriptors); + FREE(sd); + } } } diff --git a/src/util.h b/src/util.h index d87a0e82..dd564f5d 100644 --- a/src/util.h +++ b/src/util.h @@ -76,6 +76,12 @@ typedef unsigned uint; (type *)( (char *)__mptr - offsetof(type,member) );}) #endif +#define DIV_ROUND_UP(numerator, denominator) \ + (((numerator) + (denominator) - 1) / (denominator)) + +#define MODULO_NONZERO(numerator, denominator) \ + (((numerator) % (denominator)) ? ((numerator) % (denominator)) : (denominator)) + #define ARRAY_LEN(array) (sizeof(array) / sizeof((array)[0])) #define ZERO_ARRAY(array) memset(array, 0, sizeof(array)) diff --git a/src/wim.c b/src/wim.c index d4558578..0507e477 100644 --- a/src/wim.c +++ b/src/wim.c @@ -30,6 +30,8 @@ #include #include "dentry.h" +#include +#include #ifdef WITH_NTFS_3G #include @@ -214,7 +216,6 @@ int select_wim_image(WIMStruct *w, int image) return WIMLIB_ERR_INVALID_IMAGE; } - /* If a valid image is currently selected, it can be freed if it is not * modified. */ if (w->current_image != WIM_NO_IMAGE) { @@ -434,14 +435,24 @@ int open_wim_readable(WIMStruct *w, const char *path) } /* Opens a WIM writable */ -int open_wim_writable(WIMStruct *w, const char *path) +int open_wim_writable(WIMStruct *w, const char *path, + bool trunc, bool readable) { + const char *mode; + if (trunc) + if (readable) + mode = "w+b"; + else + mode = "wb"; + else + mode = "r+b"; + + DEBUG("Opening `%s' read-write", path); wimlib_assert(w->out_fp == NULL); wimlib_assert(path != NULL); - w->out_fp = fopen(path, "w+b"); + w->out_fp = fopen(path, mode); if (!w->out_fp) { - ERROR_WITH_ERRNO("Failed to open `%s' for writing", - path); + ERROR_WITH_ERRNO("Failed to open `%s' for writing", path); return WIMLIB_ERR_OPEN; } return 0; @@ -492,7 +503,7 @@ static int begin_read(WIMStruct *w, const char *in_wim_path, int open_flags) if (open_flags & WIMLIB_OPEN_FLAG_CHECK_INTEGRITY) { ret = check_wim_integrity(w, - open_flags & WIMLIB_OPEN_FLAG_SHOW_PROGRESS); + (open_flags & WIMLIB_OPEN_FLAG_SHOW_PROGRESS) != 0); if (ret == WIM_INTEGRITY_NONEXISTENT) { WARNING("No integrity information for `%s'; skipping " "integrity check.", w->filename); diff --git a/src/wimlib_internal.h b/src/wimlib_internal.h index 57c0db05..e3884c9c 100644 --- a/src/wimlib_internal.h +++ b/src/wimlib_internal.h @@ -402,12 +402,12 @@ extern int write_header(const struct wim_header *hdr, FILE *out); extern int init_header(struct wim_header *hdr, int ctype); /* integrity.c */ -extern int write_integrity_table(FILE *out, u64 end_header_offset, - u64 end_lookup_table_offset, - int show_progress, - struct resource_entry *out_res_entry); - -extern int check_wim_integrity(WIMStruct *w, int show_progress); +extern int write_integrity_table(FILE *out, + struct resource_entry *integrity_res_entry, + off_t new_lookup_table_end, + off_t old_lookup_table_end, + bool show_progress); +extern int check_wim_integrity(WIMStruct *w, bool show_progress); /* join.c */ @@ -492,10 +492,14 @@ extern int select_wim_image(WIMStruct *w, int image); extern int wim_hdr_flags_compression_type(int wim_hdr_flags); extern int for_image(WIMStruct *w, int image, int (*visitor)(WIMStruct *)); extern int open_wim_readable(WIMStruct *w, const char *path); -extern int open_wim_writable(WIMStruct *w, const char *path); +extern int open_wim_writable(WIMStruct *w, const char *path, + bool trunc, bool readable); /* Internal use only */ -#define WIMLIB_WRITE_FLAG_NO_LOOKUP_TABLE 0x80000000 +#define WIMLIB_WRITE_FLAG_NO_LOOKUP_TABLE 0x80000000 +#define WIMLIB_WRITE_FLAG_REUSE_INTEGRITY_TABLE 0x40000000 + +#define WIMLIB_WRITE_MASK_PUBLIC 0x3fffffff /* write.c */ extern int begin_write(WIMStruct *w, const char *path, int write_flags); diff --git a/src/write.c b/src/write.c index e48d3c91..7315ec90 100644 --- a/src/write.c +++ b/src/write.c @@ -1298,6 +1298,12 @@ static int write_stream_list(struct list_head *stream_list, FILE *out_fp, compression_needed = true; } + if (num_streams == 0) { + if (write_flags & WIMLIB_WRITE_FLAG_VERBOSE) + printf("No streams to write\n"); + return 0; + } + if (write_flags & WIMLIB_WRITE_FLAG_VERBOSE) { printf("Preparing to write %zu streams " "(%"PRIu64" total bytes uncompressed)\n", @@ -1369,13 +1375,6 @@ static int write_wim_streams(WIMStruct *w, int image, int write_flags, */ int finish_write(WIMStruct *w, int image, int write_flags) { - off_t lookup_table_offset; - off_t xml_data_offset; - off_t lookup_table_size; - off_t integrity_offset; - off_t xml_data_size; - off_t end_offset; - off_t integrity_size; int ret; struct wim_header hdr; FILE *out = w->out_fp; @@ -1400,11 +1399,24 @@ int finish_write(WIMStruct *w, int image, int write_flags) goto out; if (write_flags & WIMLIB_WRITE_FLAG_CHECK_INTEGRITY) { + off_t old_lookup_table_end; + off_t new_lookup_table_end; + bool show_progress; + if (write_flags & WIMLIB_WRITE_FLAG_REUSE_INTEGRITY_TABLE) { + old_lookup_table_end = w->hdr.lookup_table_res_entry.offset + + w->hdr.lookup_table_res_entry.size; + } else { + old_lookup_table_end = 0; + } + new_lookup_table_end = hdr.lookup_table_res_entry.offset + + hdr.lookup_table_res_entry.size; + show_progress = ((write_flags & WIMLIB_WRITE_FLAG_SHOW_PROGRESS) != 0); + ret = write_integrity_table(out, - WIM_HEADER_DISK_SIZE, - hdr.xml_res_entry.offset, - write_flags & WIMLIB_WRITE_FLAG_SHOW_PROGRESS, - &hdr.integrity); + &hdr.integrity, + new_lookup_table_end, + old_lookup_table_end, + show_progress); if (ret != 0) goto out; } else { @@ -1455,10 +1467,8 @@ int finish_write(WIMStruct *w, int image, int write_flags) { ERROR_WITH_ERRNO("Error flushing data to WIM file"); ret = WIMLIB_ERR_WRITE; - goto out; } } - out: if (fclose(out) != 0) { ERROR_WITH_ERRNO("Failed to close the WIM file"); @@ -1484,7 +1494,12 @@ static void close_wim_writable(WIMStruct *w) int begin_write(WIMStruct *w, const char *path, int write_flags) { int ret; - ret = open_wim_writable(w, path); + bool need_readable = false; + bool trunc = true; + if (write_flags & WIMLIB_WRITE_FLAG_CHECK_INTEGRITY) + need_readable = true; + + ret = open_wim_writable(w, path, trunc, need_readable); if (ret != 0) return ret; /* Write dummy header. It will be overwritten later. */ @@ -1500,7 +1515,7 @@ WIMLIBAPI int wimlib_write(WIMStruct *w, const char *path, if (!w || !path) return WIMLIB_ERR_INVALID_PARAM; - write_flags &= ~WIMLIB_WRITE_FLAG_NO_LOOKUP_TABLE; + write_flags &= WIMLIB_WRITE_MASK_PUBLIC; if (image != WIM_ALL_IMAGES && (image < 1 || image > w->hdr.image_count)) @@ -1550,11 +1565,11 @@ static int lte_overwrite_prepare(struct lookup_table_entry *lte, static int check_resource_offset(struct lookup_table_entry *lte, void *arg) { - u64 xml_data_offset = *(u64*)arg; + off_t end_offset = *(u64*)arg; wimlib_assert(lte->out_refcnt <= lte->refcnt); if (lte->out_refcnt < lte->refcnt) { - if (lte->resource_entry.offset > xml_data_offset) { + if (lte->resource_entry.offset + lte->resource_entry.size > end_offset) { ERROR("The following resource is after the XML data:"); print_lookup_table_entry(lte); return WIMLIB_ERR_RESOURCE_ORDER; @@ -1565,7 +1580,6 @@ static int check_resource_offset(struct lookup_table_entry *lte, void *arg) static int find_new_streams(struct lookup_table_entry *lte, void *arg) { - wimlib_assert(lte->out_refcnt <= lte->refcnt); if (lte->out_refcnt == lte->refcnt) list_add(<e->staging_list, (struct list_head*)arg); else @@ -1606,28 +1620,45 @@ static int overwrite_wim_inplace(WIMStruct *w, int write_flags, wimlib_assert(w->image_metadata[i].modified); wimlib_assert(!w->image_metadata[i].has_been_mounted_rw); wimlib_assert(w->image_metadata[i].root_dentry != NULL); + wimlib_assert(w->image_metadata[i].metadata_lte != NULL); w->private = &stream_list; for_dentry_in_tree(w->image_metadata[i].root_dentry, dentry_find_streams_to_write, w); } + if (w->hdr.integrity.offset) + old_wim_end = w->hdr.integrity.offset + w->hdr.integrity.size; + else + old_wim_end = w->hdr.xml_res_entry.offset + w->hdr.xml_res_entry.size; + ret = for_lookup_table_entry(w->lookup_table, check_resource_offset, - &w->hdr.xml_res_entry.offset); + &old_wim_end); if (ret != 0) return ret; + DEBUG("old_wim_end = %"PRIu64, old_wim_end); + INIT_LIST_HEAD(&stream_list); for_lookup_table_entry(w->lookup_table, find_new_streams, &stream_list); - if (w->hdr.integrity.offset) - old_wim_end = w->hdr.integrity.offset + w->hdr.integrity.size; - else - old_wim_end = w->hdr.xml_res_entry.offset + w->hdr.xml_res_entry.size; + { + u64 num_new_streams = 0; + struct list_head *cur; + list_for_each(cur, &stream_list) + num_new_streams++; + DEBUG("%"PRIu64" new streams to write", num_new_streams); + } - ret = open_wim_writable(w, w->filename); - if (ret != 0) - return ret; + { + bool trunc = false; + bool need_readable = false; + if (write_flags & WIMLIB_WRITE_FLAG_CHECK_INTEGRITY) + need_readable = true; + ret = open_wim_writable(w, w->filename, trunc, need_readable); + if (ret != 0) + return ret; + } if (fseeko(w->out_fp, old_wim_end, SEEK_SET) != 0) { ERROR_WITH_ERRNO("Can't seek to end of WIM"); @@ -1647,16 +1678,12 @@ static int overwrite_wim_inplace(WIMStruct *w, int write_flags, } for (int i = modified_image_idx; i < w->hdr.image_count; i++) { - wimlib_assert(w->image_metadata[i].modified); - wimlib_assert(!w->image_metadata[i].has_been_mounted_rw); - wimlib_assert(w->image_metadata[i].root_dentry != NULL); - wimlib_assert(w->image_metadata[i].metadata_lte != NULL); - ret = select_wim_image(w, i + 1); - wimlib_assert(ret == 0); + select_wim_image(w, i + 1); ret = write_metadata_resource(w); if (ret != 0) goto out_ftruncate; } + write_flags |= WIMLIB_WRITE_FLAG_REUSE_INTEGRITY_TABLE; ret = finish_write(w, WIM_ALL_IMAGES, write_flags); out_ftruncate: close_wim_writable(w); @@ -1732,12 +1759,11 @@ err: WIMLIBAPI int wimlib_overwrite(WIMStruct *w, int write_flags, unsigned num_threads) { - int ret; - if (!w) return WIMLIB_ERR_INVALID_PARAM; - write_flags &= ~WIMLIB_WRITE_FLAG_NO_LOOKUP_TABLE; + write_flags &= WIMLIB_WRITE_MASK_PUBLIC; + if (!w->filename) return WIMLIB_ERR_NO_FILENAME; @@ -1754,11 +1780,10 @@ WIMLIBAPI int wimlib_overwrite(WIMStruct *w, int write_flags, for (; i < w->hdr.image_count && w->image_metadata[i].modified && !w->image_metadata[i].has_been_mounted_rw; i++) ; - // XXX - /*if (i == w->hdr.image_count) {*/ - /*return overwrite_wim_inplace(w, write_flags, num_threads,*/ - /*modified_image_idx);*/ - /*}*/ + if (i == w->hdr.image_count) { + return overwrite_wim_inplace(w, write_flags, num_threads, + modified_image_idx); + } } return overwrite_wim_via_tmpfile(w, write_flags, num_threads); } -- 2.43.0