4 * WIM files can optionally contain a table of SHA1 message digests at the end,
5 * one digest for each chunk of the file of some specified size (often 10 MB).
6 * This file implements the checking and writing of this table.
10 * Copyright (C) 2012, 2013 Eric Biggers
12 * This file is part of wimlib, a library for working with WIM files.
14 * wimlib is free software; you can redistribute it and/or modify it under the
15 * terms of the GNU General Public License as published by the Free
16 * Software Foundation; either version 3 of the License, or (at your option)
19 * wimlib is distributed in the hope that it will be useful, but WITHOUT ANY
20 * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
21 * A PARTICULAR PURPOSE. See the GNU General Public License for more
24 * You should have received a copy of the GNU General Public License
25 * along with wimlib; if not, see http://www.gnu.org/licenses/.
32 #include "wimlib/assert.h"
33 #include "wimlib/endianness.h"
34 #include "wimlib/error.h"
35 #include "wimlib/file_io.h"
36 #include "wimlib/integrity.h"
37 #include "wimlib/resource.h"
38 #include "wimlib/sha1.h"
39 #include "wimlib/wim.h"
41 /* Size, in bytes, of each SHA1-summed chunk, when wimlib writes integrity
43 #define INTEGRITY_CHUNK_SIZE 10485760
45 /* Only use a different chunk size for compatiblity with an existing integrity
46 * table if the chunk size is between these two numbers. */
47 #define INTEGRITY_MIN_CHUNK_SIZE 4096
48 #define INTEGRITY_MAX_CHUNK_SIZE 134217728
50 struct integrity_table {
58 calculate_chunk_sha1(int in_fd, size_t this_chunk_size,
59 off_t offset, u8 sha1_md[])
63 size_t bytes_remaining;
67 bytes_remaining = this_chunk_size;
70 bytes_to_read = min(bytes_remaining, sizeof(buf));
71 bytes_read = full_pread(in_fd, buf, bytes_to_read, offset);
72 if (bytes_read != bytes_to_read) {
73 ERROR_WITH_ERRNO("Read error while calculating "
74 "integrity checksums");
75 return WIMLIB_ERR_READ;
77 sha1_update(&ctx, buf, bytes_read);
78 bytes_remaining -= bytes_read;
80 } while (bytes_remaining);
81 sha1_final(sha1_md, &ctx);
87 * read_integrity_table: - Reads the integrity table from a WIM file.
90 * The resource entry that specifies the location of the integrity table.
91 * The integrity table must exist (i.e. res_entry->offset must not be 0).
94 * File descriptor to the WIM file, opened for reading.
97 * Number of bytes of data that should be checked by the integrity table.
100 * On success, a pointer to an in-memory structure containing the integrity
101 * information is written to this location.
103 * Returns 0 on success; nonzero on failure. The possible error codes are:
105 * * WIMLIB_ERR_INVALID_INTEGRITY_TABLE: The integrity table is invalid.
106 * * WIMLIB_ERR_NOMEM: Could not allocate memory to store the integrity
108 * * WIMLIB_ERR_READ: Could not read the integrity data from the WIM file.
111 read_integrity_table(const struct resource_entry *res_entry,
113 u64 num_checked_bytes,
114 struct integrity_table **table_ret)
116 struct integrity_table *table;
119 u64 expected_num_entries;
121 if (resource_is_compressed(res_entry)) {
122 ERROR("Didn't expect a compressed integrity table");
123 return WIMLIB_ERR_INVALID_INTEGRITY_TABLE;
126 if (res_entry->size < 8 || res_entry->size > 0xffffffff) {
127 ERROR("Integrity table resource header is invalid");
128 return WIMLIB_ERR_INVALID_INTEGRITY_TABLE;
131 /* Read the integrity table into memory. */
132 table = MALLOC((size_t)res_entry->size);
134 ERROR("Can't allocate %zu bytes for integrity table",
135 (size_t)res_entry->size);
136 return WIMLIB_ERR_NOMEM;
139 if (full_pread(in_fd, table, res_entry->size,
140 res_entry->offset) != res_entry->size)
142 ERROR("Failed to read integrity table (size = %zu, "
143 " offset = %"PRIu64")",
144 (size_t)res_entry->size, res_entry->offset);
145 ret = WIMLIB_ERR_READ;
149 table->size = le32_to_cpu(table->size);
150 table->num_entries = le32_to_cpu(table->num_entries);
151 table->chunk_size = le32_to_cpu(table->chunk_size);
153 if (table->size != res_entry->size) {
154 ERROR("Inconsistent integrity table sizes: Table header says "
155 "%u bytes but resource entry says %u bytes",
156 table->size, (unsigned)res_entry->size);
157 ret = WIMLIB_ERR_INVALID_INTEGRITY_TABLE;
161 DEBUG("table->size = %u, table->num_entries = %u, "
162 "table->chunk_size = %u",
163 table->size, table->num_entries, table->chunk_size);
165 expected_size = (u64)table->num_entries * SHA1_HASH_SIZE + 12;
167 if (table->size != expected_size) {
168 ERROR("Integrity table is %u bytes, but expected %"PRIu64" "
169 "bytes to hold %u entries",
170 table->size, expected_size, table->num_entries);
171 ret = WIMLIB_ERR_INVALID_INTEGRITY_TABLE;
175 if (table->chunk_size == 0) {
176 ERROR("Cannot use integrity chunk size of 0");
177 ret = WIMLIB_ERR_INVALID_INTEGRITY_TABLE;
181 expected_num_entries = DIV_ROUND_UP(num_checked_bytes, table->chunk_size);
183 if (table->num_entries != expected_num_entries) {
184 ERROR("%"PRIu64" integrity table entries would be required "
185 "to checksum the %"PRIu64" bytes from the end of the "
187 expected_num_entries, num_checked_bytes);
188 ERROR("end of the lookup table with a chunk size of %u, but "
189 "there were only %u entries",
190 table->chunk_size, table->num_entries);
191 ret = WIMLIB_ERR_INVALID_INTEGRITY_TABLE;
204 * calculate_integrity_table():
206 * Calculates an integrity table for the data in a file beginning at offset 208
207 * (WIM_HEADER_DISK_SIZE).
210 * File descriptor for the file to be checked, opened for reading. Does
211 * not need to be at any specific location in the file.
214 * Offset of byte after the last byte to be checked.
217 * If non-NULL, a pointer to the table containing the previously calculated
218 * integrity data for a prefix of this file.
221 * If @old_table is non-NULL, the byte after the last byte that was checked
222 * in the old table. Must be less than or equal to new_check_end.
225 * If non-NULL, a progress function that will be called after every
228 * @integrity_table_ret:
229 * On success, a pointer to the calculated integrity table is written into
232 * Returns 0 on success; nonzero on failure.
235 calculate_integrity_table(int in_fd,
237 const struct integrity_table *old_table,
239 wimlib_progress_func_t progress_func,
240 struct integrity_table **integrity_table_ret)
243 size_t chunk_size = INTEGRITY_CHUNK_SIZE;
245 /* If an old table is provided, set the chunk size to be compatible with
246 * the old chunk size, unless the old chunk size was weird. */
247 if (old_table != NULL) {
248 if (old_table->num_entries == 0 ||
249 old_table->chunk_size < INTEGRITY_MIN_CHUNK_SIZE ||
250 old_table->chunk_size > INTEGRITY_MAX_CHUNK_SIZE)
253 chunk_size = old_table->chunk_size;
257 u64 old_check_bytes = old_check_end - WIM_HEADER_DISK_SIZE;
258 u64 new_check_bytes = new_check_end - WIM_HEADER_DISK_SIZE;
260 u32 old_num_chunks = DIV_ROUND_UP(old_check_bytes, chunk_size);
261 u32 new_num_chunks = DIV_ROUND_UP(new_check_bytes, chunk_size);
263 size_t old_last_chunk_size = MODULO_NONZERO(old_check_bytes, chunk_size);
264 size_t new_last_chunk_size = MODULO_NONZERO(new_check_bytes, chunk_size);
266 size_t new_table_size = 12 + new_num_chunks * SHA1_HASH_SIZE;
268 struct integrity_table *new_table = MALLOC(new_table_size);
270 return WIMLIB_ERR_NOMEM;
271 new_table->num_entries = new_num_chunks;
272 new_table->size = new_table_size;
273 new_table->chunk_size = chunk_size;
275 u64 offset = WIM_HEADER_DISK_SIZE;
276 union wimlib_progress_info progress;
279 progress.integrity.total_bytes = new_check_bytes;
280 progress.integrity.total_chunks = new_num_chunks;
281 progress.integrity.completed_chunks = 0;
282 progress.integrity.completed_bytes = 0;
283 progress.integrity.chunk_size = chunk_size;
284 progress.integrity.filename = NULL;
285 progress_func(WIMLIB_PROGRESS_MSG_CALC_INTEGRITY,
289 for (u32 i = 0; i < new_num_chunks; i++) {
290 size_t this_chunk_size;
291 if (i == new_num_chunks - 1)
292 this_chunk_size = new_last_chunk_size;
294 this_chunk_size = chunk_size;
296 ((this_chunk_size == chunk_size && i < old_num_chunks - 1) ||
297 (i == old_num_chunks - 1 && this_chunk_size == old_last_chunk_size)))
299 /* Can use SHA1 message digest from old integrity table
301 copy_hash(new_table->sha1sums[i], old_table->sha1sums[i]);
303 /* Calculate the SHA1 message digest of this chunk */
304 ret = calculate_chunk_sha1(in_fd, this_chunk_size,
305 offset, new_table->sha1sums[i]);
311 offset += this_chunk_size;
313 progress.integrity.completed_chunks++;
314 progress.integrity.completed_bytes += this_chunk_size;
315 progress_func(WIMLIB_PROGRESS_MSG_CALC_INTEGRITY,
319 *integrity_table_ret = new_table;
324 * write_integrity_table():
326 * Writes a WIM integrity table (a list of SHA1 message digests of raw 10 MiB
327 * chunks of the file).
329 * This function can optionally re-use entries from an older integrity table.
330 * To do this, make @integrity_res_entry point to the resource entry for the
331 * older table (note: this is an input-output parameter), and set
332 * @old_lookup_table_end to the offset of the byte directly following the last
333 * byte checked by the old table. If the old integrity table is invalid or
334 * cannot be read, a warning is printed and the integrity information is
338 * File descriptor to the WIM file, opened read-write, positioned at the
339 * location at which the integrity table is to be written.
341 * @integrity_res_entry:
342 * Resource entry which will be set to point to the integrity table on
343 * success. In addition, if @old_lookup_table_end != 0, this initially
344 * must point to the resource entry for the old integrity table for the
347 * @new_lookup_table_end:
348 * The offset of the byte directly following the lookup table in the WIM
351 * @old_lookup_table_end:
352 * If nonzero, the offset of the byte directly following the old lookup
356 * If non-NULL, a progress function that will be called after every
360 * 0 on success, nonzero on failure. The possible error codes are:
361 * * WIMLIB_ERR_WRITE: Could not write the integrity table.
362 * * WIMLIB_ERR_READ: Could not read a chunk of data that needed
366 write_integrity_table(int fd,
367 struct resource_entry *integrity_res_entry,
368 off_t new_lookup_table_end,
369 off_t old_lookup_table_end,
370 wimlib_progress_func_t progress_func)
372 struct integrity_table *old_table;
373 struct integrity_table *new_table;
378 wimlib_assert(old_lookup_table_end <= new_lookup_table_end);
380 cur_offset = filedes_offset(fd);
381 if (cur_offset == -1)
382 return WIMLIB_ERR_WRITE;
384 if (integrity_res_entry->offset == 0 || old_lookup_table_end == 0) {
387 ret = read_integrity_table(integrity_res_entry, fd,
388 old_lookup_table_end - WIM_HEADER_DISK_SIZE,
390 if (ret == WIMLIB_ERR_INVALID_INTEGRITY_TABLE) {
391 WARNING("Old integrity table is invalid! "
393 } else if (ret != 0) {
394 WARNING("Can't read old integrity table! "
399 ret = calculate_integrity_table(fd, new_lookup_table_end,
400 old_table, old_lookup_table_end,
401 progress_func, &new_table);
403 goto out_free_old_table;
405 new_table_size = new_table->size;
407 new_table->size = cpu_to_le32(new_table->size);
408 new_table->num_entries = cpu_to_le32(new_table->num_entries);
409 new_table->chunk_size = cpu_to_le32(new_table->chunk_size);
411 if (full_write(fd, new_table, new_table_size) != new_table_size) {
412 ERROR_WITH_ERRNO("Failed to write WIM integrity table");
413 ret = WIMLIB_ERR_WRITE;
415 integrity_res_entry->offset = cur_offset;
416 integrity_res_entry->size = new_table_size;
417 integrity_res_entry->original_size = new_table_size;
418 integrity_res_entry->flags = 0;
428 * verify_integrity():
430 * Checks a WIM for consistency with the integrity table.
433 * File descriptor to the WIM file, opened for reading.
436 * The integrity table for the WIM, read into memory.
439 * Number of bytes in the WIM that need to be checked (offset of end of the
440 * lookup table minus offset of end of the header).
443 * If non-NULL, a progress function that will be called after every
447 * > 0 (WIMLIB_ERR_*) on error
448 * 0 (WIM_INTEGRITY_OK) if the integrity was checked successfully and there
449 * were no inconsistencies.
450 * -1 (WIM_INTEGRITY_NOT_OK) if the WIM failed the integrity check.
453 verify_integrity(int in_fd, const tchar *filename,
454 const struct integrity_table *table,
456 wimlib_progress_func_t progress_func)
459 u64 offset = WIM_HEADER_DISK_SIZE;
460 u8 sha1_md[SHA1_HASH_SIZE];
461 union wimlib_progress_info progress;
464 progress.integrity.total_bytes = bytes_to_check;
465 progress.integrity.total_chunks = table->num_entries;
466 progress.integrity.completed_chunks = 0;
467 progress.integrity.completed_bytes = 0;
468 progress.integrity.chunk_size = table->chunk_size;
469 progress.integrity.filename = filename;
470 progress_func(WIMLIB_PROGRESS_MSG_VERIFY_INTEGRITY,
473 for (u32 i = 0; i < table->num_entries; i++) {
474 size_t this_chunk_size;
475 if (i == table->num_entries - 1)
476 this_chunk_size = MODULO_NONZERO(bytes_to_check,
479 this_chunk_size = table->chunk_size;
481 ret = calculate_chunk_sha1(in_fd, this_chunk_size, offset, sha1_md);
485 if (!hashes_equal(sha1_md, table->sha1sums[i]))
486 return WIM_INTEGRITY_NOT_OK;
488 offset += this_chunk_size;
490 progress.integrity.completed_chunks++;
491 progress.integrity.completed_bytes += this_chunk_size;
492 progress_func(WIMLIB_PROGRESS_MSG_VERIFY_INTEGRITY,
496 return WIM_INTEGRITY_OK;
501 * check_wim_integrity():
503 * Verifies the integrity of the WIM by making sure the SHA1 message digests of
504 * ~10 MiB chunks of the WIM match up with the values given in the integrity
508 * The WIM, opened for reading, and with the header already read.
511 * If non-NULL, a progress function that will be called after every
515 * > 0 (WIMLIB_ERR_*) on error
516 * 0 (WIM_INTEGRITY_OK) if the integrity was checked successfully and there
517 * were no inconsistencies.
518 * -1 (WIM_INTEGRITY_NOT_OK) if the WIM failed the integrity check.
519 * -2 (WIM_INTEGRITY_NONEXISTENT) if the WIM contains no integrity
523 check_wim_integrity(WIMStruct *w, wimlib_progress_func_t progress_func)
527 struct integrity_table *table;
528 u64 end_lookup_table_offset;
530 if (w->hdr.integrity.offset == 0) {
531 DEBUG("No integrity information.");
532 return WIM_INTEGRITY_NONEXISTENT;
535 end_lookup_table_offset = w->hdr.lookup_table_res_entry.offset +
536 w->hdr.lookup_table_res_entry.size;
538 if (end_lookup_table_offset < WIM_HEADER_DISK_SIZE) {
539 ERROR("WIM lookup table ends before WIM header ends!");
540 return WIMLIB_ERR_INVALID_INTEGRITY_TABLE;
543 bytes_to_check = end_lookup_table_offset - WIM_HEADER_DISK_SIZE;
545 ret = read_integrity_table(&w->hdr.integrity, w->in_fd,
546 bytes_to_check, &table);
549 ret = verify_integrity(w->in_fd, w->filename, table,
550 bytes_to_check, progress_func);