4 * WIM files can optionally contain a table of SHA1 message digests at the end,
5 * one digest for each chunk of the file of some specified size (often 10 MB).
6 * This file implements the checking and writing of this table.
10 * Copyright (C) 2012-2016 Eric Biggers
12 * This file is free software; you can redistribute it and/or modify it under
13 * the terms of the GNU Lesser General Public License as published by the Free
14 * Software Foundation; either version 3 of the License, or (at your option) any
17 * This file is distributed in the hope that it will be useful, but WITHOUT
18 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
19 * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
22 * You should have received a copy of the GNU Lesser General Public License
23 * along with this file; if not, see http://www.gnu.org/licenses/.
30 #include "wimlib/assert.h"
31 #include "wimlib/endianness.h"
32 #include "wimlib/error.h"
33 #include "wimlib/file_io.h"
34 #include "wimlib/integrity.h"
35 #include "wimlib/progress.h"
36 #include "wimlib/resource.h"
37 #include "wimlib/sha1.h"
38 #include "wimlib/wim.h"
39 #include "wimlib/write.h"
41 /* Size, in bytes, of each SHA1-summed chunk, when wimlib writes integrity
43 #define INTEGRITY_CHUNK_SIZE 10485760
45 /* Only use a different chunk size for compatibility with an existing integrity
46 * table if the chunk size is between these two numbers. */
47 #define INTEGRITY_MIN_CHUNK_SIZE 4096
48 #define INTEGRITY_MAX_CHUNK_SIZE 134217728
50 struct integrity_table {
58 calculate_chunk_sha1(struct filedes *in_fd, size_t this_chunk_size,
59 off_t offset, u8 sha1_md[])
63 size_t bytes_remaining;
67 bytes_remaining = this_chunk_size;
70 bytes_to_read = min(bytes_remaining, sizeof(buf));
71 ret = full_pread(in_fd, buf, bytes_to_read, offset);
73 ERROR_WITH_ERRNO("Read error while calculating "
74 "integrity checksums");
77 sha1_update(&ctx, buf, bytes_to_read);
78 bytes_remaining -= bytes_to_read;
79 offset += bytes_to_read;
80 } while (bytes_remaining);
81 sha1_final(sha1_md, &ctx);
87 * read_integrity_table: - Reads the integrity table from a WIM file.
90 * WIMStruct for the WIM file; @wim->hdr.integrity_table_reshdr specifies
91 * the location of the integrity table. @wim->in_fd is expected to be a
92 * seekable file descriptor to the WIM file opened for reading.
95 * Number of bytes of data that should be checked by the integrity table.
98 * On success, a pointer to an in-memory structure containing the integrity
99 * information is written to this location.
102 * WIMLIB_ERR_SUCCESS (0)
103 * WIMLIB_ERR_INVALID_INTEGRITY_TABLE
106 * WIMLIB_ERR_UNEXPECTED_END_OF_FILE
109 read_integrity_table(WIMStruct *wim, u64 num_checked_bytes,
110 struct integrity_table **table_ret)
113 struct integrity_table *table;
116 STATIC_ASSERT(sizeof(struct integrity_table) == 12);
117 if (wim->hdr.integrity_table_reshdr.uncompressed_size < 12)
118 return WIMLIB_ERR_INVALID_INTEGRITY_TABLE;
120 ret = wim_reshdr_to_data(&wim->hdr.integrity_table_reshdr, wim, &buf);
125 table->size = le32_to_cpu((_force_attr le32)table->size);
126 table->num_entries = le32_to_cpu((_force_attr le32)table->num_entries);
127 table->chunk_size = le32_to_cpu((_force_attr le32)table->chunk_size);
129 if (table->size != wim->hdr.integrity_table_reshdr.uncompressed_size ||
130 table->size != (u64)table->num_entries * SHA1_HASH_SIZE + 12 ||
131 table->chunk_size == 0 ||
132 table->num_entries != DIV_ROUND_UP(num_checked_bytes, table->chunk_size))
135 return WIMLIB_ERR_INVALID_INTEGRITY_TABLE;
143 * calculate_integrity_table():
145 * Calculates an integrity table for the data in a file beginning at offset 208
146 * (WIM_HEADER_DISK_SIZE).
149 * File descriptor for the file to be checked, opened for reading. Does
150 * not need to be at any specific location in the file.
153 * Offset of byte after the last byte to be checked.
156 * If non-NULL, a pointer to the table containing the previously calculated
157 * integrity data for a prefix of this file.
160 * If @old_table is non-NULL, the byte after the last byte that was checked
161 * in the old table. Must be less than or equal to new_check_end.
163 * @integrity_table_ret:
164 * On success, a pointer to the calculated integrity table is written into
168 * WIMLIB_ERR_SUCCESS (0)
171 * WIMLIB_ERR_UNEXPECTED_END_OF_FILE
174 calculate_integrity_table(struct filedes *in_fd,
176 const struct integrity_table *old_table,
178 struct integrity_table **integrity_table_ret,
179 wimlib_progress_func_t progfunc,
183 size_t chunk_size = INTEGRITY_CHUNK_SIZE;
185 /* If an old table is provided, set the chunk size to be compatible with
186 * the old chunk size, unless the old chunk size was weird. */
187 if (old_table != NULL) {
188 if (old_table->num_entries == 0 ||
189 old_table->chunk_size < INTEGRITY_MIN_CHUNK_SIZE ||
190 old_table->chunk_size > INTEGRITY_MAX_CHUNK_SIZE)
193 chunk_size = old_table->chunk_size;
197 u64 old_check_bytes = old_check_end - WIM_HEADER_DISK_SIZE;
198 u64 new_check_bytes = new_check_end - WIM_HEADER_DISK_SIZE;
200 u32 old_num_chunks = DIV_ROUND_UP(old_check_bytes, chunk_size);
201 u32 new_num_chunks = DIV_ROUND_UP(new_check_bytes, chunk_size);
203 size_t old_last_chunk_size = MODULO_NONZERO(old_check_bytes, chunk_size);
204 size_t new_last_chunk_size = MODULO_NONZERO(new_check_bytes, chunk_size);
206 size_t new_table_size = 12 + new_num_chunks * SHA1_HASH_SIZE;
208 struct integrity_table *new_table = MALLOC(new_table_size);
210 return WIMLIB_ERR_NOMEM;
211 new_table->num_entries = new_num_chunks;
212 new_table->size = new_table_size;
213 new_table->chunk_size = chunk_size;
215 u64 offset = WIM_HEADER_DISK_SIZE;
216 union wimlib_progress_info progress;
218 progress.integrity.total_bytes = new_check_bytes;
219 progress.integrity.total_chunks = new_num_chunks;
220 progress.integrity.completed_chunks = 0;
221 progress.integrity.completed_bytes = 0;
222 progress.integrity.chunk_size = chunk_size;
223 progress.integrity.filename = NULL;
225 ret = call_progress(progfunc, WIMLIB_PROGRESS_MSG_CALC_INTEGRITY,
228 goto out_free_new_table;
230 for (u32 i = 0; i < new_num_chunks; i++) {
231 size_t this_chunk_size;
232 if (i == new_num_chunks - 1)
233 this_chunk_size = new_last_chunk_size;
235 this_chunk_size = chunk_size;
237 ((this_chunk_size == chunk_size && i < old_num_chunks - 1) ||
238 (i == old_num_chunks - 1 && this_chunk_size == old_last_chunk_size)))
240 /* Can use SHA1 message digest from old integrity table
242 copy_hash(new_table->sha1sums[i], old_table->sha1sums[i]);
244 /* Calculate the SHA1 message digest of this chunk */
245 ret = calculate_chunk_sha1(in_fd, this_chunk_size,
246 offset, new_table->sha1sums[i]);
248 goto out_free_new_table;
250 offset += this_chunk_size;
252 progress.integrity.completed_chunks++;
253 progress.integrity.completed_bytes += this_chunk_size;
254 ret = call_progress(progfunc, WIMLIB_PROGRESS_MSG_CALC_INTEGRITY,
257 goto out_free_new_table;
259 *integrity_table_ret = new_table;
268 * write_integrity_table():
270 * Writes a WIM integrity table (a list of SHA1 message digests of raw 10 MiB
271 * chunks of the file).
273 * This function can optionally re-use entries from an older integrity table.
274 * To do this, specify old_blob_table_end and old_table.
276 * On success, @wim->out_hdr.integrity_table_reshdr will be filled in with
277 * information about the integrity table that was written.
280 * WIMStruct for the WIM file. @wim->out_fd must be a seekable descriptor
281 * to the new WIM file, opened read-write, positioned at the location at
282 * which the integrity table is to be written.
284 * @new_blob_table_end:
285 * The offset of the byte directly following the blob table in the WIM
288 * @old_blob_table_end:
289 * If nonzero, the offset of the byte directly following the old blob table
293 * Pointer to the old integrity table read into memory, or NULL if not
297 write_integrity_table(WIMStruct *wim,
298 off_t new_blob_table_end,
299 off_t old_blob_table_end,
300 struct integrity_table *old_table)
302 struct integrity_table *new_table;
306 wimlib_assert(old_blob_table_end <= new_blob_table_end);
308 ret = calculate_integrity_table(&wim->out_fd, new_blob_table_end,
309 old_table, old_blob_table_end,
310 &new_table, wim->progfunc, wim->progctx);
314 new_table_size = new_table->size;
316 new_table->size = (_force_attr u32)cpu_to_le32(new_table->size);
317 new_table->num_entries = (_force_attr u32)cpu_to_le32(new_table->num_entries);
318 new_table->chunk_size = (_force_attr u32)cpu_to_le32(new_table->chunk_size);
320 ret = write_wim_resource_from_buffer(new_table,
324 WIMLIB_COMPRESSION_TYPE_NONE,
326 &wim->out_hdr.integrity_table_reshdr,
334 * verify_integrity():
336 * Checks a WIM for consistency with the integrity table.
339 * File descriptor to the WIM file, opened for reading.
342 * The integrity table for the WIM, read into memory.
345 * Number of bytes in the WIM that need to be checked (offset of end of the
346 * blob table minus offset of end of the header).
349 * > 0 (WIMLIB_ERR_READ, WIMLIB_ERR_UNEXPECTED_END_OF_FILE) on error
350 * 0 (WIM_INTEGRITY_OK) if the integrity was checked successfully and there
351 * were no inconsistencies.
352 * -1 (WIM_INTEGRITY_NOT_OK) if the WIM failed the integrity check.
355 verify_integrity(struct filedes *in_fd, const tchar *filename,
356 const struct integrity_table *table,
358 wimlib_progress_func_t progfunc, void *progctx)
361 u64 offset = WIM_HEADER_DISK_SIZE;
362 u8 sha1_md[SHA1_HASH_SIZE];
363 union wimlib_progress_info progress;
365 progress.integrity.total_bytes = bytes_to_check;
366 progress.integrity.total_chunks = table->num_entries;
367 progress.integrity.completed_chunks = 0;
368 progress.integrity.completed_bytes = 0;
369 progress.integrity.chunk_size = table->chunk_size;
370 progress.integrity.filename = filename;
372 ret = call_progress(progfunc, WIMLIB_PROGRESS_MSG_VERIFY_INTEGRITY,
377 for (u32 i = 0; i < table->num_entries; i++) {
378 size_t this_chunk_size;
379 if (i == table->num_entries - 1)
380 this_chunk_size = MODULO_NONZERO(bytes_to_check,
383 this_chunk_size = table->chunk_size;
385 ret = calculate_chunk_sha1(in_fd, this_chunk_size, offset, sha1_md);
389 if (!hashes_equal(sha1_md, table->sha1sums[i]))
390 return WIM_INTEGRITY_NOT_OK;
392 offset += this_chunk_size;
393 progress.integrity.completed_chunks++;
394 progress.integrity.completed_bytes += this_chunk_size;
396 ret = call_progress(progfunc, WIMLIB_PROGRESS_MSG_VERIFY_INTEGRITY,
401 return WIM_INTEGRITY_OK;
406 * check_wim_integrity():
408 * Verifies the integrity of the WIM by making sure the SHA1 message digests of
409 * ~10 MiB chunks of the WIM match up with the values given in the integrity
413 * The WIM, opened for reading.
416 * > 0 (WIMLIB_ERR_INVALID_INTEGRITY_TABLE, WIMLIB_ERR_READ,
417 * WIMLIB_ERR_UNEXPECTED_END_OF_FILE) on error
418 * 0 (WIM_INTEGRITY_OK) if the integrity was checked successfully and there
419 * were no inconsistencies.
420 * -1 (WIM_INTEGRITY_NOT_OK) if the WIM failed the integrity check.
421 * -2 (WIM_INTEGRITY_NONEXISTENT) if the WIM contains no integrity
425 check_wim_integrity(WIMStruct *wim)
429 struct integrity_table *table;
430 u64 end_blob_table_offset;
432 if (!wim_has_integrity_table(wim))
433 return WIM_INTEGRITY_NONEXISTENT;
435 end_blob_table_offset = wim->hdr.blob_table_reshdr.offset_in_wim +
436 wim->hdr.blob_table_reshdr.size_in_wim;
438 if (end_blob_table_offset < WIM_HEADER_DISK_SIZE) {
439 ERROR("WIM blob table ends before WIM header ends!");
440 return WIMLIB_ERR_INVALID_INTEGRITY_TABLE;
443 bytes_to_check = end_blob_table_offset - WIM_HEADER_DISK_SIZE;
445 ret = read_integrity_table(wim, bytes_to_check, &table);
448 ret = verify_integrity(&wim->in_fd, wim->filename, table,
449 bytes_to_check, wim->progfunc, wim->progctx);