Various code cleanups
[wimlib] / src / integrity.c
1 /*
2  * integrity.c
3  *
4  * WIM files can optionally contain a table of SHA1 message digests at the end,
5  * one digest for each chunk of the file of some specified size (often 10 MB).
6  * This file implements the checking and writing of this table.
7  */
8
9 /*
10  * Copyright (C) 2012 Eric Biggers
11  *
12  * This file is part of wimlib, a library for working with WIM files.
13  *
14  * wimlib is free software; you can redistribute it and/or modify it under the
15  * terms of the GNU General Public License as published by the Free
16  * Software Foundation; either version 3 of the License, or (at your option)
17  * any later version.
18  *
19  * wimlib is distributed in the hope that it will be useful, but WITHOUT ANY
20  * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
21  * A PARTICULAR PURPOSE. See the GNU General Public License for more
22  * details.
23  *
24  * You should have received a copy of the GNU General Public License
25  * along with wimlib; if not, see http://www.gnu.org/licenses/.
26  */
27
28 #include "wimlib_internal.h"
29 #include "buffer_io.h"
30 #include "sha1.h"
31
32 /* Size, in bytes, of each SHA1-summed chunk, when wimlib writes integrity
33  * information. */
34 #define INTEGRITY_CHUNK_SIZE 10485760
35
36 /* Only use a different chunk size for compatiblity with an existing integrity
37  * table if the chunk size is between these two numbers. */
38 #define INTEGRITY_MIN_CHUNK_SIZE 4096
39 #define INTEGRITY_MAX_CHUNK_SIZE 134217728
40
41 struct integrity_table {
42         u32 size;
43         u32 num_entries;
44         u32 chunk_size;
45         u8  sha1sums[0][20];
46 };
47
48 static int calculate_chunk_sha1(FILE *fp, size_t this_chunk_size,
49                                 off_t offset, u8 sha1_md[])
50 {
51         int ret;
52         u8 buf[BUFFER_SIZE];
53         SHA_CTX ctx;
54         size_t bytes_remaining;
55         size_t bytes_to_read;
56         size_t bytes_read;
57
58         ret = fseeko(fp, offset, SEEK_SET);
59         if (ret != 0) {
60                 ERROR_WITH_ERRNO("Can't seek to offset "
61                                  "%"PRIu64" in WIM", offset);
62                 return WIMLIB_ERR_READ;
63         }
64         bytes_remaining = this_chunk_size;
65         sha1_init(&ctx);
66         do {
67                 bytes_to_read = min(bytes_remaining, sizeof(buf));
68                 bytes_read = fread(buf, 1, bytes_to_read, fp);
69                 if (bytes_read != bytes_to_read) {
70                         if (feof(fp)) {
71                                 ERROR("Unexpected EOF while calculating "
72                                       "integrity checksums");
73                         } else {
74                                 ERROR_WITH_ERRNO("File stream error while "
75                                                  "calculating integrity "
76                                                  "checksums");
77                         }
78                         return WIMLIB_ERR_READ;
79                 }
80                 sha1_update(&ctx, buf, bytes_read);
81                 bytes_remaining -= bytes_read;
82         } while (bytes_remaining);
83         sha1_final(sha1_md, &ctx);
84         return 0;
85 }
86
87
88 /*
89  * read_integrity_table: -  Reads the integrity table from a WIM file.
90  *
91  * @res_entry:
92  *      The resource entry that specifies the location of the integrity table.
93  *      The integrity table must exist (i.e. res_entry->offset must not be 0).
94  *
95  * @fp:
96  *      FILE * to the WIM file, opened for reading.
97  *
98  * @num_checked_bytes:
99  *      Number of bytes of data that should be checked by the integrity table.
100  *
101  * @table ret:
102  *      On success, a pointer to an in-memory structure containing the integrity
103  *      information is written to this location.
104  *
105  * Returns 0 on success; nonzero on failure.  The possible error codes are:
106  *
107  *     * WIMLIB_ERR_INVALID_INTEGRITY_TABLE:  The integrity table is invalid.
108  *     * WIMLIB_ERR_NOMEM:  Could not allocate memory to store the integrity
109  *                          data.
110  *     * WIMLIB_ERR_READ:   Could not read the integrity data from the WIM file.
111  */
112 static int read_integrity_table(const struct resource_entry *res_entry,
113                                 FILE *fp,
114                                 u64 num_checked_bytes,
115                                 struct integrity_table **table_ret)
116 {
117         struct integrity_table *table = NULL;
118         int ret = 0;
119         u64 expected_size;
120         u64 expected_num_entries;
121
122         if (resource_is_compressed(res_entry)) {
123                 ERROR("Didn't expect a compressed integrity table");
124                 return WIMLIB_ERR_INVALID_INTEGRITY_TABLE;
125         }
126
127         if (res_entry->size < 8 || res_entry->size  > 0xffffffff) {
128                 ERROR("Integrity table resource header is invalid");
129                 return WIMLIB_ERR_INVALID_INTEGRITY_TABLE;
130         }
131
132         /* Read the integrity table into memory. */
133         if ((table = MALLOC(res_entry->size)) == NULL) {
134                 ERROR("Can't allocate %"PRIu64" bytes for integrity table",
135                       (u64)res_entry->size);
136                 return WIMLIB_ERR_NOMEM;
137         }
138
139         ret = read_uncompressed_resource(fp, res_entry->offset,
140                                          res_entry->size, (void*)table);
141
142         if (ret != 0) {
143                 ERROR("Failed to read integrity table (size = %u, "
144                       " offset = %"PRIu64")",
145                       (unsigned)res_entry->size, res_entry->offset);
146                 goto out;
147         }
148
149         table->size        = le32_to_cpu(table->size);
150         table->num_entries = le32_to_cpu(table->num_entries);
151         table->chunk_size  = le32_to_cpu(table->chunk_size);
152
153         if (table->size != res_entry->size) {
154                 ERROR("Inconsistent integrity table sizes: Table header says "
155                       "%u bytes but resource entry says %u bytes",
156                       table->size, (unsigned)res_entry->size);
157                 ret = WIMLIB_ERR_INVALID_INTEGRITY_TABLE;
158                 goto out;
159         }
160
161         DEBUG("table->size = %u, table->num_entries = %u, "
162               "table->chunk_size = %u",
163               table->size, table->num_entries, table->chunk_size);
164
165         expected_size = (u64)table->num_entries * SHA1_HASH_SIZE + 12;
166
167         if (table->size != expected_size) {
168                 ERROR("Integrity table is %u bytes, but expected %"PRIu64" "
169                       "bytes to hold %u entries",
170                       table->size, expected_size, table->num_entries);
171                 ret = WIMLIB_ERR_INVALID_INTEGRITY_TABLE;
172                 goto out;
173         }
174
175         if (table->chunk_size == 0) {
176                 ERROR("Cannot use integrity chunk size of 0");
177                 ret = WIMLIB_ERR_INVALID_INTEGRITY_TABLE;
178                 goto out;
179         }
180
181         expected_num_entries = DIV_ROUND_UP(num_checked_bytes, table->chunk_size);
182
183         if (table->num_entries != expected_num_entries) {
184                 ERROR("%"PRIu64" integrity table entries would be required "
185                       "to checksum the %"PRIu64" bytes from the end of the "
186                       "header to the",
187                       expected_num_entries, num_checked_bytes);
188                 ERROR("end of the lookup table with a chunk size of %u, but "
189                       "there were only %u entries",
190                       table->chunk_size, table->num_entries);
191                 ret = WIMLIB_ERR_INVALID_INTEGRITY_TABLE;
192         }
193 out:
194         if (ret == 0)
195                 *table_ret = table;
196         else
197                 FREE(table);
198         return ret;
199 }
200
201 /*
202  * calculate_integrity_table():
203  *
204  * Calculates an integrity table for the data in a file beginning at offset 208
205  * (WIM_HEADER_DISK_SIZE).
206  *
207  * @fp:
208  *      FILE * for the file to be checked, opened for reading.  Does not need to
209  *      be at any specific location in the file.
210  *
211  * @new_check_end:
212  *      Offset of byte after the last byte to be checked.
213  *
214  * @old_table:
215  *      If non-NULL, a pointer to the table containing the previously calculated
216  *      integrity data for a prefix of this file.
217  *
218  * @old_check_end:
219  *      If @old_table is non-NULL, the byte after the last byte that was checked
220  *      in the old table.  Must be less than or equal to new_check_end.
221  *
222  * @progress_func:
223  *      If non-NULL, a progress function that will be called after every
224  *      calculated chunk.
225  *
226  * @integrity_table_ret:
227  *      On success, a pointer to the calculated integrity table is written into
228  *      this location.
229  *
230  * Returns 0 on success; nonzero on failure.
231  */
232 static int calculate_integrity_table(FILE *fp,
233                                      off_t new_check_end,
234                                      const struct integrity_table *old_table,
235                                      off_t old_check_end,
236                                      wimlib_progress_func_t progress_func,
237                                      struct integrity_table **integrity_table_ret)
238 {
239         int ret = 0;
240         size_t chunk_size = INTEGRITY_CHUNK_SIZE;
241
242         /* If an old table is provided, set the chunk size to be compatible with
243          * the old chunk size, unless the old chunk size was weird. */
244         if (old_table != NULL) {
245                 if (old_table->num_entries == 0 ||
246                     old_table->chunk_size < INTEGRITY_MIN_CHUNK_SIZE ||
247                     old_table->chunk_size > INTEGRITY_MAX_CHUNK_SIZE)
248                         old_table = NULL;
249                 else
250                         chunk_size = old_table->chunk_size;
251         }
252
253
254         u64 old_check_bytes = old_check_end - WIM_HEADER_DISK_SIZE;
255         u64 new_check_bytes = new_check_end - WIM_HEADER_DISK_SIZE;
256
257         u32 old_num_chunks = DIV_ROUND_UP(old_check_bytes, chunk_size);
258         u32 new_num_chunks = DIV_ROUND_UP(new_check_bytes, chunk_size);
259
260         size_t old_last_chunk_size = MODULO_NONZERO(old_check_bytes, chunk_size);
261         size_t new_last_chunk_size = MODULO_NONZERO(new_check_bytes, chunk_size);
262
263         size_t new_table_size = 12 + new_num_chunks * SHA1_HASH_SIZE;
264
265         struct integrity_table *new_table = MALLOC(new_table_size);
266         if (!new_table)
267                 return WIMLIB_ERR_NOMEM;
268         new_table->num_entries = new_num_chunks;
269         new_table->size = new_table_size;
270         new_table->chunk_size = chunk_size;
271
272         u64 offset = WIM_HEADER_DISK_SIZE;
273         union wimlib_progress_info progress;
274
275         if (progress_func) {
276                 progress.integrity.total_bytes      = new_check_bytes;
277                 progress.integrity.total_chunks     = new_num_chunks;
278                 progress.integrity.completed_chunks = 0;
279                 progress.integrity.completed_bytes  = 0;
280                 progress.integrity.chunk_size       = chunk_size;
281                 progress.integrity.filename         = NULL;
282                 progress_func(WIMLIB_PROGRESS_MSG_CALC_INTEGRITY,
283                               &progress);
284         }
285
286         for (u32 i = 0; i < new_num_chunks; i++) {
287                 size_t this_chunk_size;
288                 if (i == new_num_chunks - 1)
289                         this_chunk_size = new_last_chunk_size;
290                 else
291                         this_chunk_size = chunk_size;
292                 if (old_table &&
293                     ((this_chunk_size == chunk_size && i < old_num_chunks - 1) ||
294                       (i == old_num_chunks - 1 && this_chunk_size == old_last_chunk_size)))
295                 {
296                         /* Can use SHA1 message digest from old integrity table
297                          * */
298                         copy_hash(new_table->sha1sums[i], old_table->sha1sums[i]);
299                 } else {
300                         /* Calculate the SHA1 message digest of this chunk */
301                         ret = calculate_chunk_sha1(fp, this_chunk_size,
302                                                    offset, new_table->sha1sums[i]);
303                         if (ret != 0)
304                                 break;
305                 }
306                 offset += this_chunk_size;
307                 if (progress_func) {
308                         progress.integrity.completed_chunks++;
309                         progress.integrity.completed_bytes += this_chunk_size;
310                         progress_func(WIMLIB_PROGRESS_MSG_CALC_INTEGRITY,
311                                       &progress);
312                 }
313         }
314         if (ret == 0)
315                 *integrity_table_ret = new_table;
316         else
317                 FREE(new_table);
318         return ret;
319 }
320
321 /*
322  * write_integrity_table():
323  *
324  * Writes a WIM integrity table (a list of SHA1 message digests of raw 10 MiB
325  * chunks of the file).
326  *
327  * This function can optionally re-use entries from an older integrity table.
328  * To do this, make @integrity_res_entry point to the resource entry for the
329  * older table (note: this is an input-output parameter), and set
330  * @old_lookup_table_end to the offset of the byte directly following the last
331  * byte checked by the old table.  If the old integrity table is invalid or
332  * cannot be read, a warning is printed and the integrity information is
333  * re-calculated.
334  *
335  * @fp:
336  *      FILE * to the WIM file, opened read-write, positioned at the location at
337  *      which the integrity table is to be written.
338  *
339  * @integrity_res_entry:
340  *      Resource entry which will be set to point to the integrity table on
341  *      success.  In addition, if @old_lookup_table_end != 0, this initially
342  *      must point to the resource entry for the old integrity table for the
343  *      WIM.
344  *
345  * @new_lookup_table_end:
346  *      The offset of the byte directly following the lookup table in the WIM
347  *      being written.
348  *
349  * @old_lookup_table_end:
350  *      If nonzero, the offset of the byte directly following the old lookup
351  *      table in the WIM.
352  *
353  * @progress_func
354  *      If non-NULL, a progress function that will be called after every
355  *      calculated chunk.
356  *
357  * Returns:
358  *      0 on success, nonzero on failure.  The possible error codes are:
359  *         * WIMLIB_ERR_WRITE:  Could not write the integrity table.
360  *         * WIMLIB_ERR_READ:   Could not read a chunk of data that needed
361  *                              to be checked.
362  */
363 int write_integrity_table(FILE *fp,
364                           struct resource_entry *integrity_res_entry,
365                           off_t new_lookup_table_end,
366                           off_t old_lookup_table_end,
367                           wimlib_progress_func_t progress_func)
368 {
369         struct integrity_table *old_table;
370         struct integrity_table *new_table;
371         int ret;
372         off_t cur_offset;
373         u32 new_table_size;
374
375         wimlib_assert(old_lookup_table_end <= new_lookup_table_end);
376
377         cur_offset = ftello(fp);
378         if (cur_offset == -1)
379                 return WIMLIB_ERR_WRITE;
380
381         if (integrity_res_entry->offset == 0 || old_lookup_table_end == 0) {
382                 old_table = NULL;
383         } else {
384                 ret = read_integrity_table(integrity_res_entry, fp,
385                                            old_lookup_table_end - WIM_HEADER_DISK_SIZE,
386                                            &old_table);
387                 if (ret == WIMLIB_ERR_INVALID_INTEGRITY_TABLE) {
388                         WARNING("Old integrity table is invalid! "
389                                 "Ignoring it");
390                 } else if (ret != 0) {
391                         WARNING("Can't read old integrity table! "
392                                 "Ignoring it");
393                 }
394         }
395
396         ret = calculate_integrity_table(fp, new_lookup_table_end,
397                                         old_table, old_lookup_table_end,
398                                         progress_func, &new_table);
399         if (ret != 0)
400                 goto out_free_old_table;
401
402         new_table_size = new_table->size;
403
404         new_table->size        = cpu_to_le32(new_table->size);
405         new_table->num_entries = cpu_to_le32(new_table->num_entries);
406         new_table->chunk_size  = cpu_to_le32(new_table->chunk_size);
407
408         if (fseeko(fp, cur_offset, SEEK_SET) != 0) {
409                 ERROR_WITH_ERRNO("Failed to seek to byte %"PRIu64" of WIM to "
410                                  "write integrity table", cur_offset);
411                 ret = WIMLIB_ERR_WRITE;
412                 goto out_free_new_table;
413         }
414
415         if (fwrite(new_table, 1, new_table_size, fp) != new_table_size) {
416                 ERROR_WITH_ERRNO("Failed to write WIM integrity table");
417                 ret = WIMLIB_ERR_WRITE;
418         } else {
419                 integrity_res_entry->offset        = cur_offset;
420                 integrity_res_entry->size          = new_table_size;
421                 integrity_res_entry->original_size = new_table_size;
422                 integrity_res_entry->flags         = 0;
423                 ret = 0;
424         }
425 out_free_new_table:
426         FREE(new_table);
427 out_free_old_table:
428         FREE(old_table);
429         return ret;
430 }
431
432 /*
433  * verify_integrity():
434  *
435  * Checks a WIM for consistency with the integrity table.
436  *
437  * @fp:
438  *      FILE * to the WIM file, opened for reading.
439  *
440  * @table:
441  *      The integrity table for the WIM, read into memory.
442  *
443  * @bytes_to_check:
444  *      Number of bytes in the WIM that need to be checked (offset of end of the
445  *      lookup table minus offset of end of the header).
446  *
447  * @progress_func
448  *      If non-NULL, a progress function that will be called after every
449  *      verified chunk.
450  *
451  * Returns:
452  *      > 0 (WIMLIB_ERR_*) on error
453  *      0 (WIM_INTEGRITY_OK) if the integrity was checked successfully and there
454  *      were no inconsistencies.
455  *      -1 (WIM_INTEGRITY_NOT_OK) if the WIM failed the integrity check.
456  */
457 static int verify_integrity(FILE *fp, const char *filename,
458                             const struct integrity_table *table,
459                             u64 bytes_to_check,
460                             wimlib_progress_func_t progress_func)
461 {
462         int ret;
463         u64 offset = WIM_HEADER_DISK_SIZE;
464         u8 sha1_md[SHA1_HASH_SIZE];
465         union wimlib_progress_info progress;
466
467         if (progress_func) {
468                 progress.integrity.total_bytes      = bytes_to_check;
469                 progress.integrity.total_chunks     = table->num_entries;
470                 progress.integrity.completed_chunks = 0;
471                 progress.integrity.completed_bytes  = 0;
472                 progress.integrity.chunk_size       = table->chunk_size;
473                 progress.integrity.filename         = filename;
474                 progress_func(WIMLIB_PROGRESS_MSG_VERIFY_INTEGRITY,
475                               &progress);
476         }
477         for (u32 i = 0; i < table->num_entries; i++) {
478                 size_t this_chunk_size;
479                 if (i == table->num_entries - 1)
480                         this_chunk_size = MODULO_NONZERO(bytes_to_check,
481                                                          table->chunk_size);
482                 else
483                         this_chunk_size = table->chunk_size;
484
485                 ret = calculate_chunk_sha1(fp, this_chunk_size, offset, sha1_md);
486                 if (ret != 0)
487                         return ret;
488
489                 if (!hashes_equal(sha1_md, table->sha1sums[i]))
490                         return WIM_INTEGRITY_NOT_OK;
491
492                 offset += this_chunk_size;
493                 if (progress_func) {
494                         progress.integrity.completed_chunks++;
495                         progress.integrity.completed_bytes += this_chunk_size;
496                         progress_func(WIMLIB_PROGRESS_MSG_VERIFY_INTEGRITY,
497                                       &progress);
498                 }
499         }
500         return WIM_INTEGRITY_OK;
501 }
502
503
504 /*
505  * check_wim_integrity():
506  *
507  * Verifies the integrity of the WIM by making sure the SHA1 message digests of
508  * ~10 MiB chunks of the WIM match up with the values given in the integrity
509  * table.
510  *
511  * @w:
512  *      The WIM, opened for reading, and with the header already read.
513  *
514  * @progress_func
515  *      If non-NULL, a progress function that will be called after every
516  *      verified chunk.
517  *
518  * Returns:
519  *      > 0 (WIMLIB_ERR_*) on error
520  *      0 (WIM_INTEGRITY_OK) if the integrity was checked successfully and there
521  *      were no inconsistencies.
522  *      -1 (WIM_INTEGRITY_NOT_OK) if the WIM failed the integrity check.
523  *      -2 (WIM_INTEGRITY_NONEXISTENT) if the WIM contains no integrity
524  *      information.
525  */
526 int check_wim_integrity(WIMStruct *w, wimlib_progress_func_t progress_func)
527 {
528         int ret;
529         u64 bytes_to_check;
530         struct integrity_table *table;
531         u64 end_lookup_table_offset;
532
533         if (w->hdr.integrity.offset == 0) {
534                 DEBUG("No integrity information.");
535                 return WIM_INTEGRITY_NONEXISTENT;
536         }
537
538         end_lookup_table_offset = w->hdr.lookup_table_res_entry.offset +
539                                   w->hdr.lookup_table_res_entry.size;
540
541         if (end_lookup_table_offset < WIM_HEADER_DISK_SIZE) {
542                 ERROR("WIM lookup table ends before WIM header ends!");
543                 return WIMLIB_ERR_INVALID_INTEGRITY_TABLE;
544         }
545
546         bytes_to_check = end_lookup_table_offset - WIM_HEADER_DISK_SIZE;
547
548         ret = read_integrity_table(&w->hdr.integrity, w->fp,
549                                    bytes_to_check, &table);
550         if (ret != 0)
551                 return ret;
552         ret = verify_integrity(w->fp, w->filename, table,
553                                bytes_to_check, progress_func);
554         FREE(table);
555         return ret;
556 }