Refactor headers
[wimlib] / src / integrity.c
1 /*
2  * integrity.c
3  *
4  * WIM files can optionally contain a table of SHA1 message digests at the end,
5  * one digest for each chunk of the file of some specified size (often 10 MB).
6  * This file implements the checking and writing of this table.
7  */
8
9 /*
10  * Copyright (C) 2012, 2013 Eric Biggers
11  *
12  * This file is part of wimlib, a library for working with WIM files.
13  *
14  * wimlib is free software; you can redistribute it and/or modify it under the
15  * terms of the GNU General Public License as published by the Free
16  * Software Foundation; either version 3 of the License, or (at your option)
17  * any later version.
18  *
19  * wimlib is distributed in the hope that it will be useful, but WITHOUT ANY
20  * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
21  * A PARTICULAR PURPOSE. See the GNU General Public License for more
22  * details.
23  *
24  * You should have received a copy of the GNU General Public License
25  * along with wimlib; if not, see http://www.gnu.org/licenses/.
26  */
27
28 #ifdef HAVE_CONFIG_H
29 #  include "config.h"
30 #endif
31
32 #include "wimlib/assert.h"
33 #include "wimlib/buffer_io.h"
34 #include "wimlib/error.h"
35 #include "wimlib/file_io.h"
36 #include "wimlib/integrity.h"
37 #include "wimlib/resource.h"
38 #include "wimlib/sha1.h"
39 #include "wimlib/wim.h"
40
41 /* Size, in bytes, of each SHA1-summed chunk, when wimlib writes integrity
42  * information. */
43 #define INTEGRITY_CHUNK_SIZE 10485760
44
45 /* Only use a different chunk size for compatiblity with an existing integrity
46  * table if the chunk size is between these two numbers. */
47 #define INTEGRITY_MIN_CHUNK_SIZE 4096
48 #define INTEGRITY_MAX_CHUNK_SIZE 134217728
49
50 struct integrity_table {
51         u32 size;
52         u32 num_entries;
53         u32 chunk_size;
54         u8  sha1sums[0][20];
55 };
56
57 static int
58 calculate_chunk_sha1(int in_fd, size_t this_chunk_size,
59                      off_t offset, u8 sha1_md[])
60 {
61         u8 buf[BUFFER_SIZE];
62         SHA_CTX ctx;
63         size_t bytes_remaining;
64         size_t bytes_to_read;
65         size_t bytes_read;
66
67         bytes_remaining = this_chunk_size;
68         sha1_init(&ctx);
69         do {
70                 bytes_to_read = min(bytes_remaining, sizeof(buf));
71                 bytes_read = full_pread(in_fd, buf, bytes_to_read, offset);
72                 if (bytes_read != bytes_to_read) {
73                         ERROR_WITH_ERRNO("Read error while calculating "
74                                          "integrity checksums");
75                         return WIMLIB_ERR_READ;
76                 }
77                 sha1_update(&ctx, buf, bytes_read);
78                 bytes_remaining -= bytes_read;
79                 offset += bytes_read;
80         } while (bytes_remaining);
81         sha1_final(sha1_md, &ctx);
82         return 0;
83 }
84
85
86 /*
87  * read_integrity_table: -  Reads the integrity table from a WIM file.
88  *
89  * @res_entry:
90  *      The resource entry that specifies the location of the integrity table.
91  *      The integrity table must exist (i.e. res_entry->offset must not be 0).
92  *
93  * @in_fd:
94  *      File descriptor to the WIM file, opened for reading.
95  *
96  * @num_checked_bytes:
97  *      Number of bytes of data that should be checked by the integrity table.
98  *
99  * @table ret:
100  *      On success, a pointer to an in-memory structure containing the integrity
101  *      information is written to this location.
102  *
103  * Returns 0 on success; nonzero on failure.  The possible error codes are:
104  *
105  *     * WIMLIB_ERR_INVALID_INTEGRITY_TABLE:  The integrity table is invalid.
106  *     * WIMLIB_ERR_NOMEM:  Could not allocate memory to store the integrity
107  *                          data.
108  *     * WIMLIB_ERR_READ:   Could not read the integrity data from the WIM file.
109  */
110 static int
111 read_integrity_table(const struct resource_entry *res_entry,
112                      int in_fd,
113                      u64 num_checked_bytes,
114                      struct integrity_table **table_ret)
115 {
116         struct integrity_table *table;
117         int ret;
118         u64 expected_size;
119         u64 expected_num_entries;
120
121         if (resource_is_compressed(res_entry)) {
122                 ERROR("Didn't expect a compressed integrity table");
123                 return WIMLIB_ERR_INVALID_INTEGRITY_TABLE;
124         }
125
126         if (res_entry->size < 8 || res_entry->size  > 0xffffffff) {
127                 ERROR("Integrity table resource header is invalid");
128                 return WIMLIB_ERR_INVALID_INTEGRITY_TABLE;
129         }
130
131         /* Read the integrity table into memory. */
132         table = MALLOC((size_t)res_entry->size);
133         if (table == NULL) {
134                 ERROR("Can't allocate %zu bytes for integrity table",
135                       (size_t)res_entry->size);
136                 return WIMLIB_ERR_NOMEM;
137         }
138
139         if (full_pread(in_fd, table, res_entry->size,
140                        res_entry->offset) != res_entry->size)
141         {
142                 ERROR("Failed to read integrity table (size = %zu, "
143                       " offset = %"PRIu64")",
144                       (size_t)res_entry->size, res_entry->offset);
145                 ret = WIMLIB_ERR_READ;
146                 goto out_free_table;
147         }
148
149         table->size        = le32_to_cpu(table->size);
150         table->num_entries = le32_to_cpu(table->num_entries);
151         table->chunk_size  = le32_to_cpu(table->chunk_size);
152
153         if (table->size != res_entry->size) {
154                 ERROR("Inconsistent integrity table sizes: Table header says "
155                       "%u bytes but resource entry says %u bytes",
156                       table->size, (unsigned)res_entry->size);
157                 ret = WIMLIB_ERR_INVALID_INTEGRITY_TABLE;
158                 goto out_free_table;
159         }
160
161         DEBUG("table->size = %u, table->num_entries = %u, "
162               "table->chunk_size = %u",
163               table->size, table->num_entries, table->chunk_size);
164
165         expected_size = (u64)table->num_entries * SHA1_HASH_SIZE + 12;
166
167         if (table->size != expected_size) {
168                 ERROR("Integrity table is %u bytes, but expected %"PRIu64" "
169                       "bytes to hold %u entries",
170                       table->size, expected_size, table->num_entries);
171                 ret = WIMLIB_ERR_INVALID_INTEGRITY_TABLE;
172                 goto out_free_table;
173         }
174
175         if (table->chunk_size == 0) {
176                 ERROR("Cannot use integrity chunk size of 0");
177                 ret = WIMLIB_ERR_INVALID_INTEGRITY_TABLE;
178                 goto out_free_table;
179         }
180
181         expected_num_entries = DIV_ROUND_UP(num_checked_bytes, table->chunk_size);
182
183         if (table->num_entries != expected_num_entries) {
184                 ERROR("%"PRIu64" integrity table entries would be required "
185                       "to checksum the %"PRIu64" bytes from the end of the "
186                       "header to the",
187                       expected_num_entries, num_checked_bytes);
188                 ERROR("end of the lookup table with a chunk size of %u, but "
189                       "there were only %u entries",
190                       table->chunk_size, table->num_entries);
191                 ret = WIMLIB_ERR_INVALID_INTEGRITY_TABLE;
192                 goto out_free_table;
193         }
194         *table_ret = table;
195         ret = 0;
196         goto out;
197 out_free_table:
198         FREE(table);
199 out:
200         return ret;
201 }
202
203 /*
204  * calculate_integrity_table():
205  *
206  * Calculates an integrity table for the data in a file beginning at offset 208
207  * (WIM_HEADER_DISK_SIZE).
208  *
209  * @in_fd:
210  *      File descriptor for the file to be checked, opened for reading.  Does
211  *      not need to be at any specific location in the file.
212  *
213  * @new_check_end:
214  *      Offset of byte after the last byte to be checked.
215  *
216  * @old_table:
217  *      If non-NULL, a pointer to the table containing the previously calculated
218  *      integrity data for a prefix of this file.
219  *
220  * @old_check_end:
221  *      If @old_table is non-NULL, the byte after the last byte that was checked
222  *      in the old table.  Must be less than or equal to new_check_end.
223  *
224  * @progress_func:
225  *      If non-NULL, a progress function that will be called after every
226  *      calculated chunk.
227  *
228  * @integrity_table_ret:
229  *      On success, a pointer to the calculated integrity table is written into
230  *      this location.
231  *
232  * Returns 0 on success; nonzero on failure.
233  */
234 static int
235 calculate_integrity_table(int in_fd,
236                           off_t new_check_end,
237                           const struct integrity_table *old_table,
238                           off_t old_check_end,
239                           wimlib_progress_func_t progress_func,
240                           struct integrity_table **integrity_table_ret)
241 {
242         int ret;
243         size_t chunk_size = INTEGRITY_CHUNK_SIZE;
244
245         /* If an old table is provided, set the chunk size to be compatible with
246          * the old chunk size, unless the old chunk size was weird. */
247         if (old_table != NULL) {
248                 if (old_table->num_entries == 0 ||
249                     old_table->chunk_size < INTEGRITY_MIN_CHUNK_SIZE ||
250                     old_table->chunk_size > INTEGRITY_MAX_CHUNK_SIZE)
251                         old_table = NULL;
252                 else
253                         chunk_size = old_table->chunk_size;
254         }
255
256
257         u64 old_check_bytes = old_check_end - WIM_HEADER_DISK_SIZE;
258         u64 new_check_bytes = new_check_end - WIM_HEADER_DISK_SIZE;
259
260         u32 old_num_chunks = DIV_ROUND_UP(old_check_bytes, chunk_size);
261         u32 new_num_chunks = DIV_ROUND_UP(new_check_bytes, chunk_size);
262
263         size_t old_last_chunk_size = MODULO_NONZERO(old_check_bytes, chunk_size);
264         size_t new_last_chunk_size = MODULO_NONZERO(new_check_bytes, chunk_size);
265
266         size_t new_table_size = 12 + new_num_chunks * SHA1_HASH_SIZE;
267
268         struct integrity_table *new_table = MALLOC(new_table_size);
269         if (!new_table)
270                 return WIMLIB_ERR_NOMEM;
271         new_table->num_entries = new_num_chunks;
272         new_table->size = new_table_size;
273         new_table->chunk_size = chunk_size;
274
275         u64 offset = WIM_HEADER_DISK_SIZE;
276         union wimlib_progress_info progress;
277
278         if (progress_func) {
279                 progress.integrity.total_bytes      = new_check_bytes;
280                 progress.integrity.total_chunks     = new_num_chunks;
281                 progress.integrity.completed_chunks = 0;
282                 progress.integrity.completed_bytes  = 0;
283                 progress.integrity.chunk_size       = chunk_size;
284                 progress.integrity.filename         = NULL;
285                 progress_func(WIMLIB_PROGRESS_MSG_CALC_INTEGRITY,
286                               &progress);
287         }
288
289         for (u32 i = 0; i < new_num_chunks; i++) {
290                 size_t this_chunk_size;
291                 if (i == new_num_chunks - 1)
292                         this_chunk_size = new_last_chunk_size;
293                 else
294                         this_chunk_size = chunk_size;
295                 if (old_table &&
296                     ((this_chunk_size == chunk_size && i < old_num_chunks - 1) ||
297                       (i == old_num_chunks - 1 && this_chunk_size == old_last_chunk_size)))
298                 {
299                         /* Can use SHA1 message digest from old integrity table
300                          * */
301                         copy_hash(new_table->sha1sums[i], old_table->sha1sums[i]);
302                 } else {
303                         /* Calculate the SHA1 message digest of this chunk */
304                         ret = calculate_chunk_sha1(in_fd, this_chunk_size,
305                                                    offset, new_table->sha1sums[i]);
306                         if (ret) {
307                                 FREE(new_table);
308                                 return ret;
309                         }
310                 }
311                 offset += this_chunk_size;
312                 if (progress_func) {
313                         progress.integrity.completed_chunks++;
314                         progress.integrity.completed_bytes += this_chunk_size;
315                         progress_func(WIMLIB_PROGRESS_MSG_CALC_INTEGRITY,
316                                       &progress);
317                 }
318         }
319         *integrity_table_ret = new_table;
320         return 0;
321 }
322
323 /*
324  * write_integrity_table():
325  *
326  * Writes a WIM integrity table (a list of SHA1 message digests of raw 10 MiB
327  * chunks of the file).
328  *
329  * This function can optionally re-use entries from an older integrity table.
330  * To do this, make @integrity_res_entry point to the resource entry for the
331  * older table (note: this is an input-output parameter), and set
332  * @old_lookup_table_end to the offset of the byte directly following the last
333  * byte checked by the old table.  If the old integrity table is invalid or
334  * cannot be read, a warning is printed and the integrity information is
335  * re-calculated.
336  *
337  * @fd:
338  *      File descriptor to the WIM file, opened read-write, positioned at the
339  *      location at which the integrity table is to be written.
340  *
341  * @integrity_res_entry:
342  *      Resource entry which will be set to point to the integrity table on
343  *      success.  In addition, if @old_lookup_table_end != 0, this initially
344  *      must point to the resource entry for the old integrity table for the
345  *      WIM.
346  *
347  * @new_lookup_table_end:
348  *      The offset of the byte directly following the lookup table in the WIM
349  *      being written.
350  *
351  * @old_lookup_table_end:
352  *      If nonzero, the offset of the byte directly following the old lookup
353  *      table in the WIM.
354  *
355  * @progress_func
356  *      If non-NULL, a progress function that will be called after every
357  *      calculated chunk.
358  *
359  * Returns:
360  *      0 on success, nonzero on failure.  The possible error codes are:
361  *         * WIMLIB_ERR_WRITE:  Could not write the integrity table.
362  *         * WIMLIB_ERR_READ:   Could not read a chunk of data that needed
363  *                              to be checked.
364  */
365 int
366 write_integrity_table(int fd,
367                       struct resource_entry *integrity_res_entry,
368                       off_t new_lookup_table_end,
369                       off_t old_lookup_table_end,
370                       wimlib_progress_func_t progress_func)
371 {
372         struct integrity_table *old_table;
373         struct integrity_table *new_table;
374         int ret;
375         off_t cur_offset;
376         u32 new_table_size;
377
378         wimlib_assert(old_lookup_table_end <= new_lookup_table_end);
379
380         cur_offset = filedes_offset(fd);
381         if (cur_offset == -1)
382                 return WIMLIB_ERR_WRITE;
383
384         if (integrity_res_entry->offset == 0 || old_lookup_table_end == 0) {
385                 old_table = NULL;
386         } else {
387                 ret = read_integrity_table(integrity_res_entry, fd,
388                                            old_lookup_table_end - WIM_HEADER_DISK_SIZE,
389                                            &old_table);
390                 if (ret == WIMLIB_ERR_INVALID_INTEGRITY_TABLE) {
391                         WARNING("Old integrity table is invalid! "
392                                 "Ignoring it");
393                 } else if (ret != 0) {
394                         WARNING("Can't read old integrity table! "
395                                 "Ignoring it");
396                 }
397         }
398
399         ret = calculate_integrity_table(fd, new_lookup_table_end,
400                                         old_table, old_lookup_table_end,
401                                         progress_func, &new_table);
402         if (ret)
403                 goto out_free_old_table;
404
405         new_table_size = new_table->size;
406
407         new_table->size        = cpu_to_le32(new_table->size);
408         new_table->num_entries = cpu_to_le32(new_table->num_entries);
409         new_table->chunk_size  = cpu_to_le32(new_table->chunk_size);
410
411         if (full_write(fd, new_table, new_table_size) != new_table_size) {
412                 ERROR_WITH_ERRNO("Failed to write WIM integrity table");
413                 ret = WIMLIB_ERR_WRITE;
414         } else {
415                 integrity_res_entry->offset        = cur_offset;
416                 integrity_res_entry->size          = new_table_size;
417                 integrity_res_entry->original_size = new_table_size;
418                 integrity_res_entry->flags         = 0;
419                 ret = 0;
420         }
421         FREE(new_table);
422 out_free_old_table:
423         FREE(old_table);
424         return ret;
425 }
426
427 /*
428  * verify_integrity():
429  *
430  * Checks a WIM for consistency with the integrity table.
431  *
432  * @in_fd:
433  *      File descriptor to the WIM file, opened for reading.
434  *
435  * @table:
436  *      The integrity table for the WIM, read into memory.
437  *
438  * @bytes_to_check:
439  *      Number of bytes in the WIM that need to be checked (offset of end of the
440  *      lookup table minus offset of end of the header).
441  *
442  * @progress_func
443  *      If non-NULL, a progress function that will be called after every
444  *      verified chunk.
445  *
446  * Returns:
447  *      > 0 (WIMLIB_ERR_*) on error
448  *      0 (WIM_INTEGRITY_OK) if the integrity was checked successfully and there
449  *      were no inconsistencies.
450  *      -1 (WIM_INTEGRITY_NOT_OK) if the WIM failed the integrity check.
451  */
452 static int
453 verify_integrity(int in_fd, const tchar *filename,
454                  const struct integrity_table *table,
455                  u64 bytes_to_check,
456                  wimlib_progress_func_t progress_func)
457 {
458         int ret;
459         u64 offset = WIM_HEADER_DISK_SIZE;
460         u8 sha1_md[SHA1_HASH_SIZE];
461         union wimlib_progress_info progress;
462
463         if (progress_func) {
464                 progress.integrity.total_bytes      = bytes_to_check;
465                 progress.integrity.total_chunks     = table->num_entries;
466                 progress.integrity.completed_chunks = 0;
467                 progress.integrity.completed_bytes  = 0;
468                 progress.integrity.chunk_size       = table->chunk_size;
469                 progress.integrity.filename         = filename;
470                 progress_func(WIMLIB_PROGRESS_MSG_VERIFY_INTEGRITY,
471                               &progress);
472         }
473         for (u32 i = 0; i < table->num_entries; i++) {
474                 size_t this_chunk_size;
475                 if (i == table->num_entries - 1)
476                         this_chunk_size = MODULO_NONZERO(bytes_to_check,
477                                                          table->chunk_size);
478                 else
479                         this_chunk_size = table->chunk_size;
480
481                 ret = calculate_chunk_sha1(in_fd, this_chunk_size, offset, sha1_md);
482                 if (ret)
483                         return ret;
484
485                 if (!hashes_equal(sha1_md, table->sha1sums[i]))
486                         return WIM_INTEGRITY_NOT_OK;
487
488                 offset += this_chunk_size;
489                 if (progress_func) {
490                         progress.integrity.completed_chunks++;
491                         progress.integrity.completed_bytes += this_chunk_size;
492                         progress_func(WIMLIB_PROGRESS_MSG_VERIFY_INTEGRITY,
493                                       &progress);
494                 }
495         }
496         return WIM_INTEGRITY_OK;
497 }
498
499
500 /*
501  * check_wim_integrity():
502  *
503  * Verifies the integrity of the WIM by making sure the SHA1 message digests of
504  * ~10 MiB chunks of the WIM match up with the values given in the integrity
505  * table.
506  *
507  * @w:
508  *      The WIM, opened for reading, and with the header already read.
509  *
510  * @progress_func
511  *      If non-NULL, a progress function that will be called after every
512  *      verified chunk.
513  *
514  * Returns:
515  *      > 0 (WIMLIB_ERR_*) on error
516  *      0 (WIM_INTEGRITY_OK) if the integrity was checked successfully and there
517  *      were no inconsistencies.
518  *      -1 (WIM_INTEGRITY_NOT_OK) if the WIM failed the integrity check.
519  *      -2 (WIM_INTEGRITY_NONEXISTENT) if the WIM contains no integrity
520  *      information.
521  */
522 int
523 check_wim_integrity(WIMStruct *w, wimlib_progress_func_t progress_func)
524 {
525         int ret;
526         u64 bytes_to_check;
527         struct integrity_table *table;
528         u64 end_lookup_table_offset;
529
530         if (w->hdr.integrity.offset == 0) {
531                 DEBUG("No integrity information.");
532                 return WIM_INTEGRITY_NONEXISTENT;
533         }
534
535         end_lookup_table_offset = w->hdr.lookup_table_res_entry.offset +
536                                   w->hdr.lookup_table_res_entry.size;
537
538         if (end_lookup_table_offset < WIM_HEADER_DISK_SIZE) {
539                 ERROR("WIM lookup table ends before WIM header ends!");
540                 return WIMLIB_ERR_INVALID_INTEGRITY_TABLE;
541         }
542
543         bytes_to_check = end_lookup_table_offset - WIM_HEADER_DISK_SIZE;
544
545         ret = read_integrity_table(&w->hdr.integrity, w->in_fd,
546                                    bytes_to_check, &table);
547         if (ret)
548                 return ret;
549         ret = verify_integrity(w->in_fd, w->filename, table,
550                                bytes_to_check, progress_func);
551         FREE(table);
552         return ret;
553 }