Update progress functions
[wimlib] / src / integrity.c
1 /*
2  * integrity.c
3  *
4  * WIM files can optionally contain a table of SHA1 message digests at the end,
5  * one digest for each chunk of the file of some specified size (often 10 MB).
6  * This file implements the checking and writing of this table.
7  */
8
9 /*
10  * Copyright (C) 2012, 2013 Eric Biggers
11  *
12  * This file is part of wimlib, a library for working with WIM files.
13  *
14  * wimlib is free software; you can redistribute it and/or modify it under the
15  * terms of the GNU General Public License as published by the Free
16  * Software Foundation; either version 3 of the License, or (at your option)
17  * any later version.
18  *
19  * wimlib is distributed in the hope that it will be useful, but WITHOUT ANY
20  * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
21  * A PARTICULAR PURPOSE. See the GNU General Public License for more
22  * details.
23  *
24  * You should have received a copy of the GNU General Public License
25  * along with wimlib; if not, see http://www.gnu.org/licenses/.
26  */
27
28 #ifdef HAVE_CONFIG_H
29 #  include "config.h"
30 #endif
31
32 #include "wimlib/assert.h"
33 #include "wimlib/endianness.h"
34 #include "wimlib/error.h"
35 #include "wimlib/file_io.h"
36 #include "wimlib/integrity.h"
37 #include "wimlib/progress.h"
38 #include "wimlib/resource.h"
39 #include "wimlib/sha1.h"
40 #include "wimlib/wim.h"
41 #include "wimlib/write.h"
42
43 /* Size, in bytes, of each SHA1-summed chunk, when wimlib writes integrity
44  * information. */
45 #define INTEGRITY_CHUNK_SIZE 10485760
46
47 /* Only use a different chunk size for compatiblity with an existing integrity
48  * table if the chunk size is between these two numbers. */
49 #define INTEGRITY_MIN_CHUNK_SIZE 4096
50 #define INTEGRITY_MAX_CHUNK_SIZE 134217728
51
52 struct integrity_table {
53         u32 size;
54         u32 num_entries;
55         u32 chunk_size;
56         u8  sha1sums[][20];
57 } _packed_attribute;
58
59 static int
60 calculate_chunk_sha1(struct filedes *in_fd, size_t this_chunk_size,
61                      off_t offset, u8 sha1_md[])
62 {
63         u8 buf[BUFFER_SIZE];
64         SHA_CTX ctx;
65         size_t bytes_remaining;
66         size_t bytes_to_read;
67         int ret;
68
69         bytes_remaining = this_chunk_size;
70         sha1_init(&ctx);
71         do {
72                 bytes_to_read = min(bytes_remaining, sizeof(buf));
73                 ret = full_pread(in_fd, buf, bytes_to_read, offset);
74                 if (ret) {
75                         ERROR_WITH_ERRNO("Read error while calculating "
76                                          "integrity checksums");
77                         return ret;
78                 }
79                 sha1_update(&ctx, buf, bytes_to_read);
80                 bytes_remaining -= bytes_to_read;
81                 offset += bytes_to_read;
82         } while (bytes_remaining);
83         sha1_final(sha1_md, &ctx);
84         return 0;
85 }
86
87
88 /*
89  * read_integrity_table: -  Reads the integrity table from a WIM file.
90  *
91  * @wim:
92  *      WIMStruct for the WIM file; @wim->hdr.integrity_table_reshdr specifies
93  *      the location of the integrity table.  @wim->in_fd is expected to be a
94  *      seekable file descriptor to the WIM file opened for reading.
95  *
96  * @num_checked_bytes:
97  *      Number of bytes of data that should be checked by the integrity table.
98  *
99  * @table_ret:
100  *      On success, a pointer to an in-memory structure containing the integrity
101  *      information is written to this location.
102  *
103  * Return values:
104  *      WIMLIB_ERR_SUCCESS (0)
105  *      WIMLIB_ERR_INVALID_INTEGRITY_TABLE
106  *      WIMLIB_ERR_NOMEM
107  *      WIMLIB_ERR_READ
108  *      WIMLIB_ERR_UNEXPECTED_END_OF_FILE
109  */
110 static int
111 read_integrity_table(WIMStruct *wim, u64 num_checked_bytes,
112                      struct integrity_table **table_ret)
113 {
114         void *buf;
115         struct integrity_table *table;
116         int ret;
117
118         if (wim->hdr.integrity_table_reshdr.uncompressed_size < 8)
119                 goto invalid;
120
121         DEBUG("Reading integrity table.");
122
123         ret = wim_reshdr_to_data(&wim->hdr.integrity_table_reshdr, wim, &buf);
124         if (ret)
125                 return ret;
126         table = buf;
127
128         table->size        = le32_to_cpu(table->size);
129         table->num_entries = le32_to_cpu(table->num_entries);
130         table->chunk_size  = le32_to_cpu(table->chunk_size);
131
132         DEBUG("table->size = %u, table->num_entries = %u, "
133               "table->chunk_size = %u",
134               table->size, table->num_entries, table->chunk_size);
135
136         if (table->size != wim->hdr.integrity_table_reshdr.uncompressed_size ||
137             table->size != (u64)table->num_entries * SHA1_HASH_SIZE + 12 ||
138             table->chunk_size == 0 ||
139             table->num_entries != DIV_ROUND_UP(num_checked_bytes, table->chunk_size))
140         {
141                 FREE(table);
142                 goto invalid;
143         }
144
145         *table_ret = table;
146         return 0;
147
148 invalid:
149         ERROR("Integrity table is invalid");
150         return WIMLIB_ERR_INVALID_INTEGRITY_TABLE;
151 }
152
153 /*
154  * calculate_integrity_table():
155  *
156  * Calculates an integrity table for the data in a file beginning at offset 208
157  * (WIM_HEADER_DISK_SIZE).
158  *
159  * @in_fd:
160  *      File descriptor for the file to be checked, opened for reading.  Does
161  *      not need to be at any specific location in the file.
162  *
163  * @new_check_end:
164  *      Offset of byte after the last byte to be checked.
165  *
166  * @old_table:
167  *      If non-NULL, a pointer to the table containing the previously calculated
168  *      integrity data for a prefix of this file.
169  *
170  * @old_check_end:
171  *      If @old_table is non-NULL, the byte after the last byte that was checked
172  *      in the old table.  Must be less than or equal to new_check_end.
173  *
174  * @integrity_table_ret:
175  *      On success, a pointer to the calculated integrity table is written into
176  *      this location.
177  *
178  * Return values:
179  *      WIMLIB_ERR_SUCCESS (0)
180  *      WIMLIB_ERR_NOMEM
181  *      WIMLIB_ERR_READ
182  *      WIMLIB_ERR_UNEXPECTED_END_OF_FILE
183  */
184 static int
185 calculate_integrity_table(struct filedes *in_fd,
186                           off_t new_check_end,
187                           const struct integrity_table *old_table,
188                           off_t old_check_end,
189                           struct integrity_table **integrity_table_ret,
190                           wimlib_progress_func_t progfunc,
191                           void *progctx)
192 {
193         int ret;
194         size_t chunk_size = INTEGRITY_CHUNK_SIZE;
195
196         /* If an old table is provided, set the chunk size to be compatible with
197          * the old chunk size, unless the old chunk size was weird. */
198         if (old_table != NULL) {
199                 if (old_table->num_entries == 0 ||
200                     old_table->chunk_size < INTEGRITY_MIN_CHUNK_SIZE ||
201                     old_table->chunk_size > INTEGRITY_MAX_CHUNK_SIZE)
202                         old_table = NULL;
203                 else
204                         chunk_size = old_table->chunk_size;
205         }
206
207
208         u64 old_check_bytes = old_check_end - WIM_HEADER_DISK_SIZE;
209         u64 new_check_bytes = new_check_end - WIM_HEADER_DISK_SIZE;
210
211         u32 old_num_chunks = DIV_ROUND_UP(old_check_bytes, chunk_size);
212         u32 new_num_chunks = DIV_ROUND_UP(new_check_bytes, chunk_size);
213
214         size_t old_last_chunk_size = MODULO_NONZERO(old_check_bytes, chunk_size);
215         size_t new_last_chunk_size = MODULO_NONZERO(new_check_bytes, chunk_size);
216
217         size_t new_table_size = 12 + new_num_chunks * SHA1_HASH_SIZE;
218
219         struct integrity_table *new_table = MALLOC(new_table_size);
220         if (!new_table)
221                 return WIMLIB_ERR_NOMEM;
222         new_table->num_entries = new_num_chunks;
223         new_table->size = new_table_size;
224         new_table->chunk_size = chunk_size;
225
226         u64 offset = WIM_HEADER_DISK_SIZE;
227         union wimlib_progress_info progress;
228
229         progress.integrity.total_bytes      = new_check_bytes;
230         progress.integrity.total_chunks     = new_num_chunks;
231         progress.integrity.completed_chunks = 0;
232         progress.integrity.completed_bytes  = 0;
233         progress.integrity.chunk_size       = chunk_size;
234         progress.integrity.filename         = NULL;
235
236         ret = call_progress(progfunc, WIMLIB_PROGRESS_MSG_CALC_INTEGRITY,
237                             &progress, progctx);
238         if (ret)
239                 goto out_free_new_table;
240
241         for (u32 i = 0; i < new_num_chunks; i++) {
242                 size_t this_chunk_size;
243                 if (i == new_num_chunks - 1)
244                         this_chunk_size = new_last_chunk_size;
245                 else
246                         this_chunk_size = chunk_size;
247                 if (old_table &&
248                     ((this_chunk_size == chunk_size && i < old_num_chunks - 1) ||
249                       (i == old_num_chunks - 1 && this_chunk_size == old_last_chunk_size)))
250                 {
251                         /* Can use SHA1 message digest from old integrity table
252                          * */
253                         copy_hash(new_table->sha1sums[i], old_table->sha1sums[i]);
254                 } else {
255                         /* Calculate the SHA1 message digest of this chunk */
256                         ret = calculate_chunk_sha1(in_fd, this_chunk_size,
257                                                    offset, new_table->sha1sums[i]);
258                         if (ret)
259                                 goto out_free_new_table;
260                 }
261                 offset += this_chunk_size;
262
263                 progress.integrity.completed_chunks++;
264                 progress.integrity.completed_bytes += this_chunk_size;
265                 ret = call_progress(progfunc, WIMLIB_PROGRESS_MSG_CALC_INTEGRITY,
266                                     &progress, progctx);
267                 if (ret)
268                         goto out_free_new_table;
269         }
270         *integrity_table_ret = new_table;
271         return 0;
272
273 out_free_new_table:
274         FREE(new_table);
275         return ret;
276 }
277
278 /*
279  * write_integrity_table():
280  *
281  * Writes a WIM integrity table (a list of SHA1 message digests of raw 10 MiB
282  * chunks of the file).
283  *
284  * This function can optionally re-use entries from an older integrity table.
285  * To do this, ensure that @wim->hdr.integrity_table_reshdr is the resource
286  * header for the older table (note: this is an input-output parameter), and set
287  * @old_lookup_table_end to the offset of the byte directly following the last
288  * byte checked by the old table.  If the old integrity table is invalid or
289  * cannot be read, a warning is printed and the integrity information is
290  * re-calculated.
291  *
292  * @wim:
293  *      WIMStruct for the WIM file.  @wim->out_fd must be a seekable descriptor
294  *      to the new WIM file, opened read-write, positioned at the location at
295  *      which the integrity table is to be written.  Furthermore,
296  *      @wim->hdr.integrity is expected to be a resource entry which will be set
297  *      to the integrity table information on success.  In addition, if
298  *      @old_lookup_table_end != 0, @wim->hdr.integrity must initially contain
299  *      information about the old integrity table, and @wim->in_fd must be a
300  *      seekable descriptor to the original WIM file opened for reading.
301  *
302  * @new_lookup_table_end:
303  *      The offset of the byte directly following the lookup table in the WIM
304  *      being written.
305  *
306  * @old_lookup_table_end:
307  *      If nonzero, the offset of the byte directly following the old lookup
308  *      table in the WIM.
309  *
310  * Return values:
311  *      WIMLIB_ERR_SUCCESS (0)
312  *      WIMLIB_ERR_NOMEM
313  *      WIMLIB_ERR_UNEXPECTED_END_OF_FILE
314  *      WIMLIB_ERR_WRITE
315  */
316 int
317 write_integrity_table(WIMStruct *wim,
318                       off_t new_lookup_table_end,
319                       off_t old_lookup_table_end)
320 {
321         struct integrity_table *old_table;
322         struct integrity_table *new_table;
323         int ret;
324         u32 new_table_size;
325
326         DEBUG("Writing integrity table "
327               "(new_lookup_table_end=%"PRIu64", old_lookup_table_end=%"PRIu64")",
328               new_lookup_table_end, old_lookup_table_end);
329
330         wimlib_assert(old_lookup_table_end <= new_lookup_table_end);
331
332         old_table = NULL;
333         if (wim_has_integrity_table(wim) && old_lookup_table_end != 0) {
334                 ret = read_integrity_table(wim,
335                                            old_lookup_table_end - WIM_HEADER_DISK_SIZE,
336                                            &old_table);
337                 if (ret == WIMLIB_ERR_INVALID_INTEGRITY_TABLE) {
338                         WARNING("Old integrity table is invalid! "
339                                 "Ignoring it");
340                 } else if (ret != 0) {
341                         WARNING("Can't read old integrity table! "
342                                 "Ignoring it");
343                 }
344         }
345
346         ret = calculate_integrity_table(&wim->out_fd, new_lookup_table_end,
347                                         old_table, old_lookup_table_end,
348                                         &new_table, wim->progfunc, wim->progctx);
349         if (ret)
350                 goto out_free_old_table;
351
352         new_table_size = new_table->size;
353
354         new_table->size        = cpu_to_le32(new_table->size);
355         new_table->num_entries = cpu_to_le32(new_table->num_entries);
356         new_table->chunk_size  = cpu_to_le32(new_table->chunk_size);
357
358         ret = write_wim_resource_from_buffer(new_table,
359                                              new_table_size,
360                                              0,
361                                              &wim->out_fd,
362                                              WIMLIB_COMPRESSION_TYPE_NONE,
363                                              0,
364                                              &wim->hdr.integrity_table_reshdr,
365                                              NULL,
366                                              0);
367         FREE(new_table);
368 out_free_old_table:
369         FREE(old_table);
370         DEBUG("ret=%d", ret);
371         return ret;
372 }
373
374 /*
375  * verify_integrity():
376  *
377  * Checks a WIM for consistency with the integrity table.
378  *
379  * @in_fd:
380  *      File descriptor to the WIM file, opened for reading.
381  *
382  * @table:
383  *      The integrity table for the WIM, read into memory.
384  *
385  * @bytes_to_check:
386  *      Number of bytes in the WIM that need to be checked (offset of end of the
387  *      lookup table minus offset of end of the header).
388  *
389  * Returns:
390  *      > 0 (WIMLIB_ERR_READ, WIMLIB_ERR_UNEXPECTED_END_OF_FILE) on error
391  *      0 (WIM_INTEGRITY_OK) if the integrity was checked successfully and there
392  *      were no inconsistencies.
393  *      -1 (WIM_INTEGRITY_NOT_OK) if the WIM failed the integrity check.
394  */
395 static int
396 verify_integrity(struct filedes *in_fd, const tchar *filename,
397                  const struct integrity_table *table,
398                  u64 bytes_to_check,
399                  wimlib_progress_func_t progfunc, void *progctx)
400 {
401         int ret;
402         u64 offset = WIM_HEADER_DISK_SIZE;
403         u8 sha1_md[SHA1_HASH_SIZE];
404         union wimlib_progress_info progress;
405
406         progress.integrity.total_bytes      = bytes_to_check;
407         progress.integrity.total_chunks     = table->num_entries;
408         progress.integrity.completed_chunks = 0;
409         progress.integrity.completed_bytes  = 0;
410         progress.integrity.chunk_size       = table->chunk_size;
411         progress.integrity.filename         = filename;
412
413         ret = call_progress(progfunc, WIMLIB_PROGRESS_MSG_VERIFY_INTEGRITY,
414                             &progress, progctx);
415         if (ret)
416                 return ret;
417
418         for (u32 i = 0; i < table->num_entries; i++) {
419                 size_t this_chunk_size;
420                 if (i == table->num_entries - 1)
421                         this_chunk_size = MODULO_NONZERO(bytes_to_check,
422                                                          table->chunk_size);
423                 else
424                         this_chunk_size = table->chunk_size;
425
426                 ret = calculate_chunk_sha1(in_fd, this_chunk_size, offset, sha1_md);
427                 if (ret)
428                         return ret;
429
430                 if (!hashes_equal(sha1_md, table->sha1sums[i]))
431                         return WIM_INTEGRITY_NOT_OK;
432
433                 offset += this_chunk_size;
434                 progress.integrity.completed_chunks++;
435                 progress.integrity.completed_bytes += this_chunk_size;
436
437                 ret = call_progress(progfunc, WIMLIB_PROGRESS_MSG_VERIFY_INTEGRITY,
438                                     &progress, progctx);
439                 if (ret)
440                         return ret;
441         }
442         return WIM_INTEGRITY_OK;
443 }
444
445
446 /*
447  * check_wim_integrity():
448  *
449  * Verifies the integrity of the WIM by making sure the SHA1 message digests of
450  * ~10 MiB chunks of the WIM match up with the values given in the integrity
451  * table.
452  *
453  * @wim:
454  *      The WIM, opened for reading.
455  *
456  * Returns:
457  *      > 0 (WIMLIB_ERR_INVALID_INTEGRITY_TABLE, WIMLIB_ERR_READ,
458  *           WIMLIB_ERR_UNEXPECTED_END_OF_FILE) on error
459  *      0 (WIM_INTEGRITY_OK) if the integrity was checked successfully and there
460  *      were no inconsistencies.
461  *      -1 (WIM_INTEGRITY_NOT_OK) if the WIM failed the integrity check.
462  *      -2 (WIM_INTEGRITY_NONEXISTENT) if the WIM contains no integrity
463  *      information.
464  */
465 int
466 check_wim_integrity(WIMStruct *wim)
467 {
468         int ret;
469         u64 bytes_to_check;
470         struct integrity_table *table;
471         u64 end_lookup_table_offset;
472
473         if (!wim_has_integrity_table(wim)) {
474                 DEBUG("No integrity information.");
475                 return WIM_INTEGRITY_NONEXISTENT;
476         }
477
478         end_lookup_table_offset = wim->hdr.lookup_table_reshdr.offset_in_wim +
479                                   wim->hdr.lookup_table_reshdr.size_in_wim;
480
481         if (end_lookup_table_offset < WIM_HEADER_DISK_SIZE) {
482                 ERROR("WIM lookup table ends before WIM header ends!");
483                 return WIMLIB_ERR_INVALID_INTEGRITY_TABLE;
484         }
485
486         bytes_to_check = end_lookup_table_offset - WIM_HEADER_DISK_SIZE;
487
488         ret = read_integrity_table(wim, bytes_to_check, &table);
489         if (ret)
490                 return ret;
491         ret = verify_integrity(&wim->in_fd, wim->filename, table,
492                                bytes_to_check, wim->progfunc, wim->progctx);
493         FREE(table);
494         return ret;
495 }