Fix some typos
[wimlib] / src / integrity.c
1 /*
2  * integrity.c
3  *
4  * WIM files can optionally contain a table of SHA1 message digests at the end,
5  * one digest for each chunk of the file of some specified size (often 10 MB).
6  * This file implements the checking and writing of this table.
7  */
8
9 /*
10  * Copyright (C) 2012-2016 Eric Biggers
11  *
12  * This file is free software; you can redistribute it and/or modify it under
13  * the terms of the GNU Lesser General Public License as published by the Free
14  * Software Foundation; either version 3 of the License, or (at your option) any
15  * later version.
16  *
17  * This file is distributed in the hope that it will be useful, but WITHOUT
18  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
19  * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
20  * details.
21  *
22  * You should have received a copy of the GNU Lesser General Public License
23  * along with this file; if not, see http://www.gnu.org/licenses/.
24  */
25
26 #ifdef HAVE_CONFIG_H
27 #  include "config.h"
28 #endif
29
30 #include "wimlib/assert.h"
31 #include "wimlib/endianness.h"
32 #include "wimlib/error.h"
33 #include "wimlib/file_io.h"
34 #include "wimlib/integrity.h"
35 #include "wimlib/progress.h"
36 #include "wimlib/resource.h"
37 #include "wimlib/sha1.h"
38 #include "wimlib/wim.h"
39 #include "wimlib/write.h"
40
41 /* Size, in bytes, of each SHA1-summed chunk, when wimlib writes integrity
42  * information. */
43 #define INTEGRITY_CHUNK_SIZE 10485760
44
45 /* Only use a different chunk size for compatibility with an existing integrity
46  * table if the chunk size is between these two numbers. */
47 #define INTEGRITY_MIN_CHUNK_SIZE 4096
48 #define INTEGRITY_MAX_CHUNK_SIZE 134217728
49
50 struct integrity_table {
51         u32 size;
52         u32 num_entries;
53         u32 chunk_size;
54         u8  sha1sums[][20];
55 } _packed_attribute;
56
57 static int
58 calculate_chunk_sha1(struct filedes *in_fd, size_t this_chunk_size,
59                      off_t offset, u8 sha1_md[])
60 {
61         u8 buf[BUFFER_SIZE];
62         SHA_CTX ctx;
63         size_t bytes_remaining;
64         size_t bytes_to_read;
65         int ret;
66
67         bytes_remaining = this_chunk_size;
68         sha1_init(&ctx);
69         do {
70                 bytes_to_read = min(bytes_remaining, sizeof(buf));
71                 ret = full_pread(in_fd, buf, bytes_to_read, offset);
72                 if (ret) {
73                         ERROR_WITH_ERRNO("Read error while calculating "
74                                          "integrity checksums");
75                         return ret;
76                 }
77                 sha1_update(&ctx, buf, bytes_to_read);
78                 bytes_remaining -= bytes_to_read;
79                 offset += bytes_to_read;
80         } while (bytes_remaining);
81         sha1_final(sha1_md, &ctx);
82         return 0;
83 }
84
85
86 /*
87  * read_integrity_table: -  Reads the integrity table from a WIM file.
88  *
89  * @wim:
90  *      WIMStruct for the WIM file; @wim->hdr.integrity_table_reshdr specifies
91  *      the location of the integrity table.  @wim->in_fd is expected to be a
92  *      seekable file descriptor to the WIM file opened for reading.
93  *
94  * @num_checked_bytes:
95  *      Number of bytes of data that should be checked by the integrity table.
96  *
97  * @table_ret:
98  *      On success, a pointer to an in-memory structure containing the integrity
99  *      information is written to this location.
100  *
101  * Return values:
102  *      WIMLIB_ERR_SUCCESS (0)
103  *      WIMLIB_ERR_INVALID_INTEGRITY_TABLE
104  *      WIMLIB_ERR_NOMEM
105  *      WIMLIB_ERR_READ
106  *      WIMLIB_ERR_UNEXPECTED_END_OF_FILE
107  */
108 int
109 read_integrity_table(WIMStruct *wim, u64 num_checked_bytes,
110                      struct integrity_table **table_ret)
111 {
112         void *buf;
113         struct integrity_table *table;
114         int ret;
115
116         STATIC_ASSERT(sizeof(struct integrity_table) == 12);
117         if (wim->hdr.integrity_table_reshdr.uncompressed_size < 12)
118                 return WIMLIB_ERR_INVALID_INTEGRITY_TABLE;
119
120         ret = wim_reshdr_to_data(&wim->hdr.integrity_table_reshdr, wim, &buf);
121         if (ret)
122                 return ret;
123         table = buf;
124
125         table->size        = le32_to_cpu((_force_attr le32)table->size);
126         table->num_entries = le32_to_cpu((_force_attr le32)table->num_entries);
127         table->chunk_size  = le32_to_cpu((_force_attr le32)table->chunk_size);
128
129         if (table->size != wim->hdr.integrity_table_reshdr.uncompressed_size ||
130             table->size != (u64)table->num_entries * SHA1_HASH_SIZE + 12 ||
131             table->chunk_size == 0 ||
132             table->num_entries != DIV_ROUND_UP(num_checked_bytes, table->chunk_size))
133         {
134                 FREE(table);
135                 return WIMLIB_ERR_INVALID_INTEGRITY_TABLE;
136         }
137
138         *table_ret = table;
139         return 0;
140 }
141
142 /*
143  * calculate_integrity_table():
144  *
145  * Calculates an integrity table for the data in a file beginning at offset 208
146  * (WIM_HEADER_DISK_SIZE).
147  *
148  * @in_fd:
149  *      File descriptor for the file to be checked, opened for reading.  Does
150  *      not need to be at any specific location in the file.
151  *
152  * @new_check_end:
153  *      Offset of byte after the last byte to be checked.
154  *
155  * @old_table:
156  *      If non-NULL, a pointer to the table containing the previously calculated
157  *      integrity data for a prefix of this file.
158  *
159  * @old_check_end:
160  *      If @old_table is non-NULL, the byte after the last byte that was checked
161  *      in the old table.  Must be less than or equal to new_check_end.
162  *
163  * @integrity_table_ret:
164  *      On success, a pointer to the calculated integrity table is written into
165  *      this location.
166  *
167  * Return values:
168  *      WIMLIB_ERR_SUCCESS (0)
169  *      WIMLIB_ERR_NOMEM
170  *      WIMLIB_ERR_READ
171  *      WIMLIB_ERR_UNEXPECTED_END_OF_FILE
172  */
173 static int
174 calculate_integrity_table(struct filedes *in_fd,
175                           off_t new_check_end,
176                           const struct integrity_table *old_table,
177                           off_t old_check_end,
178                           struct integrity_table **integrity_table_ret,
179                           wimlib_progress_func_t progfunc,
180                           void *progctx)
181 {
182         int ret;
183         size_t chunk_size = INTEGRITY_CHUNK_SIZE;
184
185         /* If an old table is provided, set the chunk size to be compatible with
186          * the old chunk size, unless the old chunk size was weird. */
187         if (old_table != NULL) {
188                 if (old_table->num_entries == 0 ||
189                     old_table->chunk_size < INTEGRITY_MIN_CHUNK_SIZE ||
190                     old_table->chunk_size > INTEGRITY_MAX_CHUNK_SIZE)
191                         old_table = NULL;
192                 else
193                         chunk_size = old_table->chunk_size;
194         }
195
196
197         u64 old_check_bytes = old_check_end - WIM_HEADER_DISK_SIZE;
198         u64 new_check_bytes = new_check_end - WIM_HEADER_DISK_SIZE;
199
200         u32 old_num_chunks = DIV_ROUND_UP(old_check_bytes, chunk_size);
201         u32 new_num_chunks = DIV_ROUND_UP(new_check_bytes, chunk_size);
202
203         size_t old_last_chunk_size = MODULO_NONZERO(old_check_bytes, chunk_size);
204         size_t new_last_chunk_size = MODULO_NONZERO(new_check_bytes, chunk_size);
205
206         size_t new_table_size = 12 + new_num_chunks * SHA1_HASH_SIZE;
207
208         struct integrity_table *new_table = MALLOC(new_table_size);
209         if (!new_table)
210                 return WIMLIB_ERR_NOMEM;
211         new_table->num_entries = new_num_chunks;
212         new_table->size = new_table_size;
213         new_table->chunk_size = chunk_size;
214
215         u64 offset = WIM_HEADER_DISK_SIZE;
216         union wimlib_progress_info progress;
217
218         progress.integrity.total_bytes      = new_check_bytes;
219         progress.integrity.total_chunks     = new_num_chunks;
220         progress.integrity.completed_chunks = 0;
221         progress.integrity.completed_bytes  = 0;
222         progress.integrity.chunk_size       = chunk_size;
223         progress.integrity.filename         = NULL;
224
225         ret = call_progress(progfunc, WIMLIB_PROGRESS_MSG_CALC_INTEGRITY,
226                             &progress, progctx);
227         if (ret)
228                 goto out_free_new_table;
229
230         for (u32 i = 0; i < new_num_chunks; i++) {
231                 size_t this_chunk_size;
232                 if (i == new_num_chunks - 1)
233                         this_chunk_size = new_last_chunk_size;
234                 else
235                         this_chunk_size = chunk_size;
236                 if (old_table &&
237                     ((this_chunk_size == chunk_size && i < old_num_chunks - 1) ||
238                       (i == old_num_chunks - 1 && this_chunk_size == old_last_chunk_size)))
239                 {
240                         /* Can use SHA1 message digest from old integrity table
241                          * */
242                         copy_hash(new_table->sha1sums[i], old_table->sha1sums[i]);
243                 } else {
244                         /* Calculate the SHA1 message digest of this chunk */
245                         ret = calculate_chunk_sha1(in_fd, this_chunk_size,
246                                                    offset, new_table->sha1sums[i]);
247                         if (ret)
248                                 goto out_free_new_table;
249                 }
250                 offset += this_chunk_size;
251
252                 progress.integrity.completed_chunks++;
253                 progress.integrity.completed_bytes += this_chunk_size;
254                 ret = call_progress(progfunc, WIMLIB_PROGRESS_MSG_CALC_INTEGRITY,
255                                     &progress, progctx);
256                 if (ret)
257                         goto out_free_new_table;
258         }
259         *integrity_table_ret = new_table;
260         return 0;
261
262 out_free_new_table:
263         FREE(new_table);
264         return ret;
265 }
266
267 /*
268  * write_integrity_table():
269  *
270  * Writes a WIM integrity table (a list of SHA1 message digests of raw 10 MiB
271  * chunks of the file).
272  *
273  * This function can optionally re-use entries from an older integrity table.
274  * To do this, specify old_blob_table_end and old_table.
275  *
276  * On success, @wim->out_hdr.integrity_table_reshdr will be filled in with
277  * information about the integrity table that was written.
278  *
279  * @wim:
280  *      WIMStruct for the WIM file.  @wim->out_fd must be a seekable descriptor
281  *      to the new WIM file, opened read-write, positioned at the location at
282  *      which the integrity table is to be written.
283  *
284  * @new_blob_table_end:
285  *      The offset of the byte directly following the blob table in the WIM
286  *      being written.
287  *
288  * @old_blob_table_end:
289  *      If nonzero, the offset of the byte directly following the old blob table
290  *      in the WIM.
291  *
292  * @old_table
293  *      Pointer to the old integrity table read into memory, or NULL if not
294  *      specified.
295  */
296 int
297 write_integrity_table(WIMStruct *wim,
298                       off_t new_blob_table_end,
299                       off_t old_blob_table_end,
300                       struct integrity_table *old_table)
301 {
302         struct integrity_table *new_table;
303         int ret;
304         u32 new_table_size;
305
306         wimlib_assert(old_blob_table_end <= new_blob_table_end);
307
308         ret = calculate_integrity_table(&wim->out_fd, new_blob_table_end,
309                                         old_table, old_blob_table_end,
310                                         &new_table, wim->progfunc, wim->progctx);
311         if (ret)
312                 return ret;
313
314         new_table_size = new_table->size;
315
316         new_table->size        = (_force_attr u32)cpu_to_le32(new_table->size);
317         new_table->num_entries = (_force_attr u32)cpu_to_le32(new_table->num_entries);
318         new_table->chunk_size  = (_force_attr u32)cpu_to_le32(new_table->chunk_size);
319
320         ret = write_wim_resource_from_buffer(new_table,
321                                              new_table_size,
322                                              false,
323                                              &wim->out_fd,
324                                              WIMLIB_COMPRESSION_TYPE_NONE,
325                                              0,
326                                              &wim->out_hdr.integrity_table_reshdr,
327                                              NULL,
328                                              0);
329         FREE(new_table);
330         return ret;
331 }
332
333 /*
334  * verify_integrity():
335  *
336  * Checks a WIM for consistency with the integrity table.
337  *
338  * @in_fd:
339  *      File descriptor to the WIM file, opened for reading.
340  *
341  * @table:
342  *      The integrity table for the WIM, read into memory.
343  *
344  * @bytes_to_check:
345  *      Number of bytes in the WIM that need to be checked (offset of end of the
346  *      blob table minus offset of end of the header).
347  *
348  * Returns:
349  *      > 0 (WIMLIB_ERR_READ, WIMLIB_ERR_UNEXPECTED_END_OF_FILE) on error
350  *      0 (WIM_INTEGRITY_OK) if the integrity was checked successfully and there
351  *      were no inconsistencies.
352  *      -1 (WIM_INTEGRITY_NOT_OK) if the WIM failed the integrity check.
353  */
354 static int
355 verify_integrity(struct filedes *in_fd, const tchar *filename,
356                  const struct integrity_table *table,
357                  u64 bytes_to_check,
358                  wimlib_progress_func_t progfunc, void *progctx)
359 {
360         int ret;
361         u64 offset = WIM_HEADER_DISK_SIZE;
362         u8 sha1_md[SHA1_HASH_SIZE];
363         union wimlib_progress_info progress;
364
365         progress.integrity.total_bytes      = bytes_to_check;
366         progress.integrity.total_chunks     = table->num_entries;
367         progress.integrity.completed_chunks = 0;
368         progress.integrity.completed_bytes  = 0;
369         progress.integrity.chunk_size       = table->chunk_size;
370         progress.integrity.filename         = filename;
371
372         ret = call_progress(progfunc, WIMLIB_PROGRESS_MSG_VERIFY_INTEGRITY,
373                             &progress, progctx);
374         if (ret)
375                 return ret;
376
377         for (u32 i = 0; i < table->num_entries; i++) {
378                 size_t this_chunk_size;
379                 if (i == table->num_entries - 1)
380                         this_chunk_size = MODULO_NONZERO(bytes_to_check,
381                                                          table->chunk_size);
382                 else
383                         this_chunk_size = table->chunk_size;
384
385                 ret = calculate_chunk_sha1(in_fd, this_chunk_size, offset, sha1_md);
386                 if (ret)
387                         return ret;
388
389                 if (!hashes_equal(sha1_md, table->sha1sums[i]))
390                         return WIM_INTEGRITY_NOT_OK;
391
392                 offset += this_chunk_size;
393                 progress.integrity.completed_chunks++;
394                 progress.integrity.completed_bytes += this_chunk_size;
395
396                 ret = call_progress(progfunc, WIMLIB_PROGRESS_MSG_VERIFY_INTEGRITY,
397                                     &progress, progctx);
398                 if (ret)
399                         return ret;
400         }
401         return WIM_INTEGRITY_OK;
402 }
403
404
405 /*
406  * check_wim_integrity():
407  *
408  * Verifies the integrity of the WIM by making sure the SHA1 message digests of
409  * ~10 MiB chunks of the WIM match up with the values given in the integrity
410  * table.
411  *
412  * @wim:
413  *      The WIM, opened for reading.
414  *
415  * Returns:
416  *      > 0 (WIMLIB_ERR_INVALID_INTEGRITY_TABLE, WIMLIB_ERR_READ,
417  *           WIMLIB_ERR_UNEXPECTED_END_OF_FILE) on error
418  *      0 (WIM_INTEGRITY_OK) if the integrity was checked successfully and there
419  *      were no inconsistencies.
420  *      -1 (WIM_INTEGRITY_NOT_OK) if the WIM failed the integrity check.
421  *      -2 (WIM_INTEGRITY_NONEXISTENT) if the WIM contains no integrity
422  *      information.
423  */
424 int
425 check_wim_integrity(WIMStruct *wim)
426 {
427         int ret;
428         u64 bytes_to_check;
429         struct integrity_table *table;
430         u64 end_blob_table_offset;
431
432         if (!wim_has_integrity_table(wim))
433                 return WIM_INTEGRITY_NONEXISTENT;
434
435         end_blob_table_offset = wim->hdr.blob_table_reshdr.offset_in_wim +
436                                 wim->hdr.blob_table_reshdr.size_in_wim;
437
438         if (end_blob_table_offset < WIM_HEADER_DISK_SIZE) {
439                 ERROR("WIM blob table ends before WIM header ends!");
440                 return WIMLIB_ERR_INVALID_INTEGRITY_TABLE;
441         }
442
443         bytes_to_check = end_blob_table_offset - WIM_HEADER_DISK_SIZE;
444
445         ret = read_integrity_table(wim, bytes_to_check, &table);
446         if (ret)
447                 return ret;
448         ret = verify_integrity(&wim->in_fd, wim->filename, table,
449                                bytes_to_check, wim->progfunc, wim->progctx);
450         FREE(table);
451         return ret;
452 }