Cleanup WIM writing
[wimlib] / src / integrity.c
1 /*
2  * integrity.c
3  *
4  * WIM files can optionally contain an array of SHA1 message digests at the end,
5  * one digest for each 1 MB of the file.  This file implements the checking of
6  * the digests, and the writing of the digests for new WIM files.
7  */
8
9 /*
10  * Copyright (C) 2012 Eric Biggers
11  *
12  * This file is part of wimlib, a library for working with WIM files.
13  *
14  * wimlib is free software; you can redistribute it and/or modify it under the
15  * terms of the GNU General Public License as published by the Free
16  * Software Foundation; either version 3 of the License, or (at your option)
17  * any later version.
18  *
19  * wimlib is distributed in the hope that it will be useful, but WITHOUT ANY
20  * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
21  * A PARTICULAR PURPOSE. See the GNU General Public License for more
22  * details.
23  *
24  * You should have received a copy of the GNU General Public License
25  * along with wimlib; if not, see http://www.gnu.org/licenses/.
26  */
27
28 #include "wimlib_internal.h"
29 #include "io.h"
30 #include "sha1.h"
31
32 /* Size, in bytes, of each SHA1-summed chunk, when wimlib writes integrity
33  * information. */
34 #define INTEGRITY_CHUNK_SIZE 10485760
35
36 /*
37  * Verifies the integrity of a WIM.
38  *
39  * @fp:            FILE* of the WIM, currently positioned at the end of the header.
40  * @num_bytes:     Number of bytes to verify the integrity of.
41  * @chunk_size:    Chunk size per SHA1 message digest.
42  * @sha1sums:      Array of SHA1 message digests; 20 bytes each, one per chunk.
43  * @show_progress: Nonzero if the percent complete is to be printed after every
44  *                      chunk.
45  * @status:        On success, set to WIM_INTEGRITY_OK or WIM_INTEGRITY_NOT_OK
46  *                      based on whether the WIM is intact or not.
47  */
48 static int verify_integrity(FILE *fp, u64 num_bytes, u32 chunk_size,
49                             const u8 *sha1sums, int show_progress,
50                             int *status)
51 {
52         u8    *chunk_buf;
53         u8     resblock[SHA1_HASH_SIZE];
54         u64    bytes_remaining;
55         size_t bytes_to_read;
56         uint   percent_done;
57         int    ret;
58
59         chunk_buf = MALLOC(chunk_size);
60         if (!chunk_buf) {
61                 ERROR("Failed to allocate %u byte buffer for integrity chunks",
62                       chunk_size);
63                 return WIMLIB_ERR_NOMEM;
64         }
65         bytes_remaining = num_bytes;
66         while (bytes_remaining != 0) {
67                 if (show_progress) {
68                         percent_done = (num_bytes - bytes_remaining) * 100 /
69                                         num_bytes;
70                         printf("Verifying integrity of WIM (%"PRIu64" bytes "
71                                         "remaining, %u%% done)       \r",
72                                         bytes_remaining, percent_done);
73                         fflush(stdout);
74                 }
75                 bytes_to_read = min(chunk_size, bytes_remaining);
76                 if (fread(chunk_buf, 1, bytes_to_read, fp) != bytes_to_read) {
77                         if (feof(fp)) {
78                                 ERROR("Unexpected EOF while verifying "
79                                       "integrity of WIM");
80                         } else {
81                                 ERROR_WITH_ERRNO("File stream error while "
82                                                  "verifying integrity of WIM");
83                         }
84                         ret = WIMLIB_ERR_READ;
85                         goto verify_integrity_error;
86                 }
87                 sha1_buffer(chunk_buf, bytes_to_read, resblock);
88                 if (!hashes_equal(resblock, sha1sums)) {
89                         *status = WIM_INTEGRITY_NOT_OK;
90                         goto verify_integrity_done;
91                 }
92                 sha1sums += SHA1_HASH_SIZE;
93                 bytes_remaining -= bytes_to_read;
94         }
95         *status = WIM_INTEGRITY_OK;
96 verify_integrity_done:
97         ret = 0;
98 verify_integrity_error:
99         FREE(chunk_buf);
100         if (show_progress)
101                 putchar('\n');
102         return ret;
103 }
104
105 /*
106  * Verifies the integrity of the WIM.
107  *
108  * @show_progress: Nonzero if the percent complete is to be printed after every
109  *                      chunk.
110  * @status:        On success, set to WIM_INTEGRITY_OK, WIM_INTEGRITY_NOT_OK,
111  *                      or WIM_INTEGRITY_NONEXISTENT.
112  *
113  * Returns: 0, WIMLIB_ERR_INVALID_INTEGRITY_TABLE, WIMLIB_ERR_NOMEM, or
114  * WIMLIB_ERR_READ.  If nonzero, the boolean pointed to by @ok is not changed.
115  */
116 int check_wim_integrity(WIMStruct *w, int show_progress, int *status)
117 {
118
119         struct resource_entry *res_entry;
120         u8 *buf = NULL;
121         int ret;
122         u32 integrity_table_size;
123         u32 num_entries;
124         u32 chunk_size;
125         const u8 *p;
126         u64 expected_size;
127         u64 end_lookup_table_offset;
128         u64 bytes_to_check;
129         u64 expected_num_entries;
130
131         res_entry = &w->hdr.integrity;
132         if (res_entry->size == 0) {
133                 DEBUG("No integrity information.");
134                 *status = WIM_INTEGRITY_NONEXISTENT;
135                 return 0;
136         }
137         if (res_entry->original_size < 12) {
138                 ERROR("Integrity table is too short");
139                 return WIMLIB_ERR_INVALID_INTEGRITY_TABLE;
140         }
141         if (res_entry->flags & WIM_RESHDR_FLAG_COMPRESSED) {
142                 ERROR("Didn't expect a compressed integrity table");
143                 return WIMLIB_ERR_INVALID_INTEGRITY_TABLE;
144         }
145
146         /* Read the integrity table into memory. */
147         buf = MALLOC(res_entry->original_size);
148         if (!buf) {
149                 ERROR("Out of memory (needed %zu bytes for integrity table)",
150                       res_entry->original_size);
151                 ret = WIMLIB_ERR_NOMEM;
152                 goto out;
153         }
154         ret = read_uncompressed_resource(w->fp, res_entry->offset,
155                                          res_entry->original_size, buf);
156         if (ret != 0) {
157                 ERROR("Failed to read integrity table (size = %"PRIu64", "
158                       "original_size = %"PRIu64", offset = "
159                       "%"PRIu64")",
160                       (u64)res_entry->size, res_entry->original_size,
161                       res_entry->offset);
162                 goto out;
163         }
164
165         p = get_u32(buf, &integrity_table_size);
166         p = get_u32(p, &num_entries);
167         p = get_u32(p, &chunk_size);
168
169         /* p now points to the array of SHA1 message digests for the WIM. */
170
171         /* Make sure the integrity table is the right size. */
172         if (integrity_table_size != res_entry->original_size) {
173                 ERROR("Inconsistent integrity table sizes: header says %u "
174                       "bytes but resource entry says "
175                       "%"PRIu64" bytes",
176                       integrity_table_size, res_entry->original_size);
177                 ret = WIMLIB_ERR_INVALID_INTEGRITY_TABLE;
178                 goto out;
179         }
180
181         DEBUG("integrity_table_size = %u, num_entries = %u, chunk_size = %u",
182               integrity_table_size, num_entries, chunk_size);
183
184
185         expected_size = num_entries * SHA1_HASH_SIZE + 12;
186
187         if (integrity_table_size != expected_size) {
188                 ERROR("Integrity table is %u bytes, but expected %"PRIu64" "
189                       "bytes to hold %u entries",
190                       integrity_table_size, expected_size, num_entries);
191                 ret = WIMLIB_ERR_INVALID_INTEGRITY_TABLE;
192                 goto out;
193         }
194
195         if (chunk_size == 0) {
196                 ERROR("Cannot use integrity chunk size of 0");
197                 ret = WIMLIB_ERR_INVALID_INTEGRITY_TABLE;
198                 goto out;
199         }
200
201         end_lookup_table_offset = w->hdr.lookup_table_res_entry.offset +
202                                   w->hdr.lookup_table_res_entry.size;
203
204         if (end_lookup_table_offset < WIM_HEADER_DISK_SIZE) {
205                 ERROR("WIM lookup table ends before WIM header ends???");
206                 ret = WIMLIB_ERR_INVALID_INTEGRITY_TABLE;
207                 goto out;
208         }
209
210         bytes_to_check = end_lookup_table_offset - WIM_HEADER_DISK_SIZE;
211
212         expected_num_entries = (bytes_to_check + chunk_size - 1) / chunk_size;
213
214         if (num_entries != expected_num_entries) {
215                 ERROR("%"PRIu64" entries would be required to checksum "
216                       "the %"PRIu64" bytes from the end of the header to the",
217                       expected_num_entries, bytes_to_check);
218                 ERROR("end of the lookup table with a chunk size of %u, but "
219                       "there were only %u entries", chunk_size, num_entries);
220                 ret = WIMLIB_ERR_INVALID_INTEGRITY_TABLE;
221                 goto out;
222         }
223
224         /* The integrity checking starts after the header, so seek to the offset
225          * in the WIM after the header. */
226
227         if (fseeko(w->fp, WIM_HEADER_DISK_SIZE, SEEK_SET) != 0) {
228                 ERROR_WITH_ERRNO("Failed to seek to byte %u of WIM to check "
229                                  "integrity", WIM_HEADER_DISK_SIZE);
230                 ret = WIMLIB_ERR_READ;
231                 goto out;
232         }
233         /* call verify_integrity(), which does the actual checking of the SHA1
234          * message digests. */
235         ret = verify_integrity(w->fp, bytes_to_check, chunk_size, p,
236                                show_progress, status);
237 out:
238         FREE(buf);
239         return ret;
240 }
241
242 /*
243  * Writes integrity information to the output stream for a WIM file being
244  * written.
245  *
246  * @end_header_offset is the offset of the byte after the header, which is the
247  *      beginning of the region that is checksummed.
248  *
249  * @end_lookup_table_offset is the offset of the byte after the lookup table,
250  *      which is the end of the region that is checksummed.
251  */
252 int write_integrity_table(FILE *out, u64 end_header_offset,
253                           u64 end_lookup_table_offset, int show_progress)
254 {
255         u64  bytes_to_check;
256         u64  bytes_remaining;
257         u8  *buf;
258         u8  *p;
259         u8  *chunk_buf;
260         u32  num_entries;
261         u32  integrity_table_size;
262         int  ret;
263
264         DEBUG("Calculating integrity table");
265         if (fseeko(out, end_header_offset, SEEK_SET) != 0) {
266                 ERROR_WITH_ERRNO("Failed to seek to byte %"PRIu64" of WIM to "
267                                  "calculate integrity data", end_header_offset);
268                 return WIMLIB_ERR_WRITE;
269         }
270
271         bytes_to_check = end_lookup_table_offset - end_header_offset;
272         num_entries = (bytes_to_check + INTEGRITY_CHUNK_SIZE - 1) /
273                         INTEGRITY_CHUNK_SIZE;
274         integrity_table_size = num_entries * SHA1_HASH_SIZE + 3 * sizeof(u32);
275
276         DEBUG("integrity_table_size = %u", integrity_table_size);
277
278         buf = MALLOC(integrity_table_size);
279         if (!buf) {
280                 ERROR("Failed to allocate %u bytes for integrity table",
281                       integrity_table_size);
282                 return WIMLIB_ERR_NOMEM;
283         }
284
285         p = put_u32(buf, integrity_table_size);
286         p = put_u32(p, num_entries);
287         p = put_u32(p, INTEGRITY_CHUNK_SIZE);
288
289         chunk_buf = MALLOC(INTEGRITY_CHUNK_SIZE);
290         if (!chunk_buf) {
291                 ERROR("Failed to allocate %u bytes for integrity chunk buffer",
292                       INTEGRITY_CHUNK_SIZE);
293                 ret = WIMLIB_ERR_NOMEM;
294                 goto out_free_buf;
295         }
296
297         bytes_remaining = bytes_to_check;
298
299         DEBUG("Bytes to check = %"PRIu64, bytes_to_check);
300
301         while (bytes_remaining != 0) {
302
303                 uint percent_done = (bytes_to_check - bytes_remaining) *
304                                     100 / bytes_to_check;
305
306                 if (show_progress) {
307                         printf("Calculating integrity checksums for WIM "
308                                         "(%"PRIu64" bytes remaining, %u%% "
309                                         "done)      \r",
310                                         bytes_remaining, percent_done);
311                         fflush(stdout);
312                 }
313
314
315                 size_t bytes_to_read = min(INTEGRITY_CHUNK_SIZE, bytes_remaining);
316                 size_t bytes_read = fread(chunk_buf, 1, bytes_to_read, out);
317                 if (bytes_read != bytes_to_read) {
318                         if (feof(out)) {
319                                 ERROR("Unexpected EOF while calculating "
320                                       "integrity checksums");
321                         } else {
322                                 ERROR_WITH_ERRNO("File stream error while "
323                                                  "calculating integrity "
324                                                  "checksums");
325                         }
326                         ret = WIMLIB_ERR_READ;
327                         goto out_free_chunk_buf;
328                 }
329                 sha1_buffer(chunk_buf, bytes_read, p);
330                 p += SHA1_HASH_SIZE;
331                 bytes_remaining -= bytes_read;
332         }
333         if (show_progress)
334                 puts("Calculating integrity checksums for WIM "
335                                 "(0 bytes remaining, 100% done)"
336                                 "                       ");
337
338         if (fseeko(out, 0, SEEK_END) != 0) {
339                 ERROR_WITH_ERRNO("Failed to seek to end of WIM to write "
340                                  "integrity table");
341                 ret = WIMLIB_ERR_WRITE;
342                 goto out_free_chunk_buf;
343         }
344
345         if (fwrite(buf, 1, integrity_table_size, out) != integrity_table_size) {
346                 ERROR_WITH_ERRNO("Failed to write integrity table to end of "
347                                  "WIM");
348                 ret = WIMLIB_ERR_WRITE;
349                 goto out_free_chunk_buf;
350         }
351         ret = 0;
352 out_free_chunk_buf:
353         FREE(chunk_buf);
354 out_free_buf:
355         FREE(buf);
356         return ret;
357 }