wimlib.net Git - wimlib/blob - src/integrity.c

   1 /*
   2  * integrity.c
   3  *
   4  * WIM files can optionally contain a table of SHA1 message digests at the end,
   5  * one digest for each chunk of the file of some specified size (often 10 MB).
   6  * This file implements the checking and writing of this table.
   7  */
   8
   9 /*
  10  * Copyright (C) 2012-2016 Eric Biggers
  11  *
  12  * This file is free software; you can redistribute it and/or modify it under
  13  * the terms of the GNU Lesser General Public License as published by the Free
  14  * Software Foundation; either version 3 of the License, or (at your option) any
  15  * later version.
  16  *
  17  * This file is distributed in the hope that it will be useful, but WITHOUT
  18  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
  19  * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
  20  * details.
  21  *
  22  * You should have received a copy of the GNU Lesser General Public License
  23  * along with this file; if not, see http://www.gnu.org/licenses/.
  24  */
  25
  26 #ifdef HAVE_CONFIG_H
  27 #  include "config.h"
  28 #endif
  29
  30 #include "wimlib/assert.h"
  31 #include "wimlib/endianness.h"
  32 #include "wimlib/error.h"
  33 #include "wimlib/file_io.h"
  34 #include "wimlib/integrity.h"
  35 #include "wimlib/progress.h"
  36 #include "wimlib/resource.h"
  37 #include "wimlib/sha1.h"
  38 #include "wimlib/wim.h"
  39 #include "wimlib/write.h"
  40
  41 /* Size, in bytes, of each SHA1-summed chunk, when wimlib writes integrity
  42  * information. */
  43 #define INTEGRITY_CHUNK_SIZE 10485760
  44
  45 /* Only use a different chunk size for compatibility with an existing integrity
  46  * table if the chunk size is between these two numbers. */
  47 #define INTEGRITY_MIN_CHUNK_SIZE 4096
  48 #define INTEGRITY_MAX_CHUNK_SIZE 134217728
  49
  50 struct integrity_table {
  51         u32 size;
  52         u32 num_entries;
  53         u32 chunk_size;
  54         u8  sha1sums[][20];
  55 } _packed_attribute;
  56
  57 static int
  58 calculate_chunk_sha1(struct filedes *in_fd, size_t this_chunk_size,
  59                      off_t offset, u8 sha1_md[])
  60 {
  61         u8 buf[BUFFER_SIZE];
  62         SHA_CTX ctx;
  63         size_t bytes_remaining;
  64         size_t bytes_to_read;
  65         int ret;
  66
  67         bytes_remaining = this_chunk_size;
  68         sha1_init(&ctx);
  69         do {
  70                 bytes_to_read = min(bytes_remaining, sizeof(buf));
  71                 ret = full_pread(in_fd, buf, bytes_to_read, offset);
  72                 if (ret) {
  73                         ERROR_WITH_ERRNO("Read error while calculating "
  74                                          "integrity checksums");
  75                         return ret;
  76                 }
  77                 sha1_update(&ctx, buf, bytes_to_read);
  78                 bytes_remaining -= bytes_to_read;
  79                 offset += bytes_to_read;
  80         } while (bytes_remaining);
  81         sha1_final(sha1_md, &ctx);
  82         return 0;
  83 }
  84
  85
  86 /*
  87  * read_integrity_table: -  Reads the integrity table from a WIM file.
  88  *
  89  * @wim:
  90  *      WIMStruct for the WIM file; @wim->hdr.integrity_table_reshdr specifies
  91  *      the location of the integrity table.  @wim->in_fd is expected to be a
  92  *      seekable file descriptor to the WIM file opened for reading.
  93  *
  94  * @num_checked_bytes:
  95  *      Number of bytes of data that should be checked by the integrity table.
  96  *
  97  * @table_ret:
  98  *      On success, a pointer to an in-memory structure containing the integrity
  99  *      information is written to this location.
 100  *
 101  * Return values:
 102  *      WIMLIB_ERR_SUCCESS (0)
 103  *      WIMLIB_ERR_INVALID_INTEGRITY_TABLE
 104  *      WIMLIB_ERR_NOMEM
 105  *      WIMLIB_ERR_READ
 106  *      WIMLIB_ERR_UNEXPECTED_END_OF_FILE
 107  */
 108 int
 109 read_integrity_table(WIMStruct *wim, u64 num_checked_bytes,
 110                      struct integrity_table **table_ret)
 111 {
 112         void *buf;
 113         struct integrity_table *table;
 114         int ret;
 115
 116         STATIC_ASSERT(sizeof(struct integrity_table) == 12);
 117         if (wim->hdr.integrity_table_reshdr.uncompressed_size < 12)
 118                 return WIMLIB_ERR_INVALID_INTEGRITY_TABLE;
 119
 120         ret = wim_reshdr_to_data(&wim->hdr.integrity_table_reshdr, wim, &buf);
 121         if (ret)
 122                 return ret;
 123         table = buf;
 124
 125         table->size        = le32_to_cpu(table->size);
 126         table->num_entries = le32_to_cpu(table->num_entries);
 127         table->chunk_size  = le32_to_cpu(table->chunk_size);
 128
 129         if (table->size != wim->hdr.integrity_table_reshdr.uncompressed_size ||
 130             table->size != (u64)table->num_entries * SHA1_HASH_SIZE + 12 ||
 131             table->chunk_size == 0 ||
 132             table->num_entries != DIV_ROUND_UP(num_checked_bytes, table->chunk_size))
 133         {
 134                 FREE(table);
 135                 return WIMLIB_ERR_INVALID_INTEGRITY_TABLE;
 136         }
 137
 138         *table_ret = table;
 139         return 0;
 140 }
 141
 142 /*
 143  * calculate_integrity_table():
 144  *
 145  * Calculates an integrity table for the data in a file beginning at offset 208
 146  * (WIM_HEADER_DISK_SIZE).
 147  *
 148  * @in_fd:
 149  *      File descriptor for the file to be checked, opened for reading.  Does
 150  *      not need to be at any specific location in the file.
 151  *
 152  * @new_check_end:
 153  *      Offset of byte after the last byte to be checked.
 154  *
 155  * @old_table:
 156  *      If non-NULL, a pointer to the table containing the previously calculated
 157  *      integrity data for a prefix of this file.
 158  *
 159  * @old_check_end:
 160  *      If @old_table is non-NULL, the byte after the last byte that was checked
 161  *      in the old table.  Must be less than or equal to new_check_end.
 162  *
 163  * @integrity_table_ret:
 164  *      On success, a pointer to the calculated integrity table is written into
 165  *      this location.
 166  *
 167  * Return values:
 168  *      WIMLIB_ERR_SUCCESS (0)
 169  *      WIMLIB_ERR_NOMEM
 170  *      WIMLIB_ERR_READ
 171  *      WIMLIB_ERR_UNEXPECTED_END_OF_FILE
 172  */
 173 static int
 174 calculate_integrity_table(struct filedes *in_fd,
 175                           off_t new_check_end,
 176                           const struct integrity_table *old_table,
 177                           off_t old_check_end,
 178                           struct integrity_table **integrity_table_ret,
 179                           wimlib_progress_func_t progfunc,
 180                           void *progctx)
 181 {
 182         int ret;
 183         size_t chunk_size = INTEGRITY_CHUNK_SIZE;
 184
 185         /* If an old table is provided, set the chunk size to be compatible with
 186          * the old chunk size, unless the old chunk size was weird. */
 187         if (old_table != NULL) {
 188                 if (old_table->num_entries == 0 ||
 189                     old_table->chunk_size < INTEGRITY_MIN_CHUNK_SIZE ||
 190                     old_table->chunk_size > INTEGRITY_MAX_CHUNK_SIZE)
 191                         old_table = NULL;
 192                 else
 193                         chunk_size = old_table->chunk_size;
 194         }
 195
 196
 197         u64 old_check_bytes = old_check_end - WIM_HEADER_DISK_SIZE;
 198         u64 new_check_bytes = new_check_end - WIM_HEADER_DISK_SIZE;
 199
 200         u32 old_num_chunks = DIV_ROUND_UP(old_check_bytes, chunk_size);
 201         u32 new_num_chunks = DIV_ROUND_UP(new_check_bytes, chunk_size);
 202
 203         size_t old_last_chunk_size = MODULO_NONZERO(old_check_bytes, chunk_size);
 204         size_t new_last_chunk_size = MODULO_NONZERO(new_check_bytes, chunk_size);
 205
 206         size_t new_table_size = 12 + new_num_chunks * SHA1_HASH_SIZE;
 207
 208         struct integrity_table *new_table = MALLOC(new_table_size);
 209         if (!new_table)
 210                 return WIMLIB_ERR_NOMEM;
 211         new_table->num_entries = new_num_chunks;
 212         new_table->size = new_table_size;
 213         new_table->chunk_size = chunk_size;
 214
 215         u64 offset = WIM_HEADER_DISK_SIZE;
 216         union wimlib_progress_info progress;
 217
 218         progress.integrity.total_bytes      = new_check_bytes;
 219         progress.integrity.total_chunks     = new_num_chunks;
 220         progress.integrity.completed_chunks = 0;
 221         progress.integrity.completed_bytes  = 0;
 222         progress.integrity.chunk_size       = chunk_size;
 223         progress.integrity.filename         = NULL;
 224
 225         ret = call_progress(progfunc, WIMLIB_PROGRESS_MSG_CALC_INTEGRITY,
 226                             &progress, progctx);
 227         if (ret)
 228                 goto out_free_new_table;
 229
 230         for (u32 i = 0; i < new_num_chunks; i++) {
 231                 size_t this_chunk_size;
 232                 if (i == new_num_chunks - 1)
 233                         this_chunk_size = new_last_chunk_size;
 234                 else
 235                         this_chunk_size = chunk_size;
 236                 if (old_table &&
 237                     ((this_chunk_size == chunk_size && i < old_num_chunks - 1) ||
 238                       (i == old_num_chunks - 1 && this_chunk_size == old_last_chunk_size)))
 239                 {
 240                         /* Can use SHA1 message digest from old integrity table
 241                          * */
 242                         copy_hash(new_table->sha1sums[i], old_table->sha1sums[i]);
 243                 } else {
 244                         /* Calculate the SHA1 message digest of this chunk */
 245                         ret = calculate_chunk_sha1(in_fd, this_chunk_size,
 246                                                    offset, new_table->sha1sums[i]);
 247                         if (ret)
 248                                 goto out_free_new_table;
 249                 }
 250                 offset += this_chunk_size;
 251
 252                 progress.integrity.completed_chunks++;
 253                 progress.integrity.completed_bytes += this_chunk_size;
 254                 ret = call_progress(progfunc, WIMLIB_PROGRESS_MSG_CALC_INTEGRITY,
 255                                     &progress, progctx);
 256                 if (ret)
 257                         goto out_free_new_table;
 258         }
 259         *integrity_table_ret = new_table;
 260         return 0;
 261
 262 out_free_new_table:
 263         FREE(new_table);
 264         return ret;
 265 }
 266
 267 /*
 268  * write_integrity_table():
 269  *
 270  * Writes a WIM integrity table (a list of SHA1 message digests of raw 10 MiB
 271  * chunks of the file).
 272  *
 273  * This function can optionally re-use entries from an older integrity table.
 274  * To do this, specify old_blob_table_end and old_table.
 275  *
 276  * On success, @wim->out_hdr.integrity_table_reshdr will be filled in with
 277  * information about the integrity table that was written.
 278  *
 279  * @wim:
 280  *      WIMStruct for the WIM file.  @wim->out_fd must be a seekable descriptor
 281  *      to the new WIM file, opened read-write, positioned at the location at
 282  *      which the integrity table is to be written.
 283  *
 284  * @new_blob_table_end:
 285  *      The offset of the byte directly following the blob table in the WIM
 286  *      being written.
 287  *
 288  * @old_blob_table_end:
 289  *      If nonzero, the offset of the byte directly following the old blob table
 290  *      in the WIM.
 291  *
 292  * @old_table
 293  *      Pointer to the old integrity table read into memory, or NULL if not
 294  *      specified.
 295  */
 296 int
 297 write_integrity_table(WIMStruct *wim,
 298                       off_t new_blob_table_end,
 299                       off_t old_blob_table_end,
 300                       struct integrity_table *old_table)
 301 {
 302         struct integrity_table *new_table;
 303         int ret;
 304         u32 new_table_size;
 305
 306         wimlib_assert(old_blob_table_end <= new_blob_table_end);
 307
 308         ret = calculate_integrity_table(&wim->out_fd, new_blob_table_end,
 309                                         old_table, old_blob_table_end,
 310                                         &new_table, wim->progfunc, wim->progctx);
 311         if (ret)
 312                 return ret;
 313
 314         new_table_size = new_table->size;
 315
 316         new_table->size        = cpu_to_le32(new_table->size);
 317         new_table->num_entries = cpu_to_le32(new_table->num_entries);
 318         new_table->chunk_size  = cpu_to_le32(new_table->chunk_size);
 319
 320         ret = write_wim_resource_from_buffer(new_table,
 321                                              new_table_size,
 322                                              false,
 323                                              &wim->out_fd,
 324                                              WIMLIB_COMPRESSION_TYPE_NONE,
 325                                              0,
 326                                              &wim->out_hdr.integrity_table_reshdr,
 327                                              NULL,
 328                                              0);
 329         FREE(new_table);
 330         return ret;
 331 }
 332
 333 /*
 334  * verify_integrity():
 335  *
 336  * Checks a WIM for consistency with the integrity table.
 337  *
 338  * @in_fd:
 339  *      File descriptor to the WIM file, opened for reading.
 340  *
 341  * @table:
 342  *      The integrity table for the WIM, read into memory.
 343  *
 344  * @bytes_to_check:
 345  *      Number of bytes in the WIM that need to be checked (offset of end of the
 346  *      blob table minus offset of end of the header).
 347  *
 348  * Returns:
 349  *      > 0 (WIMLIB_ERR_READ, WIMLIB_ERR_UNEXPECTED_END_OF_FILE) on error
 350  *      0 (WIM_INTEGRITY_OK) if the integrity was checked successfully and there
 351  *      were no inconsistencies.
 352  *      -1 (WIM_INTEGRITY_NOT_OK) if the WIM failed the integrity check.
 353  */
 354 static int
 355 verify_integrity(struct filedes *in_fd, const tchar *filename,
 356                  const struct integrity_table *table,
 357                  u64 bytes_to_check,
 358                  wimlib_progress_func_t progfunc, void *progctx)
 359 {
 360         int ret;
 361         u64 offset = WIM_HEADER_DISK_SIZE;
 362         u8 sha1_md[SHA1_HASH_SIZE];
 363         union wimlib_progress_info progress;
 364
 365         progress.integrity.total_bytes      = bytes_to_check;
 366         progress.integrity.total_chunks     = table->num_entries;
 367         progress.integrity.completed_chunks = 0;
 368         progress.integrity.completed_bytes  = 0;
 369         progress.integrity.chunk_size       = table->chunk_size;
 370         progress.integrity.filename         = filename;
 371
 372         ret = call_progress(progfunc, WIMLIB_PROGRESS_MSG_VERIFY_INTEGRITY,
 373                             &progress, progctx);
 374         if (ret)
 375                 return ret;
 376
 377         for (u32 i = 0; i < table->num_entries; i++) {
 378                 size_t this_chunk_size;
 379                 if (i == table->num_entries - 1)
 380                         this_chunk_size = MODULO_NONZERO(bytes_to_check,
 381                                                          table->chunk_size);
 382                 else
 383                         this_chunk_size = table->chunk_size;
 384
 385                 ret = calculate_chunk_sha1(in_fd, this_chunk_size, offset, sha1_md);
 386                 if (ret)
 387                         return ret;
 388
 389                 if (!hashes_equal(sha1_md, table->sha1sums[i]))
 390                         return WIM_INTEGRITY_NOT_OK;
 391
 392                 offset += this_chunk_size;
 393                 progress.integrity.completed_chunks++;
 394                 progress.integrity.completed_bytes += this_chunk_size;
 395
 396                 ret = call_progress(progfunc, WIMLIB_PROGRESS_MSG_VERIFY_INTEGRITY,
 397                                     &progress, progctx);
 398                 if (ret)
 399                         return ret;
 400         }
 401         return WIM_INTEGRITY_OK;
 402 }
 403
 404
 405 /*
 406  * check_wim_integrity():
 407  *
 408  * Verifies the integrity of the WIM by making sure the SHA1 message digests of
 409  * ~10 MiB chunks of the WIM match up with the values given in the integrity
 410  * table.
 411  *
 412  * @wim:
 413  *      The WIM, opened for reading.
 414  *
 415  * Returns:
 416  *      > 0 (WIMLIB_ERR_INVALID_INTEGRITY_TABLE, WIMLIB_ERR_READ,
 417  *           WIMLIB_ERR_UNEXPECTED_END_OF_FILE) on error
 418  *      0 (WIM_INTEGRITY_OK) if the integrity was checked successfully and there
 419  *      were no inconsistencies.
 420  *      -1 (WIM_INTEGRITY_NOT_OK) if the WIM failed the integrity check.
 421  *      -2 (WIM_INTEGRITY_NONEXISTENT) if the WIM contains no integrity
 422  *      information.
 423  */
 424 int
 425 check_wim_integrity(WIMStruct *wim)
 426 {
 427         int ret;
 428         u64 bytes_to_check;
 429         struct integrity_table *table;
 430         u64 end_blob_table_offset;
 431
 432         if (!wim_has_integrity_table(wim))
 433                 return WIM_INTEGRITY_NONEXISTENT;
 434
 435         end_blob_table_offset = wim->hdr.blob_table_reshdr.offset_in_wim +
 436                                 wim->hdr.blob_table_reshdr.size_in_wim;
 437
 438         if (end_blob_table_offset < WIM_HEADER_DISK_SIZE) {
 439                 ERROR("WIM blob table ends before WIM header ends!");
 440                 return WIMLIB_ERR_INVALID_INTEGRITY_TABLE;
 441         }
 442
 443         bytes_to_check = end_blob_table_offset - WIM_HEADER_DISK_SIZE;
 444
 445         ret = read_integrity_table(wim, bytes_to_check, &table);
 446         if (ret)
 447                 return ret;
 448         ret = verify_integrity(&wim->in_fd, wim->filename, table,
 449                                bytes_to_check, wim->progfunc, wim->progctx);
 450         FREE(table);
 451         return ret;
 452 }