From db046c7191d39535dfb49e9b9bdfc0751a8c17b4 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Thu, 23 Aug 2012 09:50:42 -0500 Subject: [PATCH] Finally fixed most of the problems with the new resource code. Also, write_wim_resource() will now verify the SHA1 message digest when writing a WIM resource. This makes it possible to detect concurrent modifications of the directory tree when capturing a WIM. --- src/extract.c | 37 +++++++++++---------------- src/lookup_table.h | 9 +++++-- src/modify.c | 1 + src/mount.c | 22 +++++++++++----- src/resource.c | 58 ++++++++++++++++++++++++++++++------------- src/sha1.c | 58 ++++++++++++++++--------------------------- src/sha1.h | 32 ++++++++++++++++++------ src/split.c | 7 ++++-- src/util.h | 2 -- src/wimlib_internal.h | 3 +-- 10 files changed, 131 insertions(+), 98 deletions(-) diff --git a/src/extract.c b/src/extract.c index 64fbfa2d..803e8545 100644 --- a/src/extract.c +++ b/src/extract.c @@ -53,21 +53,19 @@ static int extract_regular_file_linked(const struct dentry *dentry, * instead either symlinks or hardlinks *all* identical files in * the WIM, even if they are in a different image (in the case * of a multi-image extraction) */ - - wimlib_assert(lte->file_on_disk); - + wimlib_assert(lte->extracted_file); if (extract_flags & WIMLIB_EXTRACT_FLAG_HARDLINK) { - if (link(lte->file_on_disk, output_path) != 0) { + if (link(lte->extracted_file, output_path) != 0) { ERROR_WITH_ERRNO("Failed to hard link " "`%s' to `%s'", - output_path, lte->file_on_disk); + output_path, lte->extracted_file); return WIMLIB_ERR_LINK; } } else { int num_path_components; int num_output_dir_path_components; - size_t file_on_disk_len; + size_t extracted_file_len; char *p; const char *p2; size_t i; @@ -83,9 +81,9 @@ static int extract_regular_file_linked(const struct dentry *dentry, num_path_components++; num_output_dir_path_components--; } - file_on_disk_len = strlen(lte->file_on_disk); + extracted_file_len = strlen(lte->extracted_file); - char buf[file_on_disk_len + 3 * num_path_components + 1]; + char buf[extracted_file_len + 3 * num_path_components + 1]; p = &buf[0]; for (i = 0; i < num_path_components; i++) { @@ -93,7 +91,7 @@ static int extract_regular_file_linked(const struct dentry *dentry, *p++ = '.'; *p++ = '/'; } - p2 = lte->file_on_disk; + p2 = lte->extracted_file; while (*p2 == '/') p2++; while (num_output_dir_path_components--) @@ -102,7 +100,7 @@ static int extract_regular_file_linked(const struct dentry *dentry, if (symlink(buf, output_path) != 0) { ERROR_WITH_ERRNO("Failed to symlink `%s' to " "`%s'", - buf, lte->file_on_disk); + buf, lte->extracted_file); return WIMLIB_ERR_LINK; } @@ -124,7 +122,9 @@ static int extract_regular_file_unlinked(WIMStruct *w, int ret; const struct list_head *head = &dentry->link_group_list; - if (head->next != head) { + if (head->next != head && + !(extract_flags & WIMLIB_EXTRACT_FLAG_MULTI_IMAGE)) + { /* This dentry is one of a hard link set of at least 2 dentries. * If one of the other dentries has already been extracted, make * a hard link to the file corresponding to this @@ -176,15 +176,6 @@ static int extract_regular_file_unlinked(WIMStruct *w, goto done; } - if (extract_flags & WIMLIB_EXTRACT_FLAG_MULTI_IMAGE) { - /* Mark the lookup table entry to indicate this file has been - * extracted. */ - lte->out_refcnt++; - FREE(lte->file_on_disk); - lte->file_on_disk = STRDUP(output_path); - if (!lte->file_on_disk) - ret = WIMLIB_ERR_NOMEM; - } done: if (close(out_fd) != 0) { ERROR_WITH_ERRNO("Failed to close file `%s'", output_path); @@ -208,12 +199,12 @@ static int extract_regular_file(WIMStruct *w, if ((extract_flags & (WIMLIB_EXTRACT_FLAG_SYMLINK | WIMLIB_EXTRACT_FLAG_HARDLINK)) && lte) { - if (lte->out_refcnt++ != 0) + if (++lte->out_refcnt != 1) return extract_regular_file_linked(dentry, output_dir, output_path, extract_flags, lte); - lte->file_on_disk = STRDUP(output_path); - if (!lte->file_on_disk) + lte->extracted_file = STRDUP(output_path); + if (!lte->extracted_file) return WIMLIB_ERR_NOMEM; } diff --git a/src/lookup_table.h b/src/lookup_table.h index 28037699..4df2c6bd 100644 --- a/src/lookup_table.h +++ b/src/lookup_table.h @@ -82,9 +82,11 @@ struct lookup_table_entry { char *file_on_disk; char *staging_file_name; u8 *attached_buffer; + }; + union { struct lookup_table_entry *next_lte_in_swm; + FILE *file_on_disk_fp; }; - FILE *file_on_disk_fp; #ifdef WITH_FUSE /* File descriptors table for this data stream */ u16 num_opened_fds; @@ -103,7 +105,10 @@ struct lookup_table_entry { * output_resource_entry is the struct resource_entry for the position of the * file resource when written to the output file. */ u32 out_refcnt; - struct resource_entry output_resource_entry; + union { + struct resource_entry output_resource_entry; + char *extracted_file; + }; /* Circular linked list of streams that share the same lookup table * entry diff --git a/src/modify.c b/src/modify.c index 2807370e..dcd0b6d3 100644 --- a/src/modify.c +++ b/src/modify.c @@ -240,6 +240,7 @@ static int add_lte_to_dest_wim(struct dentry *dentry, void *arg) dest_lte = new_lookup_table_entry(); if (!dest_lte) return WIMLIB_ERR_NOMEM; + dest_lte->resource_location = RESOURCE_IN_WIM; dest_lte->wim = src_wim; memcpy(&dest_lte->resource_entry, &src_lte->resource_entry, diff --git a/src/mount.c b/src/mount.c index f3daa237..ad001ce2 100644 --- a/src/mount.c +++ b/src/mount.c @@ -143,14 +143,17 @@ static int close_wimlib_fd(struct wimlib_fd *fd) wimlib_assert(lte); wimlib_assert(lte->num_opened_fds); - if (lte->staging_file_name) { + if (lte->resource_location == RESOURCE_IN_STAGING_FILE) { + wimlib_assert(lte->staging_file_name); wimlib_assert(fd->staging_fd != -1); if (close(fd->staging_fd) != 0) return -errno; } if (--lte->num_opened_fds == 0 && lte->refcnt == 0) { - if (lte->staging_file_name) + if (lte->resource_location == RESOURCE_IN_STAGING_FILE) { + wimlib_assert(lte->staging_file_name); unlink(lte->staging_file_name); + } free_lookup_table_entry(lte); } wimlib_assert(lte->fds[fd->idx] == fd); @@ -205,7 +208,8 @@ int dentry_to_stbuf(const struct dentry *dentry, struct stat *stbuf) /* Use the size of the unnamed (default) file stream. */ lte = dentry_first_lte_resolved(dentry); if (lte) { - if (lte->staging_file_name) { + if (lte->resource_location == RESOURCE_IN_STAGING_FILE) { + wimlib_assert(lte->staging_file_name); struct stat native_stat; if (stat(lte->staging_file_name, &native_stat) != 0) { DEBUG("Failed to stat `%s': %m", @@ -737,6 +741,8 @@ static int close_lte_fds(struct lookup_table_entry *lte) { for (u16 i = 0, j = 0; j < lte->num_opened_fds; i++) { if (lte->fds[i] && lte->fds[i]->staging_fd != -1) { + wimlib_assert(lte->resource_location == RESOURCE_IN_STAGING_FILE); + wimlib_assert(lte->staging_file_name); if (close(lte->fds[i]->staging_fd) != 0) { ERROR_WITH_ERRNO("Failed close file `%s'", lte->staging_file_name); @@ -780,13 +786,13 @@ static int update_lte_of_staging_file(struct lookup_table_entry *lte, u8 hash[SHA1_HASH_SIZE]; struct stat stbuf; + wimlib_assert(lte->resource_location == RESOURCE_IN_STAGING_FILE); wimlib_assert(lte->staging_file_name); ret = sha1sum(lte->staging_file_name, hash); if (ret != 0) return ret; - lookup_table_unlink(table, lte); duplicate_lte = __lookup_resource(table, hash); @@ -805,8 +811,11 @@ static int update_lte_of_staging_file(struct lookup_table_entry *lte, ERROR_WITH_ERRNO("Failed to stat `%s'", lte->staging_file_name); return WIMLIB_ERR_STAT; } + wimlib_assert(<e->file_on_disk == <e->staging_file_name); + lte->resource_location = RESOURCE_IN_FILE_ON_DISK; copy_hash(lte->hash, hash); lte->resource_entry.original_size = stbuf.st_size; + lte->resource_entry.size = stbuf.st_size; lookup_table_insert(table, lte); } @@ -1158,7 +1167,7 @@ static int wimfs_open(const char *path, struct fuse_file_info *fi) if (ret != 0) return ret; } - if (lte->staging_file_name) { + if (lte->resource_location == RESOURCE_IN_STAGING_FILE) { fd->staging_fd = open(lte->staging_file_name, fi->flags); if (fd->staging_fd == -1) { close_wimlib_fd(fd); @@ -1202,9 +1211,10 @@ static int wimfs_read(const char *path, char *buf, size_t size, wimlib_assert(fd->lte); - if (fd->lte->staging_file_name) { + if (fd->lte->resource_location == RESOURCE_IN_STAGING_FILE) { /* Read from staging file */ + wimlib_assert(fd->lte->staging_file_name); wimlib_assert(fd->staging_fd != -1); ssize_t ret; diff --git a/src/resource.c b/src/resource.c index f75ce7ab..7ae7e813 100644 --- a/src/resource.c +++ b/src/resource.c @@ -603,8 +603,7 @@ static int write_wim_resource_chunk(const u8 chunk[], unsigned chunk_size, const u8 *out_chunk; unsigned out_chunk_size; - if (out_ctype == WIM_COMPRESSION_TYPE_NONE) { - wimlib_assert(chunk_tab == NULL); + if (!chunk_tab) { out_chunk = chunk; out_chunk_size = chunk_size; } else { @@ -612,8 +611,6 @@ static int write_wim_resource_chunk(const u8 chunk[], unsigned chunk_size, int ret; unsigned compressed_chunk_len; - wimlib_assert(chunk_tab != NULL); - ret = compress_chunk(chunk, chunk_size, compressed_chunk, &out_chunk_size, out_ctype); if (ret == 0) { @@ -707,7 +704,8 @@ static int write_wim_resource(struct lookup_table_entry *lte, return WIMLIB_ERR_WRITE; } - raw = (wim_resource_compression_type(lte) == out_ctype); + raw = (wim_resource_compression_type(lte) == out_ctype + && out_ctype != WIM_COMPRESSION_TYPE_NONE); if (raw) bytes_remaining = old_compressed_size; else @@ -736,12 +734,17 @@ static int write_wim_resource(struct lookup_table_entry *lte, goto out; } } + SHA_CTX ctx; + if (!raw) + sha1_init(&ctx); do { u64 to_read = min(bytes_remaining, WIM_CHUNK_SIZE); ret = __read_wim_resource(lte, buf, to_read, offset, raw); if (ret != 0) goto out_fclose; + if (!raw) + sha1_update(&ctx, buf, to_read); ret = write_wim_resource_chunk(buf, to_read, out_fp, out_ctype, chunk_tab); if (ret != 0) @@ -758,6 +761,21 @@ static int write_wim_resource(struct lookup_table_entry *lte, new_compressed_size = old_compressed_size; } + if (!raw) { + u8 md[SHA1_HASH_SIZE]; + sha1_final(md, &ctx); + if (!hashes_equal(md, lte->hash)) { + ERROR("WIM resource has incorrect hash!"); + if (lte->resource_location == RESOURCE_IN_FILE_ON_DISK) { + ERROR("We were reading it from `%s'; maybe it changed " + "while we were reading it.", + lte->file_on_disk); + } + ret = WIMLIB_ERR_INTEGRITY; + goto out_fclose; + } + } + if (new_compressed_size > original_size) { /* Oops! We compressed the resource to larger than the original * size. Write the resource uncompressed instead. */ @@ -791,6 +809,8 @@ static int write_wim_resource(struct lookup_table_entry *lte, out_res_entry->flags = 0; else out_res_entry->flags = WIM_RESHDR_FLAG_COMPRESSED; + if (lte->resource_entry.flags & WIM_RESHDR_FLAG_METADATA) + out_res_entry->flags |= WIM_RESHDR_FLAG_METADATA; } out_fclose: if (lte->resource_location == RESOURCE_IN_FILE_ON_DISK @@ -804,16 +824,18 @@ out: } static int write_wim_resource_from_buffer(const u8 *buf, u64 buf_size, + u8 buf_hash[SHA1_HASH_SIZE], FILE *out_fp, int out_ctype, struct resource_entry *out_res_entry) { struct lookup_table_entry lte; - lte.resource_entry.flags = 0; + lte.resource_entry.flags = 0; lte.resource_entry.original_size = buf_size; - lte.resource_entry.size = buf_size; - lte.resource_entry.offset = 0; - lte.resource_location = RESOURCE_IN_ATTACHED_BUFFER; - lte.attached_buffer = (u8*)buf; + lte.resource_entry.size = buf_size; + lte.resource_entry.offset = 0; + lte.resource_location = RESOURCE_IN_ATTACHED_BUFFER; + lte.attached_buffer = (u8*)buf; + copy_hash(lte.hash, buf_hash); return write_wim_resource(<e, out_fp, out_ctype, out_res_entry); } @@ -869,7 +891,8 @@ int copy_resource(struct lookup_table_entry *lte, void *wim) !w->write_metadata) return 0; - ret = write_wim_resource(lte, w->out_fp, wimlib_get_compression_type(w), + ret = write_wim_resource(lte, w->out_fp, + wim_resource_compression_type(lte), <e->output_resource_entry); if (ret != 0) return ret; @@ -1100,16 +1123,17 @@ int write_metadata_resource(WIMStruct *w) lte = wim_metadata_lookup_table_entry(w); ret = write_wim_resource_from_buffer(buf, metadata_original_size, - out, metadata_ctype, + hash, out, metadata_ctype, <e->output_resource_entry); - FREE(buf); - if (ret != 0) - return ret; - lte->out_refcnt++; - lte->output_resource_entry.flags |= WIM_RESHDR_FLAG_METADATA; lookup_table_unlink(w->lookup_table, lte); copy_hash(lte->hash, hash); lookup_table_insert(w->lookup_table, lte); + lte->out_refcnt++; + lte->output_resource_entry.flags |= WIM_RESHDR_FLAG_METADATA; + FREE(buf); + if (ret != 0) + return ret; + return 0; } diff --git a/src/sha1.c b/src/sha1.c index 36ff3f15..dd4e2629 100644 --- a/src/sha1.c +++ b/src/sha1.c @@ -34,31 +34,31 @@ * Steve Reid's public domain code, or based on Intel's SSSE3 SHA1 code. */ -#ifdef WITH_LIBCRYPTO +#ifndef WITH_LIBCRYPTO -#define sha1_init SHA1_Init -#define sha1_update SHA1_Update -#define sha1_final SHA1_Final -#else /* WITH_LIBCRYPTO */ - -typedef struct { - u32 state[5]; - u32 count[2]; - u8 buffer[64]; -} SHA_CTX; +/* Initialize new context */ +void sha1_init(SHA_CTX* context) +{ + /* SHA1 initialization constants */ + context->state[0] = 0x67452301; + context->state[1] = 0xEFCDAB89; + context->state[2] = 0x98BADCFE; + context->state[3] = 0x10325476; + context->state[4] = 0xC3D2E1F0; + context->count[0] = context->count[1] = 0; +} #ifdef ENABLE_SSSE3_SHA1 extern void sha1_update_intel(int *hash, const char* input, size_t num_blocks); -static inline void sha1_update(SHA_CTX *context, const void *data, size_t len) +void sha1_update(SHA_CTX *context, const u8 data[], size_t len) { sha1_update_intel((int*)&context->state, data, len / 64); size_t j = (context->count[0] >> 3) & 63; if ((context->count[0] += len << 3) < (len << 3)) context->count[1]++; context->count[1] += (len >> 29); } - #include void ssse3_not_found() { @@ -68,21 +68,7 @@ void ssse3_not_found() "to use wimlib on this CPU.\n"); abort(); } -#endif - -/* Initialize new context */ -static void sha1_init(SHA_CTX* context) -{ - /* SHA1 initialization constants */ - context->state[0] = 0x67452301; - context->state[1] = 0xEFCDAB89; - context->state[2] = 0x98BADCFE; - context->state[3] = 0x10325476; - context->state[4] = 0xC3D2E1F0; - context->count[0] = context->count[1] = 0; -} - -#ifndef ENABLE_SSSE3_SHA1 +#else /* ENABLE_SSSE3_SHA1 */ #define rol(value, bits) (((value) << (bits)) | ((value) >> (32 - (bits)))) @@ -156,8 +142,7 @@ static void sha1_transform(u32 state[5], const u8 buffer[64]) state[4] += e; } -/* Run your data through this. */ -static void sha1_update(SHA_CTX* context, const u8* data, const size_t len) +void sha1_update(SHA_CTX* context, const u8 data[], const size_t len) { size_t i, j; @@ -177,10 +162,11 @@ static void sha1_update(SHA_CTX* context, const u8* data, const size_t len) } memcpy(&context->buffer[j], &data[i], len - i); } -#endif + +#endif /* !ENABLE_SSSE3_SHA1 */ /* Add padding and return the message digest. */ -static void sha1_final(u8 *md, SHA_CTX* context) +void sha1_final(u8 md[SHA1_HASH_SIZE], SHA_CTX* context) { u32 i; u8 finalcount[8]; @@ -199,7 +185,7 @@ static void sha1_final(u8 *md, SHA_CTX* context) } } -void sha1_buffer(const void *buffer, size_t len, void *md) +void sha1_buffer(const u8 buffer[], size_t len, u8 md[SHA1_HASH_SIZE]) { SHA_CTX ctx; sha1_init(&ctx); @@ -207,9 +193,9 @@ void sha1_buffer(const void *buffer, size_t len, void *md) sha1_final(md, &ctx); } -#endif /* WITH_LIBCRYPTO */ +#endif /* !WITH_LIBCRYPTO */ -static int sha1_stream(FILE *fp, void *md) +static int sha1_stream(FILE *fp, u8 md[SHA1_HASH_SIZE]) { char buf[BUFFER_SIZE]; size_t bytes_read; @@ -232,7 +218,7 @@ static int sha1_stream(FILE *fp, void *md) /* Calculates the SHA1 message digest given the name of a file. @md must point * to a buffer of length 20 bytes into which the message digest is written. */ -int sha1sum(const char *filename, void *md) +int sha1sum(const char *filename, u8 md[SHA1_HASH_SIZE]) { FILE *fp; int ret; diff --git a/src/sha1.h b/src/sha1.h index db524e6d..5c143617 100644 --- a/src/sha1.h +++ b/src/sha1.h @@ -32,16 +32,32 @@ static inline void print_hash(const u8 hash[]) print_byte_field(hash, SHA1_HASH_SIZE); } -extern int sha1sum(const char *filename, void *md); #ifdef WITH_LIBCRYPTO + #include -static inline void sha1_buffer(const void *buffer, size_t len, void *md) -{ - SHA1(buffer, len, md); -} -#else -extern void sha1_buffer(const void *buffer, size_t len, void *md); -#endif +#define sha1_buffer SHA1 +#define sha1_init SHA1_Init +#define sha1_update SHA1_Update +#define sha1_final SHA1_Final + +#else /* WITH_LIBCRYPTO */ + +typedef struct { + u32 state[5]; + u32 count[2]; + u8 buffer[64]; +} SHA_CTX; + +extern void sha1_buffer(const u8 buffer[], size_t len, u8 hash[SHA1_HASH_SIZE]); +extern void sha1_init(SHA_CTX *ctx); +extern void sha1_update(SHA_CTX *ctx, const u8 data[], size_t len); +extern void sha1_final(u8 hash[SHA1_HASH_SIZE], SHA_CTX *ctx); + +#endif /* !WITH_LIBCRYPTO */ + +extern int sha1sum(const char *filename, u8 hash[SHA1_HASH_SIZE]); + + #endif /* _WIMLIB_SHA1_H */ diff --git a/src/split.c b/src/split.c index 40c861a1..8370ab3b 100644 --- a/src/split.c +++ b/src/split.c @@ -53,11 +53,13 @@ static int finish_swm(WIMStruct *w, struct lookup_table_entry *lte_chain_head, lookup_table_offset); while (lte_chain_head != NULL) { + print_lookup_table_entry(lte_chain_head); + ret = write_lookup_table_entry(lte_chain_head, w->out_fp); if (ret != 0) return ret; struct lookup_table_entry *prev = lte_chain_head; - lte_chain_head = prev->next_lte_in_swm; + lte_chain_head = lte_chain_head->next_lte_in_swm; prev->next_lte_in_swm = NULL; } off_t xml_data_offset = ftello(w->out_fp); @@ -184,9 +186,10 @@ WIMLIBAPI int wimlib_split(const char *wimfile, const char *swm_name, w->write_metadata = true; for (int i = 0; i < w->hdr.image_count; i++) { - struct lookup_table_entry *metadata_lte; + DEBUG("Writing metadata resource %d", i); + metadata_lte = w->image_metadata[i].metadata_lte; ret = copy_resource(metadata_lte, w); if (ret != 0) diff --git a/src/util.h b/src/util.h index 1fb1f099..b459afd8 100644 --- a/src/util.h +++ b/src/util.h @@ -128,8 +128,6 @@ extern void randomize_byte_array(u8 *p, size_t n); extern void randomize_char_array_with_alnum(char p[], size_t n); -extern int sha1sum(const char *filename, void *buf); - extern const char *path_next_part(const char *path, size_t *first_part_len_ret); diff --git a/src/wimlib_internal.h b/src/wimlib_internal.h index d13b46e5..325b2245 100644 --- a/src/wimlib_internal.h +++ b/src/wimlib_internal.h @@ -264,14 +264,13 @@ typedef struct WIMStruct { int extract_flags; int add_flags; int write_flags; + bool write_metadata; }; /* The currently selected image, indexed starting at 1. If not 0, * subtract 1 from this to get the index of the current image in the * image_metadata array. */ int current_image; - - bool write_metadata; } WIMStruct; -- 2.43.0