From: Eric Biggers Date: Mon, 19 Nov 2012 21:26:59 +0000 (-0600) Subject: Rework WIM writing code X-Git-Tag: v1.2.0~31 X-Git-Url: https://wimlib.net/git/?p=wimlib;a=commitdiff_plain;h=e6227e4978dd41e0726ef1b54713089d343b57de Rework WIM writing code (Incomplete) --- diff --git a/programs/imagex.c b/programs/imagex.c index ec383d6c..266dc0d4 100644 --- a/programs/imagex.c +++ b/programs/imagex.c @@ -623,6 +623,8 @@ out_write: ret = wimlib_write(w, wimfile, WIM_ALL_IMAGES, write_flags, num_threads); } + if (ret == WIMLIB_ERR_REOPEN) + ret = 0; if (ret != 0) imagex_error("Failed to write the WIM file `%s'", wimfile); out: @@ -689,6 +691,8 @@ static int imagex_delete(int argc, const char **argv) } ret = wimlib_overwrite(w, write_flags, 0); + if (ret == WIMLIB_ERR_REOPEN) + ret = 0; if (ret != 0) { imagex_error("Failed to write the file `%s' with image " "deleted", wimfile); @@ -894,6 +898,8 @@ static int imagex_export(int argc, const char **argv) else ret = wimlib_overwrite(dest_w, write_flags, num_threads); out: + if (ret == WIMLIB_ERR_REOPEN) + ret = 0; wimlib_free(src_w); wimlib_free(dest_w); if (additional_swms) { @@ -1139,8 +1145,8 @@ static int imagex_info(int argc, const char **argv) } } - /* Only call wimlib_overwrite_xml_and_header() if something - * actually needs to be changed. */ + /* Only call wimlib_overwrite() if something actually needs to + * be changed. */ if (boot || new_name || new_desc || check != wimlib_has_integrity_table(w)) { @@ -1148,9 +1154,17 @@ static int imagex_info(int argc, const char **argv) if (ret != 0) return ret; - ret = wimlib_overwrite_xml_and_header(w, check ? - WIMLIB_WRITE_FLAG_CHECK_INTEGRITY | - WIMLIB_WRITE_FLAG_SHOW_PROGRESS : 0); + int write_flags; + if (check) { + write_flags = WIMLIB_WRITE_FLAG_CHECK_INTEGRITY | + WIMLIB_WRITE_FLAG_SHOW_PROGRESS; + } else { + write_flags = 0; + } + + ret = wimlib_overwrite(w, write_flags, 1); + if (ret == WIMLIB_ERR_REOPEN) + ret = 0; } else { printf("The file `%s' was not modified because nothing " "needed to be done.\n", wimfile); @@ -1158,7 +1172,7 @@ static int imagex_info(int argc, const char **argv) } } out: - /*wimlib_free(w);*/ + wimlib_free(w); return ret; } diff --git a/src/integrity.c b/src/integrity.c index f9d7aaa5..5ad55a5b 100644 --- a/src/integrity.c +++ b/src/integrity.c @@ -42,12 +42,9 @@ * @sha1sums: Array of SHA1 message digests; 20 bytes each, one per chunk. * @show_progress: Nonzero if the percent complete is to be printed after every * chunk. - * @status: On success, set to WIM_INTEGRITY_OK or WIM_INTEGRITY_NOT_OK - * based on whether the WIM is intact or not. */ static int verify_integrity(FILE *fp, u64 num_bytes, u32 chunk_size, - const u8 *sha1sums, int show_progress, - int *status) + const u8 *sha1sums, int show_progress) { u8 *chunk_buf; u8 resblock[SHA1_HASH_SIZE]; @@ -82,20 +79,18 @@ static int verify_integrity(FILE *fp, u64 num_bytes, u32 chunk_size, "verifying integrity of WIM"); } ret = WIMLIB_ERR_READ; - goto verify_integrity_error; + goto out; } sha1_buffer(chunk_buf, bytes_to_read, resblock); if (!hashes_equal(resblock, sha1sums)) { - *status = WIM_INTEGRITY_NOT_OK; - goto verify_integrity_done; + ret = WIM_INTEGRITY_NOT_OK; + goto out; } sha1sums += SHA1_HASH_SIZE; bytes_remaining -= bytes_to_read; } - *status = WIM_INTEGRITY_OK; -verify_integrity_done: - ret = 0; -verify_integrity_error: + ret = WIM_INTEGRITY_OK; +out: FREE(chunk_buf); if (show_progress) putchar('\n'); @@ -104,16 +99,8 @@ verify_integrity_error: /* * Verifies the integrity of the WIM. - * - * @show_progress: Nonzero if the percent complete is to be printed after every - * chunk. - * @status: On success, set to WIM_INTEGRITY_OK, WIM_INTEGRITY_NOT_OK, - * or WIM_INTEGRITY_NONEXISTENT. - * - * Returns: 0, WIMLIB_ERR_INVALID_INTEGRITY_TABLE, WIMLIB_ERR_NOMEM, or - * WIMLIB_ERR_READ. If nonzero, the boolean pointed to by @ok is not changed. */ -int check_wim_integrity(WIMStruct *w, int show_progress, int *status) +int check_wim_integrity(WIMStruct *w, int show_progress) { struct resource_entry *res_entry; @@ -131,8 +118,7 @@ int check_wim_integrity(WIMStruct *w, int show_progress, int *status) res_entry = &w->hdr.integrity; if (res_entry->size == 0) { DEBUG("No integrity information."); - *status = WIM_INTEGRITY_NONEXISTENT; - return 0; + return WIM_INTEGRITY_NONEXISTENT; } if (res_entry->original_size < 12) { ERROR("Integrity table is too short"); @@ -235,7 +221,7 @@ int check_wim_integrity(WIMStruct *w, int show_progress, int *status) /* call verify_integrity(), which does the actual checking of the SHA1 * message digests. */ ret = verify_integrity(w->fp, bytes_to_check, chunk_size, p, - show_progress, status); + show_progress); out: FREE(buf); return ret; @@ -252,7 +238,8 @@ out: * which is the end of the region that is checksummed. */ int write_integrity_table(FILE *out, u64 end_header_offset, - u64 end_lookup_table_offset, int show_progress) + u64 end_lookup_table_offset, int show_progress, + struct resource_entry *out_res_entry) { u64 bytes_to_check; u64 bytes_remaining; @@ -262,6 +249,11 @@ int write_integrity_table(FILE *out, u64 end_header_offset, u32 num_entries; u32 integrity_table_size; int ret; + off_t start_offset; + + start_offset = ftello(out); + if (start_offset == -1) + return WIMLIB_ERR_WRITE; DEBUG("Calculating integrity table"); if (fseeko(out, end_header_offset, SEEK_SET) != 0) { @@ -337,9 +329,8 @@ int write_integrity_table(FILE *out, u64 end_header_offset, "(0 bytes remaining, 100% done)" " "); - if (fseeko(out, 0, SEEK_END) != 0) { - ERROR_WITH_ERRNO("Failed to seek to end of WIM to write " - "integrity table"); + if (fseeko(out, start_offset, SEEK_SET) != 0) { + ERROR_WITH_ERRNO("Failed to seek to end of WIM"); ret = WIMLIB_ERR_WRITE; goto out_free_chunk_buf; } @@ -350,6 +341,11 @@ int write_integrity_table(FILE *out, u64 end_header_offset, ret = WIMLIB_ERR_WRITE; goto out_free_chunk_buf; } + + out_res_entry->offset = start_offset; + out_res_entry->size = integrity_table_size; + out_res_entry->original_size = integrity_table_size; + out_res_entry->flags = 0; ret = 0; out_free_chunk_buf: FREE(chunk_buf); diff --git a/src/join.c b/src/join.c index aae6a313..222c9a9a 100644 --- a/src/join.c +++ b/src/join.c @@ -252,7 +252,9 @@ static int join_wims(WIMStruct **swms, uint num_swms, WIMStruct *joined_wim, * table has no header, we can just concatenate the lookup tables of all * the SWM parts. */ for (i = 0; i < num_swms; i++) { - ret = write_lookup_table(swms[i]->lookup_table, out_fp); + ret = for_lookup_table_entry(swms[i]->lookup_table, + write_lookup_table_entry, + out_fp); if (ret != 0) return ret; } @@ -265,6 +267,10 @@ static int join_wims(WIMStruct **swms, uint num_swms, WIMStruct *joined_wim, swms[0]->hdr.lookup_table_res_entry.offset = lookup_table_offset; swms[0]->hdr.lookup_table_res_entry.size = xml_data_offset - lookup_table_offset; + swms[0]->hdr.lookup_table_res_entry.original_size = + xml_data_offset - lookup_table_offset; + swms[0]->hdr.lookup_table_res_entry.flags = + WIM_RESHDR_FLAG_METADATA; /* finish_write is called on the first swm, not the joined_wim, because diff --git a/src/lookup_table.c b/src/lookup_table.c index f320e2dd..d1be89b8 100644 --- a/src/lookup_table.c +++ b/src/lookup_table.c @@ -365,6 +365,14 @@ int read_lookup_table(WIMStruct *w) ret = WIMLIB_ERR_INVALID_LOOKUP_TABLE_ENTRY; goto out_free_cur_entry; } + if ((cur_entry->resource_entry.flags & WIM_RESHDR_FLAG_METADATA) + && cur_entry->refcnt != 1) + { + ERROR("Found metadata resource with refcnt != 1:"); + print_lookup_table_entry(cur_entry); + ret = WIMLIB_ERR_INVALID_LOOKUP_TABLE_ENTRY; + goto out_free_cur_entry; + } lookup_table_insert(table, cur_entry); } @@ -410,6 +418,33 @@ int write_lookup_table_entry(struct lookup_table_entry *lte, void *__out) return 0; } +/* Writes the lookup table to the output file. */ +int write_lookup_table(struct lookup_table *table, FILE *out, + struct resource_entry *out_res_entry) +{ + off_t start_offset, end_offset; + int ret; + + start_offset = ftello(out); + if (start_offset == -1) + return WIMLIB_ERR_WRITE; + + ret = for_lookup_table_entry(table, write_lookup_table_entry, out); + if (ret != 0) + return ret; + + end_offset = ftello(out); + if (end_offset == -1) + return WIMLIB_ERR_WRITE; + + out_res_entry->offset = start_offset; + out_res_entry->size = end_offset - start_offset; + out_res_entry->original_size = end_offset - start_offset; + out_res_entry->flags = WIM_RESHDR_FLAG_METADATA; + + return 0; +} + int lte_zero_real_refcnt(struct lookup_table_entry *lte, void *ignore) { diff --git a/src/lookup_table.h b/src/lookup_table.h index c870a114..2665c39c 100644 --- a/src/lookup_table.h +++ b/src/lookup_table.h @@ -198,9 +198,6 @@ struct lookup_table_entry { * been extracted to the staging directory when modifying a read-write * mounted WIM. */ struct list_head staging_list; - - /* Temporary field for creating a singly linked list. */ - struct lookup_table_entry *next_lte_in_swm; }; }; @@ -288,11 +285,8 @@ extern void free_lookup_table_entry(struct lookup_table_entry *lte); extern int dentry_resolve_ltes(struct dentry *dentry, void *__table); extern int dentry_unresolve_ltes(struct dentry *dentry, void *ignore); -/* Writes the lookup table to the output file. */ -static inline int write_lookup_table(struct lookup_table *table, FILE *out) -{ - return for_lookup_table_entry(table, write_lookup_table_entry, out); -} +int write_lookup_table(struct lookup_table *table, FILE *out, + struct resource_entry *out_res_entry); /* Unlinks and frees an entry from a lookup table. */ static inline void lookup_table_remove(struct lookup_table *table, diff --git a/src/mount.c b/src/mount.c index 8b9cab93..15d7444e 100644 --- a/src/mount.c +++ b/src/mount.c @@ -1987,8 +1987,10 @@ WIMLIBAPI int wimlib_mount(WIMStruct *wim, int image, const char *dir, #endif /* Mark dentry tree as modified if read-write mount. */ - if (flags & WIMLIB_MOUNT_FLAG_READWRITE) + if (flags & WIMLIB_MOUNT_FLAG_READWRITE) { imd->modified = true; + imd->has_been_mounted_rw = true; + } /* Resolve all the lookup table entries of the dentry tree */ DEBUG("Resolving lookup table entries"); diff --git a/src/resource.c b/src/resource.c index 51bde708..6c38a594 100644 --- a/src/resource.c +++ b/src/resource.c @@ -810,6 +810,8 @@ int read_metadata_resource(WIMStruct *w, struct image_metadata *imd) goto out_free_security_data; } + DEBUG("Reading root dentry"); + /* Allocate memory for the root dentry and read it into memory */ dentry = MALLOC(sizeof(struct dentry)); if (!dentry) { @@ -821,11 +823,18 @@ int read_metadata_resource(WIMStruct *w, struct image_metadata *imd) ret = read_dentry(buf, metadata_len, dentry_offset, dentry); - /* This is the root dentry, so set its parent to itself. */ - dentry->parent = dentry; + + if (dentry->length == 0) { + ERROR("Metadata resource cannot begin with end-of-directory entry!"); + ret = WIMLIB_ERR_INVALID_DENTRY; + } if (ret != 0) goto out_free_dentry_tree; + + /* This is the root dentry, so set its parent to itself. */ + dentry->parent = dentry; + inode_add_dentry(dentry, dentry->d_inode); /* Now read the entire directory entry tree into memory. */ @@ -892,11 +901,6 @@ int write_metadata_resource(WIMStruct *w) root = wim_root_dentry(w); sd = wim_security_data(w); - /* We do not allow the security data pointer to be NULL, although it may - * point to an empty security data with no entries. */ - wimlib_assert(root != NULL); - wimlib_assert(sd != NULL); - /* Offset of first child of the root dentry. It's equal to: * - The total length of the security data, rounded to the next 8-byte * boundary, @@ -904,7 +908,7 @@ int write_metadata_resource(WIMStruct *w) * - plus 8 bytes for an end-of-directory entry following the root * dentry (shouldn't really be needed, but just in case...) */ - subdir_offset = ((sd->total_length + 7) & ~7) + + subdir_offset = (((u64)sd->total_length + 7) & ~7) + dentry_correct_total_length(root) + 8; /* Calculate the subdirectory offsets for the entire dentry tree. */ @@ -935,8 +939,6 @@ int write_metadata_resource(WIMStruct *w) * it. */ lte = wim_metadata_lookup_table_entry(w); - wimlib_assert(lte != NULL); - /* Write the metadata resource to the output WIM using the proper * compression type. The lookup table entry for the metadata resource * is updated. */ @@ -960,12 +962,10 @@ int write_metadata_resource(WIMStruct *w) * */ lookup_table_unlink(w->lookup_table, lte); lookup_table_insert(w->lookup_table, lte); - - wimlib_assert(lte->out_refcnt == 0); lte->out_refcnt = 1; - /* Make sure that the resource entry is written marked with the metadata - * flag. */ + /* Make sure that the lookup table entry for this metadata resource is + * marked with the metadata flag. */ lte->output_resource_entry.flags |= WIM_RESHDR_FLAG_METADATA; out: /* All the data has been written to the new WIM; no need for the buffer diff --git a/src/split.c b/src/split.c index 4167ecdf..df384f82 100644 --- a/src/split.c +++ b/src/split.c @@ -33,8 +33,7 @@ struct split_args { char *swm_base_name; size_t swm_base_name_len; const char *swm_suffix; - struct lookup_table_entry *lte_chain_head; - struct lookup_table_entry *lte_chain_tail; + struct list_head *lte_list; int part_number; int write_flags; long size_remaining; @@ -43,23 +42,22 @@ struct split_args { u64 total_bytes_written; }; -static int finish_swm(WIMStruct *w, struct lookup_table_entry *lte_chain_head, +static int finish_swm(WIMStruct *w, struct list_head *lte_list, int write_flags) { off_t lookup_table_offset = ftello(w->out_fp); int ret; + struct lookup_table_entry *lte; DEBUG("Writing lookup table for SWM (offset %"PRIu64")", lookup_table_offset); - while (lte_chain_head != NULL) { - ret = write_lookup_table_entry(lte_chain_head, w->out_fp); + list_for_each_entry(lte, lte_list, staging_list) { + ret = write_lookup_table_entry(lte, w->out_fp); if (ret != 0) return ret; - struct lookup_table_entry *prev = lte_chain_head; - lte_chain_head = lte_chain_head->next_lte_in_swm; - prev->next_lte_in_swm = NULL; } + off_t xml_data_offset = ftello(w->out_fp); if (lookup_table_offset == -1 || xml_data_offset == -1) @@ -67,6 +65,9 @@ static int finish_swm(WIMStruct *w, struct lookup_table_entry *lte_chain_head, w->hdr.lookup_table_res_entry.offset = lookup_table_offset; w->hdr.lookup_table_res_entry.size = xml_data_offset - lookup_table_offset; + w->hdr.lookup_table_res_entry.original_size = + xml_data_offset - lookup_table_offset; + w->hdr.lookup_table_res_entry.flags = WIM_RESHDR_FLAG_METADATA; return finish_write(w, WIM_ALL_IMAGES, write_flags | WIMLIB_WRITE_FLAG_NO_LOOKUP_TABLE); } @@ -87,10 +88,10 @@ static int copy_resource_to_swm(struct lookup_table_entry *lte, void *__args) /* No space for this resource. Finish the previous swm and * start a new one. */ - ret = finish_swm(w, args->lte_chain_head, args->write_flags); - - args->lte_chain_tail = NULL; - args->lte_chain_head = NULL; + ret = finish_swm(w, args->lte_list, args->write_flags); + if (ret != 0) + return ret; + INIT_LIST_HEAD(args->lte_list); sprintf(args->swm_base_name + args->swm_base_name_len, "%d%s", ++args->part_number, args->swm_suffix); @@ -113,11 +114,7 @@ static int copy_resource_to_swm(struct lookup_table_entry *lte, void *__args) } args->size_remaining -= lte->resource_entry.size; args->total_bytes_written += lte->resource_entry.size; - if (args->lte_chain_tail) - args->lte_chain_tail->next_lte_in_swm = lte; - else - args->lte_chain_head = lte; - args->lte_chain_tail = lte; + list_add(<e->staging_list, args->lte_list); return copy_resource(lte, w); } @@ -134,8 +131,6 @@ WIMLIBAPI int wimlib_split(const char *wimfile, const char *swm_name, char name[swm_name_len + 20]; char *swm_suffix; - struct lookup_table_entry *lte_chain_head = NULL; - struct lookup_table_entry *lte_chain_tail = NULL; long size_remaining = part_size; u64 total_bytes_written = 0; u64 total_bytes; @@ -175,6 +170,7 @@ WIMLIBAPI int wimlib_split(const char *wimfile, const char *swm_name, (double)total_bytes * 100.0); w->write_metadata = true; + LIST_HEAD(lte_list); for (int i = 0; i < w->hdr.image_count; i++) { struct lookup_table_entry *metadata_lte; @@ -186,11 +182,7 @@ WIMLIBAPI int wimlib_split(const char *wimfile, const char *swm_name, return ret; size_remaining -= metadata_lte->resource_entry.size; total_bytes_written += metadata_lte->resource_entry.size; - if (lte_chain_tail) - lte_chain_tail->next_lte_in_swm = metadata_lte; - else - lte_chain_head = metadata_lte; - lte_chain_tail = metadata_lte; + list_add(&metadata_lte->staging_list, <e_list); } w->write_metadata = false; @@ -199,8 +191,7 @@ WIMLIBAPI int wimlib_split(const char *wimfile, const char *swm_name, .swm_base_name = name, .swm_base_name_len = swm_base_name_len, .swm_suffix = swm_suffix, - .lte_chain_head = lte_chain_head, - .lte_chain_tail = lte_chain_tail, + .lte_list = <e_list, .part_number = 1, .write_flags = write_flags, .size_remaining = size_remaining, @@ -213,7 +204,7 @@ WIMLIBAPI int wimlib_split(const char *wimfile, const char *swm_name, if (ret != 0) return ret; - ret = finish_swm(w, args.lte_chain_head, write_flags); + ret = finish_swm(w, <e_list, write_flags); if (ret != 0) return ret; diff --git a/src/util.c b/src/util.c index c0167826..faa0d0bf 100644 --- a/src/util.c +++ b/src/util.c @@ -180,6 +180,8 @@ static const char *error_strings[] = { = "Could not read the target of a symbolic link", [WIMLIB_ERR_RENAME] = "Could not rename a file", + [WIMLIB_ERR_REOPEN] + = "Could not re-open the WIM after overwriting it", [WIMLIB_ERR_RESOURCE_ORDER] = "The components of the WIM were arranged in an unexpected order", [WIMLIB_ERR_SPECIAL_FILE] diff --git a/src/wim.c b/src/wim.c index 3e9b4523..d4558578 100644 --- a/src/wim.c +++ b/src/wim.c @@ -418,6 +418,35 @@ WIMLIBAPI int wimlib_get_boot_idx(const WIMStruct *w) return w->hdr.boot_idx; } +/* Opens a WIM readable */ +int open_wim_readable(WIMStruct *w, const char *path) +{ + if (w->fp != NULL) + fclose(w->fp); + wimlib_assert(path != NULL); + w->fp = fopen(path, "rb"); + if (!w->fp) { + ERROR_WITH_ERRNO("Failed to open `%s' for reading", + path); + return WIMLIB_ERR_OPEN; + } + return 0; +} + +/* Opens a WIM writable */ +int open_wim_writable(WIMStruct *w, const char *path) +{ + wimlib_assert(w->out_fp == NULL); + wimlib_assert(path != NULL); + w->out_fp = fopen(path, "w+b"); + if (!w->out_fp) { + ERROR_WITH_ERRNO("Failed to open `%s' for writing", + path); + return WIMLIB_ERR_OPEN; + } + return 0; +} + /* * Begins the reading of a WIM file; opens the file and reads its header and * lookup table, and optionally checks the integrity. @@ -429,23 +458,20 @@ static int begin_read(WIMStruct *w, const char *in_wim_path, int open_flags) DEBUG("Reading the WIM file `%s'", in_wim_path); - w->fp = fopen(in_wim_path, "rb"); - - if (!w->fp) { - ERROR_WITH_ERRNO("Failed to open the file `%s' for reading", - in_wim_path); - return WIMLIB_ERR_OPEN; - } + ret = open_wim_readable(w, in_wim_path); + if (ret != 0) + goto out; w->filename = realpath(in_wim_path, NULL); if (!w->filename) { ERROR_WITH_ERRNO("Failed to resolve WIM filename"); - return WIMLIB_ERR_NOMEM; + ret = WIMLIB_ERR_NOMEM; + goto out_close; } ret = read_header(w->fp, &w->hdr, open_flags); if (ret != 0) - return ret; + goto out_close; DEBUG("Wim file contains %u images", w->hdr.image_count); @@ -460,35 +486,35 @@ static int begin_read(WIMStruct *w, const char *in_wim_path, int open_flags) if (wimlib_get_compression_type(w) == WIM_COMPRESSION_TYPE_INVALID) { ERROR("Invalid compression type (WIM header flags = %x)", w->hdr.flags); - return WIMLIB_ERR_INVALID_COMPRESSION_TYPE; + ret = WIMLIB_ERR_INVALID_COMPRESSION_TYPE; + goto out_close; } - if (open_flags & WIMLIB_OPEN_FLAG_CHECK_INTEGRITY) { - int integrity_status; ret = check_wim_integrity(w, - open_flags & WIMLIB_OPEN_FLAG_SHOW_PROGRESS, - &integrity_status); - if (ret != 0) - return ret; - if (integrity_status == WIM_INTEGRITY_NONEXISTENT) { + open_flags & WIMLIB_OPEN_FLAG_SHOW_PROGRESS); + if (ret == WIM_INTEGRITY_NONEXISTENT) { WARNING("No integrity information for `%s'; skipping " "integrity check.", w->filename); - } else if (integrity_status == WIM_INTEGRITY_NOT_OK) { + } else if (ret == WIM_INTEGRITY_NOT_OK) { ERROR("WIM is not intact! (Failed integrity check)"); - return WIMLIB_ERR_INTEGRITY; + ret = WIMLIB_ERR_INTEGRITY; + goto out_close; + } else if (ret != 0) { + goto out_close; } } if (resource_is_compressed(&w->hdr.lookup_table_res_entry)) { ERROR("Didn't expect a compressed lookup table!"); ERROR("Ask the author to implement support for this."); - return WIMLIB_ERR_COMPRESSED_LOOKUP_TABLE; + ret = WIMLIB_ERR_COMPRESSED_LOOKUP_TABLE; + goto out_close; } ret = read_lookup_table(w); if (ret != 0) - return ret; + goto out_close; w->image_metadata = CALLOC(w->hdr.image_count, sizeof(struct image_metadata)); @@ -496,7 +522,8 @@ static int begin_read(WIMStruct *w, const char *in_wim_path, int open_flags) if (!w->image_metadata) { ERROR("Failed to allocate memory for %u metadata structures", w->hdr.image_count); - return WIMLIB_ERR_NOMEM; + ret = WIMLIB_ERR_NOMEM; + goto out_free_lookup_table; } w->current_image = 0; @@ -507,7 +534,7 @@ static int begin_read(WIMStruct *w, const char *in_wim_path, int open_flags) append_metadata_resource_entry, w); if (ret != 0) - return ret; + goto out_free_image_metadata; /* Make sure all the expected images were found. (We already have * returned false if *extra* images were found) */ @@ -516,10 +543,10 @@ static int begin_read(WIMStruct *w, const char *in_wim_path, int open_flags) { ERROR("Only found %u images in WIM, but expected %u", w->current_image, w->hdr.image_count); - return WIMLIB_ERR_IMAGE_COUNT; + ret = WIMLIB_ERR_IMAGE_COUNT; + goto out_free_image_metadata; } - /* Sort images by the position of their metadata resources. I'm * assuming that is what determines the other of the images in the WIM * file, rather than their order in the lookup table, which is random @@ -533,10 +560,8 @@ static int begin_read(WIMStruct *w, const char *in_wim_path, int open_flags) ret = read_xml_data(w->fp, &w->hdr.xml_res_entry, &w->xml_data, &w->wim_info); - if (ret != 0) { - ERROR("Missing or invalid XML data"); - return ret; - } + if (ret != 0) + goto out_free_image_metadata; xml_num_images = wim_info_get_num_images(w->wim_info); if (xml_num_images != w->hdr.image_count) { @@ -544,11 +569,34 @@ static int begin_read(WIMStruct *w, const char *in_wim_path, int open_flags) "in the XML data,", in_wim_path, xml_num_images); ERROR("but %u images in the WIM! There must be exactly one " " element per image.", w->hdr.image_count); - return WIMLIB_ERR_IMAGE_COUNT; + ret = WIMLIB_ERR_IMAGE_COUNT; + goto out_free_xml_data; } DEBUG("Done beginning read of WIM file `%s'.", in_wim_path); return 0; + + // + // Everything is freed in wimlib_free() anyway, so no need to roll back + // changes here. + // +out_free_xml_data: + /*FREE(w->xml_data);*/ + /*w->xml_data = NULL;*/ + /*free_wim_info(w->wim_info);*/ + /*w->wim_info = NULL;*/ +out_free_image_metadata: + /*FREE(w->image_metadata);*/ + /*w->image_metadata = NULL;*/ + /*w->current_image = WIM_NO_IMAGE;*/ +out_free_lookup_table: + /*free_lookup_table(w->lookup_table);*/ + /*w->lookup_table = NULL;*/ +out_close: + /*fclose(w->fp);*/ + /*w->fp = NULL;*/ +out: + return ret; } @@ -619,4 +667,3 @@ WIMLIBAPI void wimlib_free(WIMStruct *w) #endif FREE(w); } - diff --git a/src/wimlib.h b/src/wimlib.h index 2ccbd580..abbdb703 100644 --- a/src/wimlib.h +++ b/src/wimlib.h @@ -141,14 +141,18 @@ * * After creating or modifying a WIM file, you can write it to a file using * wimlib_write(). Alternatively, if the WIM was originally read from a file, - * you can use wimlib_overwrite() to overwrite the original file. In some - * cases, wimlib_overwrite_xml_and_header() can be used instead. + * you can use wimlib_overwrite() to overwrite the original file. + * + * Please not: merely by calling wimlib_add_image() or many of the other + * functions in this library that operate on ::WIMStruct's, you are @b not + * modifing the WIM file on disk. Changes are not saved until you explicitly + * call wimlib_write() or wimlib_overwrite(). * * After you are done with the WIM file, use wimlib_free() to free all memory * associated with a ::WIMStruct and close all files associated with it. * - * To see an example of how to use wimlib, see the file - * @c programs/imagex.c in wimlib's source tree. + * To see an example of how to use wimlib, see the file @c programs/imagex.c in + * wimlib's source tree. * * wimlib supports custom memory allocators; use wimlib_set_memory_allocator() * for this. However, if wimlib calls into @c libntfs-3g, the custom memory @@ -272,6 +276,10 @@ enum wim_compression_type { /** Call fsync() when the WIM file is closed */ #define WIMLIB_WRITE_FLAG_FSYNC 0x00000008 +/** Re-build the entire WIM file rather than appending data to it, if possible. + * (Applies to wimlib_overwrite(), not wimlib_write()). */ +#define WIMLIB_WRITE_FLAG_REBUILD 0x00000010 + /** Mark the image being added as the bootable image of the WIM. */ #define WIMLIB_ADD_IMAGE_FLAG_BOOT 0x00000001 @@ -352,6 +360,7 @@ enum wimlib_error_code { WIMLIB_ERR_READLINK, WIMLIB_ERR_READ, WIMLIB_ERR_RENAME, + WIMLIB_ERR_REOPEN, WIMLIB_ERR_RESOURCE_ORDER, WIMLIB_ERR_SPECIAL_FILE, WIMLIB_ERR_SPLIT_INVALID, @@ -1074,12 +1083,33 @@ extern int wimlib_open_wim(const char *wim_file, int flags, /** * Overwrites the file that the WIM was originally read from, with changes made. * - * The new WIM is written to a temporary file and then renamed to the original - * file after it is has been completely written. The temporary file currently - * is made in the same directory as the original WIM file. - * - * After this function returns, @a wim must be freed using wimlib_free(). Any - * further actions on @a wim before doing this are undefined. + * There are two ways that a WIM may be overwritten. The first is to do a full + * rebuild: the new WIM is written to a temporary file and then renamed to the + * original file after it is has been completely written. The temporary file + * currently is made in the same directory as the original WIM file. A full + * rebuild may take a while, but can be used even if images have been modified + * or deleted, will produce a WIM with no holes, and has little chance of + * unintentional data loss because the temporary WIM is fsync()ed before being + * renamed to the original WIM. + * + * The second way to overwrite a WIM is by appending to the end of it. This can + * be much faster than a full rebuild, but it only works if the only operations + * on the WIM have been to change the header or XML data, or to add new images. + * Writing a WIM in this mode begins with writing any new file resources *after* + * everything in the old WIM, even though this will leave a hole where the old + * lookup table, XML data, and integrity were. This is done so that the WIM + * remains valid even if the operation is aborted mid-write. + * + * By default, the overwrite mode is chosen based on the past operations + * performed on the WIM. Use the flag ::WIMLIB_WRITE_FLAG_REBUILD to explicitly + * request a full rebuild. + * + * In the temporary-file overwrite mode, no changes are made to the WIM on + * failure, and the temporary file is deleted (if possible). Abnormal + * termination of the program will result in the temporary file being orphaned. + * In the direct append mode, the WIM is truncated to the original length on + * failure, while abnormal termination of the program will result in extra data + * appended to the original WIM, but it will still be a valid WIM. * * @param wim * Pointer to the ::WIMStruct for the WIM file to write. There may have @@ -1099,49 +1129,18 @@ extern int wimlib_open_wim(const char *wim_file, int flags, * @retval ::WIMLIB_ERR_RENAME * The temporary file that the WIM was written to could not be renamed to * the original filename of @a wim. + * @retval ::WIMLIB_ERR_REOPEN + * The WIM was overwritten successfully, but it could not be re-opened + * read-only. Therefore, the resources in the WIM can no longer be + * accessed, so this limits the functions that can be called on @a wim + * before calling wimlib_free(). */ extern int wimlib_overwrite(WIMStruct *wim, int write_flags, unsigned num_threads); /** - * Updates the header and XML data of the WIM file, without the need to write - * out the entire WIM to a temporary file as in wimlib_write(). - * - * This function must only be used if no files, directories, or images have been - * added, removed, or changed in the WIM. It must be used when only the boot - * index or the name or description of image(s) has been changed. - * - * After this function returns, @a wim must be freed using wimlib_free(). Any - * further actions on @a wim before doing this are undefined. - * - * @param wim - * Pointer to the ::WIMStruct for the WIM file to overwrite. - * @param write_flags - * Bitwise OR of ::WIMLIB_WRITE_FLAG_CHECK_INTEGRITY and/or - * ::WIMLIB_WRITE_FLAG_SHOW_PROGRESS. - * - * @return 0 on success; nonzero on error. - * - * @retval ::WIMLIB_ERR_NO_FILENAME - * @a wim corresponds to a WIM created with wimlib_create_new_wim() rather - * than a WIM read with wimlib_open_wim(). - * @retval ::WIMLIB_ERR_NOMEM - * Failed to allocate needed memory. - * @retval ::WIMLIB_ERR_OPEN - * The WIM file associated with @a wim could not be re-opened read-write. - * @retval ::WIMLIB_ERR_READ - * ::WIMLIB_WRITE_FLAG_CHECK_INTEGRITY was specified in @a flags, but data - * from the WIM file associated with @a wim could not be read to compute - * the SHA1 message digests, or the old integrity table (if it existed) - * could not be read. - * @retval ::WIMLIB_ERR_RESOURCE_ORDER - * Stream resources appeared in the WIM after the XML data or integrity - * table, so we could not safely overwrite the XML data and integrity - * table. Note: this error should never be received from WIMs that were - * written by this library. - * @retval ::WIMLIB_ERR_WRITE - * Failed to write the WIM header, the XML data, or the integrity table to - * the WIM file associated with @a wim. + * This function is deprecated; call wimlib_overwrite() without the + * WIMLIB_WRITE_FLAG_REBUILD flag instead. */ extern int wimlib_overwrite_xml_and_header(WIMStruct *wim, int write_flags); diff --git a/src/wimlib_internal.h b/src/wimlib_internal.h index 96136d8a..57c0db05 100644 --- a/src/wimlib_internal.h +++ b/src/wimlib_internal.h @@ -55,11 +55,9 @@ struct inode; * yet. The differences between the versions are undocumented. */ #define WIM_VERSION 0x10d00 -enum wim_integrity_status { - WIM_INTEGRITY_OK, - WIM_INTEGRITY_NOT_OK, - WIM_INTEGRITY_NONEXISTENT, -}; +#define WIM_INTEGRITY_OK 0 +#define WIM_INTEGRITY_NOT_OK -1 +#define WIM_INTEGRITY_NONEXISTENT -2 /* Metadata for a resource in a WIM file. */ struct resource_entry { @@ -249,8 +247,9 @@ struct image_metadata { /* True if the filesystem of the image has been modified. If this is * the case, the memory for the filesystem is not freed when switching * to a different WIM image. */ - bool modified; + u8 modified : 1; + u8 has_been_mounted_rw : 1; }; #define WIMLIB_RESOURCE_FLAG_RAW 0x1 @@ -260,43 +259,40 @@ struct image_metadata { typedef struct WIMStruct { /* A pointer to the file indicated by @filename, opened for reading. */ - FILE *fp; + FILE *fp; #ifdef WITH_FUSE + /* Extra file pointers to be used by concurrent readers */ FILE **fp_tab; size_t num_allocated_fps; pthread_mutex_t fp_tab_mutex; #endif /* FILE pointer for the WIM file that is being written. */ - FILE *out_fp; + FILE *out_fp; /* The name of the WIM file that has been opened. */ - char *filename; + char *filename; /* The lookup table for the WIM file. */ struct lookup_table *lookup_table; /* Pointer to the XML data read from the WIM file. */ - u8 *xml_data; + u8 *xml_data; /* Information retrieved from the XML data, arranged * in an orderly manner. */ - struct wim_info *wim_info; + struct wim_info *wim_info; /* Array of the image metadata of length image_count. Each image in the * WIM has a image metadata associated with it. */ - struct image_metadata *image_metadata; + struct image_metadata *image_metadata; /* The header of the WIM file. */ - struct wim_header hdr; + struct wim_header hdr; - /* Temporary flags to use when extracting a WIM image or adding a WIM - * image. */ + /* Temporary fields */ union { - int extract_flags; - int add_flags; - int write_flags; bool write_metadata; void *private; }; @@ -309,6 +305,7 @@ typedef struct WIMStruct { * image_metadata array. */ int current_image; + /* %true iff any images have been deleted from this WIM. */ bool deletion_occurred; } WIMStruct; @@ -407,8 +404,10 @@ extern int init_header(struct wim_header *hdr, int ctype); /* integrity.c */ extern int write_integrity_table(FILE *out, u64 end_header_offset, u64 end_lookup_table_offset, - int show_progress); -extern int check_wim_integrity(WIMStruct *w, int show_progress, int *status); + int show_progress, + struct resource_entry *out_res_entry); + +extern int check_wim_integrity(WIMStruct *w, int show_progress); /* join.c */ @@ -492,6 +491,8 @@ extern WIMStruct *new_wim_struct(); extern int select_wim_image(WIMStruct *w, int image); extern int wim_hdr_flags_compression_type(int wim_hdr_flags); extern int for_image(WIMStruct *w, int image, int (*visitor)(WIMStruct *)); +extern int open_wim_readable(WIMStruct *w, const char *path); +extern int open_wim_writable(WIMStruct *w, const char *path); /* Internal use only */ #define WIMLIB_WRITE_FLAG_NO_LOOKUP_TABLE 0x80000000 diff --git a/src/write.c b/src/write.c index bc3d93c2..e48d3c91 100644 --- a/src/write.c +++ b/src/write.c @@ -54,269 +54,32 @@ #include #endif -/* Reopens the FILE* for a WIM read-write. */ -static int reopen_rw(WIMStruct *w) +static int do_fflush(FILE *fp) { - FILE *fp; - - if (fclose(w->fp) != 0) - ERROR_WITH_ERRNO("Failed to close the file `%s'", w->filename); - w->fp = NULL; - fp = fopen(w->filename, "r+b"); - if (!fp) { - ERROR_WITH_ERRNO("Failed to open `%s' for reading and writing", - w->filename); - return WIMLIB_ERR_OPEN; - } - w->fp = fp; - return 0; -} - - - -/* - * Writes a WIM file to the original file that it was read from, overwriting it. - */ -WIMLIBAPI int wimlib_overwrite(WIMStruct *w, int write_flags, - unsigned num_threads) -{ - size_t wim_name_len; - int ret; - - if (!w) - return WIMLIB_ERR_INVALID_PARAM; - - write_flags &= ~WIMLIB_WRITE_FLAG_NO_LOOKUP_TABLE; - if (!w->filename) - return WIMLIB_ERR_NO_FILENAME; - - DEBUG("Replacing WIM file `%s'.", w->filename); - - /* Write the WIM to a temporary file. */ - /* XXX should the temporary file be somewhere else? */ - wim_name_len = strlen(w->filename); - char tmpfile[wim_name_len + 10]; - memcpy(tmpfile, w->filename, wim_name_len); - randomize_char_array_with_alnum(tmpfile + wim_name_len, 9); - tmpfile[wim_name_len + 9] = '\0'; - - ret = wimlib_write(w, tmpfile, WIM_ALL_IMAGES, - write_flags | WIMLIB_WRITE_FLAG_FSYNC, - num_threads); + int ret = fflush(fp); if (ret != 0) { - ERROR("Failed to write the WIM file `%s'", tmpfile); - if (unlink(tmpfile) != 0) - WARNING("Failed to remove `%s'", tmpfile); - return ret; - } - - DEBUG("Closing original WIM file."); - /* Close the original WIM file that was opened for reading. */ - if (w->fp) { - if (fclose(w->fp) != 0) { - WARNING("Failed to close the file `%s'", w->filename); - } - w->fp = NULL; - } - - DEBUG("Renaming `%s' to `%s'", tmpfile, w->filename); - - /* Rename the new file to the old file .*/ - if (rename(tmpfile, w->filename) != 0) { - ERROR_WITH_ERRNO("Failed to rename `%s' to `%s'", - tmpfile, w->filename); - ret = WIMLIB_ERR_RENAME; - goto err; - } - - if (write_flags & WIMLIB_WRITE_FLAG_SHOW_PROGRESS) - printf("Successfully renamed `%s' to `%s'\n", tmpfile, w->filename); - - return 0; -err: - /* Remove temporary file. */ - if (unlink(tmpfile) != 0) - ERROR_WITH_ERRNO("Failed to remove `%s'", tmpfile); - return ret; -} - -static int check_resource_offset(struct lookup_table_entry *lte, void *arg) -{ - u64 xml_data_offset = *(u64*)arg; - if (lte->resource_entry.offset > xml_data_offset) { - ERROR("The following resource is *after* the XML data:"); - print_lookup_table_entry(lte); - return WIMLIB_ERR_RESOURCE_ORDER; + ERROR_WITH_ERRNO("Failed to flush data to output WIM file"); + return WIMLIB_ERR_WRITE; } return 0; } -WIMLIBAPI int wimlib_overwrite_xml_and_header(WIMStruct *w, int write_flags) +static int fflush_and_ftruncate(FILE *fp, off_t size) { int ret; - FILE *fp; - u8 *integrity_table = NULL; - off_t xml_end; - off_t xml_size; - size_t bytes_written; - - DEBUG("Overwriting XML and header of `%s', write_flags = %#x", - w->filename, write_flags); - - if (!w->filename) - return WIMLIB_ERR_NO_FILENAME; - - write_flags &= ~WIMLIB_WRITE_FLAG_NO_LOOKUP_TABLE; - - /* Make sure that the integrity table (if present) is after the XML - * data, and that there are no stream resources, metadata resources, or - * lookup tables after the XML data. Otherwise, these data would be - * destroyed by this function. */ - if (w->hdr.integrity.offset != 0 && - w->hdr.integrity.offset < w->hdr.xml_res_entry.offset) { - ERROR("Didn't expect the integrity table to be before the XML data"); - return WIMLIB_ERR_RESOURCE_ORDER; - } - - if (w->hdr.lookup_table_res_entry.offset > - w->hdr.xml_res_entry.offset) { - ERROR("Didn't expect the lookup table to be after the XML data"); - return WIMLIB_ERR_RESOURCE_ORDER; - } - - ret = for_lookup_table_entry(w->lookup_table, check_resource_offset, - &w->hdr.xml_res_entry.offset); - if (ret != 0) - return ret; - - ret = reopen_rw(w); - if (ret != 0) - return ret; - fp = w->fp; - - /* The old integrity table is still OK, as the SHA1 message digests in - * the integrity table include neither the header nor the XML data. - * Save it for later if it exists and an integrity table was required. - * */ - if ((write_flags & WIMLIB_WRITE_FLAG_CHECK_INTEGRITY) - && w->hdr.integrity.offset != 0) - { - DEBUG("Reading existing integrity table."); - integrity_table = MALLOC(w->hdr.integrity.size); - if (!integrity_table) - return WIMLIB_ERR_NOMEM; - - ret = read_uncompressed_resource(fp, w->hdr.integrity.offset, - w->hdr.integrity.original_size, - integrity_table); - if (ret != 0) - goto err; - DEBUG("Done reading existing integrity table."); - } - - DEBUG("Overwriting XML data."); - /* Overwrite the XML data. */ - if (fseeko(fp, w->hdr.xml_res_entry.offset, SEEK_SET) != 0) { - ERROR_WITH_ERRNO("Failed to seek to byte %"PRIu64" " - "for XML data", w->hdr.xml_res_entry.offset); - ret = WIMLIB_ERR_WRITE; - goto err; - } - ret = write_xml_data(w->wim_info, WIM_ALL_IMAGES, fp, 0); - if (ret != 0) - goto err; - - DEBUG("Updating XML resource entry."); - /* Update the XML resource entry in the WIM header. */ - xml_end = ftello(fp); - if (xml_end == -1) { - ret = WIMLIB_ERR_WRITE; - goto err; - } - xml_size = xml_end - w->hdr.xml_res_entry.offset; - w->hdr.xml_res_entry.size = xml_size; - w->hdr.xml_res_entry.original_size = xml_size; - /* XML data offset is unchanged. */ - - if (write_flags & WIMLIB_WRITE_FLAG_CHECK_INTEGRITY) { - DEBUG("Writing integrity table."); - w->hdr.integrity.offset = xml_end; - if (integrity_table) { - /* The existing integrity table was saved. */ - bytes_written = fwrite(integrity_table, 1, - w->hdr.integrity.size, fp); - if (bytes_written != w->hdr.integrity.size) { - ERROR_WITH_ERRNO("Failed to write integrity " - "table"); - ret = WIMLIB_ERR_WRITE; - goto err; - } - FREE(integrity_table); - } else { - /* There was no existing integrity table, so a new one - * must be calculated. */ - ret = write_integrity_table(fp, WIM_HEADER_DISK_SIZE, - w->hdr.lookup_table_res_entry.offset + - w->hdr.lookup_table_res_entry.size, - write_flags & WIMLIB_WRITE_FLAG_SHOW_PROGRESS); - if (ret != 0) - return ret; - - off_t end_integrity = ftello(fp); - if (end_integrity == -1) - return WIMLIB_ERR_WRITE; - - off_t integrity_size = end_integrity - xml_end; - w->hdr.integrity.size = integrity_size; - w->hdr.integrity.original_size = integrity_size; - w->hdr.integrity.flags = 0; - } - } else { - DEBUG("Truncating file to end of XML data."); - /* No integrity table to write. The file should be truncated - * because it's possible that the old file was longer (due to it - * including an integrity table, or due to its XML data being - * longer) */ - if (fflush(fp) != 0) { - ERROR_WITH_ERRNO("Failed to flush stream for file `%s'", - w->filename); - return WIMLIB_ERR_WRITE; - } - if (ftruncate(fileno(fp), xml_end) != 0) { - ERROR_WITH_ERRNO("Failed to truncate `%s' to %"PRIu64" " - "bytes", w->filename, xml_end); - return WIMLIB_ERR_WRITE; - } - memset(&w->hdr.integrity, 0, sizeof(struct resource_entry)); - } - - DEBUG("Overwriting header."); - /* Overwrite the header. */ - if (fseeko(fp, 0, SEEK_SET) != 0) { - ERROR_WITH_ERRNO("Failed to seek to beginning of `%s'", - w->filename); - return WIMLIB_ERR_WRITE; - } - - ret = write_header(&w->hdr, fp); + ret = do_fflush(fp); if (ret != 0) return ret; - - DEBUG("Closing `%s'.", w->filename); - if (fclose(fp) != 0) { - ERROR_WITH_ERRNO("Failed to close `%s'", w->filename); + ret = ftruncate(fileno(fp), size); + if (ret != 0) { + ERROR_WITH_ERRNO("Failed to truncate output WIM file to " + "%"PRIu64" bytes", size); return WIMLIB_ERR_WRITE; } - w->fp = NULL; - DEBUG("Done."); return 0; -err: - FREE(integrity_table); - return ret; } - /* Chunk table that's located at the beginning of each compressed resource in * the WIM. (This is not the on-disk format; the on-disk format just has an * array of offsets.) */ @@ -376,19 +139,8 @@ out: return ret; } -typedef int (*compress_func_t)(const void *, unsigned, void *, unsigned *); - -compress_func_t get_compress_func(int out_ctype) -{ - if (out_ctype == WIM_COMPRESSION_TYPE_LZX) - return lzx_compress; - else - return xpress_compress; -} - - /* - * Compresses a chunk of a WIM resource. + * Pointer to function to compresses a chunk of a WIM resource. * * @chunk: Uncompressed data of the chunk. * @chunk_size: Size of the uncompressed chunk in bytes. @@ -397,21 +149,19 @@ compress_func_t get_compress_func(int out_ctype) * @compressed_chunk_len_ret: Pointer to an unsigned int into which the size * of the compressed chunk will be * returned. - * @ctype: Type of compression to use. Must be WIM_COMPRESSION_TYPE_LZX - * or WIM_COMPRESSION_TYPE_XPRESS. * * Returns zero if compressed succeeded, and nonzero if the chunk could not be * compressed to any smaller than @chunk_size. This function cannot fail for * any other reasons. */ -static int compress_chunk(const u8 chunk[], unsigned chunk_size, - u8 compressed_chunk[], - unsigned *compressed_chunk_len_ret, - int ctype) +typedef int (*compress_func_t)(const void *, unsigned, void *, unsigned *); + +compress_func_t get_compress_func(int out_ctype) { - compress_func_t compress = get_compress_func(ctype); - return (*compress)(chunk, chunk_size, compressed_chunk, - compressed_chunk_len_ret); + if (out_ctype == WIM_COMPRESSION_TYPE_LZX) + return lzx_compress; + else + return xpress_compress; } /* @@ -428,23 +178,17 @@ static int compress_chunk(const u8 chunk[], unsigned chunk_size, * Returns 0 on success; nonzero on failure. */ static int write_wim_resource_chunk(const u8 chunk[], unsigned chunk_size, - FILE *out_fp, int out_ctype, + FILE *out_fp, compress_func_t compress, struct chunk_table *chunk_tab) { const u8 *out_chunk; unsigned out_chunk_size; - - wimlib_assert(chunk_size <= WIM_CHUNK_SIZE); - - if (!chunk_tab) { - out_chunk = chunk; - out_chunk_size = chunk_size; - } else { + if (chunk_tab) { u8 *compressed_chunk = alloca(chunk_size); int ret; - ret = compress_chunk(chunk, chunk_size, compressed_chunk, - &out_chunk_size, out_ctype); + ret = compress(chunk, chunk_size, compressed_chunk, + &out_chunk_size); if (ret == 0) { out_chunk = compressed_chunk; } else { @@ -453,8 +197,10 @@ static int write_wim_resource_chunk(const u8 chunk[], unsigned chunk_size, } *chunk_tab->cur_offset_p++ = chunk_tab->cur_offset; chunk_tab->cur_offset += out_chunk_size; + } else { + out_chunk = chunk; + out_chunk_size = chunk_size; } - if (fwrite(out_chunk, 1, out_chunk_size, out_fp) != out_chunk_size) { ERROR_WITH_ERRNO("Failed to write WIM resource chunk"); return WIMLIB_ERR_WRITE; @@ -503,6 +249,8 @@ finish_wim_resource_chunk_tab(struct chunk_table *chunk_tab, return 0; } +/* Prepare for multiple reads to a resource by caching a FILE * or NTFS + * attribute pointer in the lookup table entry. */ static int prepare_resource_for_read(struct lookup_table_entry *lte #ifdef WITH_NTFS_3G @@ -550,6 +298,8 @@ static int prepare_resource_for_read(struct lookup_table_entry *lte return 0; } +/* Undo prepare_resource_for_read() by closing the cached FILE * or NTFS + * attribute. */ static void end_wim_resource_read(struct lookup_table_entry *lte #ifdef WITH_NTFS_3G , ntfs_inode *ni @@ -608,6 +358,7 @@ int write_wim_resource(struct lookup_table_entry *lte, struct chunk_table *chunk_tab = NULL; bool raw; off_t file_offset; + compress_func_t compress; #ifdef WITH_NTFS_3G ntfs_inode *ni = NULL; #endif @@ -673,13 +424,15 @@ int write_wim_resource(struct lookup_table_entry *lte, * hash given in the lookup table entry once we've finished reading the * resource. */ SHA_CTX ctx; - if (!raw) + if (!raw) { sha1_init(&ctx); + compress = get_compress_func(out_ctype); + } + offset = 0; /* While there are still bytes remaining in the WIM resource, read a * chunk of the resource, update SHA1, then write that chunk using the * desired compression type. */ - offset = 0; do { u64 to_read = min(bytes_remaining, WIM_CHUNK_SIZE); ret = read_wim_resource(lte, buf, to_read, offset, flags); @@ -688,7 +441,7 @@ int write_wim_resource(struct lookup_table_entry *lte, if (!raw) sha1_update(&ctx, buf, to_read); ret = write_wim_resource_chunk(buf, to_read, out_fp, - out_ctype, chunk_tab); + compress, chunk_tab); if (ret != 0) goto out_fclose; bytes_remaining -= to_read; @@ -754,16 +507,10 @@ int write_wim_resource(struct lookup_table_entry *lte, out_res_entry, flags); if (ret != 0) goto out_fclose; - if (fflush(out_fp) != 0) { - ERROR_WITH_ERRNO("Failed to flush output WIM file"); - ret = WIMLIB_ERR_WRITE; - goto out_fclose; - } - if (ftruncate(fileno(out_fp), file_offset + out_res_entry->size) != 0) { - ERROR_WITH_ERRNO("Failed to truncate output WIM file"); - ret = WIMLIB_ERR_WRITE; + + ret = fflush_and_ftruncate(out_fp, file_offset + out_res_entry->size); + if (ret != 0) goto out_fclose; - } } else { if (out_res_entry) { out_res_entry->size = new_compressed_size; @@ -787,7 +534,6 @@ out: return ret; } - #ifdef ENABLE_MULTITHREADED_COMPRESSION struct shared_queue { sem_t filled_slots; @@ -848,13 +594,6 @@ static void *shared_queue_get(struct shared_queue *q) return obj; } -static inline int shared_queue_get_filled(struct shared_queue *q) -{ - int sval; - sem_getvalue(&q->filled_slots, &sval); - return sval; -} - struct compressor_thread_params { struct shared_queue *res_to_compress_queue; struct shared_queue *compressed_res_queue; @@ -976,7 +715,7 @@ static int write_wim_chunks(struct message *msg, FILE *out_fp, if (fwrite(msg->out_compressed_chunks[i], 1, chunk_csize, out_fp) != chunk_csize) { - ERROR_WITH_ERRNO("Failed to write WIM"); + ERROR_WITH_ERRNO("Failed to write WIM chunk"); return WIMLIB_ERR_WRITE; } @@ -1465,7 +1204,6 @@ static int write_stream_list_parallel(struct list_head *stream_list, wimlib_assert(stream_list->next != stream_list); - static const double MESSAGES_PER_THREAD = 2.0; size_t queue_size = (size_t)(num_threads * MESSAGES_PER_THREAD); @@ -1536,6 +1274,10 @@ out_serial: } #endif +/* + * Write a list of streams to a WIM (@out_fp) using the compression type + * @out_ctype and up to @num_threads compressor threads. + */ static int write_stream_list(struct list_head *stream_list, FILE *out_fp, int out_ctype, int write_flags, unsigned num_threads) @@ -1597,7 +1339,7 @@ static int dentry_find_streams_to_write(struct dentry *dentry, for (unsigned i = 0; i <= dentry->d_inode->num_ads; i++) { lte = inode_stream_lte(dentry->d_inode, i, w->lookup_table); if (lte && ++lte->out_refcnt == 1) - list_add(<e->staging_list, stream_list); + list_add_tail(<e->staging_list, stream_list); } return 0; } @@ -1612,8 +1354,8 @@ static int write_wim_streams(WIMStruct *w, int image, int write_flags, unsigned num_threads) { + for_lookup_table_entry(w->lookup_table, lte_zero_out_refcnt, NULL); LIST_HEAD(stream_list); - w->private = &stream_list; for_image(w, image, find_streams_to_write); return write_stream_list(&stream_list, w->out_fp, @@ -1638,74 +1380,36 @@ int finish_write(WIMStruct *w, int image, int write_flags) struct wim_header hdr; FILE *out = w->out_fp; - if (!(write_flags & WIMLIB_WRITE_FLAG_NO_LOOKUP_TABLE)) { - /* Write the lookup table. */ - lookup_table_offset = ftello(out); - if (lookup_table_offset == -1) - return WIMLIB_ERR_WRITE; - - DEBUG("Writing lookup table (offset %"PRIu64")", - lookup_table_offset); - ret = write_lookup_table(w->lookup_table, out); - if (ret != 0) - return ret; - } - - xml_data_offset = ftello(out); - if (xml_data_offset == -1) - return WIMLIB_ERR_WRITE; - /* @hdr will be the header for the new WIM. First copy all the data * from the header in the WIMStruct; then set all the fields that may * have changed, including the resource entries, boot index, and image * count. */ memcpy(&hdr, &w->hdr, sizeof(struct wim_header)); + if (!(write_flags & WIMLIB_WRITE_FLAG_NO_LOOKUP_TABLE)) { - lookup_table_size = xml_data_offset - lookup_table_offset; - hdr.lookup_table_res_entry.offset = lookup_table_offset; - hdr.lookup_table_res_entry.size = lookup_table_size; + ret = write_lookup_table(w->lookup_table, out, &hdr.lookup_table_res_entry); + if (ret != 0) + goto out; } - hdr.lookup_table_res_entry.original_size = hdr.lookup_table_res_entry.size; - hdr.lookup_table_res_entry.flags = WIM_RESHDR_FLAG_METADATA; - DEBUG("Writing XML data (offset %"PRIu64")", xml_data_offset); ret = write_xml_data(w->wim_info, image, out, (write_flags & WIMLIB_WRITE_FLAG_NO_LOOKUP_TABLE) ? - wim_info_get_total_bytes(w->wim_info) : 0); + wim_info_get_total_bytes(w->wim_info) : 0, + &hdr.xml_res_entry); if (ret != 0) - return ret; - - integrity_offset = ftello(out); - if (integrity_offset == -1) - return WIMLIB_ERR_WRITE; - xml_data_size = integrity_offset - xml_data_offset; - - hdr.xml_res_entry.offset = xml_data_offset; - hdr.xml_res_entry.size = xml_data_size; - hdr.xml_res_entry.original_size = xml_data_size; - hdr.xml_res_entry.flags = WIM_RESHDR_FLAG_METADATA; + goto out; if (write_flags & WIMLIB_WRITE_FLAG_CHECK_INTEGRITY) { - ret = write_integrity_table(out, WIM_HEADER_DISK_SIZE, - xml_data_offset, - write_flags & WIMLIB_WRITE_FLAG_SHOW_PROGRESS); + ret = write_integrity_table(out, + WIM_HEADER_DISK_SIZE, + hdr.xml_res_entry.offset, + write_flags & WIMLIB_WRITE_FLAG_SHOW_PROGRESS, + &hdr.integrity); if (ret != 0) - return ret; - end_offset = ftello(out); - if (end_offset == -1) - return WIMLIB_ERR_WRITE; - integrity_size = end_offset - integrity_offset; - hdr.integrity.offset = integrity_offset; - hdr.integrity.size = integrity_size; - hdr.integrity.original_size = integrity_size; + goto out; } else { - hdr.integrity.offset = 0; - hdr.integrity.size = 0; - hdr.integrity.original_size = 0; + memset(&hdr.integrity, 0, sizeof(struct resource_entry)); } - hdr.integrity.flags = 0; - - DEBUG("Updating WIM header."); /* * In the WIM header, there is room for the resource entry for a @@ -1734,59 +1438,55 @@ int finish_write(WIMStruct *w, int image, int write_flags) hdr.boot_idx = 0; } - - if (fseeko(out, 0, SEEK_SET) != 0) - return WIMLIB_ERR_WRITE; + if (fseeko(out, 0, SEEK_SET) != 0) { + ret = WIMLIB_ERR_WRITE; + ERROR_WITH_ERRNO("Failed to seek to beginning of WIM " + "to overwrite header"); + goto out; + } ret = write_header(&hdr, out); if (ret != 0) - return ret; + goto out; if (write_flags & WIMLIB_WRITE_FLAG_FSYNC) { - DEBUG("fsync output WIM file"); if (fflush(out) != 0 || fsync(fileno(out)) != 0) { ERROR_WITH_ERRNO("Error flushing data to WIM file"); ret = WIMLIB_ERR_WRITE; + goto out; } } - DEBUG("Closing output WIM file."); - +out: if (fclose(out) != 0) { ERROR_WITH_ERRNO("Failed to close the WIM file"); - ret = WIMLIB_ERR_WRITE; + if (ret == 0) + ret = WIMLIB_ERR_WRITE; } w->out_fp = NULL; return ret; } -/* Open file stream and write dummy header for WIM. */ -int begin_write(WIMStruct *w, const char *path, int write_flags) +static void close_wim_writable(WIMStruct *w) { - const char *mode; - DEBUG("Opening `%s' for new WIM", path); - - /* checking the integrity requires going back over the file to read it. - * XXX - * (It also would be possible to keep a running sha1sum as the file is - * written-- this would be faster, but a bit more complicated) */ - if (write_flags & WIMLIB_WRITE_FLAG_CHECK_INTEGRITY) - mode = "w+b"; - else - mode = "wb"; - - if (w->out_fp) - fclose(w->out_fp); - - w->out_fp = fopen(path, mode); - if (!w->out_fp) { - ERROR_WITH_ERRNO("Failed to open the file `%s' for writing", - path); - return WIMLIB_ERR_OPEN; + if (w->out_fp) { + if (fclose(w->out_fp) != 0) { + WARNING("Failed to close output WIM: %s", + strerror(errno)); + } + w->out_fp = NULL; } +} +/* Open file stream and write dummy header for WIM. */ +int begin_write(WIMStruct *w, const char *path, int write_flags) +{ + int ret; + ret = open_wim_writable(w, path); + if (ret != 0) + return ret; /* Write dummy header. It will be overwritten later. */ return write_header(&w->hdr, w->out_fp); } @@ -1806,8 +1506,6 @@ WIMLIBAPI int wimlib_write(WIMStruct *w, const char *path, (image < 1 || image > w->hdr.image_count)) return WIMLIB_ERR_INVALID_IMAGE; - - if (w->hdr.total_parts != 1) { ERROR("Cannot call wimlib_write() on part of a split WIM"); return WIMLIB_ERR_SPLIT_UNSUPPORTED; @@ -1820,32 +1518,253 @@ WIMLIBAPI int wimlib_write(WIMStruct *w, const char *path, ret = begin_write(w, path, write_flags); if (ret != 0) - return ret; - - for_lookup_table_entry(w->lookup_table, lte_zero_out_refcnt, NULL); + goto out; ret = write_wim_streams(w, image, write_flags, num_threads); - - if (ret != 0) { - /*ERROR("Failed to write WIM file resources to `%s'", path);*/ - return ret; - } + if (ret != 0) + goto out; if (write_flags & WIMLIB_WRITE_FLAG_SHOW_PROGRESS) printf("Writing image metadata...\n"); ret = for_image(w, image, write_metadata_resource); + if (ret != 0) + goto out; - if (ret != 0) { - /*ERROR("Failed to write WIM image metadata to `%s'", path);*/ - return ret; + ret = finish_write(w, image, write_flags); + if (ret == 0 && (write_flags & WIMLIB_WRITE_FLAG_SHOW_PROGRESS)) + printf("Successfully wrote `%s'\n", path); +out: + close_wim_writable(w); + return ret; +} + +static int lte_overwrite_prepare(struct lookup_table_entry *lte, + void *ignore) +{ + memcpy(<e->output_resource_entry, <e->resource_entry, + sizeof(struct resource_entry)); + lte->out_refcnt = 0; + return 0; +} + +static int check_resource_offset(struct lookup_table_entry *lte, void *arg) +{ + u64 xml_data_offset = *(u64*)arg; + + wimlib_assert(lte->out_refcnt <= lte->refcnt); + if (lte->out_refcnt < lte->refcnt) { + if (lte->resource_entry.offset > xml_data_offset) { + ERROR("The following resource is after the XML data:"); + print_lookup_table_entry(lte); + return WIMLIB_ERR_RESOURCE_ORDER; + } } + return 0; +} - ret = finish_write(w, image, write_flags); +static int find_new_streams(struct lookup_table_entry *lte, void *arg) +{ + wimlib_assert(lte->out_refcnt <= lte->refcnt); + if (lte->out_refcnt == lte->refcnt) + list_add(<e->staging_list, (struct list_head*)arg); + else + lte->out_refcnt = lte->refcnt; + return 0; +} + +static int overwrite_wim_inplace(WIMStruct *w, int write_flags, + unsigned num_threads, + int modified_image_idx) +{ + int ret; + struct list_head stream_list; + off_t old_wim_end; + + DEBUG("Overwriting `%s' in-place", w->filename); + + /* Make sure that the integrity table (if present) is after the XML + * data, and that there are no stream resources, metadata resources, or + * lookup tables after the XML data. Otherwise, these data would be + * overwritten. */ + if (w->hdr.integrity.offset != 0 && + w->hdr.integrity.offset < w->hdr.xml_res_entry.offset) { + ERROR("Didn't expect the integrity table to be before the XML data"); + return WIMLIB_ERR_RESOURCE_ORDER; + } + + if (w->hdr.lookup_table_res_entry.offset > w->hdr.xml_res_entry.offset) { + ERROR("Didn't expect the lookup table to be after the XML data"); + return WIMLIB_ERR_RESOURCE_ORDER; + } + + DEBUG("Identifying newly added streams"); + for_lookup_table_entry(w->lookup_table, lte_overwrite_prepare, NULL); + INIT_LIST_HEAD(&stream_list); + for (int i = modified_image_idx; i < w->hdr.image_count; i++) { + DEBUG("Identifiying streams in image %d", i + 1); + wimlib_assert(w->image_metadata[i].modified); + wimlib_assert(!w->image_metadata[i].has_been_mounted_rw); + wimlib_assert(w->image_metadata[i].root_dentry != NULL); + w->private = &stream_list; + for_dentry_in_tree(w->image_metadata[i].root_dentry, + dentry_find_streams_to_write, w); + } + + ret = for_lookup_table_entry(w->lookup_table, check_resource_offset, + &w->hdr.xml_res_entry.offset); + if (ret != 0) + return ret; + + INIT_LIST_HEAD(&stream_list); + for_lookup_table_entry(w->lookup_table, find_new_streams, + &stream_list); + + if (w->hdr.integrity.offset) + old_wim_end = w->hdr.integrity.offset + w->hdr.integrity.size; + else + old_wim_end = w->hdr.xml_res_entry.offset + w->hdr.xml_res_entry.size; + + ret = open_wim_writable(w, w->filename); if (ret != 0) return ret; + if (fseeko(w->out_fp, old_wim_end, SEEK_SET) != 0) { + ERROR_WITH_ERRNO("Can't seek to end of WIM"); + return WIMLIB_ERR_WRITE; + } + + if (!list_empty(&stream_list)) { + DEBUG("Writing newly added streams (offset = %"PRIu64")", + old_wim_end); + ret = write_stream_list(&stream_list, w->out_fp, + wimlib_get_compression_type(w), + write_flags, num_threads); + if (ret != 0) + goto out_ftruncate; + } else { + DEBUG("No new streams were added"); + } + + for (int i = modified_image_idx; i < w->hdr.image_count; i++) { + wimlib_assert(w->image_metadata[i].modified); + wimlib_assert(!w->image_metadata[i].has_been_mounted_rw); + wimlib_assert(w->image_metadata[i].root_dentry != NULL); + wimlib_assert(w->image_metadata[i].metadata_lte != NULL); + ret = select_wim_image(w, i + 1); + wimlib_assert(ret == 0); + ret = write_metadata_resource(w); + if (ret != 0) + goto out_ftruncate; + } + ret = finish_write(w, WIM_ALL_IMAGES, write_flags); +out_ftruncate: + close_wim_writable(w); + if (ret != 0) { + WARNING("Truncating `%s' to its original size (%"PRIu64" bytes)", + w->filename, old_wim_end); + truncate(w->filename, old_wim_end); + } + return ret; +} + +static int overwrite_wim_via_tmpfile(WIMStruct *w, int write_flags, + unsigned num_threads) +{ + size_t wim_name_len; + int ret; + + DEBUG("Overwrining `%s' via a temporary file", w->filename); + + /* Write the WIM to a temporary file in the same directory as the + * original WIM. */ + wim_name_len = strlen(w->filename); + char tmpfile[wim_name_len + 10]; + memcpy(tmpfile, w->filename, wim_name_len); + randomize_char_array_with_alnum(tmpfile + wim_name_len, 9); + tmpfile[wim_name_len + 9] = '\0'; + + ret = wimlib_write(w, tmpfile, WIM_ALL_IMAGES, + write_flags | WIMLIB_WRITE_FLAG_FSYNC, + num_threads); + if (ret != 0) { + ERROR("Failed to write the WIM file `%s'", tmpfile); + goto err; + } + + /* Close the original WIM file that was opened for reading. */ + if (w->fp != NULL) { + fclose(w->fp); + w->fp = NULL; + } + + DEBUG("Renaming `%s' to `%s'", tmpfile, w->filename); + + /* Rename the new file to the old file .*/ + if (rename(tmpfile, w->filename) != 0) { + ERROR_WITH_ERRNO("Failed to rename `%s' to `%s'", + tmpfile, w->filename); + ret = WIMLIB_ERR_RENAME; + goto err; + } + if (write_flags & WIMLIB_WRITE_FLAG_SHOW_PROGRESS) - printf("Successfully wrote `%s'\n", path); - return 0; + printf("Successfully renamed `%s' to `%s'\n", tmpfile, w->filename); + + /* Re-open the WIM read-only. */ + w->fp = fopen(w->filename, "rb"); + if (w->fp == NULL) { + ret = WIMLIB_ERR_REOPEN; + WARNING("Failed to re-open `%s' read-only: %s", + w->filename, strerror(errno)); + } + return ret; +err: + /* Remove temporary file. */ + if (unlink(tmpfile) != 0) + WARNING("Failed to remove `%s': %s", tmpfile, strerror(errno)); + return ret; +} + +/* + * Writes a WIM file to the original file that it was read from, overwriting it. + */ +WIMLIBAPI int wimlib_overwrite(WIMStruct *w, int write_flags, + unsigned num_threads) +{ + int ret; + + if (!w) + return WIMLIB_ERR_INVALID_PARAM; + + write_flags &= ~WIMLIB_WRITE_FLAG_NO_LOOKUP_TABLE; + if (!w->filename) + return WIMLIB_ERR_NO_FILENAME; + + if (w->hdr.total_parts != 1) { + ERROR("Cannot modify a split WIM"); + return WIMLIB_ERR_SPLIT_UNSUPPORTED; + } + + if (!w->deletion_occurred && !(write_flags & WIMLIB_WRITE_FLAG_REBUILD)) { + int i, modified_image_idx; + for (i = 0; i < w->hdr.image_count && !w->image_metadata[i].modified; i++) + ; + modified_image_idx = i; + for (; i < w->hdr.image_count && w->image_metadata[i].modified && + !w->image_metadata[i].has_been_mounted_rw; i++) + ; + // XXX + /*if (i == w->hdr.image_count) {*/ + /*return overwrite_wim_inplace(w, write_flags, num_threads,*/ + /*modified_image_idx);*/ + /*}*/ + } + return overwrite_wim_via_tmpfile(w, write_flags, num_threads); +} + +/* Deprecated */ +WIMLIBAPI int wimlib_overwrite_xml_and_header(WIMStruct *wim, int write_flags) +{ + return wimlib_overwrite(wim, write_flags, 1); } diff --git a/src/xml.c b/src/xml.c index 76bc1713..f9e93fc3 100644 --- a/src/xml.c +++ b/src/xml.c @@ -1287,7 +1287,7 @@ out_cleanup_parser: * the offset of the XML data. */ int write_xml_data(const struct wim_info *wim_info, int image, FILE *out, - u64 total_bytes) + u64 total_bytes, struct resource_entry *out_res_entry) { xmlBuffer *buf; xmlTextWriter *writer; @@ -1297,11 +1297,16 @@ int write_xml_data(const struct wim_info *wim_info, int image, FILE *out, size_t len; size_t utf16_len; size_t bytes_written; + off_t start_offset, end_offset; wimlib_assert(image == WIM_ALL_IMAGES || (wim_info != NULL && image >= 1 && image <= wim_info->num_images)); + start_offset = ftello(out); + if (start_offset == -1) + return WIMLIB_ERR_WRITE; + /* The contents of the element in the XML data, under the * element not the element, is (for non-spit WIMs) the * size of the WIM file excluding the XML data and integrity table, @@ -1390,6 +1395,16 @@ int write_xml_data(const struct wim_info *wim_info, int image, FILE *out, DEBUG("Cleaning up."); + end_offset = ftello(out); + if (end_offset == -1) { + ret = WIMLIB_ERR_WRITE; + goto out_free_utf16_str; + } + + out_res_entry->offset = start_offset; + out_res_entry->size = end_offset - start_offset; + out_res_entry->original_size = end_offset - start_offset; + out_res_entry->flags = WIM_RESHDR_FLAG_METADATA; ret = 0; out_free_utf16_str: FREE(utf16_str); diff --git a/src/xml.h b/src/xml.h index f9999e90..5a875758 100644 --- a/src/xml.h +++ b/src/xml.h @@ -34,7 +34,7 @@ extern int read_xml_data(FILE *fp, const struct resource_entry *res, u8 **xml_data_ret, struct wim_info **info_ret); extern int write_xml_data(const struct wim_info *wim_info, int image, FILE *out, - u64 total_bytes); + u64 total_bytes, struct resource_entry *out_res_entry); static inline u64 wim_info_get_total_bytes(const struct wim_info *info) {