From f9695b9f40035f1a20968293255761a8301eaba0 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Sat, 4 May 2013 14:34:26 -0500 Subject: [PATCH] Preliminary support for native fds (UNIX only so far) --- src/dentry.c | 2 +- src/header.c | 49 +++---- src/integrity.c | 136 ++++++++---------- src/join.c | 23 ++- src/lookup_table.c | 174 ++++++++++++---------- src/lookup_table.h | 9 +- src/metadata_resource.c | 14 +- src/mount_image.c | 56 +------- src/ntfs-apply.c | 2 +- src/reparse.c | 2 +- src/resource.c | 206 +++++--------------------- src/split.c | 22 +-- src/util.c | 114 ++++++++++++++- src/util.h | 21 ++- src/wim.c | 71 +++++---- src/wimlib.h | 9 +- src/wimlib_internal.h | 43 +++--- src/write.c | 310 +++++++++++++++++++--------------------- src/xml.c | 34 +++-- src/xml.h | 4 +- 20 files changed, 609 insertions(+), 692 deletions(-) diff --git a/src/dentry.c b/src/dentry.c index 8f0ed206..73dba52b 100644 --- a/src/dentry.c +++ b/src/dentry.c @@ -1213,7 +1213,7 @@ inode_get_unix_data(const struct wim_inode *inode, if (size != sizeof(struct wimlib_unix_data)) return BAD_UNIX_DATA; - ret = read_full_resource_into_buf(lte, unix_data, true); + ret = read_full_resource_into_buf(lte, unix_data); if (ret) return ret; diff --git a/src/header.c b/src/header.c index bd2a0631..ff3bb95b 100644 --- a/src/header.c +++ b/src/header.c @@ -33,11 +33,10 @@ static const u8 wim_magic_chars[WIM_MAGIC_LEN] = { /* Reads the header for a WIM file. */ int -read_header(FILE *fp, struct wim_header *hdr, int open_flags) +read_header(filedes_t in_fd, struct wim_header *hdr, int open_flags) { size_t bytes_read; u8 buf[WIM_HEADER_DISK_SIZE]; - size_t hdr_rem_size; const u8 *p; u32 hdr_size; @@ -46,23 +45,24 @@ read_header(FILE *fp, struct wim_header *hdr, int open_flags) DEBUG("Reading WIM header."); - bytes_read = fread(buf, 1, WIM_MAGIC_LEN, fp); + bytes_read = full_read(in_fd, buf, WIM_HEADER_DISK_SIZE); - if (bytes_read != WIM_MAGIC_LEN) - goto err; + if (bytes_read != WIM_HEADER_DISK_SIZE) { + ERROR_WITH_ERRNO("Error reading WIM header"); + return WIMLIB_ERR_READ; + } /* Byte 8 */ - if (memcmp(buf, wim_magic_chars, WIM_MAGIC_LEN) != 0) { + p = buf; + + if (memcmp(p, wim_magic_chars, WIM_MAGIC_LEN)) { ERROR("Invalid magic characters in WIM header"); return WIMLIB_ERR_NOT_A_WIM_FILE; } - bytes_read = fread(&hdr_size, 1, sizeof(u32), fp); - if (bytes_read != sizeof(u32)) - goto err; - - hdr_size = le32_to_cpu(hdr_size); + p += 8; + p = get_u32(p, &hdr_size); /* Byte 12 */ @@ -72,15 +72,6 @@ read_header(FILE *fp, struct wim_header *hdr, int open_flags) return WIMLIB_ERR_INVALID_HEADER_SIZE; } - /* Read the rest of the header into a buffer. */ - - hdr_rem_size = WIM_HEADER_DISK_SIZE - WIM_MAGIC_LEN - sizeof(u32); - - bytes_read = fread(buf + WIM_MAGIC_LEN + sizeof(u32), 1, - hdr_rem_size, fp); - if (bytes_read != hdr_rem_size) - goto err; - p = get_u32(buf + WIM_MAGIC_LEN + sizeof(u32), &wim_version); if (wim_version != WIM_VERSION) { @@ -153,25 +144,17 @@ read_header(FILE *fp, struct wim_header *hdr, int open_flags) /* Byte 208 */ return 0; - -err: - if (feof(fp)) - ERROR("Unexpected EOF while reading WIM header"); - else - ERROR_WITH_ERRNO("Error reading WIM header"); - return WIMLIB_ERR_READ; } /* * Writes the header for a WIM file. * * @hdr: A pointer to a struct wim_header structure that describes the header. - * @out: The FILE* for the output file, positioned at the appropriate - * place (the beginning of the file). + * @out_fd: The file descriptor to the WIM file, opened for writing. * @return: Zero on success, nonzero on failure. */ int -write_header(const struct wim_header *hdr, FILE *out_fp) +write_header(const struct wim_header *hdr, int out_fd) { u8 buf[WIM_HEADER_DISK_SIZE]; u8 *p; @@ -197,8 +180,10 @@ write_header(const struct wim_header *hdr, FILE *out_fp) p = put_resource_entry(p, &hdr->boot_metadata_res_entry); p = put_u32(p, hdr->boot_idx); p = put_resource_entry(p, &hdr->integrity); - memset(p, 0, WIM_UNUSED_LEN); - if (fwrite(buf, 1, sizeof(buf), out_fp) != sizeof(buf)) { + p = put_zeroes(p, WIM_UNUSED_LEN); + assert(p - buf == sizeof(buf)); + + if (full_pwrite(out_fd, buf, sizeof(buf), 0) != sizeof(buf)) { ERROR_WITH_ERRNO("Failed to write WIM header"); return WIMLIB_ERR_WRITE; } diff --git a/src/integrity.c b/src/integrity.c index bbabaea2..37ff7b74 100644 --- a/src/integrity.c +++ b/src/integrity.c @@ -46,40 +46,28 @@ struct integrity_table { }; static int -calculate_chunk_sha1(FILE *fp, size_t this_chunk_size, +calculate_chunk_sha1(filedes_t in_fd, size_t this_chunk_size, off_t offset, u8 sha1_md[]) { - int ret; u8 buf[BUFFER_SIZE]; SHA_CTX ctx; size_t bytes_remaining; size_t bytes_to_read; size_t bytes_read; - ret = fseeko(fp, offset, SEEK_SET); - if (ret != 0) { - ERROR_WITH_ERRNO("Can't seek to offset " - "%"PRIu64" in WIM", offset); - return WIMLIB_ERR_READ; - } bytes_remaining = this_chunk_size; sha1_init(&ctx); do { bytes_to_read = min(bytes_remaining, sizeof(buf)); - bytes_read = fread(buf, 1, bytes_to_read, fp); + bytes_read = full_pread(in_fd, buf, bytes_to_read, offset); if (bytes_read != bytes_to_read) { - if (feof(fp)) { - ERROR("Unexpected EOF while calculating " - "integrity checksums"); - } else { - ERROR_WITH_ERRNO("File stream error while " - "calculating integrity " - "checksums"); - } + ERROR_WITH_ERRNO("Read error while calculating " + "integrity checksums"); return WIMLIB_ERR_READ; } sha1_update(&ctx, buf, bytes_read); bytes_remaining -= bytes_read; + offset += bytes_read; } while (bytes_remaining); sha1_final(sha1_md, &ctx); return 0; @@ -93,8 +81,8 @@ calculate_chunk_sha1(FILE *fp, size_t this_chunk_size, * The resource entry that specifies the location of the integrity table. * The integrity table must exist (i.e. res_entry->offset must not be 0). * - * @fp: - * FILE * to the WIM file, opened for reading. + * @in_fd: + * File descriptor to the WIM file, opened for reading. * * @num_checked_bytes: * Number of bytes of data that should be checked by the integrity table. @@ -112,12 +100,12 @@ calculate_chunk_sha1(FILE *fp, size_t this_chunk_size, */ static int read_integrity_table(const struct resource_entry *res_entry, - FILE *fp, + filedes_t in_fd, u64 num_checked_bytes, struct integrity_table **table_ret) { - struct integrity_table *table = NULL; - int ret = 0; + struct integrity_table *table; + int ret; u64 expected_size; u64 expected_num_entries; @@ -132,20 +120,21 @@ read_integrity_table(const struct resource_entry *res_entry, } /* Read the integrity table into memory. */ - if ((table = MALLOC(res_entry->size)) == NULL) { - ERROR("Can't allocate %"PRIu64" bytes for integrity table", - (u64)res_entry->size); + table = MALLOC((size_t)res_entry->size); + if (table == NULL) { + ERROR("Can't allocate %zu bytes for integrity table", + (size_t)res_entry->size); return WIMLIB_ERR_NOMEM; } - ret = read_uncompressed_resource(fp, res_entry->offset, - res_entry->size, (void*)table); - - if (ret != 0) { - ERROR("Failed to read integrity table (size = %u, " + if (full_pread(in_fd, table, res_entry->size, + res_entry->offset) != res_entry->size) + { + ERROR("Failed to read integrity table (size = %zu, " " offset = %"PRIu64")", - (unsigned)res_entry->size, res_entry->offset); - goto out; + (size_t)res_entry->size, res_entry->offset); + ret = WIMLIB_ERR_READ; + goto out_free_table; } table->size = le32_to_cpu(table->size); @@ -157,7 +146,7 @@ read_integrity_table(const struct resource_entry *res_entry, "%u bytes but resource entry says %u bytes", table->size, (unsigned)res_entry->size); ret = WIMLIB_ERR_INVALID_INTEGRITY_TABLE; - goto out; + goto out_free_table; } DEBUG("table->size = %u, table->num_entries = %u, " @@ -171,13 +160,13 @@ read_integrity_table(const struct resource_entry *res_entry, "bytes to hold %u entries", table->size, expected_size, table->num_entries); ret = WIMLIB_ERR_INVALID_INTEGRITY_TABLE; - goto out; + goto out_free_table; } if (table->chunk_size == 0) { ERROR("Cannot use integrity chunk size of 0"); ret = WIMLIB_ERR_INVALID_INTEGRITY_TABLE; - goto out; + goto out_free_table; } expected_num_entries = DIV_ROUND_UP(num_checked_bytes, table->chunk_size); @@ -191,12 +180,14 @@ read_integrity_table(const struct resource_entry *res_entry, "there were only %u entries", table->chunk_size, table->num_entries); ret = WIMLIB_ERR_INVALID_INTEGRITY_TABLE; + goto out_free_table; } + *table_ret = table; + ret = 0; + goto out; +out_free_table: + FREE(table); out: - if (ret == 0) - *table_ret = table; - else - FREE(table); return ret; } @@ -206,9 +197,9 @@ out: * Calculates an integrity table for the data in a file beginning at offset 208 * (WIM_HEADER_DISK_SIZE). * - * @fp: - * FILE * for the file to be checked, opened for reading. Does not need to - * be at any specific location in the file. + * @in_fd: + * File descriptor for the file to be checked, opened for reading. Does + * not need to be at any specific location in the file. * * @new_check_end: * Offset of byte after the last byte to be checked. @@ -232,14 +223,14 @@ out: * Returns 0 on success; nonzero on failure. */ static int -calculate_integrity_table(FILE *fp, +calculate_integrity_table(filedes_t in_fd, off_t new_check_end, const struct integrity_table *old_table, off_t old_check_end, wimlib_progress_func_t progress_func, struct integrity_table **integrity_table_ret) { - int ret = 0; + int ret; size_t chunk_size = INTEGRITY_CHUNK_SIZE; /* If an old table is provided, set the chunk size to be compatible with @@ -301,10 +292,12 @@ calculate_integrity_table(FILE *fp, copy_hash(new_table->sha1sums[i], old_table->sha1sums[i]); } else { /* Calculate the SHA1 message digest of this chunk */ - ret = calculate_chunk_sha1(fp, this_chunk_size, + ret = calculate_chunk_sha1(in_fd, this_chunk_size, offset, new_table->sha1sums[i]); - if (ret != 0) - break; + if (ret) { + FREE(new_table); + return ret; + } } offset += this_chunk_size; if (progress_func) { @@ -314,11 +307,8 @@ calculate_integrity_table(FILE *fp, &progress); } } - if (ret == 0) - *integrity_table_ret = new_table; - else - FREE(new_table); - return ret; + *integrity_table_ret = new_table; + return 0; } /* @@ -335,9 +325,9 @@ calculate_integrity_table(FILE *fp, * cannot be read, a warning is printed and the integrity information is * re-calculated. * - * @fp: - * FILE * to the WIM file, opened read-write, positioned at the location at - * which the integrity table is to be written. + * @fd: + * File descriptor to the WIM file, opened read-write, positioned at the + * location at which the integrity table is to be written. * * @integrity_res_entry: * Resource entry which will be set to point to the integrity table on @@ -364,7 +354,7 @@ calculate_integrity_table(FILE *fp, * to be checked. */ int -write_integrity_table(FILE *fp, +write_integrity_table(filedes_t fd, struct resource_entry *integrity_res_entry, off_t new_lookup_table_end, off_t old_lookup_table_end, @@ -378,14 +368,14 @@ write_integrity_table(FILE *fp, wimlib_assert(old_lookup_table_end <= new_lookup_table_end); - cur_offset = ftello(fp); + cur_offset = filedes_offset(fd); if (cur_offset == -1) return WIMLIB_ERR_WRITE; if (integrity_res_entry->offset == 0 || old_lookup_table_end == 0) { old_table = NULL; } else { - ret = read_integrity_table(integrity_res_entry, fp, + ret = read_integrity_table(integrity_res_entry, fd, old_lookup_table_end - WIM_HEADER_DISK_SIZE, &old_table); if (ret == WIMLIB_ERR_INVALID_INTEGRITY_TABLE) { @@ -397,10 +387,10 @@ write_integrity_table(FILE *fp, } } - ret = calculate_integrity_table(fp, new_lookup_table_end, + ret = calculate_integrity_table(fd, new_lookup_table_end, old_table, old_lookup_table_end, progress_func, &new_table); - if (ret != 0) + if (ret) goto out_free_old_table; new_table_size = new_table->size; @@ -409,14 +399,7 @@ write_integrity_table(FILE *fp, new_table->num_entries = cpu_to_le32(new_table->num_entries); new_table->chunk_size = cpu_to_le32(new_table->chunk_size); - if (fseeko(fp, cur_offset, SEEK_SET) != 0) { - ERROR_WITH_ERRNO("Failed to seek to byte %"PRIu64" of WIM to " - "write integrity table", cur_offset); - ret = WIMLIB_ERR_WRITE; - goto out_free_new_table; - } - - if (fwrite(new_table, 1, new_table_size, fp) != new_table_size) { + if (full_write(fd, new_table, new_table_size) != new_table_size) { ERROR_WITH_ERRNO("Failed to write WIM integrity table"); ret = WIMLIB_ERR_WRITE; } else { @@ -426,7 +409,6 @@ write_integrity_table(FILE *fp, integrity_res_entry->flags = 0; ret = 0; } -out_free_new_table: FREE(new_table); out_free_old_table: FREE(old_table); @@ -438,8 +420,8 @@ out_free_old_table: * * Checks a WIM for consistency with the integrity table. * - * @fp: - * FILE * to the WIM file, opened for reading. + * @in_fd: + * File descriptor to the WIM file, opened for reading. * * @table: * The integrity table for the WIM, read into memory. @@ -459,7 +441,7 @@ out_free_old_table: * -1 (WIM_INTEGRITY_NOT_OK) if the WIM failed the integrity check. */ static int -verify_integrity(FILE *fp, const tchar *filename, +verify_integrity(filedes_t in_fd, const tchar *filename, const struct integrity_table *table, u64 bytes_to_check, wimlib_progress_func_t progress_func) @@ -487,8 +469,8 @@ verify_integrity(FILE *fp, const tchar *filename, else this_chunk_size = table->chunk_size; - ret = calculate_chunk_sha1(fp, this_chunk_size, offset, sha1_md); - if (ret != 0) + ret = calculate_chunk_sha1(in_fd, this_chunk_size, offset, sha1_md); + if (ret) return ret; if (!hashes_equal(sha1_md, table->sha1sums[i])) @@ -551,11 +533,11 @@ check_wim_integrity(WIMStruct *w, wimlib_progress_func_t progress_func) bytes_to_check = end_lookup_table_offset - WIM_HEADER_DISK_SIZE; - ret = read_integrity_table(&w->hdr.integrity, w->fp, + ret = read_integrity_table(&w->hdr.integrity, w->in_fd, bytes_to_check, &table); - if (ret != 0) + if (ret) return ret; - ret = verify_integrity(w->fp, w->filename, table, + ret = verify_integrity(w->in_fd, w->filename, table, bytes_to_check, progress_func); FREE(table); return ret; diff --git a/src/join.c b/src/join.c index b5c7c37b..1de8b76b 100644 --- a/src/join.c +++ b/src/join.c @@ -118,24 +118,18 @@ join_wims(WIMStruct **swms, unsigned num_swms, /* Write the non-metadata resources from each SWM part */ for (i = 0; i < num_swms; i++) { - swms[i]->fp = tfopen(swms[i]->filename, T("rb")); - if (!swms[i]->fp) { - ERROR_WITH_ERRNO("Failed to reopen `%"TS"'", - swms[i]->filename); - return WIMLIB_ERR_OPEN; - } - swms[i]->out_fp = joined_wim->out_fp; + ret = reopen_wim(swms[i]); + if (ret) + return ret; + swms[i]->out_fd = joined_wim->out_fd; swms[i]->hdr.part_number = 1; ret = for_lookup_table_entry_pos_sorted(swms[i]->lookup_table, copy_resource, swms[i]); - swms[i]->out_fp = NULL; - - if (i != 0) { - fclose(swms[i]->fp); - swms[i]->fp = NULL; - } + swms[i]->out_fd = INVALID_FILEDES; + if (i != 0) + close_wim(swms[i]); if (ret) return ret; @@ -209,8 +203,7 @@ wimlib_join(const tchar * const *swm_names, /* Don't open all the parts at the same time, in case there are * a lot of them */ - fclose(swms[i]->fp); - swms[i]->fp = NULL; + close_wim(swms[i]); } qsort(swms, num_swms, sizeof(swms[0]), cmp_swms_by_part_number); diff --git a/src/lookup_table.c b/src/lookup_table.c index c6b45fed..24c13320 100644 --- a/src/lookup_table.c +++ b/src/lookup_table.c @@ -351,10 +351,16 @@ int read_lookup_table(WIMStruct *w) { u64 num_entries; - u8 buf[WIM_LOOKUP_TABLE_ENTRY_DISK_SIZE]; int ret; struct wim_lookup_table *table; struct wim_lookup_table_entry *cur_entry, *duplicate_entry; + void *table_buf; + size_t table_size; + const void *p; + + DEBUG("Reading lookup table: offset %"PRIu64", size %"PRIu64"", + w->hdr.lookup_table_res_entry.offset, + w->hdr.lookup_table_res_entry.original_size); if (resource_is_compressed(&w->hdr.lookup_table_res_entry)) { ERROR("Didn't expect a compressed lookup table!"); @@ -362,38 +368,34 @@ read_lookup_table(WIMStruct *w) return WIMLIB_ERR_COMPRESSED_LOOKUP_TABLE; } - DEBUG("Reading lookup table: offset %"PRIu64", size %"PRIu64"", - w->hdr.lookup_table_res_entry.offset, - w->hdr.lookup_table_res_entry.original_size); + table_size = w->hdr.lookup_table_res_entry.size; + if (table_size != w->hdr.lookup_table_res_entry.size) { + ERROR("Lookup table is invalid"); + return WIMLIB_ERR_INVALID_LOOKUP_TABLE_ENTRY; + } + + table_buf = MALLOC(table_size); + if (!table_buf) + return WIMLIB_ERR_NOMEM; - if (fseeko(w->fp, w->hdr.lookup_table_res_entry.offset, SEEK_SET) != 0) + if (full_pread(w->in_fd, table_buf, table_size, + w->hdr.lookup_table_res_entry.offset) != table_size) { - ERROR_WITH_ERRNO("Failed to seek to byte %"PRIu64" to read " - "lookup table", - w->hdr.lookup_table_res_entry.offset); - return WIMLIB_ERR_READ; + ret = WIMLIB_ERR_READ; + goto out_free_table_buf; } num_entries = w->hdr.lookup_table_res_entry.original_size / WIM_LOOKUP_TABLE_ENTRY_DISK_SIZE; table = new_lookup_table(num_entries * 2 + 1); - if (!table) - return WIMLIB_ERR_NOMEM; + if (!table) { + ret = WIMLIB_ERR_NOMEM; + goto out_free_table_buf; + } w->current_image = 0; + p = table_buf; while (num_entries--) { - const u8 *p; - - if (fread(buf, 1, sizeof(buf), w->fp) != sizeof(buf)) { - if (feof(w->fp)) { - ERROR("Unexpected EOF in WIM lookup table!"); - } else { - ERROR_WITH_ERRNO("Error reading WIM lookup " - "table"); - } - ret = WIMLIB_ERR_READ; - goto out_free_lookup_table; - } cur_entry = new_lookup_table_entry(); if (!cur_entry) { ret = WIMLIB_ERR_NOMEM; @@ -402,7 +404,7 @@ read_lookup_table(WIMStruct *w) cur_entry->wim = w; cur_entry->resource_location = RESOURCE_IN_WIM; - p = get_resource_entry(buf, &cur_entry->resource_entry); + p = get_resource_entry(p, &cur_entry->resource_entry); p = get_u16(p, &cur_entry->part_number); p = get_u32(p, &cur_entry->refcnt); p = get_bytes(p, SHA1_HASH_SIZE, cur_entry->hash); @@ -506,12 +508,13 @@ read_lookup_table(WIMStruct *w) DEBUG("Done reading lookup table."); w->lookup_table = table; ret = 0; - goto out; + goto out_free_table_buf; out_free_cur_entry: FREE(cur_entry); out_free_lookup_table: free_lookup_table(table); -out: +out_free_table_buf: + FREE(table_buf); w->current_image = 0; return ret; } @@ -520,51 +523,86 @@ out: /* * Writes a lookup table entry to the output file. */ +static void * +write_lookup_table_entry(struct wim_lookup_table_entry *lte, void *buf_p) +{ + buf_p = put_resource_entry(buf_p, <e->output_resource_entry); + buf_p = put_u16(buf_p, lte->part_number); + buf_p = put_u32(buf_p, lte->out_refcnt); + buf_p = put_bytes(buf_p, SHA1_HASH_SIZE, lte->hash); + return buf_p; +} + int -write_lookup_table_entry(struct wim_lookup_table_entry *lte, void *_out) +write_lookup_table_from_stream_list(struct list_head *stream_list, + filedes_t out_fd, + struct resource_entry *out_res_entry) { - FILE *out; - u8 buf[WIM_LOOKUP_TABLE_ENTRY_DISK_SIZE]; - u8 *p; + size_t num_entries; + struct list_head *pos; + size_t table_size; + void *table_buf; + void *buf_p; + struct wim_lookup_table_entry *lte; + off_t start_offset; + int ret; - out = _out; + start_offset = filedes_offset(out_fd); + if (start_offset == -1) + return WIMLIB_ERR_WRITE; - /* Don't write entries that have not had file resources or metadata - * resources written for them. */ - if (lte->out_refcnt == 0) - return 0; + num_entries = 0; + list_for_each(pos, stream_list) + num_entries++; + table_size = num_entries * WIM_LOOKUP_TABLE_ENTRY_DISK_SIZE; - if (lte->output_resource_entry.flags & WIM_RESHDR_FLAG_METADATA) { - DEBUG("Writing metadata entry at %"PRIu64" " - "(orig size = %"PRIu64")", - ftello(out), lte->output_resource_entry.original_size); + table_buf = MALLOC(table_size); + if (!table_buf) { + ERROR("Failed to allocate lookup table buffer of %zu bytes", table_size); + return WIMLIB_ERR_NOMEM; } - p = put_resource_entry(buf, <e->output_resource_entry); - p = put_u16(p, lte->part_number); - p = put_u32(p, lte->out_refcnt); - p = put_bytes(p, SHA1_HASH_SIZE, lte->hash); - if (fwrite(buf, 1, sizeof(buf), out) != sizeof(buf)) { - ERROR_WITH_ERRNO("Failed to write lookup table entry"); - return WIMLIB_ERR_WRITE; + buf_p = table_buf; + list_for_each_entry(lte, stream_list, lookup_table_list) + buf_p = write_lookup_table_entry(lte, buf_p); + + wimlib_assert(buf_p - table_buf == table_size); + + if (full_write(out_fd, table_buf, table_size) != table_size) { + ERROR_WITH_ERRNO("Failed to write lookup table"); + ret = WIMLIB_ERR_WRITE; + goto out_free_table_buf; } + + out_res_entry->offset = start_offset; + out_res_entry->size = table_size; + out_res_entry->original_size = table_size; + out_res_entry->flags = WIM_RESHDR_FLAG_METADATA; + ret = 0; +out_free_table_buf: + FREE(table_buf); + return ret; +} + +static int +append_lookup_table_entry(struct wim_lookup_table_entry *lte, void *_list) +{ + struct list_head *list = _list; + + if (lte->out_refcnt != 0) + list_add_tail(<e->lookup_table_list, list); return 0; } + /* Writes the WIM lookup table to the output file. */ int write_lookup_table(WIMStruct *w, int image, struct resource_entry *out_res_entry) { - FILE *out = w->out_fp; - off_t start_offset, end_offset; - int ret; - int start_image, end_image; + LIST_HEAD(stream_list); + int start_image; + int end_image; - start_offset = ftello(out); - if (start_offset == -1) - return WIMLIB_ERR_WRITE; - - /* Write lookup table entries for metadata resources */ if (image == WIMLIB_ALL_IMAGES) { start_image = 1; end_image = w->hdr.image_count; @@ -572,32 +610,22 @@ write_lookup_table(WIMStruct *w, int image, struct resource_entry *out_res_entry start_image = image; end_image = image; } + for (int i = start_image; i <= end_image; i++) { struct wim_lookup_table_entry *metadata_lte; metadata_lte = w->image_metadata[i - 1]->metadata_lte; metadata_lte->out_refcnt = 1; metadata_lte->output_resource_entry.flags |= WIM_RESHDR_FLAG_METADATA; - ret = write_lookup_table_entry(metadata_lte, out); - if (ret) - return ret; + append_lookup_table_entry(metadata_lte, &stream_list); } - /* Write lookup table entries for other resources */ - ret = for_lookup_table_entry(w->lookup_table, write_lookup_table_entry, out); - if (ret) - return ret; - - /* Fill in the resource entry for the lookup table itself */ - end_offset = ftello(out); - if (end_offset == -1) - return WIMLIB_ERR_WRITE; - - out_res_entry->offset = start_offset; - out_res_entry->size = end_offset - start_offset; - out_res_entry->original_size = end_offset - start_offset; - out_res_entry->flags = WIM_RESHDR_FLAG_METADATA; - return 0; + for_lookup_table_entry(w->lookup_table, + append_lookup_table_entry, + &stream_list); + return write_lookup_table_from_stream_list(&stream_list, + w->out_fd, + out_res_entry); } int diff --git a/src/lookup_table.h b/src/lookup_table.h index d9d8959c..38b52ea8 100644 --- a/src/lookup_table.h +++ b/src/lookup_table.h @@ -245,6 +245,7 @@ struct wim_lookup_table_entry { union { struct list_head unhashed_list; struct list_head swm_stream_list; + struct list_head lookup_table_list; struct list_head extraction_list; struct list_head export_stream_list; }; @@ -306,6 +307,11 @@ read_lookup_table(WIMStruct *w); extern int write_lookup_table(WIMStruct *w, int image, struct resource_entry *out_res_entry); +extern int +write_lookup_table_from_stream_list(struct list_head *stream_list, + filedes_t out_fd, + struct resource_entry *out_res_entry); + extern void free_lookup_table(struct wim_lookup_table *table); @@ -380,9 +386,6 @@ inode_resolve_ltes(struct wim_inode *inode, struct wim_lookup_table *table); extern void inode_unresolve_ltes(struct wim_inode *inode); -extern int -write_lookup_table_entry(struct wim_lookup_table_entry *lte, void *__out); - static inline struct wim_lookup_table_entry * inode_stream_lte_resolved(const struct wim_inode *inode, unsigned stream_idx) { diff --git a/src/metadata_resource.c b/src/metadata_resource.c index 9cebf9eb..8fdee61f 100644 --- a/src/metadata_resource.c +++ b/src/metadata_resource.c @@ -87,7 +87,7 @@ read_metadata_resource(WIMStruct *w, struct wim_image_metadata *imd) } /* Read the metadata resource into memory. (It may be compressed.) */ - ret = read_full_resource_into_buf(metadata_lte, buf, false); + ret = read_full_resource_into_buf(metadata_lte, buf); if (ret) goto out_free_buf; @@ -192,8 +192,8 @@ recalculate_security_data_length(struct wim_security_data *sd) * uncompressed data rather a lookup table entry; also writes the SHA1 hash of * the buffer to @hash. */ static int -write_wim_resource_from_buffer(const void *buf, u64 buf_size, - FILE *out_fp, int out_ctype, +write_wim_resource_from_buffer(const void *buf, size_t buf_size, + filedes_t out_fd, int out_ctype, struct resource_entry *out_res_entry, u8 hash[SHA1_HASH_SIZE]) { @@ -206,7 +206,7 @@ write_wim_resource_from_buffer(const void *buf, u64 buf_size, lte.resource_entry.original_size = buf_size; lte.resource_entry.flags = 0; lte.unhashed = 1; - ret = write_wim_resource(<e, out_fp, out_ctype, out_res_entry, 0); + ret = write_wim_resource(<e, out_fd, out_ctype, out_res_entry, 0); if (ret == 0) copy_hash(hash, lte.hash); return ret; @@ -225,11 +225,11 @@ write_metadata_resource(WIMStruct *w) u64 metadata_original_size; struct wim_security_data *sd; - wimlib_assert(w->out_fp != NULL); + wimlib_assert(w->out_fd != INVALID_FILEDES); wimlib_assert(w->current_image != WIMLIB_NO_IMAGE); DEBUG("Writing metadata resource for image %d (offset = %"PRIu64")", - w->current_image, ftello(w->out_fp)); + w->current_image, filedes_offset(w->out_fd)); root = wim_root_dentry(w); @@ -278,7 +278,7 @@ write_metadata_resource(WIMStruct *w) * compression type. The lookup table entry for the metadata resource * is updated. */ ret = write_wim_resource_from_buffer(buf, metadata_original_size, - w->out_fp, + w->out_fd, wimlib_get_compression_type(w), <e->output_resource_entry, lte->hash); diff --git a/src/mount_image.c b/src/mount_image.c index bd27a46a..b658effd 100644 --- a/src/mount_image.c +++ b/src/mount_image.c @@ -160,54 +160,6 @@ flags_writable(int open_flags) return open_flags & (O_RDWR | O_WRONLY); } -/* Like pread(), but keep trying until everything has been read or we know for - * sure that there was an error. */ -static ssize_t -full_pread(int fd, void *buf, size_t count, off_t offset) -{ - ssize_t bytes_remaining = count; - ssize_t bytes_read; - - while (bytes_remaining > 0) { - bytes_read = pread(fd, buf, bytes_remaining, offset); - if (bytes_read <= 0) { - if (bytes_read < 0) { - if (errno == EINTR) - continue; - } else { - errno = EIO; - } - break; - } - bytes_remaining -= bytes_read; - buf += bytes_read; - offset += bytes_read; - } - return count - bytes_remaining; -} - -/* Like pwrite(), but keep trying until everything has been written or we know - * for sure that there was an error. */ -static ssize_t -full_pwrite(int fd, const void *buf, size_t count, off_t offset) -{ - ssize_t bytes_remaining = count; - ssize_t bytes_written; - - while (bytes_remaining > 0) { - bytes_written = pwrite(fd, buf, bytes_remaining, offset); - if (bytes_written < 0) { - if (errno == EINTR) - continue; - break; - } - bytes_remaining -= bytes_written; - buf += bytes_written; - offset += bytes_written; - } - return count - bytes_remaining; -} - /* * Allocate a file descriptor for a stream. * @@ -1687,8 +1639,8 @@ wimfs_getxattr(const char *path, const char *name, char *value, if (res_size > size) return -ERANGE; - ret = read_full_resource_into_buf(lte, value, true); - if (ret != 0) + ret = read_full_resource_into_buf(lte, value); + if (ret) return -EIO; return res_size; @@ -1953,7 +1905,7 @@ wimfs_read(const char *path, char *buf, size_t size, break; case RESOURCE_IN_WIM: if (read_partial_wim_resource_into_buf(fd->f_lte, size, - offset, buf, true)) + offset, buf)) ret = -errno; else ret = size; @@ -2526,7 +2478,7 @@ wimlib_mount_image(WIMStruct *wim, int image, const char *dir, } if (mount_flags & WIMLIB_MOUNT_FLAG_READWRITE) { - ret = lock_wim(wim, wim->fp); + ret = lock_wim(wim, wim->in_fd); if (ret) goto out; } diff --git a/src/ntfs-apply.c b/src/ntfs-apply.c index f794b4a2..35760400 100644 --- a/src/ntfs-apply.c +++ b/src/ntfs-apply.c @@ -375,7 +375,7 @@ apply_reparse_data(ntfs_inode *ni, struct wim_dentry *dentry, p = put_u16(p, wim_resource_size(lte)); /* ReparseDataLength */ p = put_u16(p, 0); /* Reserved */ - ret = read_full_resource_into_buf(lte, p, false); + ret = read_full_resource_into_buf(lte, p); if (ret) return ret; diff --git a/src/reparse.c b/src/reparse.c index 4cbe69d9..13360397 100644 --- a/src/reparse.c +++ b/src/reparse.c @@ -225,7 +225,7 @@ wim_inode_get_reparse_data(const struct wim_inode *inode, u8 *rpbuf) } /* Read the data from the WIM file */ - ret = read_full_resource_into_buf(lte, rpbuf + 8, true); + ret = read_full_resource_into_buf(lte, rpbuf + 8); if (ret) return ret; diff --git a/src/resource.c b/src/resource.c index c7eedab0..464ca711 100644 --- a/src/resource.c +++ b/src/resource.c @@ -42,59 +42,13 @@ # include #endif -/* Write @n bytes from @buf to the file descriptor @fd, retrying on internupt - * and on short writes. - * - * Returns short count and set errno on failure. */ -static ssize_t -full_write(int fd, const void *buf, size_t n) -{ - const void *p = buf; - ssize_t ret; - ssize_t total = 0; - - while (total != n) { - ret = write(fd, p, n); - if (ret < 0) { - if (errno == EINTR) - continue; - else - break; - } - total += ret; - p += ret; - } - return total; -} - -/* Read @n bytes from the file descriptor @fd to the buffer @buf, retrying on - * internupt and on short reads. - * - * Returns short count and set errno on failure. */ -static size_t -full_read(int fd, void *buf, size_t n) -{ - size_t bytes_remaining = n; - while (bytes_remaining) { - ssize_t bytes_read = read(fd, buf, bytes_remaining); - if (bytes_read < 0) { - if (errno == EINTR) - continue; - break; - } - bytes_remaining -= bytes_read; - buf += bytes_read; - } - return n - bytes_remaining; -} - /* * Reads all or part of a compressed WIM resource. * * Returns zero on success, nonzero on failure. */ static int -read_compressed_resource(FILE *fp, +read_compressed_resource(filedes_t in_fd, u64 resource_compressed_size, u64 resource_uncompressed_size, u64 resource_offset, @@ -208,8 +162,6 @@ read_compressed_resource(FILE *fp, /* Skip over unneeded chunk table entries. */ u64 file_offset_of_needed_chunk_entries = resource_offset + start_table_idx * chunk_entry_size; - if (fseeko(fp, file_offset_of_needed_chunk_entries, SEEK_SET)) - goto read_error; /* Number of bytes we need to read from the chunk table. */ size_t size = num_needed_chunk_entries * chunk_entry_size; @@ -218,7 +170,8 @@ read_compressed_resource(FILE *fp, * avoid allocating another array. */ void *chunk_tab_buf = (void*)&chunk_offsets[num_needed_chunks] - size; - if (fread(chunk_tab_buf, 1, size, fp) != size) + if (full_pread(in_fd, chunk_tab_buf, size, + file_offset_of_needed_chunk_entries) != size) goto read_error; /* Now fill in chunk_offsets from the entries we have read in @@ -241,10 +194,7 @@ read_compressed_resource(FILE *fp, /* Done with the chunk table now. We must now seek to the first chunk * that is needed for the read. */ - u64 file_offset_of_first_needed_chunk = resource_offset + - chunk_table_size + chunk_offsets[0]; - if (fseeko(fp, file_offset_of_first_needed_chunk, SEEK_SET)) - goto read_error; + u64 cur_read_offset = resource_offset + chunk_table_size + chunk_offsets[0]; /* Pointer to current position in the output buffer for uncompressed * data. Alternatively, if using a callback function, we repeatedly @@ -320,19 +270,24 @@ read_compressed_resource(FILE *fp, * is equal to the uncompressed chunk size. */ if (compressed_chunk_size == uncompressed_chunk_size) { /* Uncompressed chunk */ - if (start_offset != 0) - if (fseeko(fp, start_offset, SEEK_CUR)) - goto read_error; - if (fread(cb ? out_p + start_offset : out_p, - 1, partial_chunk_size, fp) != partial_chunk_size) + if (full_pread(in_fd, + cb ? out_p + start_offset : out_p, + partial_chunk_size, + cur_read_offset + start_offset) != partial_chunk_size) + { goto read_error; + } } else { /* Compressed chunk */ /* Read the compressed data into compressed_buf. */ - if (fread(compressed_buf, 1, compressed_chunk_size, - fp) != compressed_chunk_size) + if (full_pread(in_fd, + compressed_buf, + compressed_chunk_size, + cur_read_offset) != compressed_chunk_size) + { goto read_error; + } /* For partial chunks and when writing directly to a * buffer, we must buffer the uncompressed data because @@ -374,6 +329,7 @@ read_compressed_resource(FILE *fp, * written. */ out_p += partial_chunk_size; } + cur_read_offset += compressed_chunk_size; } ret = 0; @@ -383,10 +339,7 @@ out: return ret; read_error: - if (feof(fp)) - ERROR("Unexpected EOF in compressed file resource"); - else - ERROR_WITH_ERRNO("Error reading compressed file resource"); + ERROR_WITH_ERRNO("Error reading compressed file resource"); ret = WIMLIB_ERR_READ; goto out; } @@ -460,66 +413,6 @@ put_resource_entry(void *p, const struct resource_entry *entry) return p; } -static FILE * -wim_get_fp(WIMStruct *w) -{ - FILE *fp; -#if defined(WITH_FUSE) || defined(ENABLE_MULTITHREADED_COMPRESSION) - pthread_mutex_lock(&w->fp_tab_mutex); - - wimlib_assert(w->filename != NULL); - - for (size_t i = 0; i < w->num_allocated_fps; i++) { - if (w->fp_tab[i]) { - fp = w->fp_tab[i]; - w->fp_tab[i] = NULL; - goto out_unlock; - } - } - DEBUG("Opening extra file descriptor to `%"TS"'", w->filename); - fp = tfopen(w->filename, T("rb")); - if (!fp) - ERROR_WITH_ERRNO("Failed to open `%"TS"'", w->filename); -out_unlock: - pthread_mutex_unlock(&w->fp_tab_mutex); -#else /* WITH_FUSE || ENABLE_MULTITHREADED_COMPRESSION */ - fp = w->fp; -#endif /* !WITH_FUSE && !ENABLE_MULTITHREADED_COMPRESSION */ - return fp; -} - -static int -wim_release_fp(WIMStruct *w, FILE *fp) -{ - int ret = 0; -#if defined(WITH_FUSE) || defined(ENABLE_MULTITHREADED_COMPRESSION) - FILE **fp_tab; - - pthread_mutex_lock(&w->fp_tab_mutex); - - for (size_t i = 0; i < w->num_allocated_fps; i++) { - if (w->fp_tab[i] == NULL) { - w->fp_tab[i] = fp; - goto out_unlock; - } - } - - fp_tab = REALLOC(w->fp_tab, sizeof(FILE*) * (w->num_allocated_fps + 4)); - if (!fp_tab) { - ret = WIMLIB_ERR_NOMEM; - fclose(fp); - goto out_unlock; - } - w->fp_tab = fp_tab; - memset(&w->fp_tab[w->num_allocated_fps], 0, 4 * sizeof(FILE*)); - w->fp_tab[w->num_allocated_fps] = fp; - w->num_allocated_fps += 4; -out_unlock: - pthread_mutex_unlock(&w->fp_tab_mutex); -#endif /* WITH_FUSE || ENABLE_MULTITHREADED_COMPRESSION */ - return ret; -} - static int read_partial_wim_resource(const struct wim_lookup_table_entry *lte, u64 size, @@ -528,27 +421,19 @@ read_partial_wim_resource(const struct wim_lookup_table_entry *lte, int flags, u64 offset) { - FILE *wim_fp; WIMStruct *wim; + filedes_t in_fd; int ret; wimlib_assert(lte->resource_location == RESOURCE_IN_WIM); wim = lte->wim; - if (flags & WIMLIB_RESOURCE_FLAG_THREADSAFE_READ) { - wim_fp = wim_get_fp(wim); - if (!wim_fp) { - ret = WIMLIB_ERR_READ; - goto out; - } - } else { - wim_fp = lte->wim->fp; - } + in_fd = wim->in_fd; if (lte->resource_entry.flags & WIM_RESHDR_FLAG_COMPRESSED && !(flags & WIMLIB_RESOURCE_FLAG_RAW)) { - ret = read_compressed_resource(wim_fp, + ret = read_compressed_resource(in_fd, lte->resource_entry.size, lte->resource_entry.original_size, lte->resource_entry.offset, @@ -559,47 +444,32 @@ read_partial_wim_resource(const struct wim_lookup_table_entry *lte, ctx_or_buf); } else { offset += lte->resource_entry.offset; - - if (fseeko(wim_fp, offset, SEEK_SET)) { - ERROR_WITH_ERRNO("Failed to seek to offset %"PRIu64 - " in WIM", offset); - ret = WIMLIB_ERR_READ; - goto out_release_fp; - } if (cb) { /* Send data to callback function */ u8 buf[min(WIM_CHUNK_SIZE, size)]; while (size) { size_t bytes_to_read = min(WIM_CHUNK_SIZE, size); - size_t bytes_read = fread(buf, 1, bytes_to_read, wim_fp); - + size_t bytes_read = full_pread(in_fd, buf, + bytes_to_read, offset); if (bytes_read != bytes_to_read) goto read_error; ret = cb(buf, bytes_read, ctx_or_buf); if (ret) - goto out_release_fp; + goto out; size -= bytes_read; + offset += bytes_read; } } else { /* Send data directly to a buffer */ - if (fread(ctx_or_buf, 1, size, wim_fp) != size) + if (full_pread(in_fd, ctx_or_buf, size, offset) != size) goto read_error; } ret = 0; } - goto out_release_fp; + goto out; read_error: - if (ferror(wim_fp)) - ERROR_WITH_ERRNO("Error reading data from WIM"); - else - ERROR("Unexpected EOF in WIM!"); + ERROR_WITH_ERRNO("Error reading data from WIM"); ret = WIMLIB_ERR_READ; -out_release_fp: - if (flags & WIMLIB_RESOURCE_FLAG_THREADSAFE_READ) { - int ret2 = wim_release_fp(wim, wim_fp); - if (ret == 0) - ret = ret2; - } out: if (ret) { if (errno == 0) @@ -611,12 +481,9 @@ out: int read_partial_wim_resource_into_buf(const struct wim_lookup_table_entry *lte, - size_t size, u64 offset, void *buf, - bool threadsafe) + size_t size, u64 offset, void *buf) { - return read_partial_wim_resource(lte, size, NULL, buf, - threadsafe ? WIMLIB_RESOURCE_FLAG_THREADSAFE_READ : 0, - offset); + return read_partial_wim_resource(lte, size, NULL, buf, 0, offset); } static int @@ -756,12 +623,9 @@ read_resource_prefix(const struct wim_lookup_table_entry *lte, int read_full_resource_into_buf(const struct wim_lookup_table_entry *lte, - void *buf, bool thread_safe) + void *buf) { - return read_resource_prefix(lte, - wim_resource_size(lte), - NULL, buf, - thread_safe ? WIMLIB_RESOURCE_FLAG_THREADSAFE_READ : 0); + return read_resource_prefix(lte, wim_resource_size(lte), NULL, buf, 0); } struct extract_ctx { @@ -807,8 +671,8 @@ extract_wim_resource(const struct wim_lookup_table_entry *lte, sha1_final(hash, &ctx.sha_ctx); if (!hashes_equal(hash, lte->hash)) { #ifdef ENABLE_ERROR_MESSAGES - ERROR_WITH_ERRNO("Invalid SHA1 message digest " - "on the following WIM resource:"); + ERROR("Invalid SHA1 message digest " + "on the following WIM resource:"); print_lookup_table_entry(lte, stderr); if (lte->resource_location == RESOURCE_IN_WIM) ERROR("The WIM file appears to be corrupt!"); @@ -883,7 +747,7 @@ copy_resource(struct wim_lookup_table_entry *lte, void *wim) WIMStruct *w = wim; int ret; - ret = write_wim_resource(lte, w->out_fp, + ret = write_wim_resource(lte, w->out_fd, wim_resource_compression_type(lte), <e->output_resource_entry, 0); if (ret == 0) { diff --git a/src/split.c b/src/split.c index eb8f3484..417b29ed 100644 --- a/src/split.c +++ b/src/split.c @@ -46,26 +46,12 @@ static int finish_swm(WIMStruct *w, struct list_head *lte_list, int write_flags, wimlib_progress_func_t progress_func) { - off_t lookup_table_offset = ftello(w->out_fp); int ret; - struct wim_lookup_table_entry *lte; - list_for_each_entry(lte, lte_list, swm_stream_list) { - ret = write_lookup_table_entry(lte, w->out_fp); - if (ret) - return ret; - } - - off_t xml_data_offset = ftello(w->out_fp); - - if (lookup_table_offset == -1 || xml_data_offset == -1) - return WIMLIB_ERR_WRITE; - w->hdr.lookup_table_res_entry.offset = lookup_table_offset; - w->hdr.lookup_table_res_entry.size = - xml_data_offset - lookup_table_offset; - w->hdr.lookup_table_res_entry.original_size = - xml_data_offset - lookup_table_offset; - w->hdr.lookup_table_res_entry.flags = WIM_RESHDR_FLAG_METADATA; + ret = write_lookup_table_from_stream_list(lte_list, w->out_fd, + &w->hdr.lookup_table_res_entry); + if (ret) + return ret; return finish_write(w, WIMLIB_ALL_IMAGES, write_flags | WIMLIB_WRITE_FLAG_NO_LOOKUP_TABLE, progress_func); diff --git a/src/util.c b/src/util.c index f778fbae..f1ec74f0 100644 --- a/src/util.c +++ b/src/util.c @@ -23,12 +23,14 @@ #include "config.h" + #undef _GNU_SOURCE /* Make sure the POSIX-compatible strerror_r() is declared, rather than the GNU * version, which has a different return type. */ -#define _POSIX_C_SOURCE 200112 #include + #define _GNU_SOURCE +#include #include "wimlib_internal.h" #include "endianness.h" @@ -39,8 +41,6 @@ #include #include -#include /* for getpid() */ - #ifdef __WIN32__ #include "win32.h" #endif @@ -347,8 +347,6 @@ static const tchar *error_strings[] = { = T("Could not read the target of a symbolic link"), [WIMLIB_ERR_RENAME] = T("Could not rename a file"), - [WIMLIB_ERR_REOPEN] - = T("Could not re-open the WIM after overwriting it"), [WIMLIB_ERR_REPARSE_POINT_FIXUP_FAILED] = T("Unable to complete reparse point fixup"), [WIMLIB_ERR_RESOURCE_ORDER] @@ -565,3 +563,109 @@ zap_backslashes(tchar *s) } } } + +/* Write @n bytes from @buf to the file descriptor @fd, retrying on internupt + * and on short writes. + * + * Returns short count and set errno on failure. */ +size_t +full_write(int fd, const void *buf, size_t n) +{ + const void *p = buf; + ssize_t ret; + ssize_t total = 0; + + while (total != n) { + ret = write(fd, p, n); + if (ret <= 0) { + if (errno == EINTR) + continue; + if (ret == 0) + errno = EIO; + break; + } + total += ret; + p += ret; + } + return total; +} + +/* Read @n bytes from the file descriptor @fd to the buffer @buf, retrying on + * interrupt and on short reads. + * + * Returns short count and set errno on failure. */ +size_t +full_read(int fd, void *buf, size_t n) +{ + size_t bytes_remaining = n; + while (bytes_remaining) { + ssize_t bytes_read = read(fd, buf, bytes_remaining); + if (bytes_read <= 0) { + if (errno == EINTR) + continue; + if (bytes_read == 0) + errno = EIO; + break; + } + bytes_remaining -= bytes_read; + buf += bytes_read; + } + return n - bytes_remaining; +} + +/* Read @n bytes from the file descriptor @fd at the offset @offset to the + * buffer @buf, retrying on interrupt and on short reads. + * + * Returns short count and set errno on failure. */ +size_t +full_pread(int fd, void *buf, size_t nbyte, off_t offset) +{ + size_t bytes_remaining = nbyte; + ssize_t bytes_read; + + while (bytes_remaining) { + bytes_read = pread(fd, buf, bytes_remaining, offset); + if (bytes_read <= 0) { + if (errno == EINTR) + continue; + if (bytes_read == 0) + errno = EIO; + break; + } + bytes_remaining -= bytes_read; + buf += bytes_read; + offset += bytes_read; + } + return nbyte - bytes_remaining; +} + +/* Like pwrite(), but keep trying until everything has been written or we know + * for sure that there was an error. */ +size_t +full_pwrite(int fd, const void *buf, size_t count, off_t offset) +{ + ssize_t bytes_remaining = count; + ssize_t bytes_written; + + while (bytes_remaining > 0) { + bytes_written = pwrite(fd, buf, bytes_remaining, offset); + if (bytes_written <= 0) { + if (errno == EINTR) + continue; + if (bytes_written == 0) + errno = EIO; + break; + } + bytes_remaining -= bytes_written; + buf += bytes_written; + offset += bytes_written; + } + return count - bytes_remaining; +} + + +off_t +filedes_offset(filedes_t fd) +{ + return lseek(fd, 0, SEEK_CUR); +} diff --git a/src/util.h b/src/util.h index ee9598e0..04020e14 100644 --- a/src/util.h +++ b/src/util.h @@ -129,7 +129,7 @@ tstr_to_utf8_simple(const tchar *tstr, char **out); #define ZERO_ARRAY(array) memset(array, 0, sizeof(array)) /* Used for buffering FILE IO in a few places */ -#define BUFFER_SIZE 4096 +#define BUFFER_SIZE 32768 static inline void FORMAT(printf, 1, 2) dummy_tprintf(const tchar *format, ...) @@ -285,4 +285,23 @@ hash_u64(u64 n) return n * 0x9e37fffffffc0001ULL; } +typedef int filedes_t; + +extern size_t +full_read(filedes_t fd, void *buf, size_t n); + +extern size_t +full_write(filedes_t fd, const void *buf, size_t n); + +extern size_t +full_pread(filedes_t fd, void *buf, size_t nbyte, off_t offset); + +extern size_t +full_pwrite(int fd, const void *buf, size_t count, off_t offset); + +extern off_t +filedes_offset(filedes_t fd); + +#define INVALID_FILEDES (-1) + #endif /* _WIMLIB_UTIL_H */ diff --git a/src/wim.c b/src/wim.c index 592cc440..27cf93f4 100644 --- a/src/wim.c +++ b/src/wim.c @@ -65,15 +65,10 @@ static WIMStruct * new_wim_struct() { WIMStruct *w = CALLOC(1, sizeof(WIMStruct)); -#if defined(WITH_FUSE) || defined(ENABLE_MULTITHREADED_COMPRESSION) - if (pthread_mutex_init(&w->fp_tab_mutex, NULL) != 0) { - ERROR_WITH_ERRNO("Failed to initialize mutex"); - FREE(w); - w = NULL; - } -#endif + w->in_fd = INVALID_FILEDES; + w->out_fd = INVALID_FILEDES; + w->current_image = WIMLIB_NO_IMAGE; return w; - } /* @@ -388,6 +383,35 @@ wimlib_get_boot_idx(const WIMStruct *w) return w->hdr.boot_idx; } +static int +do_open_wim(const tchar *filename, filedes_t *fd_ret) +{ + int fd; + + fd = open(filename, O_RDONLY); + if (fd == -1) { + ERROR_WITH_ERRNO("Can't open \"%"TS"\" read-only", filename); + return WIMLIB_ERR_OPEN; + } + *fd_ret = fd; + return 0; +} + +int +reopen_wim(WIMStruct *w) +{ + wimlib_assert(w->in_fd == INVALID_FILEDES); + return do_open_wim(w->filename, &w->in_fd); +} + +int +close_wim(WIMStruct *w) +{ + close(w->in_fd); + w->in_fd = INVALID_FILEDES; + return 0; +} + /* * Begins the reading of a WIM file; opens the file and reads its header and * lookup table, and optionally checks the integrity. @@ -401,12 +425,9 @@ begin_read(WIMStruct *w, const tchar *in_wim_path, int open_flags, DEBUG("Reading the WIM file `%"TS"'", in_wim_path); - w->fp = tfopen(in_wim_path, T("rb")); - if (!w->fp) { - ERROR_WITH_ERRNO("Failed to open `%"TS"' for reading", - in_wim_path); - return WIMLIB_ERR_OPEN; - } + ret = do_open_wim(in_wim_path, &w->in_fd); + if (ret) + return ret; /* The absolute path to the WIM is requested so that wimlib_overwrite() * still works even if the process changes its working directory. This @@ -429,7 +450,7 @@ begin_read(WIMStruct *w, const tchar *in_wim_path, int open_flags, return WIMLIB_ERR_OPEN; } - ret = read_header(w->fp, &w->hdr, open_flags); + ret = read_header(w->in_fd, &w->hdr, open_flags); if (ret) return ret; @@ -472,7 +493,7 @@ begin_read(WIMStruct *w, const tchar *in_wim_path, int open_flags, if (ret) return ret; - ret = read_xml_data(w->fp, &w->hdr.xml_res_entry, &w->wim_info); + ret = read_xml_data(w->in_fd, &w->hdr.xml_res_entry, &w->wim_info); if (ret) return ret; @@ -647,20 +668,10 @@ wimlib_free(WIMStruct *w) if (!w) return; - if (w->fp) - fclose(w->fp); - if (w->out_fp) - fclose(w->out_fp); - -#if defined(WITH_FUSE) || defined(ENABLE_MULTITHREADED_COMPRESSION) - if (w->fp_tab) { - for (size_t i = 0; i < w->num_allocated_fps; i++) - if (w->fp_tab[i]) - fclose(w->fp_tab[i]); - FREE(w->fp_tab); - } - pthread_mutex_destroy(&w->fp_tab_mutex); -#endif + if (w->in_fd != INVALID_FILEDES) + close(w->in_fd); + if (w->out_fd != INVALID_FILEDES) + close(w->out_fd); free_lookup_table(w->lookup_table); diff --git a/src/wimlib.h b/src/wimlib.h index 89d64569..5dbe3629 100644 --- a/src/wimlib.h +++ b/src/wimlib.h @@ -1977,6 +1977,10 @@ wimlib_open_wim(const wimlib_tchar *wim_file, * and while abnormal termination of the program will result in extra data * appended to the original WIM, it should still be a valid WIM. * + * If this function completes successfully, no functions should be called on @a + * wim other than wimlib_free(). You must use wimlib_open_wim() to read the WIM + * file anew. + * * @param wim * Pointer to the ::WIMStruct for the WIM file to write. There may have * been in-memory changes made to it, which are then reflected in the @@ -2004,11 +2008,6 @@ wimlib_open_wim(const wimlib_tchar *wim_file, * @retval ::WIMLIB_ERR_RENAME * The temporary file that the WIM was written to could not be renamed to * the original filename of @a wim. - * @retval ::WIMLIB_ERR_REOPEN - * The WIM was overwritten successfully, but it could not be re-opened - * read-only. Therefore, the resources in the WIM can no longer be - * accessed, so this limits the functions that can be called on @a wim - * before calling wimlib_free(). */ extern int wimlib_overwrite(WIMStruct *wim, int write_flags, unsigned num_threads, diff --git a/src/wimlib_internal.h b/src/wimlib_internal.h index d0b2f89c..db8d0351 100644 --- a/src/wimlib_internal.h +++ b/src/wimlib_internal.h @@ -280,18 +280,12 @@ struct wim_image_metadata { /* The opaque structure exposed to the wimlib API. */ struct WIMStruct { - /* A pointer to the file indicated by @filename, opened for reading. */ - FILE *fp; - -#if defined(WITH_FUSE) || defined(ENABLE_MULTITHREADED_COMPRESSION) - /* Extra file pointers to be used by concurrent readers */ - FILE **fp_tab; - size_t num_allocated_fps; - pthread_mutex_t fp_tab_mutex; -#endif + /* File descriptor for the WIM file, opened for reading, or -1 if it has + * not been opened or there is no associated file backing it yet. */ + filedes_t in_fd; /* FILE pointer for the WIM file (if any) currently being written. */ - FILE *out_fp; + filedes_t out_fd; /* The name of the WIM file (if any) that has been opened. */ tchar *filename; @@ -497,10 +491,10 @@ dentry_tree_fix_inodes(struct wim_dentry *root, struct list_head *inode_list); /* header.c */ extern int -read_header(FILE *fp, struct wim_header *hdr, int split_ok); +read_header(filedes_t in_fd, struct wim_header *hdr, int split_ok); extern int -write_header(const struct wim_header *hdr, FILE *out); +write_header(const struct wim_header *hdr, filedes_t out_fd); extern int init_header(struct wim_header *hdr, int ctype); @@ -512,7 +506,8 @@ init_header(struct wim_header *hdr, int ctype); #define WIM_INTEGRITY_NONEXISTENT -2 extern int -write_integrity_table(FILE *out, struct resource_entry *integrity_res_entry, +write_integrity_table(filedes_t fd, + struct resource_entry *integrity_res_entry, off_t new_lookup_table_end, off_t old_lookup_table_end, wimlib_progress_func_t progress_func); @@ -672,7 +667,6 @@ capture_fixup_absolute_symlink(tchar *dest, /* resource.c */ #define WIMLIB_RESOURCE_FLAG_RAW 0x1 -#define WIMLIB_RESOURCE_FLAG_THREADSAFE_READ 0x2 #define WIMLIB_RESOURCE_FLAG_RECOMPRESS 0x4 extern int @@ -686,19 +680,14 @@ get_resource_entry(const void *p, struct resource_entry *entry); extern void * put_resource_entry(void *p, const struct resource_entry *entry); -extern int -read_uncompressed_resource(FILE *fp, u64 offset, u64 size, void *buf); - extern int read_partial_wim_resource_into_buf(const struct wim_lookup_table_entry *lte, - size_t size, u64 offset, void *buf, - bool threadsafe); + size_t size, u64 offset, void *buf); extern int -read_full_resource_into_buf(const struct wim_lookup_table_entry *lte, - void *buf, bool thread_safe); +read_full_resource_into_buf(const struct wim_lookup_table_entry *lte, void *buf); extern int -write_wim_resource(struct wim_lookup_table_entry *lte, FILE *out_fp, +write_wim_resource(struct wim_lookup_table_entry *lte, filedes_t out_fd, int out_ctype, struct resource_entry *out_res_entry, int flags); @@ -772,6 +761,12 @@ new_image_metadata_array(unsigned num_images); extern int wim_checksum_unhashed_streams(WIMStruct *w); +extern int +reopen_wim(WIMStruct *w); + +extern int +close_wim(WIMStruct *w); + /* write.c */ /* Internal use only */ @@ -799,10 +794,10 @@ finish_write(WIMStruct *w, int image, int write_flags, #if defined(HAVE_SYS_FILE_H) && defined(HAVE_FLOCK) extern int -lock_wim(WIMStruct *w, FILE *fp); +lock_wim(WIMStruct *w, filedes_t fd); #else static inline int -lock_wim(WIMStruct *w, FILE *fp) +lock_wim(WIMStruct *w, filedes_t fd) { return 0; } diff --git a/src/write.c b/src/write.c index 8b094490..9965f5c0 100644 --- a/src/write.c +++ b/src/write.c @@ -65,6 +65,8 @@ #include +#include /* writev() */ + /* Chunk table that's located at the beginning of each compressed resource in * the WIM. (This is not the on-disk format; the on-disk format just has an * array of offsets.) */ @@ -85,7 +87,7 @@ struct chunk_table { */ static int begin_wim_resource_chunk_tab(const struct wim_lookup_table_entry *lte, - FILE *out_fp, + int out_fd, off_t file_offset, struct chunk_table **chunk_tab_ret) { @@ -110,8 +112,9 @@ begin_wim_resource_chunk_tab(const struct wim_lookup_table_entry *lte, chunk_tab->cur_offset = 0; chunk_tab->cur_offset_p = chunk_tab->offsets; - if (fwrite(chunk_tab, 1, chunk_tab->table_disk_size, out_fp) != - chunk_tab->table_disk_size) { + if (full_write(out_fd, chunk_tab, + chunk_tab->table_disk_size) != chunk_tab->table_disk_size) + { ERROR_WITH_ERRNO("Failed to write chunk table in compressed " "file resource"); FREE(chunk_tab); @@ -161,7 +164,7 @@ get_compress_func(int out_ctype) * * @chunk: Uncompressed data of the chunk. * @chunk_size: Size of the chunk (<= WIM_CHUNK_SIZE) - * @out_fp: FILE * to write the chunk to. + * @out_fd: FILE descriptor to write the chunk to. * @compress: Compression function to use (NULL if writing uncompressed * data). * @chunk_tab: Pointer to chunk table being created. It is updated with the @@ -172,7 +175,7 @@ get_compress_func(int out_ctype) static int write_wim_resource_chunk(const void * restrict chunk, unsigned chunk_size, - FILE * restrict out_fp, + filedes_t out_fd, compress_func_t compress, struct chunk_table * restrict chunk_tab) { @@ -197,7 +200,7 @@ write_wim_resource_chunk(const void * restrict chunk, out_chunk = chunk; out_chunk_size = chunk_size; } - if (fwrite(out_chunk, 1, out_chunk_size, out_fp) != out_chunk_size) { + if (full_write(out_fd, out_chunk, out_chunk_size) != out_chunk_size) { ERROR_WITH_ERRNO("Failed to write WIM resource chunk"); return WIMLIB_ERR_WRITE; } @@ -212,16 +215,10 @@ write_wim_resource_chunk(const void * restrict chunk, * @compressed_size_p. */ static int -finish_wim_resource_chunk_tab(struct chunk_table * restrict chunk_tab, - FILE * restrict out_fp, - u64 * restrict compressed_size_p) +finish_wim_resource_chunk_tab(struct chunk_table *chunk_tab, + filedes_t out_fd, u64 *compressed_size_p) { size_t bytes_written; - if (fseeko(out_fp, chunk_tab->file_offset, SEEK_SET) != 0) { - ERROR_WITH_ERRNO("Failed to seek to byte %"PRIu64" of output " - "WIM file", chunk_tab->file_offset); - return WIMLIB_ERR_WRITE; - } if (chunk_tab->bytes_per_chunk_entry == 8) { array_cpu_to_le64(chunk_tab->offsets, chunk_tab->num_chunks); @@ -230,31 +227,26 @@ finish_wim_resource_chunk_tab(struct chunk_table * restrict chunk_tab, ((u32*)chunk_tab->offsets)[i] = cpu_to_le32(chunk_tab->offsets[i]); } - bytes_written = fwrite((u8*)chunk_tab->offsets + - chunk_tab->bytes_per_chunk_entry, - 1, chunk_tab->table_disk_size, out_fp); + bytes_written = full_pwrite(out_fd, + (u8*)chunk_tab->offsets + chunk_tab->bytes_per_chunk_entry, + chunk_tab->table_disk_size, + chunk_tab->file_offset); if (bytes_written != chunk_tab->table_disk_size) { ERROR_WITH_ERRNO("Failed to write chunk table in compressed " "file resource"); return WIMLIB_ERR_WRITE; } - if (fseeko(out_fp, 0, SEEK_END) != 0) { - ERROR_WITH_ERRNO("Failed to seek to end of output WIM file"); - return WIMLIB_ERR_WRITE; - } *compressed_size_p = chunk_tab->cur_offset + chunk_tab->table_disk_size; return 0; } static int -fflush_and_ftruncate(FILE *out_fp, off_t offset) +seek_and_truncate(filedes_t out_fd, off_t offset) { - if (fseeko(out_fp, offset, SEEK_SET) || - fflush(out_fp) || - ftruncate(fileno(out_fp), offset)) + if (lseek(out_fd, offset, SEEK_SET) == -1 || + ftruncate(out_fd, offset)) { - ERROR_WITH_ERRNO("Failed to flush and/or truncate " - "output WIM file"); + ERROR_WITH_ERRNO("Failed to truncate output WIM file"); return WIMLIB_ERR_WRITE; } else { return 0; @@ -285,7 +277,7 @@ finalize_and_check_sha1(SHA_CTX * restrict sha_ctx, struct write_resource_ctx { compress_func_t compress; struct chunk_table *chunk_tab; - FILE *out_fp; + filedes_t out_fd; SHA_CTX sha_ctx; bool doing_sha; }; @@ -299,7 +291,7 @@ write_resource_cb(const void *restrict chunk, size_t chunk_size, if (ctx->doing_sha) sha1_update(&ctx->sha_ctx, chunk, chunk_size); return write_wim_resource_chunk(chunk, chunk_size, - ctx->out_fp, ctx->compress, + ctx->out_fd, ctx->compress, ctx->chunk_tab); } @@ -309,7 +301,7 @@ write_resource_cb(const void *restrict chunk, size_t chunk_size, * @lte: Lookup table entry for the resource, which could be in another WIM, * in an external file, or in another location. * - * @out_fp: FILE * opened to the output WIM. + * @out_fp: File descriptor opened to the output WIM. * * @out_ctype: One of the WIMLIB_COMPRESSION_TYPE_* constants to indicate * which compression algorithm to use. @@ -329,7 +321,7 @@ write_resource_cb(const void *restrict chunk, size_t chunk_size, */ int write_wim_resource(struct wim_lookup_table_entry *lte, - FILE *out_fp, int out_ctype, + filedes_t out_fd, int out_ctype, struct resource_entry *out_res_entry, int flags) { @@ -342,7 +334,7 @@ write_wim_resource(struct wim_lookup_table_entry *lte, flags &= ~WIMLIB_RESOURCE_FLAG_RECOMPRESS; /* Get current position in output WIM */ - offset = ftello(out_fp); + offset = filedes_offset(out_fd); if (offset == -1) { ERROR_WITH_ERRNO("Can't get position in output WIM"); return WIMLIB_ERR_WRITE; @@ -375,7 +367,7 @@ write_wim_resource(struct wim_lookup_table_entry *lte, write_ctx.chunk_tab = NULL; } else { write_ctx.compress = get_compress_func(out_ctype); - ret = begin_wim_resource_chunk_tab(lte, out_fp, + ret = begin_wim_resource_chunk_tab(lte, out_fd, offset, &write_ctx.chunk_tab); if (ret) @@ -384,7 +376,7 @@ write_wim_resource(struct wim_lookup_table_entry *lte, /* Write the entire resource by reading the entire resource and feeding * the data through the write_resource_cb function. */ - write_ctx.out_fp = out_fp; + write_ctx.out_fd = out_fd; try_write_again: ret = read_resource_prefix(lte, read_size, write_resource_cb, &write_ctx, flags); @@ -415,7 +407,7 @@ try_write_again: /* Using a different compression type: Call * finish_wim_resource_chunk_tab() and it will provide the new * compressed size. */ - ret = finish_wim_resource_chunk_tab(write_ctx.chunk_tab, out_fp, + ret = finish_wim_resource_chunk_tab(write_ctx.chunk_tab, out_fd, &new_size); if (ret) goto out_free_chunk_tab; @@ -425,7 +417,7 @@ try_write_again: DEBUG("Compressed %"PRIu64" => %"PRIu64" bytes; " "writing uncompressed instead", wim_resource_size(lte), new_size); - ret = fflush_and_ftruncate(out_fp, offset); + ret = seek_and_truncate(out_fd, offset); if (ret) goto out_free_chunk_tab; write_ctx.compress = NULL; @@ -542,10 +534,10 @@ struct compressor_thread_params { struct message { struct wim_lookup_table_entry *lte; u8 *uncompressed_chunks[MAX_CHUNKS_PER_MSG]; - u8 *out_compressed_chunks[MAX_CHUNKS_PER_MSG]; u8 *compressed_chunks[MAX_CHUNKS_PER_MSG]; unsigned uncompressed_chunk_sizes[MAX_CHUNKS_PER_MSG]; - unsigned compressed_chunk_sizes[MAX_CHUNKS_PER_MSG]; + struct iovec out_chunks[MAX_CHUNKS_PER_MSG]; + size_t total_out_bytes; unsigned num_chunks; struct list_head list; bool complete; @@ -555,20 +547,25 @@ struct message { static void compress_chunks(struct message *msg, compress_func_t compress) { + msg->total_out_bytes = 0; for (unsigned i = 0; i < msg->num_chunks; i++) { unsigned len = compress(msg->uncompressed_chunks[i], msg->uncompressed_chunk_sizes[i], msg->compressed_chunks[i]); + void *out_chunk; + unsigned out_len; if (len) { /* To be written compressed */ - msg->out_compressed_chunks[i] = msg->compressed_chunks[i]; - msg->compressed_chunk_sizes[i] = len; + out_chunk = msg->compressed_chunks[i]; + out_len = len; } else { /* To be written uncompressed */ - msg->out_compressed_chunks[i] = msg->uncompressed_chunks[i]; - msg->compressed_chunk_sizes[i] = msg->uncompressed_chunk_sizes[i]; - + out_chunk = msg->uncompressed_chunks[i]; + out_len = msg->uncompressed_chunk_sizes[i]; } + msg->out_chunks[i].iov_base = out_chunk; + msg->out_chunks[i].iov_len = out_len; + msg->total_out_bytes += out_len; } } @@ -620,7 +617,7 @@ do_write_streams_progress(union wimlib_progress_info *progress, } struct serial_write_stream_ctx { - FILE *out_fp; + filedes_t out_fd; int out_ctype; int write_resource_flags; }; @@ -629,7 +626,7 @@ static int serial_write_stream(struct wim_lookup_table_entry *lte, void *_ctx) { struct serial_write_stream_ctx *ctx = _ctx; - return write_wim_resource(lte, ctx->out_fp, + return write_wim_resource(lte, ctx->out_fd, ctx->out_ctype, <e->output_resource_entry, ctx->write_resource_flags); } @@ -715,14 +712,14 @@ do_write_stream_list(struct list_head *stream_list, static int do_write_stream_list_serial(struct list_head *stream_list, struct wim_lookup_table *lookup_table, - FILE *out_fp, + filedes_t out_fd, int out_ctype, int write_resource_flags, wimlib_progress_func_t progress_func, union wimlib_progress_info *progress) { struct serial_write_stream_ctx ctx = { - .out_fp = out_fp, + .out_fd = out_fd, .out_ctype = out_ctype, .write_resource_flags = write_resource_flags, }; @@ -747,7 +744,7 @@ write_flags_to_resource_flags(int write_flags) static int write_stream_list_serial(struct list_head *stream_list, struct wim_lookup_table *lookup_table, - FILE *out_fp, + filedes_t out_fd, int out_ctype, int write_resource_flags, wimlib_progress_func_t progress_func, @@ -759,7 +756,7 @@ write_stream_list_serial(struct list_head *stream_list, progress_func(WIMLIB_PROGRESS_MSG_WRITE_STREAMS, progress); return do_write_stream_list_serial(stream_list, lookup_table, - out_fp, + out_fd, out_ctype, write_resource_flags, progress_func, @@ -768,29 +765,59 @@ write_stream_list_serial(struct list_head *stream_list, #ifdef ENABLE_MULTITHREADED_COMPRESSION static int -write_wim_chunks(struct message *msg, FILE *out_fp, +write_wim_chunks(struct message *msg, filedes_t out_fd, struct chunk_table *chunk_tab) { - for (unsigned i = 0; i < msg->num_chunks; i++) { - unsigned chunk_csize = msg->compressed_chunk_sizes[i]; + ssize_t bytes_remaining = msg->total_out_bytes; + struct iovec *vecs = msg->out_chunks; + unsigned nvecs = msg->num_chunks; + int ret; - if (fwrite(msg->out_compressed_chunks[i], 1, chunk_csize, out_fp) - != chunk_csize) - { - ERROR_WITH_ERRNO("Failed to write WIM chunk"); - return WIMLIB_ERR_WRITE; - } + wimlib_assert(nvecs != 0); + wimlib_assert(msg->total_out_bytes != 0); + for (unsigned i = 0; i < msg->num_chunks; i++) { *chunk_tab->cur_offset_p++ = chunk_tab->cur_offset; - chunk_tab->cur_offset += chunk_csize; + chunk_tab->cur_offset += vecs[i].iov_len; + } + for (;;) { + ssize_t bytes_written; + + bytes_written = writev(out_fd, vecs, nvecs); + if (bytes_written <= 0) { + if (bytes_written < 0 && errno == EINTR) + continue; + else if (bytes_written == 0) + errno = EIO; + ERROR_WITH_ERRNO("Failed to write WIM chunks"); + ret = WIMLIB_ERR_WRITE; + break; + } + bytes_remaining -= bytes_written; + if (bytes_remaining <= 0) { + ret = 0; + break; + } + while (bytes_written >= 0) { + wimlib_assert(nvecs != 0); + if (bytes_written >= vecs[0].iov_len) { + vecs++; + nvecs--; + bytes_written -= vecs[0].iov_len; + } else { + vecs[0].iov_base += bytes_written; + vecs[0].iov_len -= bytes_written; + bytes_written = 0; + } + } } - return 0; + return ret; } struct main_writer_thread_ctx { struct list_head *stream_list; struct wim_lookup_table *lookup_table; - FILE *out_fp; + filedes_t out_fd; int out_ctype; int write_resource_flags; struct shared_queue *res_to_compress_queue; @@ -946,11 +973,11 @@ receive_compressed_chunks(struct main_writer_thread_ctx *ctx) if (msg->begin_chunk == 0) { /* This is the first set of chunks. Leave space * for the chunk table in the output file. */ - off_t cur_offset = ftello(ctx->out_fp); + off_t cur_offset = filedes_offset(ctx->out_fd); if (cur_offset == -1) return WIMLIB_ERR_WRITE; ret = begin_wim_resource_chunk_tab(cur_lte, - ctx->out_fp, + ctx->out_fd, cur_offset, &ctx->cur_chunk_tab); if (ret) @@ -958,7 +985,7 @@ receive_compressed_chunks(struct main_writer_thread_ctx *ctx) } /* Write the compressed chunks from the message. */ - ret = write_wim_chunks(msg, ctx->out_fp, ctx->cur_chunk_tab); + ret = write_wim_chunks(msg, ctx->out_fd, ctx->cur_chunk_tab); if (ret) return ret; @@ -971,7 +998,7 @@ receive_compressed_chunks(struct main_writer_thread_ctx *ctx) off_t offset; ret = finish_wim_resource_chunk_tab(ctx->cur_chunk_tab, - ctx->out_fp, + ctx->out_fd, &res_csize); if (ret) return ret; @@ -992,11 +1019,11 @@ receive_compressed_chunks(struct main_writer_thread_ctx *ctx) DEBUG("Compressed %"PRIu64" => %"PRIu64" bytes; " "writing uncompressed instead", wim_resource_size(cur_lte), res_csize); - ret = fflush_and_ftruncate(ctx->out_fp, offset); + ret = seek_and_truncate(ctx->out_fd, offset); if (ret) return ret; ret = write_wim_resource(cur_lte, - ctx->out_fp, + ctx->out_fd, WIMLIB_COMPRESSION_TYPE_NONE, &cur_lte->output_resource_entry, ctx->write_resource_flags); @@ -1030,7 +1057,7 @@ receive_compressed_chunks(struct main_writer_thread_ctx *ctx) if (!list_empty(&ctx->serial_streams)) { ret = do_write_stream_list_serial(&ctx->serial_streams, ctx->lookup_table, - ctx->out_fp, + ctx->out_fd, ctx->out_ctype, ctx->write_resource_flags, ctx->progress_func, @@ -1129,7 +1156,7 @@ main_writer_thread_finish(void *_ctx) wimlib_assert(list_empty(&ctx->outstanding_streams)); return do_write_stream_list_serial(&ctx->serial_streams, ctx->lookup_table, - ctx->out_fp, + ctx->out_fd, ctx->out_ctype, ctx->write_resource_flags, ctx->progress_func, @@ -1226,7 +1253,7 @@ get_default_num_threads() static int write_stream_list_parallel(struct list_head *stream_list, struct wim_lookup_table *lookup_table, - FILE *out_fp, + filedes_t out_fd, int out_ctype, int write_resource_flags, wimlib_progress_func_t progress_func, @@ -1299,12 +1326,12 @@ write_stream_list_parallel(struct list_head *stream_list, struct main_writer_thread_ctx ctx; ctx.stream_list = stream_list; ctx.lookup_table = lookup_table; - ctx.out_fp = out_fp; + ctx.out_fd = out_fd; ctx.out_ctype = out_ctype; ctx.res_to_compress_queue = &res_to_compress_queue; ctx.compressed_res_queue = &compressed_res_queue; ctx.num_messages = queue_size; - ctx.write_resource_flags = write_resource_flags | WIMLIB_RESOURCE_FLAG_THREADSAFE_READ; + ctx.write_resource_flags = write_resource_flags; ctx.progress_func = progress_func; ctx.progress = progress; ret = main_writer_thread_init_ctx(&ctx); @@ -1348,7 +1375,7 @@ out_serial: out_serial_quiet: return write_stream_list_serial(stream_list, lookup_table, - out_fp, + out_fd, out_ctype, write_resource_flags, progress_func, @@ -1358,13 +1385,13 @@ out_serial_quiet: #endif /* - * Write a list of streams to a WIM (@out_fp) using the compression type + * Write a list of streams to a WIM (@out_fd) using the compression type * @out_ctype and up to @num_threads compressor threads. */ static int write_stream_list(struct list_head *stream_list, struct wim_lookup_table *lookup_table, - FILE *out_fp, int out_ctype, int write_flags, + filedes_t out_fd, int out_ctype, int write_flags, unsigned num_threads, wimlib_progress_func_t progress_func) { struct wim_lookup_table_entry *lte; @@ -1406,7 +1433,7 @@ write_stream_list(struct list_head *stream_list, if (total_compression_bytes >= 1000000 && num_threads != 1) ret = write_stream_list_parallel(stream_list, lookup_table, - out_fp, + out_fd, out_ctype, write_resource_flags, progress_func, @@ -1416,7 +1443,7 @@ write_stream_list(struct list_head *stream_list, #endif ret = write_stream_list_serial(stream_list, lookup_table, - out_fp, + out_fd, out_ctype, write_resource_flags, progress_func, @@ -1676,7 +1703,7 @@ write_wim_streams(WIMStruct *wim, int image, int write_flags, return ret; return write_stream_list(&stream_list, wim->lookup_table, - wim->out_fp, + wim->out_fd, wimlib_get_compression_type(wim), write_flags, num_threads, @@ -1718,7 +1745,6 @@ finish_write(WIMStruct *w, int image, int write_flags, { int ret; struct wim_header hdr; - FILE *out = w->out_fp; /* @hdr will be the header for the new WIM. First copy all the data * from the header in the WIMStruct; then set all the fields that may @@ -1754,7 +1780,7 @@ finish_write(WIMStruct *w, int image, int write_flags, goto out_close_wim; } - ret = write_xml_data(w->wim_info, image, out, + ret = write_xml_data(w->wim_info, image, w->out_fd, (write_flags & WIMLIB_WRITE_FLAG_NO_LOOKUP_TABLE) ? wim_info_get_total_bytes(w->wim_info) : 0, &hdr.xml_res_entry); @@ -1766,28 +1792,9 @@ finish_write(WIMStruct *w, int image, int write_flags, struct wim_header checkpoint_hdr; memcpy(&checkpoint_hdr, &hdr, sizeof(struct wim_header)); zero_resource_entry(&checkpoint_hdr.integrity); - if (fseeko(out, 0, SEEK_SET)) { - ERROR_WITH_ERRNO("Failed to seek to beginning " - "of WIM being written"); - ret = WIMLIB_ERR_WRITE; - goto out_close_wim; - } - ret = write_header(&checkpoint_hdr, out); + ret = write_header(&checkpoint_hdr, w->out_fd); if (ret) goto out_close_wim; - - if (fflush(out) != 0) { - ERROR_WITH_ERRNO("Can't write data to WIM"); - ret = WIMLIB_ERR_WRITE; - goto out_close_wim; - } - - if (fseeko(out, 0, SEEK_END) != 0) { - ERROR_WITH_ERRNO("Failed to seek to end " - "of WIM being written"); - ret = WIMLIB_ERR_WRITE; - goto out_close_wim; - } } off_t old_lookup_table_end; @@ -1801,7 +1808,7 @@ finish_write(WIMStruct *w, int image, int write_flags, new_lookup_table_end = hdr.lookup_table_res_entry.offset + hdr.lookup_table_res_entry.size; - ret = write_integrity_table(out, + ret = write_integrity_table(w->out_fd, &hdr.integrity, new_lookup_table_end, old_lookup_table_end, @@ -1812,42 +1819,33 @@ finish_write(WIMStruct *w, int image, int write_flags, zero_resource_entry(&hdr.integrity); } - if (fseeko(out, 0, SEEK_SET) != 0) { - ERROR_WITH_ERRNO("Failed to seek to beginning of WIM " - "being written"); - ret = WIMLIB_ERR_WRITE; - goto out_close_wim; - } - - ret = write_header(&hdr, out); + ret = write_header(&hdr, w->out_fd); if (ret) goto out_close_wim; if (write_flags & WIMLIB_WRITE_FLAG_FSYNC) { - if (fflush(out) != 0 - || fsync(fileno(out)) != 0) - { - ERROR_WITH_ERRNO("Error flushing data to WIM file"); + if (fsync(w->out_fd)) { + ERROR_WITH_ERRNO("Error syncing data to WIM file"); ret = WIMLIB_ERR_WRITE; } } out_close_wim: - if (fclose(out) != 0) { + if (close(w->out_fd)) { ERROR_WITH_ERRNO("Failed to close the output WIM file"); if (ret == 0) ret = WIMLIB_ERR_WRITE; } - w->out_fp = NULL; + w->out_fd = INVALID_FILEDES; return ret; } #if defined(HAVE_SYS_FILE_H) && defined(HAVE_FLOCK) int -lock_wim(WIMStruct *w, FILE *fp) +lock_wim(WIMStruct *w, filedes_t fd) { int ret = 0; - if (fp && !w->wim_locked) { - ret = flock(fileno(fp), LOCK_EX | LOCK_NB); + if (fd != INVALID_FILEDES && !w->wim_locked) { + ret = flock(fd, LOCK_EX | LOCK_NB); if (ret != 0) { if (errno == EWOULDBLOCK) { ERROR("`%"TS"' is already being modified or has been " @@ -1868,37 +1866,25 @@ lock_wim(WIMStruct *w, FILE *fp) #endif static int -open_wim_writable(WIMStruct *w, const tchar *path, - bool trunc, bool also_readable) +open_wim_writable(WIMStruct *w, const tchar *path, int open_flags) { - const tchar *mode; - if (trunc) - if (also_readable) - mode = T("w+b"); - else - mode = T("wb"); - else - mode = T("r+b"); - - wimlib_assert(w->out_fp == NULL); - w->out_fp = tfopen(path, mode); - if (w->out_fp) { - return 0; - } else { + wimlib_assert(w->out_fd == INVALID_FILEDES); + w->out_fd = open(path, open_flags, 0644); + if (w->out_fd == INVALID_FILEDES) { ERROR_WITH_ERRNO("Failed to open `%"TS"' for writing", path); return WIMLIB_ERR_OPEN; } + return 0; } void close_wim_writable(WIMStruct *w) { - if (w->out_fp) { - if (fclose(w->out_fp) != 0) { + if (w->out_fd != INVALID_FILEDES) { + if (close(w->out_fd)) WARNING_WITH_ERRNO("Failed to close output WIM"); - } - w->out_fp = NULL; + w->out_fd = INVALID_FILEDES; } } @@ -1907,12 +1893,23 @@ int begin_write(WIMStruct *w, const tchar *path, int write_flags) { int ret; - ret = open_wim_writable(w, path, true, - (write_flags & WIMLIB_WRITE_FLAG_CHECK_INTEGRITY) != 0); + int open_flags = O_TRUNC | O_CREAT; + if (write_flags & WIMLIB_WRITE_FLAG_CHECK_INTEGRITY) + open_flags |= O_RDWR; + else + open_flags |= O_WRONLY; + ret = open_wim_writable(w, path, open_flags); if (ret) return ret; /* Write dummy header. It will be overwritten later. */ - return write_header(&w->hdr, w->out_fp); + ret = write_header(&w->hdr, w->out_fd); + if (ret) + return ret; + if (lseek(w->out_fd, 0, SEEK_END) == -1) { + ERROR_WITH_ERRNO("Failed to seek to end of WIM"); + return WIMLIB_ERR_WRITE; + } + return 0; } /* Writes a stand-alone WIM to a file. */ @@ -2041,6 +2038,7 @@ overwrite_wim_inplace(WIMStruct *w, int write_flags, struct list_head stream_list; off_t old_wim_end; u64 old_lookup_table_end, old_xml_begin, old_xml_end; + int open_flags; DEBUG("Overwriting `%"TS"' in-place", w->filename); @@ -2091,18 +2089,22 @@ overwrite_wim_inplace(WIMStruct *w, int write_flags, if (ret) return ret; - ret = open_wim_writable(w, w->filename, false, - (write_flags & WIMLIB_WRITE_FLAG_CHECK_INTEGRITY) != 0); + open_flags = 0; + if (write_flags & WIMLIB_WRITE_FLAG_CHECK_INTEGRITY) + open_flags |= O_RDWR; + else + open_flags |= O_WRONLY; + ret = open_wim_writable(w, w->filename, open_flags); if (ret) return ret; - ret = lock_wim(w, w->out_fp); + ret = lock_wim(w, w->out_fd); if (ret) { close_wim_writable(w); return ret; } - if (fseeko(w->out_fp, old_wim_end, SEEK_SET) != 0) { + if (lseek(w->out_fd, old_wim_end, SEEK_SET) == -1) { ERROR_WITH_ERRNO("Can't seek to end of WIM"); close_wim_writable(w); w->wim_locked = 0; @@ -2113,7 +2115,7 @@ overwrite_wim_inplace(WIMStruct *w, int write_flags, old_wim_end); ret = write_stream_list(&stream_list, w->lookup_table, - w->out_fp, + w->out_fd, wimlib_get_compression_type(w), write_flags, num_threads, @@ -2198,22 +2200,6 @@ overwrite_wim_via_tmpfile(WIMStruct *w, int write_flags, progress.rename.to = w->filename; progress_func(WIMLIB_PROGRESS_MSG_RENAME, &progress); } - - /* Close the original WIM file that was opened for reading. */ - if (w->fp != NULL) { - fclose(w->fp); - w->fp = NULL; - } - - /* Re-open the WIM read-only. */ - w->fp = tfopen(w->filename, T("rb")); - if (w->fp == NULL) { - ret = WIMLIB_ERR_REOPEN; - WARNING_WITH_ERRNO("Failed to re-open `%"TS"' read-only", - w->filename); - FREE(w->filename); - w->filename = NULL; - } goto out; out_unlink: /* Remove temporary file. */ diff --git a/src/xml.c b/src/xml.c index 72eca4b3..6f044d60 100644 --- a/src/xml.c +++ b/src/xml.c @@ -1236,7 +1236,8 @@ libxml_global_cleanup() * Reads the XML data from a WIM file. */ int -read_xml_data(FILE *fp, const struct resource_entry *res_entry, +read_xml_data(filedes_t in_fd, + const struct resource_entry *res_entry, struct wim_info **info_ret) { utf16lechar *xml_data; @@ -1265,10 +1266,13 @@ read_xml_data(FILE *fp, const struct resource_entry *res_entry, goto out; } - ret = read_uncompressed_resource(fp, res_entry->offset, - res_entry->size, xml_data); - if (ret) + if (full_pread(in_fd, xml_data, + res_entry->size, res_entry->offset) != res_entry->size) + { + ERROR_WITH_ERRNO("Error reading XML data"); + ret = WIMLIB_ERR_READ; goto out_free_xml_data; + } /* Null-terminate just in case */ ((u8*)xml_data)[res_entry->size] = 0; @@ -1324,7 +1328,7 @@ out: * the offset of the XML data. */ int -write_xml_data(const struct wim_info *wim_info, int image, FILE *out, +write_xml_data(const struct wim_info *wim_info, int image, int out_fd, u64 total_bytes, struct resource_entry *out_res_entry) { xmlCharEncodingHandler *encoding_handler; @@ -1338,7 +1342,7 @@ write_xml_data(const struct wim_info *wim_info, int image, FILE *out, (wim_info != NULL && image >= 1 && image <= wim_info->num_images)); - start_offset = ftello(out); + start_offset = filedes_offset(out_fd); if (start_offset == -1) return WIMLIB_ERR_WRITE; @@ -1347,7 +1351,8 @@ write_xml_data(const struct wim_info *wim_info, int image, FILE *out, /* 2 bytes endianness marker for UTF-16LE. This is _required_ for WIM * XML data. */ - if ((putc(0xff, out)) == EOF || (putc(0xfe, out) == EOF)) { + static u8 bom[2] = {0xff, 0xfe}; + if (full_write(out_fd, bom, 2) != 2) { ERROR_WITH_ERRNO("Error writing XML data"); return WIMLIB_ERR_WRITE; } @@ -1373,7 +1378,7 @@ write_xml_data(const struct wim_info *wim_info, int image, FILE *out, goto out; } - out_buffer = xmlOutputBufferCreateFile(out, encoding_handler); + out_buffer = xmlOutputBufferCreateFd(out_fd, encoding_handler); if (!out_buffer) { ERROR("Failed to allocate xmlOutputBuffer"); ret = WIMLIB_ERR_NOMEM; @@ -1428,7 +1433,7 @@ write_xml_data(const struct wim_info *wim_info, int image, FILE *out, xmlFreeTextWriter(writer); writer = NULL; - end_offset = ftello(out); + end_offset = filedes_offset(out_fd); if (end_offset == -1) { ret = WIMLIB_ERR_WRITE; } else { @@ -1499,10 +1504,15 @@ wimlib_extract_xml_data(WIMStruct *w, FILE *fp) if (!buf) return WIMLIB_ERR_NOMEM; - ret = read_uncompressed_resource(w->fp, w->hdr.xml_res_entry.offset, - size, buf); - if (ret) + if (full_pread(w->in_fd, + buf, + w->hdr.xml_res_entry.size, + w->hdr.xml_res_entry.offset) != w->hdr.xml_res_entry.size) + { + ERROR_WITH_ERRNO("Error reading XML data"); + ret = WIMLIB_ERR_READ; goto out_free_buf; + } if (fwrite(buf, 1, size, fp) != size) { ERROR_WITH_ERRNO("Failed to extract XML data"); diff --git a/src/xml.h b/src/xml.h index 80965faf..8840d4ea 100644 --- a/src/xml.h +++ b/src/xml.h @@ -38,11 +38,11 @@ extern void print_image_info(const struct wim_info *wim_info, int image); extern int -read_xml_data(FILE *fp, const struct resource_entry *res, +read_xml_data(filedes_t in_fd, const struct resource_entry *res, struct wim_info **info_ret); extern int -write_xml_data(const struct wim_info *wim_info, int image, FILE *out, +write_xml_data(const struct wim_info *wim_info, int image, filedes_t out_fd, u64 total_bytes, struct resource_entry *out_res_entry); extern void -- 2.43.0