From: Eric Biggers Date: Sat, 30 Mar 2013 05:27:15 +0000 (-0500) Subject: Initial rewrite of resource code X-Git-Tag: v1.3.3~96 X-Git-Url: https://wimlib.net/git/?p=wimlib;a=commitdiff_plain;h=9e2571b03cd9c71d11b3dad9ea5dcfa43f50deb4 Initial rewrite of resource code --- diff --git a/src/add_image.c b/src/add_image.c index de9a465f..02ca2f25 100644 --- a/src/add_image.c +++ b/src/add_image.c @@ -104,40 +104,20 @@ err: static int unix_capture_regular_file(const char *path, - uint64_t size, + u64 size, struct wim_inode *inode, struct wim_lookup_table *lookup_table) { - struct wim_lookup_table_entry *lte; - u8 hash[SHA1_HASH_SIZE]; - int ret; - inode->i_attributes = FILE_ATTRIBUTE_NORMAL; /* Empty files do not have to have a lookup table entry. */ - if (size == 0) - return 0; - - /* For each regular file, we must check to see if the file is in - * the lookup table already; if it is, we increment its refcnt; - * otherwise, we create a new lookup table entry and insert it. - * */ - - ret = sha1sum(path, hash); - if (ret) - return ret; + if (size != 0) { + struct wim_lookup_table_entry *lte; + char *file_on_disk; - lte = __lookup_resource(lookup_table, hash); - if (lte) { - lte->refcnt++; - DEBUG("Add lte reference %u for `%s'", lte->refcnt, - path); - } else { - char *file_on_disk = STRDUP(path); - if (!file_on_disk) { - ERROR("Failed to allocate memory for file path"); + file_on_disk = STRDUP(path); + if (!file_on_disk) return WIMLIB_ERR_NOMEM; - } lte = new_lookup_table_entry(); if (!lte) { FREE(file_on_disk); @@ -146,11 +126,9 @@ unix_capture_regular_file(const char *path, lte->file_on_disk = file_on_disk; lte->resource_location = RESOURCE_IN_FILE_ON_DISK; lte->resource_entry.original_size = size; - lte->resource_entry.size = size; - copy_hash(lte->hash, hash); - lookup_table_insert(lookup_table, lte); + lookup_table_insert_unhashed(lookup_table, lte); + inode->i_lte = lte; } - inode->i_lte = lte; return 0; } @@ -393,10 +371,12 @@ unix_build_dentry_tree_recursive(struct wim_dentry **root_ret, else ret = unix_capture_symlink(path, inode, lookup_table); out: - if (ret == 0) + if (ret == 0) { *root_ret = root; - else + } else { free_dentry_tree(root, lookup_table); + lookup_table_free_unhashed_streams(lookup_table); + } return ret; } diff --git a/src/dentry.c b/src/dentry.c index 49d5d8cb..2ed77da7 100644 --- a/src/dentry.c +++ b/src/dentry.c @@ -1158,7 +1158,7 @@ inode_get_unix_data(const struct wim_inode *inode, if (size != sizeof(struct wimlib_unix_data)) return BAD_UNIX_DATA; - ret = read_full_wim_resource(lte, unix_data, 0); + ret = read_full_resource_into_buf(lte, unix_data, true); if (ret) return ret; diff --git a/src/extract_image.c b/src/extract_image.c index 5148ddcc..a24f0ab5 100644 --- a/src/extract_image.c +++ b/src/extract_image.c @@ -328,7 +328,7 @@ extract_symlink(struct wim_dentry *dentry, { char target[4096]; ssize_t ret = inode_readlink(dentry->d_inode, target, - sizeof(target), args->w, 0); + sizeof(target), args->w, false); struct wim_lookup_table_entry *lte; if (ret <= 0) { diff --git a/src/list.h b/src/list.h index 82f08e5f..889097b5 100644 --- a/src/list.h +++ b/src/list.h @@ -119,6 +119,44 @@ static inline int list_empty(const struct list_head *head) return head->next == head; } +static inline void __list_splice(const struct list_head *list, + struct list_head *prev, + struct list_head *next) +{ + struct list_head *first = list->next; + struct list_head *last = list->prev; + + first->prev = prev; + prev->next = first; + + last->next = next; + next->prev = last; +} + +/** + * list_splice - join two lists, this is designed for stacks + * @list: the new list to add. + * @head: the place to add it in the first list. + */ +static inline void list_splice(const struct list_head *list, + struct list_head *head) +{ + if (!list_empty(list)) + __list_splice(list, head, head->next); +} + +/** + * list_splice_tail - join two lists, each list being a queue + * @list: the new list to add. + * @head: the place to add it in the first list. + */ +static inline void list_splice_tail(struct list_head *list, + struct list_head *head) +{ + if (!list_empty(list)) + __list_splice(list, head->prev, head); +} + /** * list_entry - get the struct for this entry * @ptr: the &struct list_head pointer. diff --git a/src/lookup_table.c b/src/lookup_table.c index 064215ec..9502db47 100644 --- a/src/lookup_table.c +++ b/src/lookup_table.c @@ -40,7 +40,7 @@ new_lookup_table(size_t capacity) struct wim_lookup_table *table; struct hlist_head *array; - table = MALLOC(sizeof(struct wim_lookup_table)); + table = CALLOC(1, sizeof(struct wim_lookup_table)); if (table) { array = CALLOC(capacity, sizeof(array[0])); if (array) { @@ -920,3 +920,13 @@ lookup_table_total_stream_size(struct wim_lookup_table *table) for_lookup_table_entry(table, lte_add_stream_size, &total_size); return total_size; } + +void +lookup_table_free_unhashed_streams(struct wim_lookup_table *table) +{ + struct wim_lookup_table_entry *lte, *tmp; + + list_for_each_entry_safe(lte, tmp, table->unhashed_streams, staging_list) + free_lookup_table_entry(lte); + INIT_LIST_HEAD(table->unhashed_streams); +} diff --git a/src/lookup_table.h b/src/lookup_table.h index eacd9aa4..ab545f2d 100644 --- a/src/lookup_table.h +++ b/src/lookup_table.h @@ -32,6 +32,7 @@ struct wim_lookup_table { struct hlist_head *array; u64 num_entries; u64 capacity; + struct list_head *unhashed_streams; }; #ifdef WITH_NTFS_3G @@ -68,23 +69,35 @@ enum resource_location { * */ RESOURCE_IN_FILE_ON_DISK, - /* The stream resource is located in an external file in the staging - * directory for a read-write mount. */ - RESOURCE_IN_STAGING_FILE, - /* The stream resource is directly attached in an in-memory buffer * pointed to by @attached_buffer. */ RESOURCE_IN_ATTACHED_BUFFER, +#ifdef WITH_FUSE + /* The stream resource is located in an external file in the staging + * directory for a read-write mount. */ + RESOURCE_IN_STAGING_FILE, +#endif + +#ifdef WITH_NTFS_3G /* The stream resource is located in an NTFS volume. It is identified * by volume, filename, data stream name, and by whether it is a reparse * point or not. @ntfs_loc points to a structure containing this * information. */ RESOURCE_IN_NTFS_VOLUME, +#endif +#ifdef __WIN32__ /* Resource must be accessed using Win32 API (may be a named data * stream) */ RESOURCE_WIN32, + + /* Windows only: the file is on disk in the file named @file_on_disk, + * but the file is encrypted and must be read using special functions. + * */ + RESOURCE_WIN32_ENCRYPTED, +#endif + }; /* @@ -168,27 +181,11 @@ struct wim_lookup_table_entry { struct ntfs_location *ntfs_loc; #endif }; - union { - /* @file_on_disk_fp and @attr are both used to cache file/stream - * handles so we don't have re-open them on every read */ - - /* Valid iff resource_location == RESOURCE_IN_FILE_ON_DISK */ - FILE *file_on_disk_fp; - #ifdef WITH_NTFS_3G - /* Valid iff resource_location == RESOURCE_IN_NTFS_VOLUME */ - struct _ntfs_attr *attr; - #endif - - #ifdef __WIN32__ - HANDLE win32_file_on_disk_fp; - #endif - - /* Pointer to inode that contains the opened file descriptors to - * this stream (valid iff resource_location == - * RESOURCE_IN_STAGING_FILE) */ - struct wim_inode *lte_inode; - }; + /* Pointer to inode that contains the opened file descriptors to + * this stream (valid iff resource_location == + * RESOURCE_IN_STAGING_FILE) */ + struct wim_inode *lte_inode; /* When a WIM file is written, out_refcnt starts at 0 and is incremented * whenever the file resource pointed to by this lookup table entry @@ -253,6 +250,19 @@ wim_resource_compression_type(const struct wim_lookup_table_entry *lte) return wimlib_get_compression_type(lte->wim); } +static inline bool +lte_filename_valid(const struct wim_lookup_table_entry *lte) +{ + return lte->resource_location == RESOURCE_IN_FILE_ON_DISK + #ifdef __WIN32__ + || lte->resource_location == RESOURCE_WIN32 + || lte->resource_location == RESOURCE_WIN32_ENCRYPTED + #endif + #ifdef WITH_FUSE + || lte->resource_location == RESOURCE_IN_STAGING_FILE + #endif + ; +} extern struct wim_lookup_table * new_lookup_table(size_t capacity); @@ -461,4 +471,15 @@ inode_unnamed_lte(const struct wim_inode *inode, const struct wim_lookup_table * extern u64 lookup_table_total_stream_size(struct wim_lookup_table *table); + +static inline void +lookup_table_insert_unhashed(struct wim_lookup_table *table, + struct wim_lookup_table_entry *lte) +{ + list_add_tail(<e->staging_list, table->unhashed_streams); +} + +extern void +lookup_table_free_unhashed_streams(struct wim_lookup_table *table); + #endif diff --git a/src/metadata_resource.c b/src/metadata_resource.c index db35ba64..9b5a8305 100644 --- a/src/metadata_resource.c +++ b/src/metadata_resource.c @@ -86,8 +86,8 @@ read_metadata_resource(WIMStruct *w, struct wim_image_metadata *imd) } /* Read the metadata resource into memory. (It may be compressed.) */ - ret = read_full_wim_resource(metadata_lte, buf, 0); - if (ret != 0) + ret = read_full_resource_into_buf(metadata_lte, buf, false); + if (ret) goto out_free_buf; DEBUG("Finished reading metadata resource into memory."); @@ -208,16 +208,12 @@ write_wim_resource_from_buffer(const u8 *buf, u64 buf_size, * write_wim_resource(). */ struct wim_lookup_table_entry lte; int ret; - lte.resource_entry.flags = 0; lte.resource_entry.original_size = buf_size; - lte.resource_entry.size = buf_size; - lte.resource_entry.offset = 0; lte.resource_location = RESOURCE_IN_ATTACHED_BUFFER; lte.attached_buffer = (void*)buf; - zero_out_hash(lte.hash); ret = write_wim_resource(<e, out_fp, out_ctype, out_res_entry, 0); - if (ret != 0) + if (ret) return ret; copy_hash(hash, lte.hash); return 0; @@ -293,8 +289,8 @@ write_metadata_resource(WIMStruct *w) wimlib_get_compression_type(w), <e->output_resource_entry, lte->hash); - if (ret != 0) - goto out; + if (ret) + goto out_free_buf; /* Note that although the SHA1 message digest of the metadata resource * is very likely to have changed, the corresponding lookup table entry @@ -302,7 +298,7 @@ write_metadata_resource(WIMStruct *w) * re-inserted in the hash table. */ lte->out_refcnt = 1; lte->output_resource_entry.flags |= WIM_RESHDR_FLAG_METADATA; -out: +out_free_buf: /* All the data has been written to the new WIM; no need for the buffer * anymore */ FREE(buf); diff --git a/src/mount_image.c b/src/mount_image.c index 024c82b7..128e1736 100644 --- a/src/mount_image.c +++ b/src/mount_image.c @@ -422,22 +422,10 @@ inode_to_stbuf(const struct wim_inode *inode, } stbuf->st_ino = (ino_t)inode->i_ino; stbuf->st_nlink = inode->i_nlink; - if (lte) { - if (lte->resource_location == RESOURCE_IN_STAGING_FILE) { - struct stat native_stat; - if (stat(lte->staging_file_name, &native_stat) != 0) { - DEBUG("Failed to stat `%s': %m", - lte->staging_file_name); - return -errno; - } - stbuf->st_size = native_stat.st_size; - } else { - stbuf->st_size = wim_resource_size(lte); - } - } else { + if (lte) + stbuf->st_size = wim_resource_size(lte); + else stbuf->st_size = 0; - } - #ifdef HAVE_STAT_NANOSECOND_PRECISION stbuf->st_atim = wim_timestamp_to_timespec(inode->i_last_access_time); stbuf->st_mtim = wim_timestamp_to_timespec(inode->i_last_write_time); @@ -658,11 +646,10 @@ extract_resource_to_staging_dir(struct wim_inode *inode, } } - new_lte->refcnt = inode->i_nlink; - new_lte->resource_location = RESOURCE_IN_STAGING_FILE; - new_lte->staging_file_name = staging_file_name; - new_lte->lte_inode = inode; - random_hash(new_lte->hash); + new_lte->refcnt = inode->i_nlink; + new_lte->resource_location = RESOURCE_IN_STAGING_FILE; + new_lte->staging_file_name = staging_file_name; + new_lte->lte_inode = inode; if (stream_id == 0) inode->i_lte = new_lte; @@ -671,8 +658,8 @@ extract_resource_to_staging_dir(struct wim_inode *inode, if (inode->i_ads_entries[i].stream_id == stream_id) inode->i_ads_entries[i].lte = new_lte; - lookup_table_insert(ctx->wim->lookup_table, new_lte); - list_add(&new_lte->staging_list, &ctx->staging_list); + lookup_table_insert_unhashed(ctx->wim->lookup_table, new_lte); + list_add_tail(&new_lte->staging_list, &ctx->staging_list); *lte = new_lte; return 0; out_revert_fd_changes: @@ -807,45 +794,15 @@ inode_update_lte_ptr(struct wim_inode *inode, } } -static int -update_lte_of_staging_file(struct wim_lookup_table_entry *lte, - struct wim_lookup_table *table) +static void +free_lte_if_unneeded(struct wim_lookup_table_entry *lte) { - struct wim_lookup_table_entry *duplicate_lte; - int ret; - u8 hash[SHA1_HASH_SIZE]; - struct stat stbuf; - ret = sha1sum(lte->staging_file_name, hash); - if (ret != 0) - return ret; - lookup_table_unlink(table, lte); - duplicate_lte = __lookup_resource(table, hash); - if (duplicate_lte) { - /* Merge duplicate lookup table entries */ - duplicate_lte->refcnt += lte->refcnt; - inode_update_lte_ptr(lte->lte_inode, lte, duplicate_lte); + if (wim_resource_size(lte) == 0) { + /* Zero-length stream. No lookup table entry needed. */ + inode_update_lte_ptr(lte->lte_inode, lte, NULL); free_lookup_table_entry(lte); - } else { - if (stat(lte->staging_file_name, &stbuf) != 0) { - ERROR_WITH_ERRNO("Failed to stat `%s'", lte->staging_file_name); - return WIMLIB_ERR_STAT; - } - if (stbuf.st_size == 0) { - /* Zero-length stream. No lookup table entry needed. */ - inode_update_lte_ptr(lte->lte_inode, lte, NULL); - free_lookup_table_entry(lte); - } else { - BUILD_BUG_ON(<e->file_on_disk != <e->staging_file_name); - lte->resource_entry.original_size = stbuf.st_size; - lte->resource_entry.size = stbuf.st_size; - lte->resource_location = RESOURCE_IN_FILE_ON_DISK; - lte->file_on_disk_fp = NULL; - copy_hash(lte->hash, hash); - lookup_table_insert(table, lte); - } } - return 0; } static int @@ -881,14 +838,13 @@ rebuild_wim(struct wimfs_context *ctx, int write_flags, return ret; } - DEBUG("Calculating SHA1 checksums for all new staging files."); - list_for_each_entry_safe(lte, tmp, &ctx->staging_list, staging_list) { - ret = update_lte_of_staging_file(lte, w->lookup_table); - if (ret != 0) - return ret; - } + DEBUG("Freeing entries for zero-length streams"); + list_for_each_entry_safe(lte, tmp, &ctx->staging_list, staging_list) + free_lte_if_unneeded(lte); xml_update_image_info(w, w->current_image); + list_splice(&ctx->staging_list, + &w->image_metadata[w->current_image - 1].unhashed_streams); ret = wimlib_overwrite(w, write_flags, 0, progress_func); if (ret != 0) ERROR("Failed to commit changes to mounted WIM image"); @@ -1660,10 +1616,10 @@ wimfs_ftruncate(const char *path, off_t size, struct fuse_file_info *fi) { struct wimfs_fd *fd = (struct wimfs_fd*)(uintptr_t)fi->fh; int ret = ftruncate(fd->staging_fd, size); - if (ret != 0) + if (ret) return -errno; - if (fd->f_lte && size < fd->f_lte->resource_entry.original_size) - fd->f_lte->resource_entry.original_size = size; + touch_inode(fd->f_inode); + fd->f_lte->resource_entry.original_size = size; return 0; } @@ -1723,8 +1679,7 @@ wimfs_getxattr(const char *path, const char *name, char *value, if (res_size > size) return -ERANGE; - ret = read_full_wim_resource(lte, (u8*)value, - WIMLIB_RESOURCE_FLAG_MULTITHREADED); + ret = read_full_resource_into_buf(lte, value, true); if (ret != 0) return -EIO; @@ -1861,8 +1816,7 @@ wimfs_mknod(const char *path, mode_t mode, dev_t rdev) inode = wim_pathname_to_inode(wimfs_ctx->wim, path); if (!inode) return -errno; - if (inode->i_attributes & - (FILE_ATTRIBUTE_REPARSE_POINT | FILE_ATTRIBUTE_DIRECTORY)) + if (inode->i_attributes & FILE_ATTRIBUTE_REPARSE_POINT) return -ENOENT; if (inode_get_ads_entry(inode, stream_name, NULL)) return -EEXIST; @@ -1965,6 +1919,7 @@ wimfs_read(const char *path, char *buf, size_t size, { struct wimfs_fd *fd = (struct wimfs_fd*)(uintptr_t)fi->fh; ssize_t ret; + u64 res_size; if (!fd) return -EBADF; @@ -1972,32 +1927,34 @@ wimfs_read(const char *path, char *buf, size_t size, if (!fd->f_lte) /* Empty stream with no lookup table entry */ return 0; - if (fd->f_lte->resource_location == RESOURCE_IN_STAGING_FILE) { - /* Read from staging file */ - - wimlib_assert(fd->f_lte->staging_file_name); - wimlib_assert(fd->staging_fd != -1); - - DEBUG("Seek to offset %"PRIu64, offset); + res_size = wim_resource_size(fd->f_lte); + if (offset > res_size) + return -EOVERFLOW; + size = min(size, INT_MAX); + size = min(size, res_size - offset); - if (lseek(fd->staging_fd, offset, SEEK_SET) == -1) - return -errno; - ret = read(fd->staging_fd, buf, size); - if (ret == -1) - return -errno; - return ret; - } else { - /* Read from WIM */ - u64 res_size = wim_resource_size(fd->f_lte); - if (offset > res_size) - return -EOVERFLOW; - size = min(size, res_size - offset); - if (read_wim_resource(fd->f_lte, buf, - size, offset, - WIMLIB_RESOURCE_FLAG_MULTITHREADED) != 0) - return -EIO; - return size; + switch (fd->f_lte->resource_location) { + case RESOURCE_IN_STAGING_FILE: + ret = pread(fd->staging_fd, buf, size, offset); + if (ret < 0) + ret = -errno; + break; + case RESOURCE_IN_WIM: + if (read_partial_wim_resource_into_buf(fd->f_lte, size, + offset, buf, true)) + ret = -errno; + ret = size; + break; + case RESOURCE_IN_ATTACHED_BUFFER: + memcpy(buf, fd->f_lte->attached_buffer + offset, size); + ret = size; + break; + default: + ERROR("Invalid resource location"); + ret = -EIO; + break; } + return ret; } struct fill_params { @@ -2064,8 +2021,7 @@ wimfs_readlink(const char *path, char *buf, size_t buf_len) if (!inode_is_symlink(inode)) return -EINVAL; - ret = inode_readlink(inode, buf, buf_len, ctx->wim, - WIMLIB_RESOURCE_FLAG_MULTITHREADED); + ret = inode_readlink(inode, buf, buf_len, ctx->wim, true); if (ret > 0) ret = 0; return ret; @@ -2299,6 +2255,8 @@ wimfs_truncate(const char *path, off_t size) ret = extract_resource_to_staging_dir(inode, stream_id, <e, size, ctx); } + if (ret == 0) + lte->resource_entry.original_size = size; return ret; } @@ -2390,20 +2348,20 @@ wimfs_write(const char *path, const char *buf, size_t size, if (!fd) return -EBADF; - wimlib_assert(fd->f_lte); - wimlib_assert(fd->f_lte->staging_file_name); + wimlib_assert(fd->f_lte != NULL); + wimlib_assert(fd->f_lte->staging_file_name != NULL); wimlib_assert(fd->staging_fd != -1); - wimlib_assert(fd->f_inode); - - /* Seek to the requested position */ - if (lseek(fd->staging_fd, offset, SEEK_SET) == -1) - return -errno; + wimlib_assert(fd->f_inode != NULL); /* Write the data. */ - ret = write(fd->staging_fd, buf, size); + ret = pwrite(fd->staging_fd, buf, size, offset); if (ret == -1) return -errno; + /* Update file size */ + if (offset + size > fd->f_lte->resource_entry.original_size) + fd->f_lte->resource_entry.original_size = offset + size; + /* Update timestamps */ touch_inode(fd->f_inode); return ret; diff --git a/src/ntfs-apply.c b/src/ntfs-apply.c index ca73db2f..994bbd27 100644 --- a/src/ntfs-apply.c +++ b/src/ntfs-apply.c @@ -43,12 +43,17 @@ #include #include +struct ntfs_attr_extract_ctx { + u64 offset; + ntfs_attr *na; +}; + static int -extract_wim_chunk_to_ntfs_attr(const void *buf, size_t len, - u64 offset, void *arg) +extract_wim_chunk_to_ntfs_attr(const void *buf, size_t len, void *_ctx) { - ntfs_attr *na = arg; - if (ntfs_attr_pwrite(na, offset, len, buf) == len) { + struct ntfs_attr_extract_ctx *ctx = _ctx; + if (ntfs_attr_pwrite(ctx->na, ctx->offset, len, buf) == len) { + ctx->offset += len; return 0; } else { ERROR_WITH_ERRNO("Error extracting WIM resource to NTFS attribute"); @@ -63,8 +68,11 @@ static int extract_wim_resource_to_ntfs_attr(const struct wim_lookup_table_entry *lte, ntfs_attr *na) { + struct ntfs_attr_extract_ctx ctx; + ctx.na = na; + ctx.offset = 0; return extract_wim_resource(lte, wim_resource_size(lte), - extract_wim_chunk_to_ntfs_attr, na); + extract_wim_chunk_to_ntfs_attr, &ctx); } /* Writes the data streams of a WIM inode to the data attributes of a NTFS @@ -360,13 +368,13 @@ apply_reparse_data(ntfs_inode *ni, const struct wim_dentry *dentry, p = put_u16(p, wim_resource_size(lte)); /* ReparseDataLength */ p = put_u16(p, 0); /* Reserved */ - ret = read_full_wim_resource(lte, p, 0); - if (ret != 0) + ret = read_full_resource_into_buf(lte, p, false); + if (ret) return ret; ret = ntfs_set_ntfs_reparse_data(ni, (char*)reparse_data_buf, wim_resource_size(lte) + 8, 0); - if (ret != 0) { + if (ret) { ERROR_WITH_ERRNO("Failed to set NTFS reparse data on `%s'", dentry->full_path); return WIMLIB_ERR_NTFS_3G; diff --git a/src/ntfs-capture.c b/src/ntfs-capture.c index 94e74cee..5c96c1b2 100644 --- a/src/ntfs-capture.c +++ b/src/ntfs-capture.c @@ -58,62 +58,79 @@ attr_record_name(ATTR_RECORD *ar) return (ntfschar*)((u8*)ar + le16_to_cpu(ar->name_offset)); } -/* Calculates the SHA1 message digest of a NTFS attribute. - * - * @ni: The NTFS inode containing the attribute. - * @ar: The ATTR_RECORD describing the attribute. - * @md: If successful, the returned SHA1 message digest. - * @reparse_tag_ret: Optional pointer into which the first 4 bytes of the - * attribute will be written (to get the reparse - * point ID) - * - * Return 0 on success or nonzero on error. - */ -static int -ntfs_attr_sha1sum(ntfs_inode *ni, ATTR_RECORD *ar, - u8 md[SHA1_HASH_SIZE], - bool is_reparse_point, - u32 *reparse_tag_ret) +int +read_ntfs_file_prefix(const struct wim_lookup_table_entry *lte, + u64 size, + consume_data_callback_t cb, + void *ctx_or_buf, + int _ignored_flags) { - s64 pos = 0; - s64 bytes_remaining; - char buf[BUFFER_SIZE]; + struct ntfs_location *loc = lte->ntfs_loc; + ntfs_volume *vol = *loc->ntfs_vol_p; + ntfs_inode *ni; ntfs_attr *na; - SHA_CTX ctx; + s64 pos; + s64 bytes_remaining; + void *out_buf; + int ret; - na = ntfs_attr_open(ni, ar->type, attr_record_name(ar), - ar->name_length); - if (!na) { - ERROR_WITH_ERRNO("Failed to open NTFS attribute"); - return WIMLIB_ERR_NTFS_3G; + ni = ntfs_pathname_to_inode(vol, NULL, loc->path); + if (!ni) { + ERROR_WITH_ERRNO("Can't find NTFS inode for \"%"TS"\"", loc->path); + ret = WIMLIB_ERR_NTFS_3G; + goto out; } - bytes_remaining = na->data_size; - - if (is_reparse_point) { - if (ntfs_attr_pread(na, 0, 8, buf) != 8) - goto out_error; - *reparse_tag_ret = le32_to_cpu(*(u32*)buf); - DEBUG("ReparseTag = %#x", *reparse_tag_ret); - pos = 8; - bytes_remaining -= 8; + na = ntfs_attr_open(ni, + loc->is_reparse_point ? AT_REPARSE_POINT : AT_DATA, + loc->stream_name, + loc->stream_name_nchars); + if (!na) { + ERROR_WITH_ERRNO("Failed to open attribute of \"%"TS"\" in " + "NTFS volume", loc->path); + ret = WIMLIB_ERR_NTFS_3G; + goto out_close_ntfs_inode; } - sha1_init(&ctx); + /*if (is_reparse_point) {*/ + /*if (ntfs_attr_pread(na, 0, 8, buf) != 8)*/ + /*goto out_error;*/ + /**reparse_tag_ret = le32_to_cpu(*(u32*)buf);*/ + /*DEBUG("ReparseTag = %#x", *reparse_tag_ret);*/ + /*pos = 8;*/ + /*bytes_remaining -= 8;*/ + /*}*/ + + if (cb) + out_buf = alloca(WIM_CHUNK_SIZE); + else + out_buf = ctx_or_buf; + pos = 0; + bytes_remaining = na->data_size; while (bytes_remaining) { - s64 to_read = min(bytes_remaining, sizeof(buf)); - if (ntfs_attr_pread(na, pos, to_read, buf) != to_read) - goto out_error; - sha1_update(&ctx, buf, to_read); + s64 to_read = min(bytes_remaining, WIM_CHUNK_SIZE); + if (ntfs_attr_pread(na, pos, to_read, out_buf) != to_read) { + ERROR_WITH_ERRNO("Error reading \"%"TS"\"", loc->path); + ret = WIMLIB_ERR_NTFS_3G; + goto out_close_ntfs_attr; + } pos += to_read; bytes_remaining -= to_read; + if (cb) { + ret = cb(out_buf, to_read, ctx_or_buf); + if (ret) + goto out_close_ntfs_attr; + } else { + out_buf += to_read; + } } - sha1_final(md, &ctx); + ret = 0; +out_close_ntfs_attr: ntfs_attr_close(na); - return 0; -out_error: - ERROR_WITH_ERRNO("Error reading NTFS attribute"); - return WIMLIB_ERR_NTFS_3G; +out_close_ntfs_inode: + ntfs_inode_close(ni); +out: + return ret; } /* Load the streams from a file or reparse point in the NTFS volume into the WIM @@ -128,7 +145,6 @@ capture_ntfs_streams(struct wim_inode *inode, ATTR_TYPES type) { ntfs_attr_search_ctx *actx; - u8 attr_hash[SHA1_HASH_SIZE]; struct ntfs_location *ntfs_loc = NULL; int ret = 0; struct wim_lookup_table_entry *lte; @@ -147,7 +163,6 @@ capture_ntfs_streams(struct wim_inode *inode, while (!ntfs_attr_lookup(type, NULL, 0, CASE_SENSITIVE, 0, NULL, 0, actx)) { - u32 reparse_tag; u64 data_size = ntfs_get_attribute_value_length(actx->attr); u64 name_length = actx->attr->name_length; if (data_size == 0) { @@ -160,64 +175,44 @@ capture_ntfs_streams(struct wim_inode *inode, /* Empty stream. No lookup table entry is needed. */ lte = NULL; } else { - if (type == AT_REPARSE_POINT && data_size < 8) { - ERROR("`%s': reparse point buffer too small", - path); - ret = WIMLIB_ERR_NTFS_3G; + ntfs_loc = CALLOC(1, sizeof(*ntfs_loc)); + if (!ntfs_loc) goto out_put_actx; + ntfs_loc->ntfs_vol_p = ntfs_vol_p; + ntfs_loc->path = MALLOC(path_len + 1); + if (!ntfs_loc->path) + goto out_free_ntfs_loc; + memcpy(ntfs_loc->path, path, path_len + 1); + if (name_length) { + ntfs_loc->stream_name = MALLOC(name_length * 2); + if (!ntfs_loc->stream_name) + goto out_free_ntfs_loc; + memcpy(ntfs_loc->stream_name, + attr_record_name(actx->attr), + actx->attr->name_length * 2); + ntfs_loc->stream_name_nchars = name_length; } - /* Checksum the stream. */ - ret = ntfs_attr_sha1sum(ni, actx->attr, attr_hash, - type == AT_REPARSE_POINT, &reparse_tag); - if (ret != 0) - goto out_put_actx; - - if (type == AT_REPARSE_POINT) - inode->i_reparse_tag = reparse_tag; - /* Make a lookup table entry for the stream, or use an existing - * one if there's already an identical stream. */ - lte = __lookup_resource(lookup_table, attr_hash); - ret = WIMLIB_ERR_NOMEM; - if (lte) { - lte->refcnt++; + lte = new_lookup_table_entry(); + if (!lte) + goto out_free_ntfs_loc; + lte->ntfs_loc = ntfs_loc; + lte->resource_location = RESOURCE_IN_NTFS_VOLUME; + #if 0 + if (type == AT_REPARSE_POINT) { + ntfs_loc->is_reparse_point = true; + lte->resource_entry.original_size = data_size - 8; + lte->resource_entry.size = data_size - 8; } else { - ntfs_loc = CALLOC(1, sizeof(*ntfs_loc)); - if (!ntfs_loc) - goto out_put_actx; - ntfs_loc->ntfs_vol_p = ntfs_vol_p; - ntfs_loc->path = MALLOC(path_len + 1); - if (!ntfs_loc->path) - goto out_free_ntfs_loc; - memcpy(ntfs_loc->path, path, path_len + 1); - if (name_length) { - ntfs_loc->stream_name = MALLOC(name_length * 2); - if (!ntfs_loc->stream_name) - goto out_free_ntfs_loc; - memcpy(ntfs_loc->stream_name, - attr_record_name(actx->attr), - actx->attr->name_length * 2); - ntfs_loc->stream_name_nchars = name_length; - } - - lte = new_lookup_table_entry(); - if (!lte) - goto out_free_ntfs_loc; - lte->ntfs_loc = ntfs_loc; - lte->resource_location = RESOURCE_IN_NTFS_VOLUME; - if (type == AT_REPARSE_POINT) { - ntfs_loc->is_reparse_point = true; - lte->resource_entry.original_size = data_size - 8; - lte->resource_entry.size = data_size - 8; - } else { - ntfs_loc->is_reparse_point = false; - lte->resource_entry.original_size = data_size; - lte->resource_entry.size = data_size; - } - ntfs_loc = NULL; - copy_hash(lte->hash, attr_hash); - lookup_table_insert(lookup_table, lte); + ntfs_loc->is_reparse_point = false; + lte->resource_entry.original_size = data_size; + lte->resource_entry.size = data_size; } + #else + ntfs_loc->is_reparse_point = (type == AT_REPARSE_POINT); + lte->resource_entry.original_size = data_size; + #endif + ntfs_loc = NULL; } if (name_length == 0) { /* Unnamed data stream. Put the reference to it in the @@ -242,6 +237,8 @@ capture_ntfs_streams(struct wim_inode *inode, wimlib_assert(new_ads_entry->stream_name_nbytes == name_length * 2); new_ads_entry->lte = lte; } + if (lte) + lookup_table_insert_unhashed(lookup_table, lte); } ret = 0; goto out_put_actx; diff --git a/src/resource.c b/src/resource.c index a8e98ca4..90e45ca1 100644 --- a/src/resource.c +++ b/src/resource.c @@ -36,33 +36,56 @@ #include #include #include +#include -#ifdef WITH_NTFS_3G -# include -# include -# include -# include -#endif +/* Write @n bytes from @buf to the file descriptor @fd, retrying on internupt + * and on short writes. + * + * Returns short count and set errno on failure. */ +static ssize_t +full_write(int fd, const void *buf, size_t n) +{ + const void *p = buf; + ssize_t ret; + ssize_t total = 0; -#if defined(__WIN32__) && !defined(INVALID_HANDLE_VALUE) -# define INVALID_HANDLE_VALUE ((HANDLE)(-1)) -#endif + while (total != n) { + ret = write(fd, p, n); + if (ret < 0) { + if (errno == EINTR) + continue; + else + break; + } + total += ret; + p += ret; + } + return total; +} -/* - * Reads all or part of a compressed resource into an in-memory buffer. +/* Read @n bytes from the file descriptor @fd to the buffer @buf, retrying on + * internupt and on short reads. * - * @fp: The FILE* for the WIM file. - * @resource_compressed_size: The compressed size of the resource. - * @resource_uncompressed_size: The uncompressed size of the resource. - * @resource_offset: The offset of the start of the resource from - * the start of the stream @fp. - * @resource_ctype: The compression type of the resource. - * @len: The number of bytes of uncompressed data to read from - * the resource. - * @offset: The offset of the bytes to read within the uncompressed - * resource. - * @contents_len: An array into which the uncompressed data is written. - * It must be at least @len bytes long. + * Returns short count and set errno on failure. */ +static size_t +full_read(int fd, void *buf, size_t n) +{ + size_t bytes_remaining = n; + while (bytes_remaining) { + ssize_t bytes_read = read(fd, buf, bytes_remaining); + if (bytes_read < 0) { + if (errno == EINTR) + continue; + break; + } + bytes_remaining -= bytes_read; + buf += bytes_read; + } + return n - bytes_remaining; +} + +/* + * Reads all or part of a compressed WIM resource. * * Returns zero on success, nonzero on failure. */ @@ -70,16 +93,12 @@ static int read_compressed_resource(FILE *fp, u64 resource_compressed_size, u64 resource_uncompressed_size, u64 resource_offset, int resource_ctype, - u64 len, u64 offset, void *contents_ret) + u64 len, u64 offset, + consume_data_callback_t cb, + void *ctx_or_buf) { + int ret; - DEBUG2("comp size = %"PRIu64", uncomp size = %"PRIu64", " - "res offset = %"PRIu64"", - resource_compressed_size, - resource_uncompressed_size, - resource_offset); - DEBUG2("resource_ctype = %"TS", len = %"PRIu64", offset = %"PRIu64"", - wimlib_get_compression_type_string(resource_ctype), len, offset); /* Trivial case */ if (len == 0) return 0; @@ -168,36 +187,34 @@ read_compressed_resource(FILE *fp, u64 resource_compressed_size, /* Skip over unneeded chunk table entries. */ u64 file_offset_of_needed_chunk_entries = resource_offset + start_table_idx * chunk_entry_size; - if (fseeko(fp, file_offset_of_needed_chunk_entries, SEEK_SET) != 0) { - ERROR_WITH_ERRNO("Failed to seek to byte %"PRIu64" to read " - "chunk table of compressed resource", - file_offset_of_needed_chunk_entries); - return WIMLIB_ERR_READ; - } + if (fseeko(fp, file_offset_of_needed_chunk_entries, SEEK_SET)) + goto read_error; /* Number of bytes we need to read from the chunk table. */ size_t size = num_needed_chunk_entries * chunk_entry_size; - u8 chunk_tab_buf[size]; + { + u8 chunk_tab_buf[size]; - if (fread(chunk_tab_buf, 1, size, fp) != size) - goto err; + if (fread(chunk_tab_buf, 1, size, fp) != size) + goto read_error; - /* Now fill in chunk_offsets from the entries we have read in - * chunk_tab_buf. */ + /* Now fill in chunk_offsets from the entries we have read in + * chunk_tab_buf. */ - u64 *chunk_tab_p = chunk_offsets; - if (start_chunk == 0) - chunk_tab_p++; + u64 *chunk_tab_p = chunk_offsets; + if (start_chunk == 0) + chunk_tab_p++; - if (chunk_entry_size == 4) { - u32 *entries = (u32*)chunk_tab_buf; - while (num_needed_chunk_entries--) - *chunk_tab_p++ = le32_to_cpu(*entries++); - } else { - u64 *entries = (u64*)chunk_tab_buf; - while (num_needed_chunk_entries--) - *chunk_tab_p++ = le64_to_cpu(*entries++); + if (chunk_entry_size == 4) { + u32 *entries = (u32*)chunk_tab_buf; + while (num_needed_chunk_entries--) + *chunk_tab_p++ = le32_to_cpu(*entries++); + } else { + u64 *entries = (u64*)chunk_tab_buf; + while (num_needed_chunk_entries--) + *chunk_tab_p++ = le64_to_cpu(*entries++); + } } /* Done with the chunk table now. We must now seek to the first chunk @@ -205,16 +222,16 @@ read_compressed_resource(FILE *fp, u64 resource_compressed_size, u64 file_offset_of_first_needed_chunk = resource_offset + chunk_table_size + chunk_offsets[0]; - if (fseeko(fp, file_offset_of_first_needed_chunk, SEEK_SET) != 0) { - ERROR_WITH_ERRNO("Failed to seek to byte %"PRIu64" to read " - "first chunk of compressed resource", - file_offset_of_first_needed_chunk); - return WIMLIB_ERR_READ; - } + if (fseeko(fp, file_offset_of_first_needed_chunk, SEEK_SET)) + goto read_error; /* Pointer to current position in the output buffer for uncompressed * data. */ - u8 *out_p = contents_ret; + u8 *out_p; + if (cb) + out_p = alloca(32768); + else + out_p = ctx_or_buf; /* Buffer for compressed data. While most compressed chunks will have a * size much less than WIM_CHUNK_SIZE, WIM_CHUNK_SIZE - 1 is the maximum @@ -222,15 +239,11 @@ read_compressed_resource(FILE *fp, u64 resource_compressed_size, * happen to compress to more than the uncompressed size (i.e. a * sequence of random bytes) are always stored uncompressed. But this seems * to be the case in M$'s WIM files, even though it is undocumented. */ - u8 compressed_buf[WIM_CHUNK_SIZE - 1]; - + void *compressed_buf = alloca(WIM_CHUNK_SIZE - 1); /* Decompress all the chunks. */ for (u64 i = start_chunk; i <= end_chunk; i++) { - DEBUG2("Chunk %"PRIu64" (start %"PRIu64", end %"PRIu64").", - i, start_chunk, end_chunk); - /* Calculate the sizes of the compressed chunk and of the * uncompressed chunk. */ unsigned compressed_chunk_size; @@ -265,11 +278,6 @@ read_compressed_resource(FILE *fp, u64 resource_compressed_size, uncompressed_chunk_size = WIM_CHUNK_SIZE; } - DEBUG2("compressed_chunk_size = %u, " - "uncompressed_chunk_size = %u", - compressed_chunk_size, uncompressed_chunk_size); - - /* Figure out how much of this chunk we actually need to read */ u64 start_offset; if (i == start_chunk) @@ -282,75 +290,80 @@ read_compressed_resource(FILE *fp, u64 resource_compressed_size, else end_offset = WIM_CHUNK_SIZE - 1; - u64 partial_chunk_size = end_offset + 1 - start_offset; - bool is_partial_chunk = (partial_chunk_size != - uncompressed_chunk_size); - - DEBUG2("start_offset = %"PRIu64", end_offset = %"PRIu64"", - start_offset, end_offset); - DEBUG2("partial_chunk_size = %"PRIu64"", partial_chunk_size); + unsigned partial_chunk_size = end_offset + 1 - start_offset; + bool is_partial_chunk = (partial_chunk_size != uncompressed_chunk_size); /* This is undocumented, but chunks can be uncompressed. This * appears to always be the case when the compressed chunk size * is equal to the uncompressed chunk size. */ if (compressed_chunk_size == uncompressed_chunk_size) { - /* Probably an uncompressed chunk */ + /* Uncompressed chunk */ - if (start_offset != 0) { - if (fseeko(fp, start_offset, SEEK_CUR) != 0) { - ERROR_WITH_ERRNO("Uncompressed partial " - "chunk fseek() error"); - return WIMLIB_ERR_READ; - } - } - if (fread(out_p, 1, partial_chunk_size, fp) != - partial_chunk_size) - goto err; + if (start_offset != 0) + if (fseeko(fp, start_offset, SEEK_CUR)) + goto read_error; + if (fread(out_p, 1, partial_chunk_size, fp) != partial_chunk_size) + goto read_error; } else { /* Compressed chunk */ - int ret; /* Read the compressed data into compressed_buf. */ if (fread(compressed_buf, 1, compressed_chunk_size, fp) != compressed_chunk_size) - goto err; + goto read_error; - /* For partial chunks we must buffer the uncompressed - * data because we don't need all of it. */ - if (is_partial_chunk) { + /* For partial chunks and when writing directly to a + * buffer, we must buffer the uncompressed data because + * we don't need all of it. */ + if (is_partial_chunk && !cb) { u8 uncompressed_buf[uncompressed_chunk_size]; ret = decompress(compressed_buf, - compressed_chunk_size, - uncompressed_buf, - uncompressed_chunk_size); - if (ret != 0) - return WIMLIB_ERR_DECOMPRESSION; + compressed_chunk_size, + uncompressed_buf, + uncompressed_chunk_size); + if (ret) { + ret = WIMLIB_ERR_DECOMPRESSION; + goto out; + } memcpy(out_p, uncompressed_buf + start_offset, - partial_chunk_size); + partial_chunk_size); } else { ret = decompress(compressed_buf, - compressed_chunk_size, - out_p, - uncompressed_chunk_size); - if (ret != 0) - return WIMLIB_ERR_DECOMPRESSION; + compressed_chunk_size, + out_p, + uncompressed_chunk_size); + if (ret) { + ret = WIMLIB_ERR_DECOMPRESSION; + goto out; + } } } - - /* Advance the pointer into the uncompressed output data by the - * number of uncompressed bytes that were written. */ - out_p += partial_chunk_size; + if (cb) { + /* Feed the data to the callback function */ + ret = cb(out_p, partial_chunk_size, ctx_or_buf); + if (ret) + goto out; + } else { + /* No callback function provided; we are writing + * directly to a buffer. Advance the pointer into this + * buffer by the number of uncompressed bytes that were + * written. */ + out_p += partial_chunk_size; + } } - return 0; + ret = 0; +out: + return ret; -err: +read_error: if (feof(fp)) ERROR("Unexpected EOF in compressed file resource"); else ERROR_WITH_ERRNO("Error reading compressed file resource"); - return WIMLIB_ERR_READ; + ret = WIMLIB_ERR_READ; + goto out; } /* @@ -381,8 +394,8 @@ read_uncompressed_resource(FILE *fp, u64 offset, u64 len, void *contents_ret) /* Reads the contents of a struct resource_entry, as represented in the on-disk * format, from the memory pointed to by @p, and fills in the fields of @entry. * A pointer to the byte after the memory read at @p is returned. */ -const u8 * -get_resource_entry(const u8 *p, struct resource_entry *entry) +const void * +get_resource_entry(const void *p, struct resource_entry *entry) { u64 size; u8 flags; @@ -412,8 +425,8 @@ get_resource_entry(const u8 *p, struct resource_entry *entry) /* Copies the struct resource_entry @entry to the memory pointed to by @p in the * on-disk format. A pointer to the byte after the memory written at @p is * returned. */ -u8 * -put_resource_entry(u8 *p, const struct resource_entry *entry) +void * +put_resource_entry(void *p, const struct resource_entry *entry) { p = put_u56(p, entry->size); p = put_u8(p, entry->flags); @@ -422,10 +435,10 @@ put_resource_entry(u8 *p, const struct resource_entry *entry) return p; } -#ifdef WITH_FUSE static FILE * wim_get_fp(WIMStruct *w) { +#ifdef WITH_FUSE pthread_mutex_lock(&w->fp_tab_mutex); FILE *fp; @@ -435,15 +448,18 @@ wim_get_fp(WIMStruct *w) if (w->fp_tab[i]) { fp = w->fp_tab[i]; w->fp_tab[i] = NULL; - goto out; + goto out_unlock; } } DEBUG("Opening extra file descriptor to `%"TS"'", w->filename); fp = tfopen(w->filename, T("rb")); if (!fp) ERROR_WITH_ERRNO("Failed to open `%"TS"'", w->filename); -out: +out_unlock: pthread_mutex_unlock(&w->fp_tab_mutex); +#else /* WITH_FUSE */ + fp = w->fp; +#endif /* !WITH_FUSE */ return fp; } @@ -451,6 +467,7 @@ static int wim_release_fp(WIMStruct *w, FILE *fp) { int ret = 0; +#ifdef WITH_FUSE FILE **fp_tab; pthread_mutex_lock(&w->fp_tab_mutex); @@ -458,251 +475,251 @@ wim_release_fp(WIMStruct *w, FILE *fp) for (size_t i = 0; i < w->num_allocated_fps; i++) { if (w->fp_tab[i] == NULL) { w->fp_tab[i] = fp; - goto out; + goto out_unlock; } } fp_tab = REALLOC(w->fp_tab, sizeof(FILE*) * (w->num_allocated_fps + 4)); if (!fp_tab) { ret = WIMLIB_ERR_NOMEM; - goto out; + fclose(fp); + goto out_unlock; } w->fp_tab = fp_tab; memset(&w->fp_tab[w->num_allocated_fps], 0, 4 * sizeof(FILE*)); w->fp_tab[w->num_allocated_fps] = fp; w->num_allocated_fps += 4; -out: +out_unlock: pthread_mutex_unlock(&w->fp_tab_mutex); +#endif /* WITH_FUSE */ return ret; } -#endif /* !WITH_FUSE */ -/* - * Reads some data from the resource corresponding to a WIM lookup table entry. - * - * @lte: The WIM lookup table entry for the resource. - * @buf: Buffer into which to write the data. - * @size: Number of bytes to read. - * @offset: Offset at which to start reading the resource. - * - * Returns zero on success, nonzero on failure. - */ -int -read_wim_resource(const struct wim_lookup_table_entry *lte, void *buf, - size_t size, u64 offset, int flags) +static int +read_partial_wim_resource(const struct wim_lookup_table_entry *lte, + u64 size, + consume_data_callback_t cb, + void *ctx_or_buf, + int flags, + u64 offset) { - int ctype; - int ret = 0; - FILE *fp; + FILE *wim_fp; + WIMStruct *wim; + int ret; - /* We shouldn't be allowing read over-runs in any part of the library. - * */ - if (flags & WIMLIB_RESOURCE_FLAG_RAW) - wimlib_assert(offset + size <= lte->resource_entry.size); - else - wimlib_assert(offset + size <= lte->resource_entry.original_size); - - switch (lte->resource_location) { - case RESOURCE_IN_WIM: - /* The resource is in a WIM file, and its WIMStruct is given by - * the lte->wim member. The resource may be either compressed - * or uncompressed. */ - wimlib_assert(lte->wim != NULL); - - #ifdef WITH_FUSE - if (flags & WIMLIB_RESOURCE_FLAG_MULTITHREADED) { - fp = wim_get_fp(lte->wim); - if (!fp) - return WIMLIB_ERR_OPEN; - } else - #endif - { - wimlib_assert(!(flags & WIMLIB_RESOURCE_FLAG_MULTITHREADED)); - wimlib_assert(lte->wim->fp != NULL); - fp = lte->wim->fp; - } + wimlib_assert(lte->resource_location == RESOURCE_IN_WIM); + wimlib_assert(offset + size <= lte->resource_entry.original_size); - ctype = wim_resource_compression_type(lte); + wim = lte->wim; - wimlib_assert(ctype != WIMLIB_COMPRESSION_TYPE_NONE || - (lte->resource_entry.original_size == - lte->resource_entry.size)); + if (flags & WIMLIB_RESOURCE_FLAG_MULTITHREADED) { + wim_fp = wim_get_fp(wim); + if (!wim_fp) { + ret = -1; + goto out; + } + } else { + wim_fp = lte->wim->fp; + } - if ((flags & WIMLIB_RESOURCE_FLAG_RAW) - || ctype == WIMLIB_COMPRESSION_TYPE_NONE) - ret = read_uncompressed_resource(fp, - lte->resource_entry.offset + offset, - size, buf); - else - ret = read_compressed_resource(fp, - lte->resource_entry.size, - lte->resource_entry.original_size, - lte->resource_entry.offset, - ctype, size, offset, buf); - #ifdef WITH_FUSE - if (flags & WIMLIB_RESOURCE_FLAG_MULTITHREADED) { - int ret2 = wim_release_fp(lte->wim, fp); - if (ret == 0) - ret = ret2; + wimlib_assert(wim_fp != NULL); + + if (lte->resource_entry.flags & WIM_RESHDR_FLAG_COMPRESSED && + !(flags & WIMLIB_RESOURCE_FLAG_RAW)) + { + ret = read_compressed_resource(wim_fp, + lte->resource_entry.size, + lte->resource_entry.original_size, + lte->resource_entry.offset, + wimlib_get_compression_type(wim), + size, + offset, + cb, + ctx_or_buf); + } else { + if (fseeko(wim_fp, offset, SEEK_SET)) { + ERROR_WITH_ERRNO("Failed to seek to offset %"PRIu64 + " in WIM", offset); + ret = WIMLIB_ERR_READ; + goto out_release_fp; } - #endif - break; - case RESOURCE_IN_STAGING_FILE: - case RESOURCE_IN_FILE_ON_DISK: - /* The resource is in some file on the external filesystem and - * needs to be read uncompressed */ - wimlib_assert(lte->file_on_disk != NULL); - BUILD_BUG_ON(<e->file_on_disk != <e->staging_file_name); - /* Use existing file pointer if available; otherwise open one - * temporarily */ - if (lte->file_on_disk_fp) { - fp = lte->file_on_disk_fp; - } else { - fp = tfopen(lte->file_on_disk, T("rb")); - if (!fp) { - ERROR_WITH_ERRNO("Failed to open the file " - "`%"TS"'", lte->file_on_disk); - ret = WIMLIB_ERR_OPEN; - break; + if (cb) { + char buf[min(32768, size)]; + while (size) { + size_t bytes_to_read = min(32768, size); + size_t bytes_read = fread(buf, 1, bytes_to_read, wim_fp); + + if (bytes_read != bytes_to_read) + goto read_error; + ret = cb(buf, bytes_read, ctx_or_buf); + if (ret) + goto out_release_fp; } + } else { + if (fread(ctx_or_buf, 1, size, wim_fp) != size) + goto read_error; } - ret = read_uncompressed_resource(fp, offset, size, buf); - if (fp != lte->file_on_disk_fp) - fclose(fp); - break; -#ifdef __WIN32__ - case RESOURCE_WIN32: - wimlib_assert(lte->win32_file_on_disk_fp != INVALID_HANDLE_VALUE); - ret = win32_read_file(lte->file_on_disk, - lte->win32_file_on_disk_fp, offset, - size, buf); - break; -#endif - case RESOURCE_IN_ATTACHED_BUFFER: - /* The resource is directly attached uncompressed in an - * in-memory buffer. */ - wimlib_assert(lte->attached_buffer != NULL); - memcpy(buf, lte->attached_buffer + offset, size); - break; -#ifdef WITH_NTFS_3G - case RESOURCE_IN_NTFS_VOLUME: - wimlib_assert(lte->ntfs_loc != NULL); - wimlib_assert(lte->attr != NULL); - if (lte->ntfs_loc->is_reparse_point) - offset += 8; - if (ntfs_attr_pread(lte->attr, offset, size, buf) != size) { - ERROR_WITH_ERRNO("Error reading NTFS attribute " - "at `%"TS"'", - lte->ntfs_loc->path); - ret = WIMLIB_ERR_NTFS_3G; - } - break; -#endif - default: - wimlib_assert(0); + ret = 0; + } + goto out_release_fp; +read_error: + ERROR_WITH_ERRNO("Error reading data from WIM"); + ret = WIMLIB_ERR_READ; +out_release_fp: + if (flags & WIMLIB_RESOURCE_FLAG_MULTITHREADED) + ret |= wim_release_fp(wim, wim_fp); +out: + if (ret) { + if (errno == 0) + errno = EIO; ret = -1; - break; } return ret; } -/* - * Reads all the data from the resource corresponding to a WIM lookup table - * entry. - * - * @lte: The WIM lookup table entry for the resource. - * @buf: Buffer into which to write the data. It must be at least - * wim_resource_size(lte) bytes long. - * - * Returns 0 on success; nonzero on failure. - */ + int -read_full_wim_resource(const struct wim_lookup_table_entry *lte, - void *buf, int flags) +read_partial_wim_resource_into_buf(const struct wim_lookup_table_entry *lte, + size_t size, u64 offset, void *buf, + bool threadsafe) { - return read_wim_resource(lte, buf, wim_resource_size(lte), 0, flags); + return read_partial_wim_resource(lte, size, NULL, buf, + threadsafe ? WIMLIB_RESOURCE_FLAG_MULTITHREADED : 0, + offset); } -/* Extracts the first @size bytes of a WIM resource to somewhere. In the - * process, the SHA1 message digest of the resource is checked if the full - * resource is being extracted. - * - * @extract_chunk is a function that is called to extract each chunk of the - * resource. */ -int -extract_wim_resource(const struct wim_lookup_table_entry *lte, - u64 size, - extract_chunk_func_t extract_chunk, - void *extract_chunk_arg) +static int +read_wim_resource_prefix(const struct wim_lookup_table_entry *lte, + u64 size, + consume_data_callback_t cb, + void *ctx_or_buf, + int flags) { - u64 bytes_remaining = size; - u8 buf[min(WIM_CHUNK_SIZE, bytes_remaining)]; - u64 offset = 0; - int ret = 0; - u8 hash[SHA1_HASH_SIZE]; - bool check_hash = (size == wim_resource_size(lte)); - SHA_CTX ctx; + return read_partial_wim_resource(lte, size, cb, ctx_or_buf, flags, 0); +} - if (check_hash) - sha1_init(&ctx); - while (bytes_remaining) { - u64 to_read = min(bytes_remaining, sizeof(buf)); - ret = read_wim_resource(lte, buf, to_read, offset, 0); - if (ret != 0) - return ret; - if (check_hash) - sha1_update(&ctx, buf, to_read); - ret = extract_chunk(buf, to_read, offset, extract_chunk_arg); - if (ret != 0) { - ERROR_WITH_ERRNO("Error extracting WIM resource"); - return ret; - } - bytes_remaining -= to_read; - offset += to_read; +static int +read_file_on_disk_prefix(const struct wim_lookup_table_entry *lte, + u64 size, + consume_data_callback_t cb, + void *ctx_or_buf, + int _ignored_flags) +{ + const tchar *filename = lte->file_on_disk; + int ret; + int fd; + size_t bytes_read; + + fd = open(filename, O_RDONLY); + if (fd < 0) { + ERROR_WITH_ERRNO("Can't open \"%"TS"\"", filename); + return WIMLIB_ERR_OPEN; } - if (check_hash) { - sha1_final(hash, &ctx); - if (!hashes_equal(hash, lte->hash)) { - #ifdef ENABLE_ERROR_MESSAGES - ERROR("Invalid checksum on the following WIM resource:"); - print_lookup_table_entry(lte, stderr); - #endif - return WIMLIB_ERR_INVALID_RESOURCE_HASH; + if (cb) { + /* Send data to callback function */ + char buf[min(32768, size)]; + size_t bytes_to_read; + while (size) { + bytes_to_read = min(32768, size); + bytes_read = full_read(fd, buf, bytes_to_read); + if (bytes_read != bytes_to_read) + goto read_error; + ret = cb(buf, bytes_read, ctx_or_buf); + if (ret) + goto out_close; } + } else { + /* Send data directly to a buffer */ + bytes_read = full_read(fd, ctx_or_buf, size); + if (bytes_read != size) + goto read_error; } - return 0; + ret = 0; + goto out_close; +read_error: + ERROR_WITH_ERRNO("Error reading \"%"TS"\"", filename); + ret = WIMLIB_ERR_READ; +out_close: + close(fd); + return ret; } -/* Write @n bytes from @buf to the file descriptor @fd, retrying on internupt - * and on short writes. - * - * Returns short count and set errno on failure. */ -static ssize_t -full_write(int fd, const void *buf, size_t n) +static int +read_buffer_prefix(const struct wim_lookup_table_entry *lte, + u64 size, consume_data_callback_t cb, + void *ctx_or_buf, int _ignored_flags) { - const void *p = buf; - ssize_t ret; - ssize_t total = 0; - - while (total != n) { - ret = write(fd, p, n); - if (ret < 0) { - if (errno == EINTR) - continue; - else - break; - } - total += ret; - p += ret; + const void *inbuf = lte->attached_buffer; + if (cb) { + return cb(inbuf, size, ctx_or_buf); + } else { + memcpy(ctx_or_buf, inbuf, size); + return 0; } - return total; } +typedef int (*read_resource_prefix_handler_t)(const struct wim_lookup_table_entry *lte, + u64 size, + consume_data_callback_t cb, + void *ctx_or_buf, + int flags); + int -extract_wim_chunk_to_fd(const void *buf, size_t len, u64 offset, void *arg) +read_resource_prefix(const struct wim_lookup_table_entry *lte, + u64 size, consume_data_callback_t cb, void *ctx_or_buf, + int flags) { - int fd = *(int*)arg; + static const read_resource_prefix_handler_t handlers[] = { + [RESOURCE_IN_WIM] = read_wim_resource_prefix, + [RESOURCE_IN_FILE_ON_DISK] = read_file_on_disk_prefix, + [RESOURCE_IN_ATTACHED_BUFFER] = read_buffer_prefix, + #ifdef WITH_FUSE + [RESOURCE_IN_STAGING_FILE] = read_file_on_disk_prefix, + #endif + #ifdef WITH_NTFS_3G + [RESOURCE_IN_NTFS_VOLUME] = read_ntfs_file_prefix, + #endif + #ifdef __WIN32__ + [RESOURCE_WIN32] = read_win32_file_prefix, + [RESOURCE_WIN32_ENCRYPTED] = read_win32_encrypted_file_prefix, + #endif + }; + wimlib_assert(lte->resource_location < ARRAY_LEN(handlers) + && handlers[lte->resource_location] != NULL); + return handlers[lte->resource_location](lte, size, cb, ctx_or_buf, flags); +} + +int +read_full_resource_into_buf(const struct wim_lookup_table_entry *lte, + void *buf, bool thread_safe) +{ + return read_resource_prefix(lte, + wim_resource_size(lte), + NULL, buf, + thread_safe ? WIMLIB_RESOURCE_FLAG_MULTITHREADED : 0); +} + +/* Extracts the first @size bytes of a WIM resource to somewhere. In the + * process, the SHA1 message digest of the resource is checked if the full + * resource is being extracted. + * + * @extract_chunk is a function that is called to extract each chunk of the + * resource. */ +int +extract_wim_resource(const struct wim_lookup_table_entry *lte, + u64 size, + consume_data_callback_t extract_chunk, + void *extract_chunk_arg) +{ + return read_resource_prefix(lte, size, extract_chunk, + extract_chunk_arg, 0); +} + +static int +extract_wim_chunk_to_fd(const void *buf, size_t len, void *_fd_p) +{ + int fd = *(int*)_fd_p; ssize_t ret = full_write(fd, buf, len); if (ret < len) { ERROR_WITH_ERRNO("Error writing to file descriptor"); @@ -712,6 +729,13 @@ extract_wim_chunk_to_fd(const void *buf, size_t len, u64 offset, void *arg) } } +int +extract_wim_resource_to_fd(const struct wim_lookup_table_entry *lte, + int fd, u64 size) +{ + return extract_wim_resource(lte, size, extract_wim_chunk_to_fd, &fd); +} + /* * Copies the file resource specified by the lookup table entry @lte from the * input WIM to the output WIM that has its FILE * given by diff --git a/src/symlink.c b/src/symlink.c index 70301b35..bd370e30 100644 --- a/src/symlink.c +++ b/src/symlink.c @@ -171,7 +171,7 @@ make_symlink_reparse_data_buf(const char *symlink_target, */ ssize_t inode_readlink(const struct wim_inode *inode, char *buf, size_t buf_len, - const WIMStruct *w, int read_resource_flags) + const WIMStruct *w, bool threadsafe) { const struct wim_lookup_table_entry *lte; int ret; @@ -186,7 +186,7 @@ inode_readlink(const struct wim_inode *inode, char *buf, size_t buf_len, return -EIO; u8 res_buf[wim_resource_size(lte)]; - ret = read_full_wim_resource(lte, res_buf, read_resource_flags); + ret = read_full_resource_into_buf(lte, res_buf, threadsafe); if (ret != 0) return -EIO; return get_symlink_name(res_buf, wim_resource_size(lte), buf, diff --git a/src/wimlib_internal.h b/src/wimlib_internal.h index 3d7fb488..61a70f1c 100644 --- a/src/wimlib_internal.h +++ b/src/wimlib_internal.h @@ -256,6 +256,8 @@ struct wim_image_metadata { /* Linked list of inodes of this image */ struct hlist_head inode_list; + struct list_head unhashed_streams; + /* 1 iff the dentry tree has been modified. If this is the case, the * memory for the dentry tree should not be freed when switching to a * different WIM image. */ @@ -495,6 +497,16 @@ extern void libntfs3g_global_init(); /* ntfs-capture.c */ + +/* The types of these two callbacks are intentionally the same. */ +typedef int (*consume_data_callback_t)(const void *buf, size_t len, void *ctx); + +extern int +read_ntfs_file_prefix(const struct wim_lookup_table_entry *lte, + u64 size, + consume_data_callback_t cb, + void *ctx_or_buf, + int _ignored_flags); extern int build_dentry_tree_ntfs(struct wim_dentry **root_p, const tchar *device, @@ -512,61 +524,46 @@ build_dentry_tree_ntfs(struct wim_dentry **root_p, #define WIMLIB_RESOURCE_FLAG_MULTITHREADED 0x2 #define WIMLIB_RESOURCE_FLAG_RECOMPRESS 0x4 -extern const u8 * -get_resource_entry(const u8 *p, struct resource_entry *entry); +extern int +read_resource_prefix(const struct wim_lookup_table_entry *lte, + u64 size, consume_data_callback_t cb, void *ctx_or_buf, + int flags); -extern u8 * -put_resource_entry(u8 *p, const struct resource_entry *entry); +extern const void * +get_resource_entry(const void *p, struct resource_entry *entry); + +extern void * +put_resource_entry(void *p, const struct resource_entry *entry); extern int read_uncompressed_resource(FILE *fp, u64 offset, u64 size, void *buf); extern int -read_wim_resource(const struct wim_lookup_table_entry *lte, void *buf, - size_t size, u64 offset, int flags); - +read_partial_wim_resource_into_buf(const struct wim_lookup_table_entry *lte, + size_t size, u64 offset, void *buf, + bool threadsafe); extern int -read_full_wim_resource(const struct wim_lookup_table_entry *lte, - void *buf, int flags); +read_full_resource_into_buf(const struct wim_lookup_table_entry *lte, + void *buf, bool thread_safe); extern int write_wim_resource(struct wim_lookup_table_entry *lte, FILE *out_fp, int out_ctype, struct resource_entry *out_res_entry, int flags); - -typedef int (*extract_chunk_func_t)(const void *, size_t, u64, void *); - -extern int -extract_wim_chunk_to_fd(const void *buf, size_t len, u64 offset, void *arg); - extern int extract_wim_resource(const struct wim_lookup_table_entry *lte, - u64 size, extract_chunk_func_t extract_chunk, + u64 size, + consume_data_callback_t extract_chunk, void *extract_chunk_arg); -/* - * Extracts the first @size bytes of the WIM resource specified by @lte to the - * open file descriptor @fd. - * - * Returns 0 on success; nonzero on failure. - */ -static inline int -extract_wim_resource_to_fd(const struct wim_lookup_table_entry *lte, - int fd, u64 size) -{ - return extract_wim_resource(lte, size, - extract_wim_chunk_to_fd, &fd); -} - - extern int -write_dentry_resources(struct wim_dentry *dentry, void *wim_p); +extract_wim_resource_to_fd(const struct wim_lookup_table_entry *lte, + int fd, u64 size); extern int copy_resource(struct wim_lookup_table_entry *lte, void *w); - /* security.c */ extern int read_security_data(const u8 metadata_resource[], @@ -585,7 +582,7 @@ free_security_data(struct wim_security_data *sd); #ifndef __WIN32__ ssize_t inode_readlink(const struct wim_inode *inode, char *buf, size_t buf_len, - const WIMStruct *w, int read_resource_flags); + const WIMStruct *w, bool threadsafe); extern int inode_set_symlink(struct wim_inode *inode, const char *target, diff --git a/src/win32.c b/src/win32.c index 05f457a8..b3aec1cb 100644 --- a/src/win32.c +++ b/src/win32.c @@ -53,6 +53,28 @@ struct win32_capture_state { #define MAX_SET_SD_ACCESS_DENIED_WARNINGS 1 #define MAX_SET_SACL_PRIV_NOTHELD_WARNINGS 1 +#ifdef ENABLE_ERROR_MESSAGES +static void +win32_error(u32 err_code) +{ + wchar_t *buffer; + DWORD nchars; + nchars = FormatMessageW(FORMAT_MESSAGE_FROM_SYSTEM | + FORMAT_MESSAGE_ALLOCATE_BUFFER, + NULL, err_code, 0, + (wchar_t*)&buffer, 0, NULL); + if (nchars == 0) { + ERROR("Error printing error message! " + "Computer will self-destruct in 3 seconds."); + } else { + ERROR("Win32 error: %ls", buffer); + LocalFree(buffer); + } +} +#else /* ENABLE_ERROR_MESSAGES */ +# define win32_error(err_code) +#endif /* !ENABLE_ERROR_MESSAGES */ + /* Pointers to functions that are not available on all targetted versions of * Windows (XP and later). NOTE: The WINAPI annotations seem to be important; I * assume it specifies a certain calling convention. */ @@ -128,32 +150,6 @@ L"If you are not running this program as the administrator, you may\n" " `wimlib-imagex apply' with the --no-acls flag instead.\n" ; -#ifdef ENABLE_ERROR_MESSAGES -void -win32_error(u32 err_code) -{ - wchar_t *buffer; - DWORD nchars; - nchars = FormatMessageW(FORMAT_MESSAGE_FROM_SYSTEM | - FORMAT_MESSAGE_ALLOCATE_BUFFER, - NULL, err_code, 0, - (wchar_t*)&buffer, 0, NULL); - if (nchars == 0) { - ERROR("Error printing error message! " - "Computer will self-destruct in 3 seconds."); - } else { - ERROR("Win32 error: %ls", buffer); - LocalFree(buffer); - } -} - -void -win32_error_last() -{ - win32_error(GetLastError()); -} -#endif - static HANDLE win32_open_existing_file(const wchar_t *path, DWORD dwDesiredAccess) { @@ -174,29 +170,58 @@ win32_open_file_data_only(const wchar_t *path) } int -win32_read_file(const wchar_t *filename, - void *handle, u64 offset, size_t size, void *buf) +read_win32_file_prefix(const struct lookup_table_entry *lte, + u64 size, + consume_data_callback_t cb, + void *ctx_or_buf, + int _ignored_flags) { - HANDLE h = handle; + int ret; + void *out_buf; DWORD err; - DWORD bytesRead; - LARGE_INTEGER liOffset = {.QuadPart = offset}; + u64 bytes_remaining; - wimlib_assert(size <= 0xffffffff); + HANDLE hFile = win32_open_file_data_only(lte->file_on_disk); + if (hFile == INVALID_HANDLE_VALUE) { + err = GetLastError(); + ERROR("Failed to open \"%ls\"", lte->file_on_disk); + win32_error(err); + ret = WIMLIB_ERR_OPEN; + goto out; + } - if (SetFilePointerEx(h, liOffset, NULL, FILE_BEGIN)) - if (ReadFile(h, buf, size, &bytesRead, NULL) && bytesRead == size) - return 0; - err = GetLastError(); - ERROR("Error reading \"%ls\"", filename); - win32_error(err); - return WIMLIB_ERR_READ; -} + if (cb) + out_buf = alloca(WIM_CHUNK_SIZE); + else + out_buf = ctx_or_buf; -void -win32_close_file(void *handle) -{ - CloseHandle((HANDLE)handle); + bytes_remaining = size; + while (bytes_remaining) { + DWORD bytesToRead, bytesRead; + + bytesToRead = min(WIM_CHUNK_SIZE, bytes_remaining); + if (!ReadFile(h, out_buf, bytesToRead, &bytesRead, NULL) || + bytesRead != bytesToRead) + { + err = GetLastError(); + ERROR("Failed to read data from \"%ls\"", lte->file_on_disk); + win32_error(err); + ret = WIMLIB_ERR_READ; + goto out_close_handle; + } + bytes_remaining -= bytesRead; + if (cb) { + ret = cb(out_buf, bytesRead, ctx_or_buf); + if (ret) + goto out_close_handle; + } else { + out_buf += bytesRead; + } + } +out_close_handle: + CloseHandle(hFile); +out: + return ret; } static u64 @@ -451,47 +476,6 @@ win32_capture_reparse_point(HANDLE hFile, bytesReturned - 8, lookup_table); } -/* Calculate the SHA1 message digest of a Win32 data stream, which may be either - * an unnamed or named data stream. - * - * @path: Path to the file, with the stream noted at the end for named - * streams. UTF-16LE encoding. - * - * @hash: On success, the SHA1 message digest of the stream is written to - * this location. - * - * Returns 0 on success; nonzero on failure. - */ -static int -win32_sha1sum(const wchar_t *path, u8 hash[SHA1_HASH_SIZE]) -{ - HANDLE hFile; - SHA_CTX ctx; - u8 buf[32768]; - DWORD bytesRead; - int ret; - - hFile = win32_open_file_data_only(path); - if (hFile == INVALID_HANDLE_VALUE) - return WIMLIB_ERR_OPEN; - - sha1_init(&ctx); - for (;;) { - if (!ReadFile(hFile, buf, sizeof(buf), &bytesRead, NULL)) { - ret = WIMLIB_ERR_READ; - goto out_close_handle; - } - if (bytesRead == 0) - break; - sha1_update(&ctx, buf, bytesRead); - } - ret = 0; - sha1_final(hash, &ctx); -out_close_handle: - CloseHandle(hFile); - return ret; -} - /* Scans an unnamed or named stream of a Win32 file (not a reparse point * stream); calculates its SHA1 message digest and either creates a `struct * wim_lookup_table_entry' in memory for it, or uses an existing 'struct @@ -591,35 +575,18 @@ win32_capture_stream(const wchar_t *path, swprintf(spath, L"%ls%ls%ls%ls", relpath_prefix, path, colonchar, stream_name); - ret = win32_sha1sum(spath, hash); - if (ret) { - err = GetLastError(); - ERROR("Failed to read \"%ls\" to calculate SHA1sum", spath); - win32_error(err); + /* Make a new wim_lookup_table_entry */ + lte = new_lookup_table_entry(); + if (!lte) { + ret = WIMLIB_ERR_NOMEM; goto out_free_spath; } + lte->file_on_disk = spath; + spath = NULL; + lte->resource_location = RESOURCE_WIN32; + lte->resource_entry.original_size = (u64)dat->StreamSize.QuadPart; + lookup_table_insert_unhashed(lookup_table, lte); - lte = __lookup_resource(lookup_table, hash); - if (lte) { - /* Use existing wim_lookup_table_entry that has the same SHA1 - * message digest */ - lte->refcnt++; - } else { - /* Make a new wim_lookup_table_entry */ - lte = new_lookup_table_entry(); - if (!lte) { - ret = WIMLIB_ERR_NOMEM; - goto out_free_spath; - } - lte->file_on_disk = spath; - lte->win32_file_on_disk_fp = INVALID_HANDLE_VALUE; - spath = NULL; - lte->resource_location = RESOURCE_WIN32; - lte->resource_entry.original_size = (uint64_t)dat->StreamSize.QuadPart; - lte->resource_entry.size = (uint64_t)dat->StreamSize.QuadPart; - copy_hash(lte->hash, hash); - lookup_table_insert(lookup_table, lte); - } if (is_named_stream) ads_entry->lte = lte; else diff --git a/src/win32.h b/src/win32.h index 1df793f1..0ef410a8 100644 --- a/src/win32.h +++ b/src/win32.h @@ -20,19 +20,14 @@ extern int win32_read_file(const tchar *filename, HANDLE handle, u64 offset, size_t size, void *buf); -extern HANDLE -win32_open_file_data_only(const wchar_t *path_utf16); - -extern void -win32_close_file(void *handle); - -#ifdef ENABLE_ERROR_MESSAGES -extern void win32_error(u32 err); -extern void win32_error_last(); -#else -# define win32_error(err) -# define win32_error_last() -#endif +extern int +read_win32_file_prefix(const struct lookup_table_entry *lte, + u64 size, + consume_data_callback_t cb, + void *ctx_or_buf, + int _ignored_flags) +{ +} #define FNM_PATHNAME 0x1 #define FNM_NOESCAPE 0x2 diff --git a/src/write.c b/src/write.c index e5dbbdf9..9835dba0 100644 --- a/src/write.c +++ b/src/write.c @@ -196,7 +196,7 @@ get_compress_func(int out_ctype) * Returns 0 on success; nonzero on failure. */ static int -write_wim_resource_chunk(const u8 chunk[], unsigned chunk_size, +write_wim_resource_chunk(const void *chunk, unsigned chunk_size, FILE *out_fp, compress_func_t compress, struct chunk_table *chunk_tab) { @@ -269,107 +269,6 @@ finish_wim_resource_chunk_tab(struct chunk_table *chunk_tab, return 0; } -/* Prepare for multiple reads to a resource by caching a FILE * or NTFS - * attribute pointer in the lookup table entry. */ -static int -prepare_resource_for_read(struct wim_lookup_table_entry *lte - - #ifdef WITH_NTFS_3G - , ntfs_inode **ni_ret - #endif - ) -{ - switch (lte->resource_location) { - case RESOURCE_IN_FILE_ON_DISK: - if (!lte->file_on_disk_fp) { - lte->file_on_disk_fp = tfopen(lte->file_on_disk, T("rb")); - if (!lte->file_on_disk_fp) { - ERROR_WITH_ERRNO("Failed to open the file " - "`%"TS"'", lte->file_on_disk); - return WIMLIB_ERR_OPEN; - } - } - break; -#ifdef WITH_NTFS_3G - case RESOURCE_IN_NTFS_VOLUME: - if (!lte->attr) { - struct ntfs_location *loc = lte->ntfs_loc; - ntfs_inode *ni; - wimlib_assert(loc); - ni = ntfs_pathname_to_inode(*loc->ntfs_vol_p, NULL, loc->path); - if (!ni) { - ERROR_WITH_ERRNO("Failed to open inode `%"TS"' in NTFS " - "volume", loc->path); - return WIMLIB_ERR_NTFS_3G; - } - lte->attr = ntfs_attr_open(ni, - loc->is_reparse_point ? AT_REPARSE_POINT : AT_DATA, - loc->stream_name, - loc->stream_name_nchars); - if (!lte->attr) { - ERROR_WITH_ERRNO("Failed to open attribute of `%"TS"' in " - "NTFS volume", loc->path); - ntfs_inode_close(ni); - return WIMLIB_ERR_NTFS_3G; - } - *ni_ret = ni; - } - break; -#endif -#ifdef __WIN32__ - case RESOURCE_WIN32: - if (lte->win32_file_on_disk_fp == INVALID_HANDLE_VALUE) { - lte->win32_file_on_disk_fp = - win32_open_file_data_only(lte->file_on_disk); - if (lte->win32_file_on_disk_fp == INVALID_HANDLE_VALUE) { - ERROR("Win32 API: Can't open %"TS, lte->file_on_disk); - win32_error_last(); - return WIMLIB_ERR_OPEN; - } - } - break; -#endif - default: - break; - } - return 0; -} - -/* Undo prepare_resource_for_read() by closing the cached FILE * or NTFS - * attribute. */ -static void -end_wim_resource_read(struct wim_lookup_table_entry *lte - #ifdef WITH_NTFS_3G - , ntfs_inode *ni - #endif - ) -{ - if (lte->resource_location == RESOURCE_IN_FILE_ON_DISK - && lte->file_on_disk_fp) - { - fclose(lte->file_on_disk_fp); - lte->file_on_disk_fp = NULL; - } -#ifdef WITH_NTFS_3G - else if (lte->resource_location == RESOURCE_IN_NTFS_VOLUME) { - if (lte->attr) { - ntfs_attr_close(lte->attr); - lte->attr = NULL; - } - if (ni) - ntfs_inode_close(ni); - } -#endif -#ifdef __WIN32__ - else if (lte->resource_location == RESOURCE_WIN32 - && lte->win32_file_on_disk_fp != INVALID_HANDLE_VALUE) - { - win32_close_file(lte->win32_file_on_disk_fp); - lte->win32_file_on_disk_fp = INVALID_HANDLE_VALUE; - } -#endif -} - static int write_uncompressed_resource_and_truncate(struct wim_lookup_table_entry *lte, FILE *out_fp, @@ -382,217 +281,158 @@ write_uncompressed_resource_and_truncate(struct wim_lookup_table_entry *lte, "output WIM file", file_offset); return WIMLIB_ERR_WRITE; } - ret = write_wim_resource(lte, out_fp, WIMLIB_COMPRESSION_TYPE_NONE, - out_res_entry, 0); - if (ret != 0) + ret = write_wim_resource(lte, out_fp, + WIMLIB_COMPRESSION_TYPE_NONE, + out_res_entry, + 0); + if (ret) return ret; return fflush_and_ftruncate(out_fp, file_offset + wim_resource_size(lte)); } -/* - * Writes a WIM resource to a FILE * opened for writing. The resource may be - * written uncompressed or compressed depending on the @out_ctype parameter. - * - * If by chance the resource compresses to more than the original size (this may - * happen with random data or files than are pre-compressed), the resource is - * instead written uncompressed (and this is reflected in the @out_res_entry by - * removing the WIM_RESHDR_FLAG_COMPRESSED flag). - * - * @lte: The lookup table entry for the WIM resource. - * @out_fp: The FILE * to write the resource to. - * @out_ctype: The compression type of the resource to write. Note: if this is - * the same as the compression type of the WIM resource we - * need to read, we simply copy the data (i.e. we do not - * uncompress it, then compress it again). - * @out_res_entry: If non-NULL, a resource entry that is filled in with the - * offset, original size, compressed size, and compression flag - * of the output resource. - * - * Returns 0 on success; nonzero on failure. - */ +struct write_resource_ctx { + compress_func_t compress; + struct chunk_table *chunk_tab; + FILE *out_fp; + SHA_CTX sha_ctx; + bool doing_sha; +}; + +static int +write_resource_cb(const void *chunk, size_t chunk_size, void *_ctx) +{ + struct write_resource_ctx *ctx = _ctx; + + if (ctx->doing_sha) + sha1_update(&ctx->sha_ctx, chunk, chunk_size); + + if (ctx->compress) { + return write_wim_resource_chunk(chunk, chunk_size, + ctx->out_fp, ctx->compress, + ctx->chunk_tab); + } else { + if (fwrite(chunk, 1, chunk_size, ctx->out_fp) != chunk_size) { + ERROR_WITH_ERRNO("Error writing to output WIM"); + return WIMLIB_ERR_WRITE; + } else { + return 0; + } + } +} + int write_wim_resource(struct wim_lookup_table_entry *lte, FILE *out_fp, int out_ctype, struct resource_entry *out_res_entry, int flags) { - u64 bytes_remaining; - u64 original_size; - u64 old_compressed_size; - u64 new_compressed_size; - u64 offset; + struct write_resource_ctx write_ctx; + u64 new_size; + off_t offset; int ret; - struct chunk_table *chunk_tab = NULL; - bool raw; - off_t file_offset; - compress_func_t compress = NULL; -#ifdef WITH_NTFS_3G - ntfs_inode *ni = NULL; -#endif - - wimlib_assert(lte); - - /* Original size of the resource */ - original_size = wim_resource_size(lte); - /* Compressed size of the resource (as it exists now) */ - old_compressed_size = wim_resource_compressed_size(lte); + if (wim_resource_size(lte) == 0) { + /* Empty resource; nothing needs to be done, so just return + * success. */ + return 0; + } - /* Current offset in output file */ - file_offset = ftello(out_fp); - if (file_offset == -1) { - ERROR_WITH_ERRNO("Failed to get offset in output " - "stream"); + offset = ftello(out_fp); + if (offset == -1) { + ERROR_WITH_ERRNO("Can't get position in output WIM"); return WIMLIB_ERR_WRITE; } - /* Are the compression types the same? If so, do a raw copy (copy - * without decompressing and recompressing the data). */ - raw = (wim_resource_compression_type(lte) == out_ctype - && out_ctype != WIMLIB_COMPRESSION_TYPE_NONE - && !(flags & WIMLIB_RESOURCE_FLAG_RECOMPRESS)); + /* Can we simply copy the compressed data without recompressing it? */ - if (raw) { + if (!(flags & WIMLIB_RESOURCE_FLAG_RECOMPRESS) && + lte->resource_location == RESOURCE_IN_WIM && + wimlib_get_compression_type(lte->wim) == out_ctype) + { flags |= WIMLIB_RESOURCE_FLAG_RAW; - bytes_remaining = old_compressed_size; + write_ctx.doing_sha = false; } else { - flags &= ~WIMLIB_RESOURCE_FLAG_RAW; - bytes_remaining = original_size; - } - - /* Empty resource; nothing needs to be done, so just return success. */ - if (bytes_remaining == 0) - return 0; - - /* Buffer for reading chunks for the resource */ - u8 buf[min(WIM_CHUNK_SIZE, bytes_remaining)]; - - /* If we are writing a compressed resource and not doing a raw copy, we - * need to initialize the chunk table */ - if (out_ctype != WIMLIB_COMPRESSION_TYPE_NONE && !raw) { - ret = begin_wim_resource_chunk_tab(lte, out_fp, file_offset, - &chunk_tab); - if (ret != 0) - goto out; + write_ctx.doing_sha = true; + sha1_init(&write_ctx.sha_ctx); } - /* If the WIM resource is in an external file, open a FILE * to it so we - * don't have to open a temporary one in read_wim_resource() for each - * chunk. */ -#ifdef WITH_NTFS_3G - ret = prepare_resource_for_read(lte, &ni); -#else - ret = prepare_resource_for_read(lte); -#endif - if (ret != 0) - goto out; - - /* If we aren't doing a raw copy, we will compute the SHA1 message - * digest of the resource as we read it, and verify it's the same as the - * hash given in the lookup table entry once we've finished reading the - * resource. */ - SHA_CTX ctx; - if (!raw) { - sha1_init(&ctx); - compress = get_compress_func(out_ctype); - } - offset = 0; - - /* While there are still bytes remaining in the WIM resource, read a - * chunk of the resource, update SHA1, then write that chunk using the - * desired compression type. */ - do { - u64 to_read = min(bytes_remaining, WIM_CHUNK_SIZE); - ret = read_wim_resource(lte, buf, to_read, offset, flags); - if (ret != 0) - goto out_fclose; - if (!raw) - sha1_update(&ctx, buf, to_read); - ret = write_wim_resource_chunk(buf, to_read, out_fp, - compress, chunk_tab); - if (ret != 0) - goto out_fclose; - bytes_remaining -= to_read; - offset += to_read; - } while (bytes_remaining); - - /* Raw copy: The new compressed size is the same as the old compressed - * size - * - * Using WIMLIB_COMPRESSION_TYPE_NONE: The new compressed size is the - * original size - * - * Using a different compression type: Call - * finish_wim_resource_chunk_tab() and it will provide the new - * compressed size. - */ - if (raw) { - new_compressed_size = old_compressed_size; + /* Initialize the chunk table and set the compression function if + * compressing the resource. */ + if (out_ctype == WIMLIB_COMPRESSION_TYPE_NONE || + (flags & WIMLIB_RESOURCE_FLAG_RAW)) { + write_ctx.compress = NULL; + write_ctx.chunk_tab = NULL; } else { - if (out_ctype == WIMLIB_COMPRESSION_TYPE_NONE) - new_compressed_size = original_size; - else { - ret = finish_wim_resource_chunk_tab(chunk_tab, out_fp, - &new_compressed_size); - if (ret != 0) - goto out_fclose; - } + write_ctx.compress = get_compress_func(out_ctype); + ret = begin_wim_resource_chunk_tab(lte, out_fp, + offset, + &write_ctx.chunk_tab); + if (ret) + return ret; } - /* Verify SHA1 message digest of the resource, unless we are doing a raw - * write (in which case we never even saw the uncompressed data). Or, - * if the hash we had before is all 0's, just re-set it to be the new - * hash. */ - if (!raw) { + /* Write the data */ + write_ctx.out_fp = out_fp; + ret = read_resource_prefix(lte, wim_resource_size(lte), + write_resource_cb, &write_ctx, 0); + + /* Verify SHA1 message digest of the resource, Or, if the hash we had + * before is all 0's, just re-set it to be the new hash. */ + if (write_ctx.doing_sha) { u8 md[SHA1_HASH_SIZE]; - sha1_final(md, &ctx); + sha1_final(md, &write_ctx.sha_ctx); if (is_zero_hash(lte->hash)) { copy_hash(lte->hash, md); } else if (!hashes_equal(md, lte->hash)) { ERROR("WIM resource has incorrect hash!"); - if (lte->resource_location == RESOURCE_IN_FILE_ON_DISK) { - ERROR("We were reading it from `%"TS"'; maybe " + if (lte_filename_valid(lte)) { + ERROR("We were reading it from \"%"TS"\"; maybe " "it changed while we were reading it.", lte->file_on_disk); } ret = WIMLIB_ERR_INVALID_RESOURCE_HASH; - goto out_fclose; + goto out_free_chunk_tab; } } - if (!raw && new_compressed_size >= original_size && - out_ctype != WIMLIB_COMPRESSION_TYPE_NONE) - { - /* Oops! We compressed the resource to larger than the original - * size. Write the resource uncompressed instead. */ - ret = write_uncompressed_resource_and_truncate(lte, - out_fp, - file_offset, - out_res_entry); - if (ret != 0) - goto out_fclose; + out_res_entry->flags = lte->resource_entry.flags; + out_res_entry->original_size = wim_resource_size(lte); + out_res_entry->offset = offset; + if (flags & WIMLIB_RESOURCE_FLAG_RAW) { + /* Doing a raw write: The new compressed size is the same as + * the compressed size in the other WIM. */ + new_size = lte->resource_entry.size; + out_res_entry->flags = lte->resource_entry.flags; + } else if (out_ctype == WIMLIB_COMPRESSION_TYPE_NONE) { + /* Using WIMLIB_COMPRESSION_TYPE_NONE: The new compressed size + * is the original size. */ + new_size = lte->resource_entry.original_size; + out_res_entry->flags &= ~WIM_RESHDR_FLAG_COMPRESSED; } else { - if (out_res_entry) { - out_res_entry->size = new_compressed_size; - out_res_entry->original_size = original_size; - out_res_entry->offset = file_offset; - out_res_entry->flags = lte->resource_entry.flags - & ~WIM_RESHDR_FLAG_COMPRESSED; - if (out_ctype != WIMLIB_COMPRESSION_TYPE_NONE) - out_res_entry->flags |= WIM_RESHDR_FLAG_COMPRESSED; + /* Using a different compression type: Call + * finish_wim_resource_chunk_tab() and it will provide the new + * compressed size. */ + ret = finish_wim_resource_chunk_tab(write_ctx.chunk_tab, out_fp, + &new_size); + if (ret) + goto out_free_chunk_tab; + if (new_size >= wim_resource_size(lte)) { + /* Oops! We compressed the resource to larger than the original + * size. Write the resource uncompressed instead. */ + ret = write_uncompressed_resource_and_truncate(lte, + out_fp, + offset, + out_res_entry); + goto out_free_chunk_tab; } + out_res_entry->flags |= WIM_RESHDR_FLAG_COMPRESSED; } + out_res_entry->size = new_size; ret = 0; -out_fclose: -#ifdef WITH_NTFS_3G - end_wim_resource_read(lte, ni); -#else - end_wim_resource_read(lte); -#endif -out: - FREE(chunk_tab); +out_free_chunk_tab: + FREE(write_ctx.chunk_tab); return ret; }