From: Eric Biggers Date: Sun, 31 Mar 2013 01:35:25 +0000 (-0500) Subject: Fixes; comments X-Git-Tag: v1.3.3~92 X-Git-Url: https://wimlib.net/git/?p=wimlib;a=commitdiff_plain;h=f3e97b29c4a8c564d54b0fd11cd43a9b4cd6a8ad Fixes; comments --- diff --git a/src/export_image.c b/src/export_image.c index 784938e9..deeaddb4 100644 --- a/src/export_image.c +++ b/src/export_image.c @@ -47,7 +47,8 @@ inode_allocate_needed_ltes(struct wim_inode *inode, dest_lte = clone_lookup_table_entry(src_lte); if (!dest_lte) return WIMLIB_ERR_NOMEM; - list_add_tail(&dest_lte->staging_list, lte_list_head); + list_add_tail(&dest_lte->new_stream_list, + lte_list_head); } } } @@ -76,7 +77,7 @@ inode_move_ltes_to_table(struct wim_inode *inode, list_del(next); dest_lte = container_of(next, struct wim_lookup_table_entry, - staging_list); + new_stream_list); dest_lte->part_number = 1; dest_lte->refcnt = 0; wimlib_assert(hashes_equal(dest_lte->hash, src_lte->hash)); @@ -250,7 +251,7 @@ out_xml_delete_image: out_free_ltes: { struct wim_lookup_table_entry *lte, *tmp; - list_for_each_entry_safe(lte, tmp, <e_list_head, staging_list) + list_for_each_entry_safe(lte, tmp, <e_list_head, new_stream_list) free_lookup_table_entry(lte); } out: diff --git a/src/lookup_table.c b/src/lookup_table.c index 4d62ca5b..8ab54b6e 100644 --- a/src/lookup_table.c +++ b/src/lookup_table.c @@ -634,7 +634,7 @@ print_lookup_table_entry(const struct wim_lookup_table_entry *lte, FILE *out) tfprintf(out, T("Reference Count = %u\n"), lte->refcnt); if (lte->unhashed) { - tfprintf(out, T("(Unhashed, back ptr at %p)\n"), lte->my_ptr); + tfprintf(out, T("(Unhashed, back ptr at %p)\n"), lte->back_ptr); } else { tfprintf(out, T("Hash = 0x")); print_hash(lte->hash, out); @@ -891,15 +891,3 @@ lookup_table_total_stream_size(struct wim_lookup_table *table) for_lookup_table_entry(table, lte_add_stream_size, &total_size); return total_size; } - -void -free_lte_list(struct list_head *list) -{ - struct wim_lookup_table_entry *lte, *tmp; - - list_for_each_entry_safe(lte, tmp, list, staging_list) { - DEBUG("%p", lte); - free_lookup_table_entry(lte); -} - INIT_LIST_HEAD(list); -} diff --git a/src/lookup_table.h b/src/lookup_table.h index 633c874e..81cb50a5 100644 --- a/src/lookup_table.h +++ b/src/lookup_table.h @@ -163,7 +163,7 @@ struct wim_lookup_table_entry { * to the pointer to this 'struct wim_lookup_table_entry' * contained in a 'struct wim_ads_entry' or 'struct wim_inode'. * */ - struct wim_lookup_table_entry **my_ptr; + struct wim_lookup_table_entry **back_ptr; }; /* When a WIM file is written, out_refcnt starts at 0 and is incremented @@ -176,10 +176,13 @@ struct wim_lookup_table_entry { /* Pointers to somewhere where the stream is actually located. See the * comments for the @resource_location field above. */ union { + void *resource_loc_private; WIMStruct *wim; tchar *file_on_disk; - tchar *staging_file_name; void *attached_buffer; + #ifdef WITH_FUSE + tchar *staging_file_name; + #endif #ifdef WITH_NTFS_3G struct ntfs_location *ntfs_loc; #endif @@ -219,20 +222,17 @@ struct wim_lookup_table_entry { struct { struct hlist_node hash_list_2; + struct list_head write_streams_list; }; }; - /* List of lookup table entries that correspond to streams that have - * been extracted to the staging directory when modifying a read-write - * mounted WIM. - * - * This field is also used to make other lists of lookup table entries. - * */ + /* Temporary list fields */ union { struct list_head unhashed_list; - struct list_head staging_list; + struct list_head swm_stream_list; struct list_head extraction_list; + struct list_head new_stream_list; }; }; @@ -485,15 +485,12 @@ lookup_table_total_stream_size(struct wim_lookup_table *table); static inline void lookup_table_insert_unhashed(struct wim_lookup_table *table, struct wim_lookup_table_entry *lte, - struct wim_lookup_table_entry **my_ptr) + struct wim_lookup_table_entry **back_ptr) { lte->unhashed = 1; list_add_tail(<e->unhashed_list, table->unhashed_streams); - lte->my_ptr = my_ptr; - *my_ptr = lte; + lte->back_ptr = back_ptr; + *back_ptr = lte; } -extern void -free_lte_list(struct list_head *list); - #endif diff --git a/src/lzx-decompress.c b/src/lzx-decompress.c index 18fa3d00..45660267 100644 --- a/src/lzx-decompress.c +++ b/src/lzx-decompress.c @@ -327,7 +327,7 @@ lzx_read_block_header(struct input_bitstream *istream, unsigned len; ret = bitstream_ensure_bits(istream, 4); - if (ret != 0) { + if (ret) { ERROR("LZX input stream overrun"); return ret; } @@ -345,7 +345,7 @@ lzx_read_block_header(struct input_bitstream *istream, block_size = 32768; } else { ret = bitstream_read_bits(istream, 16, &block_size); - if (ret != 0) + if (ret) return ret; block_size = le16_to_cpu(block_size); } @@ -359,7 +359,7 @@ lzx_read_block_header(struct input_bitstream *istream, ret = bitstream_read_bits(istream, LZX_ALIGNEDTREE_ELEMENT_SIZE, &len); - if (ret != 0) + if (ret) return ret; tables->alignedtree_lens[i] = len; } @@ -370,7 +370,7 @@ lzx_read_block_header(struct input_bitstream *istream, LZX_ALIGNEDTREE_TABLEBITS, tables->alignedtree_lens, 8); - if (ret != 0) { + if (ret) { ERROR("lzx_decompress(): Failed to make the decode " "table for the aligned offset tree"); return ret; @@ -388,7 +388,7 @@ lzx_read_block_header(struct input_bitstream *istream, * tree. */ ret = lzx_read_code_lens(istream, tables->maintree_lens, LZX_NUM_CHARS); - if (ret != 0) { + if (ret) { ERROR("lzx_decompress(): Failed to read the code " "lengths for the first 256 elements of the " "main tree"); @@ -403,7 +403,7 @@ lzx_read_block_header(struct input_bitstream *istream, ret = lzx_read_code_lens(istream, tables->maintree_lens + LZX_NUM_CHARS, LZX_MAINTREE_NUM_SYMBOLS - LZX_NUM_CHARS); - if (ret != 0) { + if (ret) { ERROR("lzx_decompress(): Failed to read the path " "lengths for the remaining elements of the main " "tree"); @@ -418,7 +418,7 @@ lzx_read_block_header(struct input_bitstream *istream, LZX_MAINTREE_TABLEBITS, tables->maintree_lens, LZX_MAX_CODEWORD_LEN); - if (ret != 0) { + if (ret) { ERROR("lzx_decompress(): Failed to make the decode " "table for the main tree"); return ret; @@ -427,7 +427,7 @@ lzx_read_block_header(struct input_bitstream *istream, LZX_DEBUG("Reading path lengths for the length tree."); ret = lzx_read_code_lens(istream, tables->lentree_lens, LZX_LENTREE_NUM_SYMBOLS); - if (ret != 0) { + if (ret) { ERROR("lzx_decompress(): Failed to read the path " "lengths for the length tree"); return ret; @@ -439,7 +439,7 @@ lzx_read_block_header(struct input_bitstream *istream, LZX_LENTREE_TABLEBITS, tables->lentree_lens, LZX_MAX_CODEWORD_LEN); - if (ret != 0) { + if (ret) { ERROR("lzx_decompress(): Failed to build the length " "Huffman tree"); return ret; @@ -456,13 +456,19 @@ lzx_read_block_header(struct input_bitstream *istream, * *already* aligned, the correct thing to do is to throw away * the next 16 bits. */ if (istream->bitsleft == 0) { - if (istream->data_bytes_left < 14) + if (istream->data_bytes_left < 14) { + ERROR("lzx_decompress(): Insufficient length in " + "uncompressed block"); return -1; + } istream->data += 2; istream->data_bytes_left -= 2; } else { - if (istream->data_bytes_left < 12) + if (istream->data_bytes_left < 12) { + ERROR("lzx_decompress(): Insufficient length in " + "uncompressed block"); return -1; + } istream->bitsleft = 0; istream->bitbuf = 0; } @@ -763,7 +769,7 @@ lzx_decompress_block(int block_type, unsigned block_size, while (window_pos < end) { ret = read_huffsym_using_maintree(istream, tables, &main_element); - if (ret != 0) + if (ret) return ret; if (main_element < LZX_NUM_CHARS) { @@ -830,7 +836,7 @@ wimlib_lzx_decompress(const void *compressed_data, unsigned compressed_len, LZX_DEBUG("Reading block header."); ret = lzx_read_block_header(&istream, &block_size, &block_type, &tables, &queue); - if (ret != 0) + if (ret) return ret; LZX_DEBUG("block_size = %u, window_pos = %u", @@ -857,7 +863,7 @@ wimlib_lzx_decompress(const void *compressed_data, unsigned compressed_len, &tables, &queue, &istream); - if (ret != 0) + if (ret) return ret; if (tables.maintree_lens[0xe8] != 0) e8_preprocessing_done = true; diff --git a/src/metadata_resource.c b/src/metadata_resource.c index 91b1121f..9cebf9eb 100644 --- a/src/metadata_resource.c +++ b/src/metadata_resource.c @@ -55,8 +55,10 @@ read_metadata_resource(WIMStruct *w, struct wim_image_metadata *imd) metadata_lte = imd->metadata_lte; metadata_len = wim_resource_size(metadata_lte); - DEBUG("Reading metadata resource: length = %"PRIu64", " - "offset = %"PRIu64"", metadata_len, + DEBUG("Reading metadata resource: original_size = %"PRIu64", " + "size = %"PRIu64", offset = %"PRIu64"", + metadata_lte->resource_entry.original_size, + metadata_lte->resource_entry.size, metadata_lte->resource_entry.offset); /* There is no way the metadata resource could possibly be less than (8 @@ -280,16 +282,11 @@ write_metadata_resource(WIMStruct *w) wimlib_get_compression_type(w), <e->output_resource_entry, lte->hash); - if (ret) - goto out_free_buf; - /* Note that although the SHA1 message digest of the metadata resource * is very likely to have changed, the corresponding lookup table entry * is not actually located in the hash table, so it need not be * re-inserted in the hash table. */ - lte->out_refcnt = 1; - lte->output_resource_entry.flags |= WIM_RESHDR_FLAG_METADATA; -out_free_buf: + /* All the data has been written to the new WIM; no need for the buffer * anymore */ FREE(buf); diff --git a/src/mount_image.c b/src/mount_image.c index 2d0d56c6..82b80b9a 100644 --- a/src/mount_image.c +++ b/src/mount_image.c @@ -647,21 +647,21 @@ extract_resource_to_staging_dir(struct wim_inode *inode, new_lte->staging_file_name = staging_file_name; new_lte->lte_inode = inode; - struct wim_lookup_table_entry **my_ptr; + struct wim_lookup_table_entry **back_ptr; if (stream_id == 0) { - my_ptr = &inode->i_lte; + back_ptr = &inode->i_lte; } else { for (u16 i = 0; ; i++) { wimlib_assert(i < inode->i_num_ads); if (inode->i_ads_entries[i].stream_id == stream_id) { - my_ptr = &inode->i_ads_entries[i].lte; + back_ptr = &inode->i_ads_entries[i].lte; break; } } } - lookup_table_insert_unhashed(ctx->wim->lookup_table, new_lte, my_ptr); + lookup_table_insert_unhashed(ctx->wim->lookup_table, new_lte, back_ptr); *lte = new_lte; return 0; out_revert_fd_changes: @@ -816,7 +816,7 @@ rebuild_wim(struct wimfs_context *ctx, int write_flags, DEBUG("Freeing entries for zero-length streams"); image_for_each_unhashed_stream(lte, imd) { if (wim_resource_size(lte) == 0) { - *lte->my_ptr = NULL; + *lte->back_ptr = NULL; free_lookup_table_entry(lte); } } diff --git a/src/ntfs-capture.c b/src/ntfs-capture.c index 43d7ca2e..7f4f2bb4 100644 --- a/src/ntfs-capture.c +++ b/src/ntfs-capture.c @@ -165,7 +165,7 @@ capture_ntfs_streams(struct wim_inode *inode, { u64 data_size = ntfs_get_attribute_value_length(actx->attr); u64 name_length = actx->attr->name_length; - struct wim_lookup_table_entry **my_ptr; + struct wim_lookup_table_entry **back_ptr; if (data_size == 0) { if (errno != 0) { ERROR_WITH_ERRNO("Failed to get size of attribute of " @@ -224,7 +224,7 @@ capture_ntfs_streams(struct wim_inode *inode, ret = WIMLIB_ERR_NTFS_3G; goto out_free_lte; } - my_ptr = &inode->i_lte; + back_ptr = &inode->i_lte; } else { /* Named data stream. Put the reference to it in the * alternate data stream entries */ @@ -236,9 +236,9 @@ capture_ntfs_streams(struct wim_inode *inode, if (!new_ads_entry) goto out_free_lte; wimlib_assert(new_ads_entry->stream_name_nbytes == name_length * 2); - my_ptr = &new_ads_entry->lte; + back_ptr = &new_ads_entry->lte; } - lookup_table_insert_unhashed(lookup_table, lte, my_ptr); + lookup_table_insert_unhashed(lookup_table, lte, back_ptr); } ret = 0; goto out_put_actx; diff --git a/src/resource.c b/src/resource.c index 7a5d1718..5237b88a 100644 --- a/src/resource.c +++ b/src/resource.c @@ -38,6 +38,10 @@ #include #include +#ifdef HAVE_ALLOCA_H +# include +#endif + /* Write @n bytes from @buf to the file descriptor @fd, retrying on internupt * and on short writes. * @@ -90,10 +94,13 @@ full_read(int fd, void *buf, size_t n) * Returns zero on success, nonzero on failure. */ static int -read_compressed_resource(FILE *fp, u64 resource_compressed_size, +read_compressed_resource(FILE *fp, + u64 resource_compressed_size, u64 resource_uncompressed_size, - u64 resource_offset, int resource_ctype, - u64 len, u64 offset, + u64 resource_offset, + int resource_ctype, + u64 len, + u64 offset, consume_data_callback_t cb, void *ctx_or_buf) { @@ -122,7 +129,8 @@ read_compressed_resource(FILE *fp, u64 resource_compressed_size, * follows the chunk table and therefore must have an offset of 0. */ - /* Calculate how many chunks the resource conists of in its entirety. */ + /* Calculate how many chunks the resource consists of in its entirety. + * */ u64 num_chunks = (resource_uncompressed_size + WIM_CHUNK_SIZE - 1) / WIM_CHUNK_SIZE; /* As mentioned, the first chunk has no entry in the chunk table. */ @@ -149,9 +157,22 @@ read_compressed_resource(FILE *fp, u64 resource_compressed_size, if (end_chunk != num_chunks - 1) num_needed_chunks++; - /* Declare the chunk table. It will only contain offsets for the chunks - * that are actually needed for this read. */ - u64 chunk_offsets[num_needed_chunks]; + /* Allocate the chunk table. It will only contain offsets for the + * chunks that are actually needed for this read. */ + u64 *chunk_offsets; + bool chunk_offsets_malloced; + if (num_needed_chunks < 1000) { + chunk_offsets = alloca(num_needed_chunks * sizeof(u64)); + chunk_offsets_malloced = false; + } else { + chunk_offsets = malloc(num_needed_chunks * sizeof(u64)); + if (!chunk_offsets) { + ERROR("Failed to allocate chunk table " + "with %"PRIu64" entries", num_needed_chunks); + return WIMLIB_ERR_NOMEM; + } + chunk_offsets_malloced = true; + } /* Set the implicit offset of the first chunk if it is included in the * needed chunks. @@ -193,28 +214,28 @@ read_compressed_resource(FILE *fp, u64 resource_compressed_size, /* Number of bytes we need to read from the chunk table. */ size_t size = num_needed_chunk_entries * chunk_entry_size; - { - u8 chunk_tab_buf[size]; + /* Read the raw data into the end of the chunk_offsets array to + * avoid allocating another array. */ + void *chunk_tab_buf = (void*)&chunk_offsets[num_needed_chunks] - size; - if (fread(chunk_tab_buf, 1, size, fp) != size) - goto read_error; + if (fread(chunk_tab_buf, 1, size, fp) != size) + goto read_error; - /* Now fill in chunk_offsets from the entries we have read in - * chunk_tab_buf. */ + /* Now fill in chunk_offsets from the entries we have read in + * chunk_tab_buf. */ - u64 *chunk_tab_p = chunk_offsets; - if (start_chunk == 0) - chunk_tab_p++; + u64 *chunk_tab_p = chunk_offsets; + if (start_chunk == 0) + chunk_tab_p++; - if (chunk_entry_size == 4) { - u32 *entries = (u32*)chunk_tab_buf; - while (num_needed_chunk_entries--) - *chunk_tab_p++ = le32_to_cpu(*entries++); - } else { - u64 *entries = (u64*)chunk_tab_buf; - while (num_needed_chunk_entries--) - *chunk_tab_p++ = le64_to_cpu(*entries++); - } + if (chunk_entry_size == 4) { + u32 *entries = (u32*)chunk_tab_buf; + while (num_needed_chunk_entries--) + *chunk_tab_p++ = le32_to_cpu(*entries++); + } else { + u64 *entries = (u64*)chunk_tab_buf; + while (num_needed_chunk_entries--) + *chunk_tab_p++ = le64_to_cpu(*entries++); } /* Done with the chunk table now. We must now seek to the first chunk @@ -226,10 +247,11 @@ read_compressed_resource(FILE *fp, u64 resource_compressed_size, goto read_error; /* Pointer to current position in the output buffer for uncompressed - * data. */ + * data. Alternatively, if using a callback function, we repeatedly + * fill a temporary buffer to feed data into the callback function. */ u8 *out_p; if (cb) - out_p = alloca(32768); + out_p = alloca(WIM_CHUNK_SIZE); else out_p = ctx_or_buf; @@ -341,7 +363,8 @@ read_compressed_resource(FILE *fp, u64 resource_compressed_size, } if (cb) { /* Feed the data to the callback function */ - ret = cb(out_p, partial_chunk_size, ctx_or_buf); + ret = cb(out_p + start_offset, + partial_chunk_size, ctx_or_buf); if (ret) goto out; } else { @@ -355,6 +378,8 @@ read_compressed_resource(FILE *fp, u64 resource_compressed_size, ret = 0; out: + if (chunk_offsets_malloced) + FREE(chunk_offsets); return ret; read_error: @@ -512,7 +537,7 @@ read_partial_wim_resource(const struct wim_lookup_table_entry *lte, wim = lte->wim; - if (flags & WIMLIB_RESOURCE_FLAG_MULTITHREADED) { + if (flags & WIMLIB_RESOURCE_FLAG_THREADSAFE_READ) { wim_fp = wim_get_fp(wim); if (!wim_fp) { ret = -1; @@ -571,7 +596,7 @@ read_error: ERROR_WITH_ERRNO("Error reading data from WIM"); ret = WIMLIB_ERR_READ; out_release_fp: - if (flags & WIMLIB_RESOURCE_FLAG_MULTITHREADED) + if (flags & WIMLIB_RESOURCE_FLAG_THREADSAFE_READ) ret |= wim_release_fp(wim, wim_fp); out: if (ret) { @@ -588,7 +613,7 @@ read_partial_wim_resource_into_buf(const struct wim_lookup_table_entry *lte, bool threadsafe) { return read_partial_wim_resource(lte, size, NULL, buf, - threadsafe ? WIMLIB_RESOURCE_FLAG_MULTITHREADED : 0, + threadsafe ? WIMLIB_RESOURCE_FLAG_THREADSAFE_READ : 0, offset); } @@ -691,10 +716,12 @@ typedef int (*read_resource_prefix_handler_t)(const struct wim_lookup_table_entr * When using a callback function, it is called with chunks up to 32768 bytes in * size until the resource is exhausted. * - * If the resource is located in a WIM file, @flags can be - * WIMLIB_RESOURCE_FLAG_MULTITHREADED if it must be safe to access the resource - * concurrently by multiple threads, or WIMLIB_RESOURCE_FLAG_RAW if the raw - * compressed data is to be supplied instead of the uncompressed data. + * If the resource is located in a WIM file, @flags can be: + * * WIMLIB_RESOURCE_FLAG_THREADSAFE_READ if it must be safe to access the resource + * concurrently by multiple threads. + * * WIMLIB_RESOURCE_FLAG_RAW if the raw compressed data is to be supplied + * instead of the uncompressed data. + * Otherwise, the @flags are ignored. */ int read_resource_prefix(const struct wim_lookup_table_entry *lte, @@ -728,7 +755,7 @@ read_full_resource_into_buf(const struct wim_lookup_table_entry *lte, return read_resource_prefix(lte, wim_resource_size(lte), NULL, buf, - thread_safe ? WIMLIB_RESOURCE_FLAG_MULTITHREADED : 0); + thread_safe ? WIMLIB_RESOURCE_FLAG_THREADSAFE_READ : 0); } struct extract_ctx { diff --git a/src/sha1.c b/src/sha1.c index 6fb18c4d..f09008d1 100644 --- a/src/sha1.c +++ b/src/sha1.c @@ -210,47 +210,3 @@ sha1_buffer(const void *buffer, size_t len, u8 md[SHA1_HASH_SIZE]) } #endif /* !WITH_LIBCRYPTO */ - -static int -sha1_stream(FILE *fp, u8 md[SHA1_HASH_SIZE]) -{ - char buf[BUFFER_SIZE]; - size_t bytes_read; - SHA_CTX ctx; - sha1_init(&ctx); - while (1) { - bytes_read = fread(buf, 1, sizeof(buf), fp); - sha1_update(&ctx, buf, bytes_read); - if (bytes_read < sizeof(buf)) { - if (ferror(fp)) - return WIMLIB_ERR_READ; - break; - } - } - sha1_final(md, &ctx); - return 0; - -} - -/* Calculates the SHA1 message digest of a file. @md must point to a buffer of - * length 20 bytes into which the message digest is written. */ -int -sha1sum(const tchar *filename, u8 md[SHA1_HASH_SIZE]) -{ - FILE *fp; - int ret; - - fp = tfopen(filename, T("rb")); - if (!fp) { - ERROR_WITH_ERRNO("Cannot open the file `%"TS"' for reading", - filename); - return WIMLIB_ERR_OPEN; - } - ret = sha1_stream(fp, md); - if (ret != 0) { - ERROR_WITH_ERRNO("Error calculating SHA1 message digest of " - "`%"TS"'", filename); - } - fclose(fp); - return ret; -} diff --git a/src/sha1.h b/src/sha1.h index 5c231e37..d278aa92 100644 --- a/src/sha1.h +++ b/src/sha1.h @@ -93,7 +93,4 @@ sha1_final(u8 hash[SHA1_HASH_SIZE], SHA_CTX *ctx); #endif /* !WITH_LIBCRYPTO */ -extern int -sha1sum(const tchar *filename, u8 hash[SHA1_HASH_SIZE]); - #endif /* _WIMLIB_SHA1_H */ diff --git a/src/split.c b/src/split.c index 660ffaf7..a24ac4b4 100644 --- a/src/split.c +++ b/src/split.c @@ -50,9 +50,9 @@ finish_swm(WIMStruct *w, struct list_head *lte_list, int ret; struct wim_lookup_table_entry *lte; - list_for_each_entry(lte, lte_list, staging_list) { + list_for_each_entry(lte, lte_list, swm_stream_list) { ret = write_lookup_table_entry(lte, w->out_fp); - if (ret != 0) + if (ret) return ret; } @@ -90,7 +90,7 @@ copy_resource_to_swm(struct wim_lookup_table_entry *lte, void *__args) ret = finish_swm(w, &args->lte_list, args->write_flags, args->progress_func); - if (ret != 0) + if (ret) return ret; if (args->progress_func) { @@ -113,13 +113,13 @@ copy_resource_to_swm(struct wim_lookup_table_entry *lte, void *__args) } ret = begin_write(w, args->swm_base_name, args->write_flags); - if (ret != 0) + if (ret) return ret; args->size_remaining = args->part_size; } args->size_remaining -= lte->resource_entry.size; args->progress.split.completed_bytes += lte->resource_entry.size; - list_add_tail(<e->staging_list, &args->lte_list); + list_add_tail(<e->swm_stream_list, &args->lte_list); return copy_resource(lte, w); } @@ -153,7 +153,7 @@ wimlib_split(WIMStruct *w, const tchar *swm_name, w->hdr.boot_idx = 0; randomize_byte_array(w->hdr.guid, WIM_GID_LEN); ret = begin_write(w, swm_name, write_flags); - if (ret != 0) + if (ret) goto out; tmemcpy(swm_base_name, swm_name, swm_name_len + 1); @@ -197,17 +197,17 @@ wimlib_split(WIMStruct *w, const tchar *swm_name, args.progress.split.completed_bytes += metadata_lte->resource_entry.size; /* Careful: The metadata lookup table entries must be added in * order of the images. */ - list_add_tail(&metadata_lte->staging_list, &args.lte_list); + list_add_tail(&metadata_lte->swm_stream_list, &args.lte_list); } ret = for_lookup_table_entry_pos_sorted(w->lookup_table, copy_resource_to_swm, &args); - if (ret != 0) + if (ret) goto out; ret = finish_swm(w, &args.lte_list, write_flags, progress_func); - if (ret != 0) + if (ret) goto out; if (progress_func) { diff --git a/src/wim.c b/src/wim.c index 96fba59f..4060fa58 100644 --- a/src/wim.c +++ b/src/wim.c @@ -199,6 +199,7 @@ select_wim_image(WIMStruct *w, int image) if (w->current_image != WIMLIB_NO_IMAGE) { imd = wim_get_current_image_metadata(w); if (!imd->modified) { + wimlib_assert(list_empty(&imd->unhashed_streams)); DEBUG("Freeing image %u", w->current_image); destroy_image_metadata(imd, NULL, false); } @@ -532,6 +533,11 @@ destroy_image_metadata(struct wim_image_metadata *imd, free_lookup_table_entry(imd->metadata_lte); imd->metadata_lte = NULL; } + if (!table) { + struct wim_lookup_table_entry *lte, *tmp; + list_for_each_entry_safe(lte, tmp, &imd->unhashed_streams, unhashed_list) + free_lookup_table_entry(lte); + } INIT_LIST_HEAD(&imd->unhashed_streams); INIT_LIST_HEAD(&imd->inode_list); } diff --git a/src/wimlib_internal.h b/src/wimlib_internal.h index 91f99ebf..a7ac5dc2 100644 --- a/src/wimlib_internal.h +++ b/src/wimlib_internal.h @@ -369,6 +369,21 @@ resource_is_compressed(const struct resource_entry *entry) #define image_for_each_unhashed_stream(lte, imd) \ list_for_each_entry(lte, &imd->unhashed_streams, unhashed_list) +#if 1 +# define copy_resource_entry(dst, src) memcpy(dst, src, sizeof(struct resource_entry)) +#else +static inline void +copy_resource_entry(struct resource_entry *dst, + const struct resource_entry *src) +{ + memcpy(dst, src, sizeof(struct resource_entry)); + BUILD_BUG_ON(sizeof(struct resource_entry) != 24); + ((u64*)dst)[0] = ((u64*)src)[0]; + ((u64*)dst)[1] = ((u64*)src)[1]; + ((u64*)dst)[2] = ((u64*)src)[2]; +} +#endif + /* add_image.c */ extern bool @@ -537,8 +552,9 @@ build_dentry_tree_ntfs(struct wim_dentry **root_p, /* resource.c */ #define WIMLIB_RESOURCE_FLAG_RAW 0x1 -#define WIMLIB_RESOURCE_FLAG_MULTITHREADED 0x2 +#define WIMLIB_RESOURCE_FLAG_THREADSAFE_READ 0x2 #define WIMLIB_RESOURCE_FLAG_RECOMPRESS 0x4 +//#define WIMLIB_RESOURCE_FLAG_OVERWRITE_INPLACE 0x8 extern int read_resource_prefix(const struct wim_lookup_table_entry *lte, @@ -650,7 +666,8 @@ new_image_metadata_array(unsigned num_images); #define WIMLIB_WRITE_FLAG_NO_LOOKUP_TABLE 0x80000000 #define WIMLIB_WRITE_FLAG_REUSE_INTEGRITY_TABLE 0x40000000 #define WIMLIB_WRITE_FLAG_CHECKPOINT_AFTER_XML 0x20000000 -#define WIMLIB_WRITE_MASK_PUBLIC 0x1fffffff +//#define WIMLIB_WRITE_FLAG_OVERWRITE_INPLACE 0x10000000 +#define WIMLIB_WRITE_MASK_PUBLIC 0x0fffffff /* We are capturing a tree to be placed in the root of the WIM image */ #define WIMLIB_ADD_IMAGE_FLAG_ROOT 0x80000000 diff --git a/src/win32.c b/src/win32.c index 8110d4fb..e8ef17a1 100644 --- a/src/win32.c +++ b/src/win32.c @@ -586,13 +586,13 @@ win32_capture_stream(const wchar_t *path, lte->resource_location = RESOURCE_WIN32; lte->resource_entry.original_size = (u64)dat->StreamSize.QuadPart; - struct wim_lookup_table_entry **my_ptr; + struct wim_lookup_table_entry **back_ptr; if (is_named_stream) - my_ptr = &ads_entry->lte; + back_ptr = &ads_entry->lte; else - my_ptr = &inode->i_lte; + back_ptr = &inode->i_lte; - lookup_table_insert_unhashed(lookup_table, lte, my_ptr); + lookup_table_insert_unhashed(lookup_table, lte, back_ptr); out_free_spath: FREE(spath); out: diff --git a/src/write.c b/src/write.c index 1891c89b..159fd57d 100644 --- a/src/write.c +++ b/src/write.c @@ -69,25 +69,6 @@ # define INVALID_HANDLE_VALUE ((HANDLE)(-1)) #endif -static int -fflush_and_ftruncate(FILE *fp, off_t size) -{ - int ret; - - ret = fflush(fp); - if (ret != 0) { - ERROR_WITH_ERRNO("Failed to flush data to output WIM file"); - return WIMLIB_ERR_WRITE; - } - ret = ftruncate(fileno(fp), size); - if (ret != 0) { - ERROR_WITH_ERRNO("Failed to truncate output WIM file to " - "%"PRIu64" bytes", size); - return WIMLIB_ERR_WRITE; - } - return 0; -} - /* Chunk table that's located at the beginning of each compressed resource in * the WIM. (This is not the on-disk format; the on-disk format just has an * array of offsets.) */ @@ -137,13 +118,14 @@ begin_wim_resource_chunk_tab(const struct wim_lookup_table_entry *lte, chunk_tab->table_disk_size) { ERROR_WITH_ERRNO("Failed to write chunk table in compressed " "file resource"); + FREE(chunk_tab); ret = WIMLIB_ERR_WRITE; goto out; } ret = 0; -out: *chunk_tab_ret = chunk_tab; +out: return ret; } @@ -269,29 +251,6 @@ finish_wim_resource_chunk_tab(struct chunk_table *chunk_tab, return 0; } -static int -write_uncompressed_resource_and_truncate(struct wim_lookup_table_entry *lte, - FILE *out_fp, - off_t file_offset, - struct resource_entry *out_res_entry) -{ - int ret; - if (fseeko(out_fp, file_offset, SEEK_SET) != 0) { - ERROR_WITH_ERRNO("Failed to seek to byte %"PRIu64" of " - "output WIM file", file_offset); - return WIMLIB_ERR_WRITE; - } - ret = write_wim_resource(lte, out_fp, - WIMLIB_COMPRESSION_TYPE_NONE, - out_res_entry, - 0); - if (ret) - return ret; - - return fflush_and_ftruncate(out_fp, - file_offset + wim_resource_size(lte)); -} - struct write_resource_ctx { compress_func_t compress; struct chunk_table *chunk_tab; @@ -322,6 +281,30 @@ write_resource_cb(const void *chunk, size_t chunk_size, void *_ctx) } } +/* + * Write a resource to an output WIM. + * + * @lte: Lookup table entry for the resource, which could be in another WIM, + * in an external file, or in another location. + * + * @out_fp: FILE * opened to the output WIM. + * + * @out_ctype: One of the WIMLIB_COMPRESSION_TYPE_* constants to indicate + * which compression algorithm to use. + * + * @out_res_entry: On success, this is filled in with the offset, flags, + * compressed size, and uncompressed size of the resource + * in the output WIM. + * + * @flags: WIMLIB_RESOURCE_FLAG_RECOMPRESS to force data to be recompressed + * even if it could otherwise be copied directly from the input. + * + * Additional notes: The SHA1 message digest of the uncompressed data is + * calculated (except when doing a raw copy --- see below). If the @unhashed + * flag is set on the lookup table entry, this message digest is simply copied + * to it; otherwise, the message digest is compared with the existing one, and + * the function will fail if they do not match. + */ int write_wim_resource(struct wim_lookup_table_entry *lte, FILE *out_fp, int out_ctype, @@ -333,22 +316,29 @@ write_wim_resource(struct wim_lookup_table_entry *lte, off_t offset; int ret; + flags &= ~WIMLIB_RESOURCE_FLAG_RECOMPRESS; + if (wim_resource_size(lte) == 0) { /* Empty resource; nothing needs to be done, so just return * success. */ return 0; } + /* Get current position in output WIM */ offset = ftello(out_fp); if (offset == -1) { ERROR_WITH_ERRNO("Can't get position in output WIM"); return WIMLIB_ERR_WRITE; } - /* Can we simply copy the compressed data without recompressing it? */ - + /* If we are not forcing the data to be recompressed, and the input + * resource is located in a WIM with the same compression type as that + * desired other than no compression, we can simply copy the compressed + * data without recompressing it. This also means we must skip + * calculating the SHA1, as we never will see the uncompressed data. */ if (!(flags & WIMLIB_RESOURCE_FLAG_RECOMPRESS) && lte->resource_location == RESOURCE_IN_WIM && + out_ctype != WIMLIB_COMPRESSION_TYPE_NONE && wimlib_get_compression_type(lte->wim) == out_ctype) { flags |= WIMLIB_RESOURCE_FLAG_RAW; @@ -373,10 +363,12 @@ write_wim_resource(struct wim_lookup_table_entry *lte, return ret; } - /* Write the data */ + /* Write the entire resource by reading the entire resource and feeding + * the data through the write_resource_cb function. */ write_ctx.out_fp = out_fp; +try_write_again: ret = read_resource_prefix(lte, wim_resource_size(lte), - write_resource_cb, &write_ctx, 0); + write_resource_cb, &write_ctx, flags); /* Verify SHA1 message digest of the resource, or set the hash for the * first time. */ @@ -420,11 +412,23 @@ write_wim_resource(struct wim_lookup_table_entry *lte, if (new_size >= wim_resource_size(lte)) { /* Oops! We compressed the resource to larger than the original * size. Write the resource uncompressed instead. */ - ret = write_uncompressed_resource_and_truncate(lte, - out_fp, - offset, - out_res_entry); - goto out_free_chunk_tab; + if (fseeko(out_fp, offset, SEEK_SET) || + fflush(out_fp) || + ftruncate(fileno(out_fp), + offset + wim_resource_size(lte))) + { + ERROR_WITH_ERRNO("Failed to flush and/or truncate " + "output WIM file"); + ret = WIMLIB_ERR_WRITE; + goto out_free_chunk_tab; + } + DEBUG("Compressed %"PRIu64" => %"PRIu64" bytes; " + "writing uncompressed instead", + wim_resource_size(lte), new_size); + write_ctx.compress = NULL; + write_ctx.doing_sha = false; + out_ctype = WIMLIB_COMPRESSION_TYPE_NONE; + goto try_write_again; } out_res_entry->flags |= WIM_RESHDR_FLAG_COMPRESSED; } @@ -625,7 +629,7 @@ enum { }; static int -do_write_stream_list(struct list_head *my_resources, +do_write_stream_list(struct list_head *stream_list, struct wim_lookup_table *lookup_table, FILE *out_fp, int out_ctype, @@ -636,44 +640,86 @@ do_write_stream_list(struct list_head *my_resources, int ret; struct wim_lookup_table_entry *lte; - while (!list_empty(my_resources)) { - lte = container_of(my_resources->next, + /* For each stream in @stream_list ... */ + while (!list_empty(stream_list)) { + lte = container_of(stream_list->next, struct wim_lookup_table_entry, write_streams_list); list_del(<e->write_streams_list); if (lte->unhashed && !lte->unique_size) { + + /* Unhashed stream that shares a size with some other + * stream in the WIM we are writing. The stream must be + * checksummed to know if we need to write it or not. */ struct wim_lookup_table_entry *duplicate_lte; - struct wim_lookup_table_entry **my_ptr; + struct wim_lookup_table_entry **back_ptr; - my_ptr = lte->my_ptr; + /* back_ptr must be saved because it's in union with the + * SHA1 message digest and will no longer be valid once + * the SHA1 has been calculated. */ + back_ptr = lte->back_ptr; + + /* Checksum the stream */ ret = sha1_resource(lte); if (ret) return ret; + + /* Look for a duplicate stream */ duplicate_lte = __lookup_resource(lookup_table, lte->hash); if (duplicate_lte) { - bool new_stream = (duplicate_lte->out_refcnt == 0); + /* We have a duplicate stream. Transfer the + * reference counts from this stream to the + * duplicate, update the reference to this + * stream (in an inode or ads_entry) to point to + * the duplicate, then free this stream. */ + wimlib_assert(!(duplicate_lte->unhashed)); + bool is_new_stream = (duplicate_lte->out_refcnt == 0); duplicate_lte->refcnt += lte->refcnt; duplicate_lte->out_refcnt += lte->refcnt; - *my_ptr = duplicate_lte; + *back_ptr = duplicate_lte; + list_del(<e->unhashed_list); free_lookup_table_entry(lte); lte = duplicate_lte; - if (new_stream) { - DEBUG("Stream of length %"PRIu64" is duplicate " - "with one already in WIM", - wim_resource_size(lte)); + + if (is_new_stream) { + /* The duplicate stream is one we + * weren't already planning to write. + * But, now we must write it. + * + * XXX: Currently, the copy of the + * stream in the WIM is always chosen + * for writing, rather than the extra + * copy we just read (which may be in an + * external file). This may not always + * be fastest. */ } else { + /* We have already written, or are going + * to write, the duplicate stream. So + * just skip to the next stream. */ DEBUG("Discarding duplicate stream of length %"PRIu64, wim_resource_size(lte)); goto skip_to_progress; } } else { + /* No duplicate stream, so we need to insert + * this stream into the lookup table and treat + * it as a hashed stream. */ + list_del(<e->unhashed_list); lookup_table_insert(lookup_table, lte); lte->out_refcnt = lte->refcnt; lte->unhashed = 0; } } + /* Here, @lte either a hashed stream or an unhashed stream with + * a unique size. In either case we know that the stream has to + * be written. In either case the SHA1 message digest will be + * calculated over the stream while writing it; however, in the + * former case this is done merely to check the data, while in + * the latter case this is done because we do not have the SHA1 + * message digest yet. */ + wimlib_assert(lte->out_refcnt != 0); ret = write_wim_resource(lte, @@ -684,6 +730,7 @@ do_write_stream_list(struct list_head *my_resources, if (ret) return ret; if (lte->unhashed) { + list_del(<e->unhashed_list); lookup_table_insert(lookup_table, lte); lte->unhashed = 0; } @@ -704,12 +751,10 @@ write_stream_list_serial(struct list_head *stream_list, wimlib_progress_func_t progress_func, union wimlib_progress_info *progress) { - int write_resource_flags; - + int write_resource_flags = 0; if (write_flags & WIMLIB_WRITE_FLAG_RECOMPRESS) - write_resource_flags = WIMLIB_RESOURCE_FLAG_RECOMPRESS; - else - write_resource_flags = 0; + write_resource_flags |= WIMLIB_RESOURCE_FLAG_RECOMPRESS; + progress->write_streams.num_threads = 1; if (progress_func) progress_func(WIMLIB_PROGRESS_MSG_WRITE_STREAMS, progress); @@ -903,16 +948,16 @@ main_writer_thread_proc(struct list_head *stream_list, } next_lte = container_of(next_resource, struct wim_lookup_table_entry, - staging_list); + write_streams_list); next_resource = next_resource->next; if ((!(write_flags & WIMLIB_WRITE_FLAG_RECOMPRESS) && wim_resource_compression_type(next_lte) == out_ctype) || wim_resource_size(next_lte) == 0) { - list_add_tail(&next_lte->staging_list, + list_add_tail(&next_lte->write_streams_list, &my_resources); } else { - list_add_tail(&next_lte->staging_list, + list_add_tail(&next_lte->write_streams_list, &outstanding_resources); next_chunk = 0; next_num_chunks = wim_resource_chunks(next_lte); @@ -1094,15 +1139,15 @@ main_writer_thread_proc(struct list_head *stream_list, FREE(cur_chunk_tab); cur_chunk_tab = NULL; - struct list_head *next = cur_lte->staging_list.next; - list_del(&cur_lte->staging_list); + struct list_head *next = cur_lte->write_streams_list.next; + list_del(&cur_lte->write_streams_list); if (next == &outstanding_resources) cur_lte = NULL; else - cur_lte = container_of(cur_lte->staging_list.next, + cur_lte = container_of(cur_lte->write_streams_list.next, struct wim_lookup_table_entry, - staging_list); + write_streams_list); // Since we just finished writing a stream, // write any streams that have been added to the @@ -1398,8 +1443,8 @@ stream_size_table_insert(struct wim_lookup_table_entry *lte, void *_tab) struct lte_overwrite_prepare_args { WIMStruct *wim; off_t end_offset; - struct list_head *stream_list; - struct stream_size_table *stream_size_tab; + struct list_head stream_list; + struct stream_size_table stream_size_tab; }; static int @@ -1422,22 +1467,20 @@ lte_overwrite_prepare(struct wim_lookup_table_entry *lte, void *arg) return WIMLIB_ERR_RESOURCE_ORDER; } } else { - if (!(lte->resource_entry.flags & WIM_RESHDR_FLAG_METADATA)) - list_add_tail(<e->write_streams_list, args->stream_list); + wimlib_assert(!(lte->resource_entry.flags & WIM_RESHDR_FLAG_METADATA)); + list_add_tail(<e->write_streams_list, &args->stream_list); } lte->out_refcnt = lte->refcnt; - stream_size_table_insert(lte, args->stream_size_tab); + stream_size_table_insert(lte, &args->stream_size_tab); return 0; } static int lte_set_output_res_entry(struct wim_lookup_table_entry *lte, void *_wim) { - if (lte->resource_location == RESOURCE_IN_WIM && - lte->wim == _wim) - { - memcpy(<e->output_resource_entry, <e->resource_entry, - sizeof(struct resource_entry)); + if (lte->resource_location == RESOURCE_IN_WIM && lte->wim == _wim) { + copy_resource_entry(<e->output_resource_entry, + <e->resource_entry); } return 0; } @@ -1460,19 +1503,16 @@ prepare_streams_for_overwrite(WIMStruct *wim, off_t end_offset, struct list_head *stream_list) { int ret; - struct stream_size_table stream_size_tab; - struct lte_overwrite_prepare_args args = { - .wim = wim, - .end_offset = end_offset, - .stream_list = stream_list, - .stream_size_tab = &stream_size_tab, - }; - - ret = init_stream_size_table(&stream_size_tab, 9001); + struct lte_overwrite_prepare_args args; + + args.wim = wim; + args.end_offset = end_offset; + ret = init_stream_size_table(&args.stream_size_tab, + wim->lookup_table->capacity); if (ret) return ret; - INIT_LIST_HEAD(stream_list); + INIT_LIST_HEAD(&args.stream_list); for (int i = 0; i < wim->hdr.image_count; i++) { struct wim_image_metadata *imd; struct wim_lookup_table_entry *lte; @@ -1492,10 +1532,11 @@ prepare_streams_for_overwrite(WIMStruct *wim, off_t end_offset, for (int i = 0; i < wim->hdr.image_count; i++) lte_set_output_res_entry(wim->image_metadata[i]->metadata_lte, wim); - ret = for_lookup_table_entry(wim->lookup_table, - lte_set_output_res_entry, wim); + for_lookup_table_entry(wim->lookup_table, lte_set_output_res_entry, wim); + INIT_LIST_HEAD(stream_list); + list_splice(&args.stream_list, stream_list); out_destroy_stream_size_table: - destroy_stream_size_table(&stream_size_tab); + destroy_stream_size_table(&args.stream_size_tab); return ret; } @@ -1505,7 +1546,7 @@ struct find_streams_ctx { struct stream_size_table stream_size_tab; }; -static int +static void inode_find_streams_to_write(struct wim_inode *inode, struct wim_lookup_table *table, struct list_head *stream_list, @@ -1523,7 +1564,6 @@ inode_find_streams_to_write(struct wim_inode *inode, lte->out_refcnt += inode->i_nlink; } } - return 0; } static int @@ -1540,7 +1580,7 @@ image_find_streams_to_write(WIMStruct *w) image_for_each_unhashed_stream(lte, imd) { lte->out_refcnt = 0; wimlib_assert(lte->unhashed); - wimlib_assert(lte->my_ptr != NULL); + wimlib_assert(lte->back_ptr != NULL); } /* Go through this image's inodes to find any streams that have not been @@ -1573,19 +1613,21 @@ prepare_stream_list(WIMStruct *wim, int image, struct list_head *stream_list) struct find_streams_ctx ctx; for_lookup_table_entry(wim->lookup_table, lte_zero_out_refcnt, NULL); - ret = init_stream_size_table(&ctx.stream_size_tab, 9001); + ret = init_stream_size_table(&ctx.stream_size_tab, + wim->lookup_table->capacity); if (ret) return ret; for_lookup_table_entry(wim->lookup_table, stream_size_table_insert, &ctx.stream_size_tab); INIT_LIST_HEAD(&ctx.stream_list); wim->private = &ctx; - for_image(wim, image, image_find_streams_to_write); + ret = for_image(wim, image, image_find_streams_to_write); destroy_stream_size_table(&ctx.stream_size_tab); - - INIT_LIST_HEAD(stream_list); - list_splice(&ctx.stream_list, stream_list); - return 0; + if (ret == 0) { + INIT_LIST_HEAD(stream_list); + list_splice(&ctx.stream_list, stream_list); + } + return ret; } /* Writes the streams for the specified @image in @wim to @wim->out_fp. @@ -1841,7 +1883,7 @@ begin_write(WIMStruct *w, const tchar *path, int write_flags) int ret; ret = open_wim_writable(w, path, true, (write_flags & WIMLIB_WRITE_FLAG_CHECK_INTEGRITY) != 0); - if (ret != 0) + if (ret) return ret; /* Write dummy header. It will be overwritten later. */ return write_header(&w->hdr, w->out_fp); @@ -2031,7 +2073,8 @@ overwrite_wim_inplace(WIMStruct *w, int write_flags, w->lookup_table, w->out_fp, wimlib_get_compression_type(w), - write_flags, num_threads, + write_flags, + num_threads, progress_func); if (ret) goto out_ftruncate; diff --git a/src/xpress-decompress.c b/src/xpress-decompress.c index 380f017d..6756b159 100644 --- a/src/xpress-decompress.c +++ b/src/xpress-decompress.c @@ -180,7 +180,7 @@ xpress_decompress_block(struct input_bitstream *istream, ret = read_huffsym(istream, decode_table, lens, XPRESS_NUM_SYMBOLS, XPRESS_TABLEBITS, &huffsym, XPRESS_MAX_CODEWORD_LEN); - if (ret != 0) + if (ret) return ret; if (huffsym < XPRESS_NUM_CHARS) { @@ -223,8 +223,10 @@ wimlib_xpress_decompress(const void *__compressed_data, unsigned compressed_len, * code lengths of these symbols are given literally as 4-bit integers * in the first 256 bytes of the compressed data. */ - if (compressed_len < XPRESS_NUM_SYMBOLS / 2) + if (compressed_len < XPRESS_NUM_SYMBOLS / 2) { + ERROR("xpress_decompress(): Compressed length too short!"); return -1; + } for (i = 0; i < XPRESS_NUM_SYMBOLS / 2; i++) { *lens_p++ = compressed_data[i] & 0xf; @@ -234,7 +236,7 @@ wimlib_xpress_decompress(const void *__compressed_data, unsigned compressed_len, ret = make_huffman_decode_table(decode_table, XPRESS_NUM_SYMBOLS, XPRESS_TABLEBITS, lens, XPRESS_MAX_CODEWORD_LEN); - if (ret != 0) + if (ret) return ret; init_input_bitstream(&istream, compressed_data + XPRESS_NUM_SYMBOLS / 2,