dest_lte = clone_lookup_table_entry(src_lte);
if (!dest_lte)
return WIMLIB_ERR_NOMEM;
- list_add_tail(&dest_lte->staging_list, lte_list_head);
+ list_add_tail(&dest_lte->new_stream_list,
+ lte_list_head);
}
}
}
list_del(next);
dest_lte = container_of(next,
struct wim_lookup_table_entry,
- staging_list);
+ new_stream_list);
dest_lte->part_number = 1;
dest_lte->refcnt = 0;
wimlib_assert(hashes_equal(dest_lte->hash, src_lte->hash));
out_free_ltes:
{
struct wim_lookup_table_entry *lte, *tmp;
- list_for_each_entry_safe(lte, tmp, <e_list_head, staging_list)
+ list_for_each_entry_safe(lte, tmp, <e_list_head, new_stream_list)
free_lookup_table_entry(lte);
}
out:
tfprintf(out, T("Reference Count = %u\n"), lte->refcnt);
if (lte->unhashed) {
- tfprintf(out, T("(Unhashed, back ptr at %p)\n"), lte->my_ptr);
+ tfprintf(out, T("(Unhashed, back ptr at %p)\n"), lte->back_ptr);
} else {
tfprintf(out, T("Hash = 0x"));
print_hash(lte->hash, out);
for_lookup_table_entry(table, lte_add_stream_size, &total_size);
return total_size;
}
-
-void
-free_lte_list(struct list_head *list)
-{
- struct wim_lookup_table_entry *lte, *tmp;
-
- list_for_each_entry_safe(lte, tmp, list, staging_list) {
- DEBUG("%p", lte);
- free_lookup_table_entry(lte);
-}
- INIT_LIST_HEAD(list);
-}
* to the pointer to this 'struct wim_lookup_table_entry'
* contained in a 'struct wim_ads_entry' or 'struct wim_inode'.
* */
- struct wim_lookup_table_entry **my_ptr;
+ struct wim_lookup_table_entry **back_ptr;
};
/* When a WIM file is written, out_refcnt starts at 0 and is incremented
/* Pointers to somewhere where the stream is actually located. See the
* comments for the @resource_location field above. */
union {
+ void *resource_loc_private;
WIMStruct *wim;
tchar *file_on_disk;
- tchar *staging_file_name;
void *attached_buffer;
+ #ifdef WITH_FUSE
+ tchar *staging_file_name;
+ #endif
#ifdef WITH_NTFS_3G
struct ntfs_location *ntfs_loc;
#endif
struct {
struct hlist_node hash_list_2;
+
struct list_head write_streams_list;
};
};
- /* List of lookup table entries that correspond to streams that have
- * been extracted to the staging directory when modifying a read-write
- * mounted WIM.
- *
- * This field is also used to make other lists of lookup table entries.
- * */
+ /* Temporary list fields */
union {
struct list_head unhashed_list;
- struct list_head staging_list;
+ struct list_head swm_stream_list;
struct list_head extraction_list;
+ struct list_head new_stream_list;
};
};
static inline void
lookup_table_insert_unhashed(struct wim_lookup_table *table,
struct wim_lookup_table_entry *lte,
- struct wim_lookup_table_entry **my_ptr)
+ struct wim_lookup_table_entry **back_ptr)
{
lte->unhashed = 1;
list_add_tail(<e->unhashed_list, table->unhashed_streams);
- lte->my_ptr = my_ptr;
- *my_ptr = lte;
+ lte->back_ptr = back_ptr;
+ *back_ptr = lte;
}
-extern void
-free_lte_list(struct list_head *list);
-
#endif
unsigned len;
ret = bitstream_ensure_bits(istream, 4);
- if (ret != 0) {
+ if (ret) {
ERROR("LZX input stream overrun");
return ret;
}
block_size = 32768;
} else {
ret = bitstream_read_bits(istream, 16, &block_size);
- if (ret != 0)
+ if (ret)
return ret;
block_size = le16_to_cpu(block_size);
}
ret = bitstream_read_bits(istream,
LZX_ALIGNEDTREE_ELEMENT_SIZE,
&len);
- if (ret != 0)
+ if (ret)
return ret;
tables->alignedtree_lens[i] = len;
}
LZX_ALIGNEDTREE_TABLEBITS,
tables->alignedtree_lens,
8);
- if (ret != 0) {
+ if (ret) {
ERROR("lzx_decompress(): Failed to make the decode "
"table for the aligned offset tree");
return ret;
* tree. */
ret = lzx_read_code_lens(istream, tables->maintree_lens,
LZX_NUM_CHARS);
- if (ret != 0) {
+ if (ret) {
ERROR("lzx_decompress(): Failed to read the code "
"lengths for the first 256 elements of the "
"main tree");
ret = lzx_read_code_lens(istream,
tables->maintree_lens + LZX_NUM_CHARS,
LZX_MAINTREE_NUM_SYMBOLS - LZX_NUM_CHARS);
- if (ret != 0) {
+ if (ret) {
ERROR("lzx_decompress(): Failed to read the path "
"lengths for the remaining elements of the main "
"tree");
LZX_MAINTREE_TABLEBITS,
tables->maintree_lens,
LZX_MAX_CODEWORD_LEN);
- if (ret != 0) {
+ if (ret) {
ERROR("lzx_decompress(): Failed to make the decode "
"table for the main tree");
return ret;
LZX_DEBUG("Reading path lengths for the length tree.");
ret = lzx_read_code_lens(istream, tables->lentree_lens,
LZX_LENTREE_NUM_SYMBOLS);
- if (ret != 0) {
+ if (ret) {
ERROR("lzx_decompress(): Failed to read the path "
"lengths for the length tree");
return ret;
LZX_LENTREE_TABLEBITS,
tables->lentree_lens,
LZX_MAX_CODEWORD_LEN);
- if (ret != 0) {
+ if (ret) {
ERROR("lzx_decompress(): Failed to build the length "
"Huffman tree");
return ret;
* *already* aligned, the correct thing to do is to throw away
* the next 16 bits. */
if (istream->bitsleft == 0) {
- if (istream->data_bytes_left < 14)
+ if (istream->data_bytes_left < 14) {
+ ERROR("lzx_decompress(): Insufficient length in "
+ "uncompressed block");
return -1;
+ }
istream->data += 2;
istream->data_bytes_left -= 2;
} else {
- if (istream->data_bytes_left < 12)
+ if (istream->data_bytes_left < 12) {
+ ERROR("lzx_decompress(): Insufficient length in "
+ "uncompressed block");
return -1;
+ }
istream->bitsleft = 0;
istream->bitbuf = 0;
}
while (window_pos < end) {
ret = read_huffsym_using_maintree(istream, tables,
&main_element);
- if (ret != 0)
+ if (ret)
return ret;
if (main_element < LZX_NUM_CHARS) {
LZX_DEBUG("Reading block header.");
ret = lzx_read_block_header(&istream, &block_size,
&block_type, &tables, &queue);
- if (ret != 0)
+ if (ret)
return ret;
LZX_DEBUG("block_size = %u, window_pos = %u",
&tables,
&queue,
&istream);
- if (ret != 0)
+ if (ret)
return ret;
if (tables.maintree_lens[0xe8] != 0)
e8_preprocessing_done = true;
metadata_lte = imd->metadata_lte;
metadata_len = wim_resource_size(metadata_lte);
- DEBUG("Reading metadata resource: length = %"PRIu64", "
- "offset = %"PRIu64"", metadata_len,
+ DEBUG("Reading metadata resource: original_size = %"PRIu64", "
+ "size = %"PRIu64", offset = %"PRIu64"",
+ metadata_lte->resource_entry.original_size,
+ metadata_lte->resource_entry.size,
metadata_lte->resource_entry.offset);
/* There is no way the metadata resource could possibly be less than (8
wimlib_get_compression_type(w),
<e->output_resource_entry,
lte->hash);
- if (ret)
- goto out_free_buf;
-
/* Note that although the SHA1 message digest of the metadata resource
* is very likely to have changed, the corresponding lookup table entry
* is not actually located in the hash table, so it need not be
* re-inserted in the hash table. */
- lte->out_refcnt = 1;
- lte->output_resource_entry.flags |= WIM_RESHDR_FLAG_METADATA;
-out_free_buf:
+
/* All the data has been written to the new WIM; no need for the buffer
* anymore */
FREE(buf);
new_lte->staging_file_name = staging_file_name;
new_lte->lte_inode = inode;
- struct wim_lookup_table_entry **my_ptr;
+ struct wim_lookup_table_entry **back_ptr;
if (stream_id == 0) {
- my_ptr = &inode->i_lte;
+ back_ptr = &inode->i_lte;
} else {
for (u16 i = 0; ; i++) {
wimlib_assert(i < inode->i_num_ads);
if (inode->i_ads_entries[i].stream_id == stream_id) {
- my_ptr = &inode->i_ads_entries[i].lte;
+ back_ptr = &inode->i_ads_entries[i].lte;
break;
}
}
}
- lookup_table_insert_unhashed(ctx->wim->lookup_table, new_lte, my_ptr);
+ lookup_table_insert_unhashed(ctx->wim->lookup_table, new_lte, back_ptr);
*lte = new_lte;
return 0;
out_revert_fd_changes:
DEBUG("Freeing entries for zero-length streams");
image_for_each_unhashed_stream(lte, imd) {
if (wim_resource_size(lte) == 0) {
- *lte->my_ptr = NULL;
+ *lte->back_ptr = NULL;
free_lookup_table_entry(lte);
}
}
{
u64 data_size = ntfs_get_attribute_value_length(actx->attr);
u64 name_length = actx->attr->name_length;
- struct wim_lookup_table_entry **my_ptr;
+ struct wim_lookup_table_entry **back_ptr;
if (data_size == 0) {
if (errno != 0) {
ERROR_WITH_ERRNO("Failed to get size of attribute of "
ret = WIMLIB_ERR_NTFS_3G;
goto out_free_lte;
}
- my_ptr = &inode->i_lte;
+ back_ptr = &inode->i_lte;
} else {
/* Named data stream. Put the reference to it in the
* alternate data stream entries */
if (!new_ads_entry)
goto out_free_lte;
wimlib_assert(new_ads_entry->stream_name_nbytes == name_length * 2);
- my_ptr = &new_ads_entry->lte;
+ back_ptr = &new_ads_entry->lte;
}
- lookup_table_insert_unhashed(lookup_table, lte, my_ptr);
+ lookup_table_insert_unhashed(lookup_table, lte, back_ptr);
}
ret = 0;
goto out_put_actx;
#include <unistd.h>
#include <fcntl.h>
+#ifdef HAVE_ALLOCA_H
+# include <alloca.h>
+#endif
+
/* Write @n bytes from @buf to the file descriptor @fd, retrying on internupt
* and on short writes.
*
* Returns zero on success, nonzero on failure.
*/
static int
-read_compressed_resource(FILE *fp, u64 resource_compressed_size,
+read_compressed_resource(FILE *fp,
+ u64 resource_compressed_size,
u64 resource_uncompressed_size,
- u64 resource_offset, int resource_ctype,
- u64 len, u64 offset,
+ u64 resource_offset,
+ int resource_ctype,
+ u64 len,
+ u64 offset,
consume_data_callback_t cb,
void *ctx_or_buf)
{
* follows the chunk table and therefore must have an offset of 0.
*/
- /* Calculate how many chunks the resource conists of in its entirety. */
+ /* Calculate how many chunks the resource consists of in its entirety.
+ * */
u64 num_chunks = (resource_uncompressed_size + WIM_CHUNK_SIZE - 1) /
WIM_CHUNK_SIZE;
/* As mentioned, the first chunk has no entry in the chunk table. */
if (end_chunk != num_chunks - 1)
num_needed_chunks++;
- /* Declare the chunk table. It will only contain offsets for the chunks
- * that are actually needed for this read. */
- u64 chunk_offsets[num_needed_chunks];
+ /* Allocate the chunk table. It will only contain offsets for the
+ * chunks that are actually needed for this read. */
+ u64 *chunk_offsets;
+ bool chunk_offsets_malloced;
+ if (num_needed_chunks < 1000) {
+ chunk_offsets = alloca(num_needed_chunks * sizeof(u64));
+ chunk_offsets_malloced = false;
+ } else {
+ chunk_offsets = malloc(num_needed_chunks * sizeof(u64));
+ if (!chunk_offsets) {
+ ERROR("Failed to allocate chunk table "
+ "with %"PRIu64" entries", num_needed_chunks);
+ return WIMLIB_ERR_NOMEM;
+ }
+ chunk_offsets_malloced = true;
+ }
/* Set the implicit offset of the first chunk if it is included in the
* needed chunks.
/* Number of bytes we need to read from the chunk table. */
size_t size = num_needed_chunk_entries * chunk_entry_size;
- {
- u8 chunk_tab_buf[size];
+ /* Read the raw data into the end of the chunk_offsets array to
+ * avoid allocating another array. */
+ void *chunk_tab_buf = (void*)&chunk_offsets[num_needed_chunks] - size;
- if (fread(chunk_tab_buf, 1, size, fp) != size)
- goto read_error;
+ if (fread(chunk_tab_buf, 1, size, fp) != size)
+ goto read_error;
- /* Now fill in chunk_offsets from the entries we have read in
- * chunk_tab_buf. */
+ /* Now fill in chunk_offsets from the entries we have read in
+ * chunk_tab_buf. */
- u64 *chunk_tab_p = chunk_offsets;
- if (start_chunk == 0)
- chunk_tab_p++;
+ u64 *chunk_tab_p = chunk_offsets;
+ if (start_chunk == 0)
+ chunk_tab_p++;
- if (chunk_entry_size == 4) {
- u32 *entries = (u32*)chunk_tab_buf;
- while (num_needed_chunk_entries--)
- *chunk_tab_p++ = le32_to_cpu(*entries++);
- } else {
- u64 *entries = (u64*)chunk_tab_buf;
- while (num_needed_chunk_entries--)
- *chunk_tab_p++ = le64_to_cpu(*entries++);
- }
+ if (chunk_entry_size == 4) {
+ u32 *entries = (u32*)chunk_tab_buf;
+ while (num_needed_chunk_entries--)
+ *chunk_tab_p++ = le32_to_cpu(*entries++);
+ } else {
+ u64 *entries = (u64*)chunk_tab_buf;
+ while (num_needed_chunk_entries--)
+ *chunk_tab_p++ = le64_to_cpu(*entries++);
}
/* Done with the chunk table now. We must now seek to the first chunk
goto read_error;
/* Pointer to current position in the output buffer for uncompressed
- * data. */
+ * data. Alternatively, if using a callback function, we repeatedly
+ * fill a temporary buffer to feed data into the callback function. */
u8 *out_p;
if (cb)
- out_p = alloca(32768);
+ out_p = alloca(WIM_CHUNK_SIZE);
else
out_p = ctx_or_buf;
}
if (cb) {
/* Feed the data to the callback function */
- ret = cb(out_p, partial_chunk_size, ctx_or_buf);
+ ret = cb(out_p + start_offset,
+ partial_chunk_size, ctx_or_buf);
if (ret)
goto out;
} else {
ret = 0;
out:
+ if (chunk_offsets_malloced)
+ FREE(chunk_offsets);
return ret;
read_error:
wim = lte->wim;
- if (flags & WIMLIB_RESOURCE_FLAG_MULTITHREADED) {
+ if (flags & WIMLIB_RESOURCE_FLAG_THREADSAFE_READ) {
wim_fp = wim_get_fp(wim);
if (!wim_fp) {
ret = -1;
ERROR_WITH_ERRNO("Error reading data from WIM");
ret = WIMLIB_ERR_READ;
out_release_fp:
- if (flags & WIMLIB_RESOURCE_FLAG_MULTITHREADED)
+ if (flags & WIMLIB_RESOURCE_FLAG_THREADSAFE_READ)
ret |= wim_release_fp(wim, wim_fp);
out:
if (ret) {
bool threadsafe)
{
return read_partial_wim_resource(lte, size, NULL, buf,
- threadsafe ? WIMLIB_RESOURCE_FLAG_MULTITHREADED : 0,
+ threadsafe ? WIMLIB_RESOURCE_FLAG_THREADSAFE_READ : 0,
offset);
}
* When using a callback function, it is called with chunks up to 32768 bytes in
* size until the resource is exhausted.
*
- * If the resource is located in a WIM file, @flags can be
- * WIMLIB_RESOURCE_FLAG_MULTITHREADED if it must be safe to access the resource
- * concurrently by multiple threads, or WIMLIB_RESOURCE_FLAG_RAW if the raw
- * compressed data is to be supplied instead of the uncompressed data.
+ * If the resource is located in a WIM file, @flags can be:
+ * * WIMLIB_RESOURCE_FLAG_THREADSAFE_READ if it must be safe to access the resource
+ * concurrently by multiple threads.
+ * * WIMLIB_RESOURCE_FLAG_RAW if the raw compressed data is to be supplied
+ * instead of the uncompressed data.
+ * Otherwise, the @flags are ignored.
*/
int
read_resource_prefix(const struct wim_lookup_table_entry *lte,
return read_resource_prefix(lte,
wim_resource_size(lte),
NULL, buf,
- thread_safe ? WIMLIB_RESOURCE_FLAG_MULTITHREADED : 0);
+ thread_safe ? WIMLIB_RESOURCE_FLAG_THREADSAFE_READ : 0);
}
struct extract_ctx {
}
#endif /* !WITH_LIBCRYPTO */
-
-static int
-sha1_stream(FILE *fp, u8 md[SHA1_HASH_SIZE])
-{
- char buf[BUFFER_SIZE];
- size_t bytes_read;
- SHA_CTX ctx;
- sha1_init(&ctx);
- while (1) {
- bytes_read = fread(buf, 1, sizeof(buf), fp);
- sha1_update(&ctx, buf, bytes_read);
- if (bytes_read < sizeof(buf)) {
- if (ferror(fp))
- return WIMLIB_ERR_READ;
- break;
- }
- }
- sha1_final(md, &ctx);
- return 0;
-
-}
-
-/* Calculates the SHA1 message digest of a file. @md must point to a buffer of
- * length 20 bytes into which the message digest is written. */
-int
-sha1sum(const tchar *filename, u8 md[SHA1_HASH_SIZE])
-{
- FILE *fp;
- int ret;
-
- fp = tfopen(filename, T("rb"));
- if (!fp) {
- ERROR_WITH_ERRNO("Cannot open the file `%"TS"' for reading",
- filename);
- return WIMLIB_ERR_OPEN;
- }
- ret = sha1_stream(fp, md);
- if (ret != 0) {
- ERROR_WITH_ERRNO("Error calculating SHA1 message digest of "
- "`%"TS"'", filename);
- }
- fclose(fp);
- return ret;
-}
#endif /* !WITH_LIBCRYPTO */
-extern int
-sha1sum(const tchar *filename, u8 hash[SHA1_HASH_SIZE]);
-
#endif /* _WIMLIB_SHA1_H */
int ret;
struct wim_lookup_table_entry *lte;
- list_for_each_entry(lte, lte_list, staging_list) {
+ list_for_each_entry(lte, lte_list, swm_stream_list) {
ret = write_lookup_table_entry(lte, w->out_fp);
- if (ret != 0)
+ if (ret)
return ret;
}
ret = finish_swm(w, &args->lte_list, args->write_flags,
args->progress_func);
- if (ret != 0)
+ if (ret)
return ret;
if (args->progress_func) {
}
ret = begin_write(w, args->swm_base_name, args->write_flags);
- if (ret != 0)
+ if (ret)
return ret;
args->size_remaining = args->part_size;
}
args->size_remaining -= lte->resource_entry.size;
args->progress.split.completed_bytes += lte->resource_entry.size;
- list_add_tail(<e->staging_list, &args->lte_list);
+ list_add_tail(<e->swm_stream_list, &args->lte_list);
return copy_resource(lte, w);
}
w->hdr.boot_idx = 0;
randomize_byte_array(w->hdr.guid, WIM_GID_LEN);
ret = begin_write(w, swm_name, write_flags);
- if (ret != 0)
+ if (ret)
goto out;
tmemcpy(swm_base_name, swm_name, swm_name_len + 1);
args.progress.split.completed_bytes += metadata_lte->resource_entry.size;
/* Careful: The metadata lookup table entries must be added in
* order of the images. */
- list_add_tail(&metadata_lte->staging_list, &args.lte_list);
+ list_add_tail(&metadata_lte->swm_stream_list, &args.lte_list);
}
ret = for_lookup_table_entry_pos_sorted(w->lookup_table,
copy_resource_to_swm,
&args);
- if (ret != 0)
+ if (ret)
goto out;
ret = finish_swm(w, &args.lte_list, write_flags, progress_func);
- if (ret != 0)
+ if (ret)
goto out;
if (progress_func) {
if (w->current_image != WIMLIB_NO_IMAGE) {
imd = wim_get_current_image_metadata(w);
if (!imd->modified) {
+ wimlib_assert(list_empty(&imd->unhashed_streams));
DEBUG("Freeing image %u", w->current_image);
destroy_image_metadata(imd, NULL, false);
}
free_lookup_table_entry(imd->metadata_lte);
imd->metadata_lte = NULL;
}
+ if (!table) {
+ struct wim_lookup_table_entry *lte, *tmp;
+ list_for_each_entry_safe(lte, tmp, &imd->unhashed_streams, unhashed_list)
+ free_lookup_table_entry(lte);
+ }
INIT_LIST_HEAD(&imd->unhashed_streams);
INIT_LIST_HEAD(&imd->inode_list);
}
#define image_for_each_unhashed_stream(lte, imd) \
list_for_each_entry(lte, &imd->unhashed_streams, unhashed_list)
+#if 1
+# define copy_resource_entry(dst, src) memcpy(dst, src, sizeof(struct resource_entry))
+#else
+static inline void
+copy_resource_entry(struct resource_entry *dst,
+ const struct resource_entry *src)
+{
+ memcpy(dst, src, sizeof(struct resource_entry));
+ BUILD_BUG_ON(sizeof(struct resource_entry) != 24);
+ ((u64*)dst)[0] = ((u64*)src)[0];
+ ((u64*)dst)[1] = ((u64*)src)[1];
+ ((u64*)dst)[2] = ((u64*)src)[2];
+}
+#endif
+
/* add_image.c */
extern bool
/* resource.c */
#define WIMLIB_RESOURCE_FLAG_RAW 0x1
-#define WIMLIB_RESOURCE_FLAG_MULTITHREADED 0x2
+#define WIMLIB_RESOURCE_FLAG_THREADSAFE_READ 0x2
#define WIMLIB_RESOURCE_FLAG_RECOMPRESS 0x4
+//#define WIMLIB_RESOURCE_FLAG_OVERWRITE_INPLACE 0x8
extern int
read_resource_prefix(const struct wim_lookup_table_entry *lte,
#define WIMLIB_WRITE_FLAG_NO_LOOKUP_TABLE 0x80000000
#define WIMLIB_WRITE_FLAG_REUSE_INTEGRITY_TABLE 0x40000000
#define WIMLIB_WRITE_FLAG_CHECKPOINT_AFTER_XML 0x20000000
-#define WIMLIB_WRITE_MASK_PUBLIC 0x1fffffff
+//#define WIMLIB_WRITE_FLAG_OVERWRITE_INPLACE 0x10000000
+#define WIMLIB_WRITE_MASK_PUBLIC 0x0fffffff
/* We are capturing a tree to be placed in the root of the WIM image */
#define WIMLIB_ADD_IMAGE_FLAG_ROOT 0x80000000
lte->resource_location = RESOURCE_WIN32;
lte->resource_entry.original_size = (u64)dat->StreamSize.QuadPart;
- struct wim_lookup_table_entry **my_ptr;
+ struct wim_lookup_table_entry **back_ptr;
if (is_named_stream)
- my_ptr = &ads_entry->lte;
+ back_ptr = &ads_entry->lte;
else
- my_ptr = &inode->i_lte;
+ back_ptr = &inode->i_lte;
- lookup_table_insert_unhashed(lookup_table, lte, my_ptr);
+ lookup_table_insert_unhashed(lookup_table, lte, back_ptr);
out_free_spath:
FREE(spath);
out:
# define INVALID_HANDLE_VALUE ((HANDLE)(-1))
#endif
-static int
-fflush_and_ftruncate(FILE *fp, off_t size)
-{
- int ret;
-
- ret = fflush(fp);
- if (ret != 0) {
- ERROR_WITH_ERRNO("Failed to flush data to output WIM file");
- return WIMLIB_ERR_WRITE;
- }
- ret = ftruncate(fileno(fp), size);
- if (ret != 0) {
- ERROR_WITH_ERRNO("Failed to truncate output WIM file to "
- "%"PRIu64" bytes", size);
- return WIMLIB_ERR_WRITE;
- }
- return 0;
-}
-
/* Chunk table that's located at the beginning of each compressed resource in
* the WIM. (This is not the on-disk format; the on-disk format just has an
* array of offsets.) */
chunk_tab->table_disk_size) {
ERROR_WITH_ERRNO("Failed to write chunk table in compressed "
"file resource");
+ FREE(chunk_tab);
ret = WIMLIB_ERR_WRITE;
goto out;
}
ret = 0;
-out:
*chunk_tab_ret = chunk_tab;
+out:
return ret;
}
return 0;
}
-static int
-write_uncompressed_resource_and_truncate(struct wim_lookup_table_entry *lte,
- FILE *out_fp,
- off_t file_offset,
- struct resource_entry *out_res_entry)
-{
- int ret;
- if (fseeko(out_fp, file_offset, SEEK_SET) != 0) {
- ERROR_WITH_ERRNO("Failed to seek to byte %"PRIu64" of "
- "output WIM file", file_offset);
- return WIMLIB_ERR_WRITE;
- }
- ret = write_wim_resource(lte, out_fp,
- WIMLIB_COMPRESSION_TYPE_NONE,
- out_res_entry,
- 0);
- if (ret)
- return ret;
-
- return fflush_and_ftruncate(out_fp,
- file_offset + wim_resource_size(lte));
-}
-
struct write_resource_ctx {
compress_func_t compress;
struct chunk_table *chunk_tab;
}
}
+/*
+ * Write a resource to an output WIM.
+ *
+ * @lte: Lookup table entry for the resource, which could be in another WIM,
+ * in an external file, or in another location.
+ *
+ * @out_fp: FILE * opened to the output WIM.
+ *
+ * @out_ctype: One of the WIMLIB_COMPRESSION_TYPE_* constants to indicate
+ * which compression algorithm to use.
+ *
+ * @out_res_entry: On success, this is filled in with the offset, flags,
+ * compressed size, and uncompressed size of the resource
+ * in the output WIM.
+ *
+ * @flags: WIMLIB_RESOURCE_FLAG_RECOMPRESS to force data to be recompressed
+ * even if it could otherwise be copied directly from the input.
+ *
+ * Additional notes: The SHA1 message digest of the uncompressed data is
+ * calculated (except when doing a raw copy --- see below). If the @unhashed
+ * flag is set on the lookup table entry, this message digest is simply copied
+ * to it; otherwise, the message digest is compared with the existing one, and
+ * the function will fail if they do not match.
+ */
int
write_wim_resource(struct wim_lookup_table_entry *lte,
FILE *out_fp, int out_ctype,
off_t offset;
int ret;
+ flags &= ~WIMLIB_RESOURCE_FLAG_RECOMPRESS;
+
if (wim_resource_size(lte) == 0) {
/* Empty resource; nothing needs to be done, so just return
* success. */
return 0;
}
+ /* Get current position in output WIM */
offset = ftello(out_fp);
if (offset == -1) {
ERROR_WITH_ERRNO("Can't get position in output WIM");
return WIMLIB_ERR_WRITE;
}
- /* Can we simply copy the compressed data without recompressing it? */
-
+ /* If we are not forcing the data to be recompressed, and the input
+ * resource is located in a WIM with the same compression type as that
+ * desired other than no compression, we can simply copy the compressed
+ * data without recompressing it. This also means we must skip
+ * calculating the SHA1, as we never will see the uncompressed data. */
if (!(flags & WIMLIB_RESOURCE_FLAG_RECOMPRESS) &&
lte->resource_location == RESOURCE_IN_WIM &&
+ out_ctype != WIMLIB_COMPRESSION_TYPE_NONE &&
wimlib_get_compression_type(lte->wim) == out_ctype)
{
flags |= WIMLIB_RESOURCE_FLAG_RAW;
return ret;
}
- /* Write the data */
+ /* Write the entire resource by reading the entire resource and feeding
+ * the data through the write_resource_cb function. */
write_ctx.out_fp = out_fp;
+try_write_again:
ret = read_resource_prefix(lte, wim_resource_size(lte),
- write_resource_cb, &write_ctx, 0);
+ write_resource_cb, &write_ctx, flags);
/* Verify SHA1 message digest of the resource, or set the hash for the
* first time. */
if (new_size >= wim_resource_size(lte)) {
/* Oops! We compressed the resource to larger than the original
* size. Write the resource uncompressed instead. */
- ret = write_uncompressed_resource_and_truncate(lte,
- out_fp,
- offset,
- out_res_entry);
- goto out_free_chunk_tab;
+ if (fseeko(out_fp, offset, SEEK_SET) ||
+ fflush(out_fp) ||
+ ftruncate(fileno(out_fp),
+ offset + wim_resource_size(lte)))
+ {
+ ERROR_WITH_ERRNO("Failed to flush and/or truncate "
+ "output WIM file");
+ ret = WIMLIB_ERR_WRITE;
+ goto out_free_chunk_tab;
+ }
+ DEBUG("Compressed %"PRIu64" => %"PRIu64" bytes; "
+ "writing uncompressed instead",
+ wim_resource_size(lte), new_size);
+ write_ctx.compress = NULL;
+ write_ctx.doing_sha = false;
+ out_ctype = WIMLIB_COMPRESSION_TYPE_NONE;
+ goto try_write_again;
}
out_res_entry->flags |= WIM_RESHDR_FLAG_COMPRESSED;
}
};
static int
-do_write_stream_list(struct list_head *my_resources,
+do_write_stream_list(struct list_head *stream_list,
struct wim_lookup_table *lookup_table,
FILE *out_fp,
int out_ctype,
int ret;
struct wim_lookup_table_entry *lte;
- while (!list_empty(my_resources)) {
- lte = container_of(my_resources->next,
+ /* For each stream in @stream_list ... */
+ while (!list_empty(stream_list)) {
+ lte = container_of(stream_list->next,
struct wim_lookup_table_entry,
write_streams_list);
list_del(<e->write_streams_list);
if (lte->unhashed && !lte->unique_size) {
+
+ /* Unhashed stream that shares a size with some other
+ * stream in the WIM we are writing. The stream must be
+ * checksummed to know if we need to write it or not. */
struct wim_lookup_table_entry *duplicate_lte;
- struct wim_lookup_table_entry **my_ptr;
+ struct wim_lookup_table_entry **back_ptr;
- my_ptr = lte->my_ptr;
+ /* back_ptr must be saved because it's in union with the
+ * SHA1 message digest and will no longer be valid once
+ * the SHA1 has been calculated. */
+ back_ptr = lte->back_ptr;
+
+ /* Checksum the stream */
ret = sha1_resource(lte);
if (ret)
return ret;
+
+ /* Look for a duplicate stream */
duplicate_lte = __lookup_resource(lookup_table, lte->hash);
if (duplicate_lte) {
- bool new_stream = (duplicate_lte->out_refcnt == 0);
+ /* We have a duplicate stream. Transfer the
+ * reference counts from this stream to the
+ * duplicate, update the reference to this
+ * stream (in an inode or ads_entry) to point to
+ * the duplicate, then free this stream. */
+ wimlib_assert(!(duplicate_lte->unhashed));
+ bool is_new_stream = (duplicate_lte->out_refcnt == 0);
duplicate_lte->refcnt += lte->refcnt;
duplicate_lte->out_refcnt += lte->refcnt;
- *my_ptr = duplicate_lte;
+ *back_ptr = duplicate_lte;
+ list_del(<e->unhashed_list);
free_lookup_table_entry(lte);
lte = duplicate_lte;
- if (new_stream) {
- DEBUG("Stream of length %"PRIu64" is duplicate "
- "with one already in WIM",
- wim_resource_size(lte));
+
+ if (is_new_stream) {
+ /* The duplicate stream is one we
+ * weren't already planning to write.
+ * But, now we must write it.
+ *
+ * XXX: Currently, the copy of the
+ * stream in the WIM is always chosen
+ * for writing, rather than the extra
+ * copy we just read (which may be in an
+ * external file). This may not always
+ * be fastest. */
} else {
+ /* We have already written, or are going
+ * to write, the duplicate stream. So
+ * just skip to the next stream. */
DEBUG("Discarding duplicate stream of length %"PRIu64,
wim_resource_size(lte));
goto skip_to_progress;
}
} else {
+ /* No duplicate stream, so we need to insert
+ * this stream into the lookup table and treat
+ * it as a hashed stream. */
+ list_del(<e->unhashed_list);
lookup_table_insert(lookup_table, lte);
lte->out_refcnt = lte->refcnt;
lte->unhashed = 0;
}
}
+ /* Here, @lte either a hashed stream or an unhashed stream with
+ * a unique size. In either case we know that the stream has to
+ * be written. In either case the SHA1 message digest will be
+ * calculated over the stream while writing it; however, in the
+ * former case this is done merely to check the data, while in
+ * the latter case this is done because we do not have the SHA1
+ * message digest yet. */
+
wimlib_assert(lte->out_refcnt != 0);
ret = write_wim_resource(lte,
if (ret)
return ret;
if (lte->unhashed) {
+ list_del(<e->unhashed_list);
lookup_table_insert(lookup_table, lte);
lte->unhashed = 0;
}
wimlib_progress_func_t progress_func,
union wimlib_progress_info *progress)
{
- int write_resource_flags;
-
+ int write_resource_flags = 0;
if (write_flags & WIMLIB_WRITE_FLAG_RECOMPRESS)
- write_resource_flags = WIMLIB_RESOURCE_FLAG_RECOMPRESS;
- else
- write_resource_flags = 0;
+ write_resource_flags |= WIMLIB_RESOURCE_FLAG_RECOMPRESS;
+
progress->write_streams.num_threads = 1;
if (progress_func)
progress_func(WIMLIB_PROGRESS_MSG_WRITE_STREAMS, progress);
}
next_lte = container_of(next_resource,
struct wim_lookup_table_entry,
- staging_list);
+ write_streams_list);
next_resource = next_resource->next;
if ((!(write_flags & WIMLIB_WRITE_FLAG_RECOMPRESS)
&& wim_resource_compression_type(next_lte) == out_ctype)
|| wim_resource_size(next_lte) == 0)
{
- list_add_tail(&next_lte->staging_list,
+ list_add_tail(&next_lte->write_streams_list,
&my_resources);
} else {
- list_add_tail(&next_lte->staging_list,
+ list_add_tail(&next_lte->write_streams_list,
&outstanding_resources);
next_chunk = 0;
next_num_chunks = wim_resource_chunks(next_lte);
FREE(cur_chunk_tab);
cur_chunk_tab = NULL;
- struct list_head *next = cur_lte->staging_list.next;
- list_del(&cur_lte->staging_list);
+ struct list_head *next = cur_lte->write_streams_list.next;
+ list_del(&cur_lte->write_streams_list);
if (next == &outstanding_resources)
cur_lte = NULL;
else
- cur_lte = container_of(cur_lte->staging_list.next,
+ cur_lte = container_of(cur_lte->write_streams_list.next,
struct wim_lookup_table_entry,
- staging_list);
+ write_streams_list);
// Since we just finished writing a stream,
// write any streams that have been added to the
struct lte_overwrite_prepare_args {
WIMStruct *wim;
off_t end_offset;
- struct list_head *stream_list;
- struct stream_size_table *stream_size_tab;
+ struct list_head stream_list;
+ struct stream_size_table stream_size_tab;
};
static int
return WIMLIB_ERR_RESOURCE_ORDER;
}
} else {
- if (!(lte->resource_entry.flags & WIM_RESHDR_FLAG_METADATA))
- list_add_tail(<e->write_streams_list, args->stream_list);
+ wimlib_assert(!(lte->resource_entry.flags & WIM_RESHDR_FLAG_METADATA));
+ list_add_tail(<e->write_streams_list, &args->stream_list);
}
lte->out_refcnt = lte->refcnt;
- stream_size_table_insert(lte, args->stream_size_tab);
+ stream_size_table_insert(lte, &args->stream_size_tab);
return 0;
}
static int
lte_set_output_res_entry(struct wim_lookup_table_entry *lte, void *_wim)
{
- if (lte->resource_location == RESOURCE_IN_WIM &&
- lte->wim == _wim)
- {
- memcpy(<e->output_resource_entry, <e->resource_entry,
- sizeof(struct resource_entry));
+ if (lte->resource_location == RESOURCE_IN_WIM && lte->wim == _wim) {
+ copy_resource_entry(<e->output_resource_entry,
+ <e->resource_entry);
}
return 0;
}
struct list_head *stream_list)
{
int ret;
- struct stream_size_table stream_size_tab;
- struct lte_overwrite_prepare_args args = {
- .wim = wim,
- .end_offset = end_offset,
- .stream_list = stream_list,
- .stream_size_tab = &stream_size_tab,
- };
-
- ret = init_stream_size_table(&stream_size_tab, 9001);
+ struct lte_overwrite_prepare_args args;
+
+ args.wim = wim;
+ args.end_offset = end_offset;
+ ret = init_stream_size_table(&args.stream_size_tab,
+ wim->lookup_table->capacity);
if (ret)
return ret;
- INIT_LIST_HEAD(stream_list);
+ INIT_LIST_HEAD(&args.stream_list);
for (int i = 0; i < wim->hdr.image_count; i++) {
struct wim_image_metadata *imd;
struct wim_lookup_table_entry *lte;
for (int i = 0; i < wim->hdr.image_count; i++)
lte_set_output_res_entry(wim->image_metadata[i]->metadata_lte,
wim);
- ret = for_lookup_table_entry(wim->lookup_table,
- lte_set_output_res_entry, wim);
+ for_lookup_table_entry(wim->lookup_table, lte_set_output_res_entry, wim);
+ INIT_LIST_HEAD(stream_list);
+ list_splice(&args.stream_list, stream_list);
out_destroy_stream_size_table:
- destroy_stream_size_table(&stream_size_tab);
+ destroy_stream_size_table(&args.stream_size_tab);
return ret;
}
struct stream_size_table stream_size_tab;
};
-static int
+static void
inode_find_streams_to_write(struct wim_inode *inode,
struct wim_lookup_table *table,
struct list_head *stream_list,
lte->out_refcnt += inode->i_nlink;
}
}
- return 0;
}
static int
image_for_each_unhashed_stream(lte, imd) {
lte->out_refcnt = 0;
wimlib_assert(lte->unhashed);
- wimlib_assert(lte->my_ptr != NULL);
+ wimlib_assert(lte->back_ptr != NULL);
}
/* Go through this image's inodes to find any streams that have not been
struct find_streams_ctx ctx;
for_lookup_table_entry(wim->lookup_table, lte_zero_out_refcnt, NULL);
- ret = init_stream_size_table(&ctx.stream_size_tab, 9001);
+ ret = init_stream_size_table(&ctx.stream_size_tab,
+ wim->lookup_table->capacity);
if (ret)
return ret;
for_lookup_table_entry(wim->lookup_table, stream_size_table_insert,
&ctx.stream_size_tab);
INIT_LIST_HEAD(&ctx.stream_list);
wim->private = &ctx;
- for_image(wim, image, image_find_streams_to_write);
+ ret = for_image(wim, image, image_find_streams_to_write);
destroy_stream_size_table(&ctx.stream_size_tab);
-
- INIT_LIST_HEAD(stream_list);
- list_splice(&ctx.stream_list, stream_list);
- return 0;
+ if (ret == 0) {
+ INIT_LIST_HEAD(stream_list);
+ list_splice(&ctx.stream_list, stream_list);
+ }
+ return ret;
}
/* Writes the streams for the specified @image in @wim to @wim->out_fp.
int ret;
ret = open_wim_writable(w, path, true,
(write_flags & WIMLIB_WRITE_FLAG_CHECK_INTEGRITY) != 0);
- if (ret != 0)
+ if (ret)
return ret;
/* Write dummy header. It will be overwritten later. */
return write_header(&w->hdr, w->out_fp);
w->lookup_table,
w->out_fp,
wimlib_get_compression_type(w),
- write_flags, num_threads,
+ write_flags,
+ num_threads,
progress_func);
if (ret)
goto out_ftruncate;
ret = read_huffsym(istream, decode_table, lens,
XPRESS_NUM_SYMBOLS, XPRESS_TABLEBITS,
&huffsym, XPRESS_MAX_CODEWORD_LEN);
- if (ret != 0)
+ if (ret)
return ret;
if (huffsym < XPRESS_NUM_CHARS) {
* code lengths of these symbols are given literally as 4-bit integers
* in the first 256 bytes of the compressed data.
*/
- if (compressed_len < XPRESS_NUM_SYMBOLS / 2)
+ if (compressed_len < XPRESS_NUM_SYMBOLS / 2) {
+ ERROR("xpress_decompress(): Compressed length too short!");
return -1;
+ }
for (i = 0; i < XPRESS_NUM_SYMBOLS / 2; i++) {
*lens_p++ = compressed_data[i] & 0xf;
ret = make_huffman_decode_table(decode_table, XPRESS_NUM_SYMBOLS,
XPRESS_TABLEBITS, lens,
XPRESS_MAX_CODEWORD_LEN);
- if (ret != 0)
+ if (ret)
return ret;
init_input_bitstream(&istream, compressed_data + XPRESS_NUM_SYMBOLS / 2,
git://wimlib.net / wimlib / commitdiff