*/
/*
- * Copyright (C) 2012, 2013 Eric Biggers
+ * Copyright (C) 2012, 2013, 2014 Eric Biggers
*
* This file is part of wimlib, a library for working with WIM files.
*
#include "wimlib/integrity.h"
#include "wimlib/lookup_table.h"
#include "wimlib/metadata.h"
+#include "wimlib/paths.h"
#include "wimlib/progress.h"
#include "wimlib/resource.h"
#ifdef __WIN32__
#define WRITE_RESOURCE_FLAG_RECOMPRESS 0x00000001
#define WRITE_RESOURCE_FLAG_PIPABLE 0x00000002
#define WRITE_RESOURCE_FLAG_PACK_STREAMS 0x00000004
+#define WRITE_RESOURCE_FLAG_SEND_DONE_WITH_FILE 0x00000008
static inline int
write_flags_to_resource_flags(int write_flags)
write_resource_flags |= WRITE_RESOURCE_FLAG_PIPABLE;
if (write_flags & WIMLIB_WRITE_FLAG_PACK_STREAMS)
write_resource_flags |= WRITE_RESOURCE_FLAG_PACK_STREAMS;
+ if (write_flags & WIMLIB_WRITE_FLAG_SEND_DONE_WITH_FILE_MESSAGES)
+ write_resource_flags |= WRITE_RESOURCE_FLAG_SEND_DONE_WITH_FILE;
return write_resource_flags;
}
stream_filtered(const struct wim_lookup_table_entry *lte,
const struct filter_context *ctx)
{
- int write_flags = ctx->write_flags;
- WIMStruct *wim = ctx->wim;
+ int write_flags;
+ WIMStruct *wim;
if (ctx == NULL)
return 0;
+ write_flags = ctx->write_flags;
+ wim = ctx->wim;
+
if (write_flags & WIMLIB_WRITE_FLAG_OVERWRITE &&
lte->resource_location == RESOURCE_IN_WIM &&
lte->rspec->wim == wim)
if (progress_data->next_progress == progress->write_streams.total_bytes) {
progress_data->next_progress = ~(uint64_t)0;
} else {
+ /* Handle rate-limiting of messages */
+
+ /* Send new message as soon as another 1/128 of the
+ * total has been written. (Arbitrary number.) */
progress_data->next_progress =
- min(progress->write_streams.total_bytes,
- progress->write_streams.completed_bytes +
- progress->write_streams.total_bytes / 100);
+ progress->write_streams.completed_bytes +
+ progress->write_streams.total_bytes / 128;
+
+ /* ... Unless that would be more than 5000000 bytes, in
+ * which case send the next after the next 5000000
+ * bytes. (Another arbitrary number.) */
+ if (progress->write_streams.completed_bytes + 5000000 <
+ progress_data->next_progress)
+ progress_data->next_progress =
+ progress->write_streams.completed_bytes + 5000000;
+
+ /* ... But always send a message as soon as we're
+ * completely done. */
+ if (progress->write_streams.total_bytes <
+ progress_data->next_progress)
+ progress_data->next_progress =
+ progress->write_streams.total_bytes;
}
}
return 0;
* pending_streams rather than the entry being read.) */
bool stream_was_duplicate;
+ struct wim_inode *stream_inode;
+
/* Current uncompressed offset in the stream being read. */
u64 cur_read_stream_offset;
return 0;
}
+/* No more data streams of the file at @path are needed. */
+static int
+done_with_file(const tchar *path, wimlib_progress_func_t progfunc, void *progctx)
+{
+ union wimlib_progress_info info;
+
+ info.done_with_file.path_to_file = path;
+
+ return call_progress(progfunc, WIMLIB_PROGRESS_MSG_DONE_WITH_FILE,
+ &info, progctx);
+}
+
+/* Handle WIMLIB_WRITE_FLAG_SEND_DONE_WITH_FILE_MESSAGES mode. */
+static int
+done_with_stream(struct wim_lookup_table_entry *stream,
+ wimlib_progress_func_t progfunc, void *progctx,
+ struct wim_inode *inode)
+{
+ if (stream->resource_location == RESOURCE_IN_FILE_ON_DISK
+#ifdef __WIN32__
+ || stream->resource_location == RESOURCE_IN_WINNT_FILE_ON_DISK
+ || stream->resource_location == RESOURCE_WIN32_ENCRYPTED
+#endif
+ )
+ {
+ int ret;
+
+ wimlib_assert(inode->num_unread_streams > 0);
+ if (--inode->num_unread_streams > 0)
+ return 0;
+
+ #ifdef __WIN32__
+ /* XXX: This logic really should be somewhere else. */
+
+ /* We want the path to the file, but stream->file_on_disk might
+ * actually refer to a named data stream. Temporarily strip the
+ * named data stream from the path. */
+ wchar_t *p_colon = NULL;
+ wchar_t *p_question_mark = NULL;
+ const wchar_t *p_stream_name;
+
+ p_stream_name = path_stream_name(stream->file_on_disk);
+ if (unlikely(p_stream_name)) {
+ p_colon = (wchar_t *)(p_stream_name - 1);
+ wimlib_assert(*p_colon == L':');
+ *p_colon = L'\0';
+ }
+
+ /* We also should use a fake Win32 path instead of a NT path */
+ if (!wcsncmp(stream->file_on_disk, L"\\??\\", 4)) {
+ p_question_mark = &stream->file_on_disk[1];
+ *p_question_mark = L'\\';
+ }
+ #endif
+
+ ret = done_with_file(stream->file_on_disk, progfunc, progctx);
+
+ #ifdef __WIN32__
+ if (p_colon)
+ *p_colon = L':';
+ if (p_question_mark)
+ *p_question_mark = L'?';
+ #endif
+ return ret;
+ }
+ return 0;
+}
+
/* Begin processing a stream for writing. */
static int
-write_stream_begin_read(struct wim_lookup_table_entry *lte,
- u32 flags, void *_ctx)
+write_stream_begin_read(struct wim_lookup_table_entry *lte, void *_ctx)
{
struct write_streams_ctx *ctx = _ctx;
int ret;
ctx->cur_read_stream_offset = 0;
ctx->cur_read_stream_size = lte->size;
+ if (lte->unhashed)
+ ctx->stream_inode = lte->back_inode;
+ else
+ ctx->stream_inode = NULL;
+
/* As an optimization, we allow some streams to be "unhashed", meaning
* their SHA1 message digests are unknown. This is the case with
* streams that are added by scanning a directry tree with
ctx->stream_was_duplicate = false;
if (ctx->lookup_table != NULL && lte->unhashed && !lte->unique_size) {
- wimlib_assert(!(flags & BEGIN_STREAM_FLAG_PARTIAL_RESOURCE));
-
struct wim_lookup_table_entry *lte_new;
ret = hash_unhashed_stream(lte, ctx->lookup_table, <e_new);
lte_new->out_refcnt += lte->out_refcnt;
if (ctx->write_resource_flags & WRITE_RESOURCE_FLAG_PACK_STREAMS)
ctx->cur_write_res_size -= lte->size;
+ if (!ret && unlikely(ctx->write_resource_flags &
+ WRITE_RESOURCE_FLAG_SEND_DONE_WITH_FILE))
+ {
+ ret = done_with_stream(lte,
+ ctx->progress_data.progfunc,
+ ctx->progress_data.progctx,
+ ctx->stream_inode);
+ }
free_lookup_table_entry(lte);
if (ret)
return ret;
if (ctx->compressor != NULL &&
lte->out_reshdr.size_in_wim >= lte->out_reshdr.uncompressed_size &&
- !(ctx->write_resource_flags & WRITE_RESOURCE_FLAG_PIPABLE) &&
+ !(ctx->write_resource_flags & (WRITE_RESOURCE_FLAG_PIPABLE |
+ WRITE_RESOURCE_FLAG_SEND_DONE_WITH_FILE)) &&
!(lte->flags & WIM_RESHDR_FLAG_PACKED_STREAMS))
{
/* Stream did not compress to less than its original
write_stream_end_read(struct wim_lookup_table_entry *lte, int status, void *_ctx)
{
struct write_streams_ctx *ctx = _ctx;
- if (status == 0)
- wimlib_assert(ctx->cur_read_stream_offset == ctx->cur_read_stream_size);
- if (ctx->stream_was_duplicate) {
- free_lookup_table_entry(lte);
- } else if (lte->unhashed && ctx->lookup_table != NULL) {
+ if (status)
+ goto out;
+
+ wimlib_assert(ctx->cur_read_stream_offset == ctx->cur_read_stream_size);
+
+ if (unlikely(ctx->write_resource_flags &
+ WRITE_RESOURCE_FLAG_SEND_DONE_WITH_FILE) &&
+ ctx->stream_inode != NULL)
+ {
+ status = done_with_stream(lte,
+ ctx->progress_data.progfunc,
+ ctx->progress_data.progctx,
+ ctx->stream_inode);
+ }
+
+ if (!ctx->stream_was_duplicate && lte->unhashed &&
+ ctx->lookup_table != NULL)
+ {
list_del(<e->unhashed_list);
lookup_table_insert(ctx->lookup_table, lte);
lte->unhashed = 0;
}
+out:
+ if (ctx->stream_was_duplicate)
+ free_lookup_table_entry(lte);
return status;
}
}
}
+ /* If needed, set auxiliary information so that we can detect when
+ * wimlib has finished using each external file. */
+ if (unlikely(write_flags & WIMLIB_WRITE_FLAG_SEND_DONE_WITH_FILE_MESSAGES)) {
+ list_for_each_entry(lte, stream_list, write_streams_list)
+ if (lte->unhashed)
+ lte->back_inode->num_unread_streams = 0;
+ list_for_each_entry(lte, stream_list, write_streams_list)
+ if (lte->unhashed)
+ lte->back_inode->num_unread_streams++;
+ }
+
return wim_write_stream_list(wim,
stream_list,
write_flags,
* (private) WIMLIB_WRITE_FLAG_NO_LOOKUP_TABLE:
* Don't write the lookup table.
*
- * (private) WIMLIB_WRITE_FLAG_REUSE_INTEGRITY_TABLE:
- * When (if) writing the integrity table, re-use entries from the
- * existing integrity table, if possible.
- *
* (private) WIMLIB_WRITE_FLAG_CHECKPOINT_AFTER_XML:
* After writing the XML data but before writing the integrity
* table, write a temporary WIM header and flush the stream so that
* Use the existing <TOTALBYTES> stored in the in-memory XML
* information, rather than setting it to the offset of the XML
* data being written.
+ * (private) WIMLIB_WRITE_FLAG_OVERWRITE
+ * The existing WIM file is being updated in-place. The entries
+ * from its integrity table may be re-used.
*/
static int
finish_write(WIMStruct *wim, int image, int write_flags,
int ret;
off_t hdr_offset;
int write_resource_flags;
- off_t old_lookup_table_end;
+ off_t old_lookup_table_end = 0;
off_t new_lookup_table_end;
u64 xml_totalbytes;
+ struct integrity_table *old_integrity_table = NULL;
DEBUG("image=%d, write_flags=%08x", image, write_flags);
wim->hdr.boot_idx - 1]->metadata_lte->out_reshdr);
}
- /* Write lookup table. (Save old position first.) */
- old_lookup_table_end = wim->hdr.lookup_table_reshdr.offset_in_wim +
- wim->hdr.lookup_table_reshdr.size_in_wim;
+ /* If overwriting the WIM file containing an integrity table in-place,
+ * we'd like to re-use the information in the old integrity table
+ * instead of recalculating it. But we might overwrite the old
+ * integrity table when we expand the XML data. Read it into memory
+ * just in case. */
+ if ((write_flags & (WIMLIB_WRITE_FLAG_OVERWRITE |
+ WIMLIB_WRITE_FLAG_CHECK_INTEGRITY)) ==
+ (WIMLIB_WRITE_FLAG_OVERWRITE |
+ WIMLIB_WRITE_FLAG_CHECK_INTEGRITY)
+ && wim_has_integrity_table(wim))
+ {
+ old_lookup_table_end = wim->hdr.lookup_table_reshdr.offset_in_wim +
+ wim->hdr.lookup_table_reshdr.size_in_wim;
+ (void)read_integrity_table(wim,
+ old_lookup_table_end - WIM_HEADER_DISK_SIZE,
+ &old_integrity_table);
+ /* If we couldn't read the old integrity table, we can still
+ * re-calculate the full integrity table ourselves. Hence the
+ * ignoring of the return value. */
+ }
+
+ /* Write lookup table. */
if (!(write_flags & WIMLIB_WRITE_FLAG_NO_LOOKUP_TABLE)) {
ret = write_wim_lookup_table(wim, image, write_flags,
&wim->hdr.lookup_table_reshdr,
lookup_table_list);
- if (ret)
+ if (ret) {
+ free_integrity_table(old_integrity_table);
return ret;
+ }
}
/* Write XML data. */
ret = write_wim_xml_data(wim, image, xml_totalbytes,
&wim->hdr.xml_data_reshdr,
write_resource_flags);
- if (ret)
+ if (ret) {
+ free_integrity_table(old_integrity_table);
return ret;
+ }
/* Write integrity table (optional). */
if (write_flags & WIMLIB_WRITE_FLAG_CHECK_INTEGRITY) {
checkpoint_hdr.flags |= WIM_HDR_FLAG_WRITE_IN_PROGRESS;
ret = write_wim_header_at_offset(&checkpoint_hdr,
&wim->out_fd, 0);
- if (ret)
+ if (ret) {
+ free_integrity_table(old_integrity_table);
return ret;
- }
-
- if (!(write_flags & WIMLIB_WRITE_FLAG_REUSE_INTEGRITY_TABLE))
- old_lookup_table_end = 0;
-
- if (wim->hdr.integrity_table_reshdr.offset_in_wim <
- wim->hdr.xml_data_reshdr.offset_in_wim +
- wim->hdr.xml_data_reshdr.size_in_wim)
- {
- /* Old integrity table was partially overwritten by the
- * XML data. */
- old_lookup_table_end = 0;
+ }
}
new_lookup_table_end = wim->hdr.lookup_table_reshdr.offset_in_wim +
ret = write_integrity_table(wim,
new_lookup_table_end,
- old_lookup_table_end);
+ old_lookup_table_end,
+ old_integrity_table);
+ free_integrity_table(old_integrity_table);
if (ret)
return ret;
} else {
if (ret)
goto out_truncate;
- write_flags |= WIMLIB_WRITE_FLAG_REUSE_INTEGRITY_TABLE;
ret = finish_write(wim, WIMLIB_ALL_IMAGES, write_flags,
&lookup_table_list);
if (ret)