*/
/*
- * Copyright (C) 2012, 2013 Eric Biggers
+ * Copyright (C) 2012, 2013, 2014 Eric Biggers
*
* This file is part of wimlib, a library for working with WIM files.
*
#include "wimlib/integrity.h"
#include "wimlib/lookup_table.h"
#include "wimlib/metadata.h"
+#include "wimlib/paths.h"
#include "wimlib/progress.h"
#include "wimlib/resource.h"
#ifdef __WIN32__
#define WRITE_RESOURCE_FLAG_RECOMPRESS 0x00000001
#define WRITE_RESOURCE_FLAG_PIPABLE 0x00000002
#define WRITE_RESOURCE_FLAG_PACK_STREAMS 0x00000004
+#define WRITE_RESOURCE_FLAG_SEND_DONE_WITH_FILE 0x00000008
static inline int
write_flags_to_resource_flags(int write_flags)
write_resource_flags |= WRITE_RESOURCE_FLAG_PIPABLE;
if (write_flags & WIMLIB_WRITE_FLAG_PACK_STREAMS)
write_resource_flags |= WRITE_RESOURCE_FLAG_PACK_STREAMS;
+ if (write_flags & WIMLIB_WRITE_FLAG_SEND_DONE_WITH_FILE_MESSAGES)
+ write_resource_flags |= WRITE_RESOURCE_FLAG_SEND_DONE_WITH_FILE;
return write_resource_flags;
}
stream_filtered(const struct wim_lookup_table_entry *lte,
const struct filter_context *ctx)
{
- int write_flags = ctx->write_flags;
- WIMStruct *wim = ctx->wim;
+ int write_flags;
+ WIMStruct *wim;
if (ctx == NULL)
return 0;
+ write_flags = ctx->write_flags;
+ wim = ctx->wim;
+
if (write_flags & WIMLIB_WRITE_FLAG_OVERWRITE &&
lte->resource_location == RESOURCE_IN_WIM &&
lte->rspec->wim == wim)
if (progress_data->next_progress == progress->write_streams.total_bytes) {
progress_data->next_progress = ~(uint64_t)0;
} else {
+ /* Handle rate-limiting of messages */
+
+ /* Send new message as soon as another 1/128 of the
+ * total has been written. (Arbitrary number.) */
progress_data->next_progress =
- min(progress->write_streams.total_bytes,
- progress->write_streams.completed_bytes +
- progress->write_streams.total_bytes / 100);
+ progress->write_streams.completed_bytes +
+ progress->write_streams.total_bytes / 128;
+
+ /* ... Unless that would be more than 5000000 bytes, in
+ * which case send the next after the next 5000000
+ * bytes. (Another arbitrary number.) */
+ if (progress->write_streams.completed_bytes + 5000000 <
+ progress_data->next_progress)
+ progress_data->next_progress =
+ progress->write_streams.completed_bytes + 5000000;
+
+ /* ... But always send a message as soon as we're
+ * completely done. */
+ if (progress->write_streams.total_bytes <
+ progress_data->next_progress)
+ progress_data->next_progress =
+ progress->write_streams.total_bytes;
}
}
return 0;
* pending_streams rather than the entry being read.) */
bool stream_was_duplicate;
+ struct wim_inode *stream_inode;
+
/* Current uncompressed offset in the stream being read. */
u64 cur_read_stream_offset;
reserve_size = expected_num_chunk_entries *
get_chunk_entry_size(res_expected_size,
0 != (ctx->write_resource_flags &
- WIM_RESHDR_FLAG_PACKED_STREAMS));
+ WRITE_RESOURCE_FLAG_PACK_STREAMS));
if (ctx->write_resource_flags & WRITE_RESOURCE_FLAG_PACK_STREAMS)
reserve_size += sizeof(struct alt_chunk_table_header_disk);
memset(ctx->chunk_csizes, 0, reserve_size);
chunk_entry_size = get_chunk_entry_size(res_actual_size,
0 != (ctx->write_resource_flags &
- WIM_RESHDR_FLAG_PACKED_STREAMS));
+ WRITE_RESOURCE_FLAG_PACK_STREAMS));
typedef le64 __attribute__((may_alias)) aliased_le64_t;
typedef le32 __attribute__((may_alias)) aliased_le32_t;
return 0;
}
+/* No more data streams of the file at @path are needed. */
+static int
+done_with_file(const tchar *path, wimlib_progress_func_t progfunc, void *progctx)
+{
+ union wimlib_progress_info info;
+
+ info.done_with_file.path_to_file = path;
+
+ return call_progress(progfunc, WIMLIB_PROGRESS_MSG_DONE_WITH_FILE,
+ &info, progctx);
+}
+
+/* Handle WIMLIB_WRITE_FLAG_SEND_DONE_WITH_FILE_MESSAGES mode. */
+static int
+done_with_stream(struct wim_lookup_table_entry *stream,
+ wimlib_progress_func_t progfunc, void *progctx,
+ struct wim_inode *inode)
+{
+ if (stream->resource_location == RESOURCE_IN_FILE_ON_DISK
+#ifdef __WIN32__
+ || stream->resource_location == RESOURCE_IN_WINNT_FILE_ON_DISK
+ || stream->resource_location == RESOURCE_WIN32_ENCRYPTED
+#endif
+ )
+ {
+ int ret;
+
+ wimlib_assert(inode->num_unread_streams > 0);
+ if (--inode->num_unread_streams > 0)
+ return 0;
+
+ #ifdef __WIN32__
+ /* XXX: This logic really should be somewhere else. */
+
+ /* We want the path to the file, but stream->file_on_disk might
+ * actually refer to a named data stream. Temporarily strip the
+ * named data stream from the path. */
+ wchar_t *p_colon = NULL;
+ wchar_t *p_question_mark = NULL;
+ const wchar_t *p_stream_name;
+
+ p_stream_name = path_stream_name(stream->file_on_disk);
+ if (unlikely(p_stream_name)) {
+ p_colon = (wchar_t *)(p_stream_name - 1);
+ wimlib_assert(*p_colon == L':');
+ *p_colon = L'\0';
+ }
+
+ /* We also should use a fake Win32 path instead of a NT path */
+ if (!wcsncmp(stream->file_on_disk, L"\\??\\", 4)) {
+ p_question_mark = &stream->file_on_disk[1];
+ *p_question_mark = L'\\';
+ }
+ #endif
+
+ ret = done_with_file(stream->file_on_disk, progfunc, progctx);
+
+ #ifdef __WIN32__
+ if (p_colon)
+ *p_colon = L':';
+ if (p_question_mark)
+ *p_question_mark = L'?';
+ #endif
+ return ret;
+ }
+ return 0;
+}
+
/* Begin processing a stream for writing. */
static int
-write_stream_begin_read(struct wim_lookup_table_entry *lte,
- u32 flags, void *_ctx)
+write_stream_begin_read(struct wim_lookup_table_entry *lte, void *_ctx)
{
struct write_streams_ctx *ctx = _ctx;
int ret;
ctx->cur_read_stream_offset = 0;
ctx->cur_read_stream_size = lte->size;
+ if (lte->unhashed)
+ ctx->stream_inode = lte->back_inode;
+ else
+ ctx->stream_inode = NULL;
+
/* As an optimization, we allow some streams to be "unhashed", meaning
* their SHA1 message digests are unknown. This is the case with
* streams that are added by scanning a directry tree with
ctx->stream_was_duplicate = false;
if (ctx->lookup_table != NULL && lte->unhashed && !lte->unique_size) {
- wimlib_assert(!(flags & BEGIN_STREAM_FLAG_PARTIAL_RESOURCE));
-
struct wim_lookup_table_entry *lte_new;
ret = hash_unhashed_stream(lte, ctx->lookup_table, <e_new);
lte_new->out_refcnt += lte->out_refcnt;
if (ctx->write_resource_flags & WRITE_RESOURCE_FLAG_PACK_STREAMS)
ctx->cur_write_res_size -= lte->size;
+ if (!ret && unlikely(ctx->write_resource_flags &
+ WRITE_RESOURCE_FLAG_SEND_DONE_WITH_FILE))
+ {
+ ret = done_with_stream(lte,
+ ctx->progress_data.progfunc,
+ ctx->progress_data.progctx,
+ ctx->stream_inode);
+ }
free_lookup_table_entry(lte);
if (ret)
return ret;
if (ctx->compressor != NULL &&
lte->out_reshdr.size_in_wim >= lte->out_reshdr.uncompressed_size &&
- !(ctx->write_resource_flags & WRITE_RESOURCE_FLAG_PIPABLE) &&
+ !(ctx->write_resource_flags & (WRITE_RESOURCE_FLAG_PIPABLE |
+ WRITE_RESOURCE_FLAG_SEND_DONE_WITH_FILE)) &&
!(lte->flags & WIM_RESHDR_FLAG_PACKED_STREAMS))
{
/* Stream did not compress to less than its original
write_stream_end_read(struct wim_lookup_table_entry *lte, int status, void *_ctx)
{
struct write_streams_ctx *ctx = _ctx;
- if (status == 0)
- wimlib_assert(ctx->cur_read_stream_offset == ctx->cur_read_stream_size);
- if (ctx->stream_was_duplicate) {
- free_lookup_table_entry(lte);
- } else if (lte->unhashed && ctx->lookup_table != NULL) {
+ if (status)
+ goto out;
+
+ wimlib_assert(ctx->cur_read_stream_offset == ctx->cur_read_stream_size);
+
+ if (unlikely(ctx->write_resource_flags &
+ WRITE_RESOURCE_FLAG_SEND_DONE_WITH_FILE) &&
+ ctx->stream_inode != NULL)
+ {
+ status = done_with_stream(lte,
+ ctx->progress_data.progfunc,
+ ctx->progress_data.progctx,
+ ctx->stream_inode);
+ }
+
+ if (!ctx->stream_was_duplicate && lte->unhashed &&
+ ctx->lookup_table != NULL)
+ {
list_del(<e->unhashed_list);
lookup_table_insert(ctx->lookup_table, lte);
lte->unhashed = 0;
}
+out:
+ if (ctx->stream_was_duplicate)
+ free_lookup_table_entry(lte);
return status;
}
*
* @out_ctype
* Compression format to use to write the output streams, specified as one
- * of the WIMLIB_COMPRESSION_TYPE_* constants, excepting
- * WIMLIB_COMPRESSION_TYPE_INVALID but including
- * WIMLIB_COMPRESSION_TYPE_NONE.
+ * of the WIMLIB_COMPRESSION_TYPE_* constants.
+ * WIMLIB_COMPRESSION_TYPE_NONE is allowed.
*
* @out_chunk_size
* Chunk size to use to write the streams. It must be a valid chunk size
write_resource_flags = write_flags_to_resource_flags(write_flags);
- /* wimlib v1.6.3: pack streams by default if the WIM version has been
+ /* wimlib v1.7.0: pack streams by default if the WIM version has been
* set to WIM_VERSION_PACKED_STREAMS and at least one stream in the
* WIM's lookup table is located in a packed resource (may be the same
* WIM, or a different one in the case of export). */
}
}
+ /* If needed, set auxiliary information so that we can detect when
+ * wimlib has finished using each external file. */
+ if (unlikely(write_flags & WIMLIB_WRITE_FLAG_SEND_DONE_WITH_FILE_MESSAGES)) {
+ list_for_each_entry(lte, stream_list, write_streams_list)
+ if (lte->unhashed)
+ lte->back_inode->num_unread_streams = 0;
+ list_for_each_entry(lte, stream_list, write_streams_list)
+ if (lte->unhashed)
+ lte->back_inode->num_unread_streams++;
+ }
+
return wim_write_stream_list(wim,
stream_list,
write_flags,
* (private) WIMLIB_WRITE_FLAG_NO_LOOKUP_TABLE:
* Don't write the lookup table.
*
- * (private) WIMLIB_WRITE_FLAG_REUSE_INTEGRITY_TABLE:
- * When (if) writing the integrity table, re-use entries from the
- * existing integrity table, if possible.
- *
* (private) WIMLIB_WRITE_FLAG_CHECKPOINT_AFTER_XML:
* After writing the XML data but before writing the integrity
* table, write a temporary WIM header and flush the stream so that
* Use the existing <TOTALBYTES> stored in the in-memory XML
* information, rather than setting it to the offset of the XML
* data being written.
+ * (private) WIMLIB_WRITE_FLAG_OVERWRITE
+ * The existing WIM file is being updated in-place. The entries
+ * from its integrity table may be re-used.
*/
static int
finish_write(WIMStruct *wim, int image, int write_flags,
int ret;
off_t hdr_offset;
int write_resource_flags;
- off_t old_lookup_table_end;
+ off_t old_lookup_table_end = 0;
off_t new_lookup_table_end;
u64 xml_totalbytes;
+ struct integrity_table *old_integrity_table = NULL;
DEBUG("image=%d, write_flags=%08x", image, write_flags);
wim->hdr.boot_idx - 1]->metadata_lte->out_reshdr);
}
- /* Write lookup table. (Save old position first.) */
- old_lookup_table_end = wim->hdr.lookup_table_reshdr.offset_in_wim +
- wim->hdr.lookup_table_reshdr.size_in_wim;
+ /* If overwriting the WIM file containing an integrity table in-place,
+ * we'd like to re-use the information in the old integrity table
+ * instead of recalculating it. But we might overwrite the old
+ * integrity table when we expand the XML data. Read it into memory
+ * just in case. */
+ if ((write_flags & (WIMLIB_WRITE_FLAG_OVERWRITE |
+ WIMLIB_WRITE_FLAG_CHECK_INTEGRITY)) ==
+ (WIMLIB_WRITE_FLAG_OVERWRITE |
+ WIMLIB_WRITE_FLAG_CHECK_INTEGRITY)
+ && wim_has_integrity_table(wim))
+ {
+ old_lookup_table_end = wim->hdr.lookup_table_reshdr.offset_in_wim +
+ wim->hdr.lookup_table_reshdr.size_in_wim;
+ (void)read_integrity_table(wim,
+ old_lookup_table_end - WIM_HEADER_DISK_SIZE,
+ &old_integrity_table);
+ /* If we couldn't read the old integrity table, we can still
+ * re-calculate the full integrity table ourselves. Hence the
+ * ignoring of the return value. */
+ }
+
+ /* Write lookup table. */
if (!(write_flags & WIMLIB_WRITE_FLAG_NO_LOOKUP_TABLE)) {
ret = write_wim_lookup_table(wim, image, write_flags,
&wim->hdr.lookup_table_reshdr,
lookup_table_list);
- if (ret)
+ if (ret) {
+ free_integrity_table(old_integrity_table);
return ret;
+ }
}
/* Write XML data. */
ret = write_wim_xml_data(wim, image, xml_totalbytes,
&wim->hdr.xml_data_reshdr,
write_resource_flags);
- if (ret)
+ if (ret) {
+ free_integrity_table(old_integrity_table);
return ret;
+ }
/* Write integrity table (optional). */
if (write_flags & WIMLIB_WRITE_FLAG_CHECK_INTEGRITY) {
checkpoint_hdr.flags |= WIM_HDR_FLAG_WRITE_IN_PROGRESS;
ret = write_wim_header_at_offset(&checkpoint_hdr,
&wim->out_fd, 0);
- if (ret)
+ if (ret) {
+ free_integrity_table(old_integrity_table);
return ret;
+ }
}
- if (!(write_flags & WIMLIB_WRITE_FLAG_REUSE_INTEGRITY_TABLE))
- old_lookup_table_end = 0;
-
new_lookup_table_end = wim->hdr.lookup_table_reshdr.offset_in_wim +
wim->hdr.lookup_table_reshdr.size_in_wim;
ret = write_integrity_table(wim,
new_lookup_table_end,
- old_lookup_table_end);
+ old_lookup_table_end,
+ old_integrity_table);
+ free_integrity_table(old_integrity_table);
if (ret)
return ret;
} else {
}
#if defined(HAVE_SYS_FILE_H) && defined(HAVE_FLOCK)
+
+/* Set advisory lock on WIM file (if not already done so) */
int
-lock_wim(WIMStruct *wim, int fd)
+lock_wim_for_append(WIMStruct *wim)
{
- int ret = 0;
- if (fd != -1 && !wim->wim_locked) {
- ret = flock(fd, LOCK_EX | LOCK_NB);
- if (ret != 0) {
- if (errno == EWOULDBLOCK) {
- ERROR("`%"TS"' is already being modified or has been "
- "mounted read-write\n"
- " by another process!", wim->filename);
- ret = WIMLIB_ERR_ALREADY_LOCKED;
- } else {
- WARNING_WITH_ERRNO("Failed to lock `%"TS"'",
- wim->filename);
- ret = 0;
- }
- } else {
- wim->wim_locked = 1;
- }
+ if (wim->locked_for_append)
+ return 0;
+ if (!flock(wim->in_fd.fd, LOCK_EX | LOCK_NB)) {
+ wim->locked_for_append = 1;
+ return 0;
+ }
+ if (errno != EWOULDBLOCK)
+ return 0;
+ return WIMLIB_ERR_ALREADY_LOCKED;
+}
+
+/* Remove advisory lock on WIM file (if present) */
+void
+unlock_wim_for_append(WIMStruct *wim)
+{
+ if (wim->locked_for_append) {
+ flock(wim->in_fd.fd, LOCK_UN);
+ wim->locked_for_append = 0;
}
- return ret;
}
#endif
if (ret)
goto out_restore_memory_hdr;
- ret = lock_wim(wim, wim->out_fd.fd);
+ ret = lock_wim_for_append(wim);
if (ret)
goto out_close_wim;
if (ret)
goto out_truncate;
- write_flags |= WIMLIB_WRITE_FLAG_REUSE_INTEGRITY_TABLE;
ret = finish_write(wim, WIMLIB_ALL_IMAGES, write_flags,
&lookup_table_list);
if (ret)
goto out_truncate;
- wim->wim_locked = 0;
+ unlock_wim_for_append(wim);
return 0;
out_truncate:
out_restore_physical_hdr:
(void)write_wim_header_flags(hdr_save.flags, &wim->out_fd);
out_unlock_wim:
- wim->wim_locked = 0;
+ unlock_wim_for_append(wim);
out_close_wim:
(void)close_wim_writable(wim, write_flags);
out_restore_memory_hdr:
git://wimlib.net / wimlib / blobdiff