Add experimental support for WIMLIB_PROGRESS_MSG_DONE_WITH_FILE

diff --git a/NEWS b/NEWS
index 5b205d3cbc5b15d9d60d3146fb0161c64f7d1cc2..a903d01f033a8a05a296cc80ff6a624890691782 100644 (file)
--- a/NEWS
+++ b/NEWS
@@ -6,7 +6,10 @@ Version 1.7.1-BETA:
        times in one run of an application program.
 
        Library users will now receive WIMLIB_PROGRESS_MSG_WRITE_STREAMS
        times in one run of an application program.
 
        Library users will now receive WIMLIB_PROGRESS_MSG_WRITE_STREAMS

-       messages more frequency when writing large WIM files.
+       messages more frequently when writing large WIM files.
+
+       Added experimental new write flag:
+       WIMLIB_WRITE_FLAG_SEND_DONE_WITH_FILE_MESSAGES.

 
 Version 1.7.0:
        Improved compression, decompression, and extraction performance.
 
 Version 1.7.0:
        Improved compression, decompression, and extraction performance.

diff --git a/include/wimlib.h b/include/wimlib.h
index ecb7635304489b5f755267dc179fb0712b7ba7f9..f56e72cf0047585db6eb4995d5d2b282ab46953f 100644 (file)
--- a/include/wimlib.h
+++ b/include/wimlib.h
@@ -620,6 +620,12 @@ enum wimlib_progress_msg {
        /** Starting to unmount a WIM image.  @p info will point to
         * ::wimlib_progress_info.unmount.  */
        WIMLIB_PROGRESS_MSG_UNMOUNT_BEGIN = 25,
        /** Starting to unmount a WIM image.  @p info will point to
         * ::wimlib_progress_info.unmount.  */
        WIMLIB_PROGRESS_MSG_UNMOUNT_BEGIN = 25,

+
+       /** wimlib has used a file's data for the last time (including all data
+        * streams, if it has multiple).  @p info will point to
+        * ::wimlib_progress_info.done_with_file.  This message is only received
+        * if ::WIMLIB_WRITE_FLAG_SEND_DONE_WITH_FILE_MESSAGES was provided.  */
+       WIMLIB_PROGRESS_MSG_DONE_WITH_FILE = 26,

 };
 
 /** Valid return values from user-provided progress functions
 };
 
 /** Valid return values from user-provided progress functions


@@ -1024,6 +1030,28 @@ union wimlib_progress_info {
                /** Flags passed to wimlib_unmount_image().  */
                uint32_t unmount_flags;
        } unmount;
                /** Flags passed to wimlib_unmount_image().  */
                uint32_t unmount_flags;
        } unmount;

+
+       /** Valid on messages ::WIMLIB_PROGRESS_MSG_DONE_WITH_FILE.  */
+       struct wimlib_progress_info_done_with_file {
+               /* Path to the file whose data has been written to the WIM file,
+                * or is currently being asynchronously compressed in memory,
+                * and therefore is no longer needed by wimlib.
+                *
+                * WARNING: The file data will not actually be accessible in the
+                * WIM file until the WIM file has been completely written.
+                * Ordinarily you should <b>not</b> treat this message as a
+                * green light to go ahead and delete the specified file, since
+                * that would result in data loss if the WIM file cannot be
+                * successfully created for any reason.
+                *
+                * If a file has multiple names (hard links),
+                * ::WIMLIB_PROGRESS_MSG_DONE_WITH_FILE will only be received
+                * for one name.  Also, this message will not be received for
+                * empty files or reparse points (or symbolic links), unless
+                * they have nonempty named data streams.
+                */
+               const wimlib_tchar *path_to_file;
+       } done_with_file;

 };
 
 /**
 };
 
 /**


@@ -1936,6 +1964,13 @@ typedef int (*wimlib_iterate_lookup_table_callback_t)(const struct wimlib_resour
  */
 #define WIMLIB_WRITE_FLAG_PACK_STREAMS                 0x00001000
 
  */
 #define WIMLIB_WRITE_FLAG_PACK_STREAMS                 0x00001000
 

+/**
+ * Send ::WIMLIB_PROGRESS_MSG_DONE_WITH_FILE messages while writing the WIM
+ * file.  This is only needed in the unusual case that the library user needs to
+ * know exactly when wimlib has read each file for the last time.
+ */
+#define WIMLIB_WRITE_FLAG_SEND_DONE_WITH_FILE_MESSAGES 0x00002000
+

 /** @} */
 /** @addtogroup G_general
  * @{ */
 /** @} */
 /** @addtogroup G_general
  * @{ */

diff --git a/include/wimlib/inode.h b/include/wimlib/inode.h
index 4d39c1ba9904fd0f034392e614a7c662d2be07c1..3b0222990bf77a8476766b1fda097990e5a4740d 100644 (file)
--- a/include/wimlib/inode.h
+++ b/include/wimlib/inode.h
@@ -176,6 +176,12 @@ struct wim_inode {
                #endif
                };
 
                #endif
                };
 

+               /* Used during WIM writing with
+                * WIMLIB_WRITE_FLAG_SEND_DONE_WITH_FILE_MESSAGES:  the number
+                * of data streams this inode has that have not yet been fully
+                * read.  */
+               u32 num_unread_streams;
+

 #ifdef WITH_FUSE
                struct {
                        /* Used only during image mount:  Table of file
 #ifdef WITH_FUSE
                struct {
                        /* Used only during image mount:  Table of file

diff --git a/include/wimlib/write.h b/include/wimlib/write.h
index d6957ace4b5ffeabb30f89fd6bf4c89efcdfe398..61f3bc670e88627f783a336006cdb29e3867b4ff 100644 (file)
--- a/include/wimlib/write.h
+++ b/include/wimlib/write.h
@@ -27,7 +27,8 @@
        WIMLIB_WRITE_FLAG_SKIP_EXTERNAL_WIMS            | \
        WIMLIB_WRITE_FLAG_STREAMS_OK                    | \
        WIMLIB_WRITE_FLAG_RETAIN_GUID                   | \
        WIMLIB_WRITE_FLAG_SKIP_EXTERNAL_WIMS            | \
        WIMLIB_WRITE_FLAG_STREAMS_OK                    | \
        WIMLIB_WRITE_FLAG_RETAIN_GUID                   | \

-       WIMLIB_WRITE_FLAG_PACK_STREAMS)
+       WIMLIB_WRITE_FLAG_PACK_STREAMS                  | \
+       WIMLIB_WRITE_FLAG_SEND_DONE_WITH_FILE_MESSAGES)

 
 #if defined(HAVE_SYS_FILE_H) && defined(HAVE_FLOCK)
 extern int
 
 #if defined(HAVE_SYS_FILE_H) && defined(HAVE_FLOCK)
 extern int

diff --git a/src/write.c b/src/write.c
index 56eb5dd46e5dc8db1ee2a9148c57da427dae0c8a..c45d0b7fdd450c739a1bde7ac80c568e0a35c70e 100644 (file)
--- a/src/write.c
+++ b/src/write.c
@@ -43,6 +43,7 @@
 #include "wimlib/integrity.h"
 #include "wimlib/lookup_table.h"
 #include "wimlib/metadata.h"
 #include "wimlib/integrity.h"
 #include "wimlib/lookup_table.h"
 #include "wimlib/metadata.h"

+#include "wimlib/paths.h"

 #include "wimlib/progress.h"
 #include "wimlib/resource.h"
 #ifdef __WIN32__
 #include "wimlib/progress.h"
 #include "wimlib/resource.h"
 #ifdef __WIN32__


@@ -64,6 +65,7 @@
 #define WRITE_RESOURCE_FLAG_RECOMPRESS         0x00000001
 #define WRITE_RESOURCE_FLAG_PIPABLE            0x00000002
 #define WRITE_RESOURCE_FLAG_PACK_STREAMS       0x00000004
 #define WRITE_RESOURCE_FLAG_RECOMPRESS         0x00000001
 #define WRITE_RESOURCE_FLAG_PIPABLE            0x00000002
 #define WRITE_RESOURCE_FLAG_PACK_STREAMS       0x00000004

+#define WRITE_RESOURCE_FLAG_SEND_DONE_WITH_FILE        0x00000008

 
 static inline int
 write_flags_to_resource_flags(int write_flags)
 
 static inline int
 write_flags_to_resource_flags(int write_flags)


@@ -76,6 +78,8 @@ write_flags_to_resource_flags(int write_flags)
                write_resource_flags |= WRITE_RESOURCE_FLAG_PIPABLE;
        if (write_flags & WIMLIB_WRITE_FLAG_PACK_STREAMS)
                write_resource_flags |= WRITE_RESOURCE_FLAG_PACK_STREAMS;
                write_resource_flags |= WRITE_RESOURCE_FLAG_PIPABLE;
        if (write_flags & WIMLIB_WRITE_FLAG_PACK_STREAMS)
                write_resource_flags |= WRITE_RESOURCE_FLAG_PACK_STREAMS;

+       if (write_flags & WIMLIB_WRITE_FLAG_SEND_DONE_WITH_FILE_MESSAGES)
+               write_resource_flags |= WRITE_RESOURCE_FLAG_SEND_DONE_WITH_FILE;

        return write_resource_flags;
 }
 
        return write_resource_flags;
 }
 


@@ -390,6 +394,8 @@ struct write_streams_ctx {
         * pending_streams rather than the entry being read.)  */
        bool stream_was_duplicate;
 
         * pending_streams rather than the entry being read.)  */
        bool stream_was_duplicate;
 

+       struct wim_inode *stream_inode;
+

        /* Current uncompressed offset in the stream being read.  */
        u64 cur_read_stream_offset;
 
        /* Current uncompressed offset in the stream being read.  */
        u64 cur_read_stream_offset;
 


@@ -636,6 +642,74 @@ end_write_resource(struct write_streams_ctx *ctx, struct wim_reshdr *out_reshdr)
        return 0;
 }
 
        return 0;
 }
 

+/* No more data streams of the file at @path are needed.  */
+static int
+done_with_file(const tchar *path, wimlib_progress_func_t progfunc, void *progctx)
+{
+       union wimlib_progress_info info;
+
+       info.done_with_file.path_to_file = path;
+
+       return call_progress(progfunc, WIMLIB_PROGRESS_MSG_DONE_WITH_FILE,
+                            &info, progctx);
+}
+
+/* Handle WIMLIB_WRITE_FLAG_SEND_DONE_WITH_FILE_MESSAGES mode.  */
+static int
+done_with_stream(struct wim_lookup_table_entry *stream,
+                wimlib_progress_func_t progfunc, void *progctx,
+                struct wim_inode *inode)
+{
+       if (stream->resource_location == RESOURCE_IN_FILE_ON_DISK
+#ifdef __WIN32__
+           || stream->resource_location == RESOURCE_IN_WINNT_FILE_ON_DISK
+           || stream->resource_location == RESOURCE_WIN32_ENCRYPTED
+#endif
+          )
+       {
+               int ret;
+
+               wimlib_assert(inode->num_unread_streams > 0);
+               if (--inode->num_unread_streams > 0)
+                       return 0;
+
+       #ifdef __WIN32__
+               /* XXX: This logic really should be somewhere else.  */
+
+               /* We want the path to the file, but stream->file_on_disk might
+                * actually refer to a named data stream.  Temporarily strip the
+                * named data stream from the path.  */
+               wchar_t *p_colon = NULL;
+               wchar_t *p_question_mark = NULL;
+               const wchar_t *p_stream_name;
+
+               p_stream_name = path_stream_name(stream->file_on_disk);
+               if (unlikely(p_stream_name)) {
+                       p_colon = (wchar_t *)(p_stream_name - 1);
+                       wimlib_assert(*p_colon == L':');
+                       *p_colon = L'\0';
+               }
+
+               /* We also should use a fake Win32 path instead of a NT path  */
+               if (!wcsncmp(stream->file_on_disk, L"\\??\\", 4)) {
+                       p_question_mark = &stream->file_on_disk[1];
+                       *p_question_mark = L'\\';
+               }
+       #endif
+
+               ret = done_with_file(stream->file_on_disk, progfunc, progctx);
+
+       #ifdef __WIN32__
+               if (p_colon)
+                       *p_colon = L':';
+               if (p_question_mark)
+                       *p_question_mark = L'?';
+       #endif
+               return ret;
+       }
+       return 0;
+}
+

 /* Begin processing a stream for writing.  */
 static int
 write_stream_begin_read(struct wim_lookup_table_entry *lte,
 /* Begin processing a stream for writing.  */
 static int
 write_stream_begin_read(struct wim_lookup_table_entry *lte,


@@ -649,6 +723,11 @@ write_stream_begin_read(struct wim_lookup_table_entry *lte,
        ctx->cur_read_stream_offset = 0;
        ctx->cur_read_stream_size = lte->size;
 
        ctx->cur_read_stream_offset = 0;
        ctx->cur_read_stream_size = lte->size;
 

+       if (lte->unhashed)
+               ctx->stream_inode = lte->back_inode;
+       else
+               ctx->stream_inode = NULL;
+

        /* As an optimization, we allow some streams to be "unhashed", meaning
         * their SHA1 message digests are unknown.  This is the case with
         * streams that are added by scanning a directry tree with
        /* As an optimization, we allow some streams to be "unhashed", meaning
         * their SHA1 message digests are unknown.  This is the case with
         * streams that are added by scanning a directry tree with


@@ -693,6 +772,14 @@ write_stream_begin_read(struct wim_lookup_table_entry *lte,
                                        lte_new->out_refcnt += lte->out_refcnt;
                                if (ctx->write_resource_flags & WRITE_RESOURCE_FLAG_PACK_STREAMS)
                                        ctx->cur_write_res_size -= lte->size;
                                        lte_new->out_refcnt += lte->out_refcnt;
                                if (ctx->write_resource_flags & WRITE_RESOURCE_FLAG_PACK_STREAMS)
                                        ctx->cur_write_res_size -= lte->size;

+                               if (!ret && unlikely(ctx->write_resource_flags &
+                                                    WRITE_RESOURCE_FLAG_SEND_DONE_WITH_FILE))
+                               {
+                                       ret = done_with_stream(lte,
+                                                              ctx->progress_data.progfunc,
+                                                              ctx->progress_data.progctx,
+                                                              ctx->stream_inode);
+                               }

                                free_lookup_table_entry(lte);
                                if (ret)
                                        return ret;
                                free_lookup_table_entry(lte);
                                if (ret)
                                        return ret;


@@ -881,7 +968,8 @@ write_chunk(struct write_streams_ctx *ctx, const void *cchunk,
 
                        if (ctx->compressor != NULL &&
                            lte->out_reshdr.size_in_wim >= lte->out_reshdr.uncompressed_size &&
 
                        if (ctx->compressor != NULL &&
                            lte->out_reshdr.size_in_wim >= lte->out_reshdr.uncompressed_size &&

-                           !(ctx->write_resource_flags & WRITE_RESOURCE_FLAG_PIPABLE) &&
+                           !(ctx->write_resource_flags & (WRITE_RESOURCE_FLAG_PIPABLE |
+                                                          WRITE_RESOURCE_FLAG_SEND_DONE_WITH_FILE)) &&

                            !(lte->flags & WIM_RESHDR_FLAG_PACKED_STREAMS))
                        {
                                /* Stream did not compress to less than its original
                            !(lte->flags & WIM_RESHDR_FLAG_PACKED_STREAMS))
                        {
                                /* Stream did not compress to less than its original


@@ -1024,15 +1112,31 @@ static int
 write_stream_end_read(struct wim_lookup_table_entry *lte, int status, void *_ctx)
 {
        struct write_streams_ctx *ctx = _ctx;
 write_stream_end_read(struct wim_lookup_table_entry *lte, int status, void *_ctx)
 {
        struct write_streams_ctx *ctx = _ctx;

-       if (status == 0)
-               wimlib_assert(ctx->cur_read_stream_offset == ctx->cur_read_stream_size);
-       if (ctx->stream_was_duplicate) {
-               free_lookup_table_entry(lte);
-       } else if (lte->unhashed && ctx->lookup_table != NULL) {
+       if (status)
+               goto out;
+
+       wimlib_assert(ctx->cur_read_stream_offset == ctx->cur_read_stream_size);
+
+       if (unlikely(ctx->write_resource_flags &
+                    WRITE_RESOURCE_FLAG_SEND_DONE_WITH_FILE) &&
+           ctx->stream_inode != NULL)
+       {
+               status = done_with_stream(lte,
+                                         ctx->progress_data.progfunc,
+                                         ctx->progress_data.progctx,
+                                         ctx->stream_inode);
+       }
+
+       if (!ctx->stream_was_duplicate && lte->unhashed &&
+           ctx->lookup_table != NULL)
+       {

                list_del(&lte->unhashed_list);
                lookup_table_insert(ctx->lookup_table, lte);
                lte->unhashed = 0;
        }
                list_del(&lte->unhashed_list);
                lookup_table_insert(ctx->lookup_table, lte);
                lte->unhashed = 0;
        }

+out:
+       if (ctx->stream_was_duplicate)
+               free_lookup_table_entry(lte);

        return status;
 }
 
        return status;
 }
 


@@ -2068,6 +2172,17 @@ write_wim_streams(WIMStruct *wim, int image, int write_flags,
                }
        }
 
                }
        }
 

+       /* If needed, set auxiliary information so that we can detect when
+        * wimlib has finished using each external file.  */
+       if (unlikely(write_flags & WIMLIB_WRITE_FLAG_SEND_DONE_WITH_FILE_MESSAGES)) {
+               list_for_each_entry(lte, stream_list, write_streams_list)
+                       if (lte->unhashed)
+                               lte->back_inode->num_unread_streams = 0;
+               list_for_each_entry(lte, stream_list, write_streams_list)
+                       if (lte->unhashed)
+                               lte->back_inode->num_unread_streams++;
+       }
+

        return wim_write_stream_list(wim,
                                     stream_list,
                                     write_flags,
        return wim_write_stream_list(wim,
                                     stream_list,
                                     write_flags,