Allow in-place overwrites when unmounting read-write mounted WIM
authorEric Biggers <ebiggers3@gmail.com>
Tue, 18 Dec 2012 00:22:00 +0000 (18:22 -0600)
committerEric Biggers <ebiggers3@gmail.com>
Tue, 18 Dec 2012 01:48:27 +0000 (19:48 -0600)
- Add WIMLIB_UNMOUNT_FLAG_REBUILD and WIMLIB_UNMOUNT_FLAG_RECOMPRESS
- Make lock_wim() set a flag WIMStruct.wim_locked to prevent double-locks
- Maintain the inode list for each image through a read-write mount (not
  strictly necessary yet, but makes sense to keep it correct)
- overwrite_wim_inplace():  Search for new streams by searching directly through
  the lookup table for entries that are not located in the WIM being
  overwritten.

src/add_image.c
src/dentry.c
src/metadata_resource.c
src/mount_image.c
src/wimlib.h
src/wimlib_internal.h
src/write.c

index 99a1ca7..d09dd19 100644 (file)
@@ -633,7 +633,7 @@ WIMLIBAPI int wimlib_add_image(WIMStruct *w, const char *source,
        struct dentry *root_dentry = NULL;
        struct wim_security_data *sd;
        struct capture_config config;
-       struct hlist_head inode_list;
+       struct image_metadata *imd;
        int ret;
 
        if (add_image_flags & WIMLIB_ADD_IMAGE_FLAG_NTFS) {
@@ -729,14 +729,14 @@ WIMLIBAPI int wimlib_add_image(WIMStruct *w, const char *source,
        if (ret != 0)
                goto out_free_dentry_tree;
 
+       imd = &w->image_metadata[w->hdr.image_count - 1];
 
-       ret = dentry_tree_fix_inodes(root_dentry, &inode_list);
+       ret = dentry_tree_fix_inodes(root_dentry, &imd->inode_list);
        if (ret != 0)
                goto out_destroy_imd;
 
        DEBUG("Assigning hard link group IDs");
-       assign_inode_numbers(&inode_list);
-       w->image_metadata[w->hdr.image_count - 1].inode_list = inode_list;
+       assign_inode_numbers(&imd->inode_list);
 
        ret = xml_add_image(w, name);
        if (ret != 0)
index daab2fe..fb57c93 100644 (file)
@@ -890,6 +890,8 @@ void free_inode(struct inode *inode)
                wimlib_assert(inode->num_opened_fds == 0);
                FREE(inode->fds);
                pthread_mutex_destroy(&inode->i_mutex);
+               if (inode->hlist.next)
+                       hlist_del(&inode->hlist);
        #endif
                FREE(inode->extracted_file);
                FREE(inode);
index 1972326..726deb5 100644 (file)
@@ -172,6 +172,8 @@ int read_metadata_resource(WIMStruct *w, struct image_metadata *imd)
 
        imd->root_dentry = dentry;
        imd->inode_list  = inode_list;
+       if (imd->inode_list.first)
+               imd->inode_list.first->pprev = &imd->inode_list.first;
        goto out_free_buf;
 out_free_dentry_tree:
        free_dentry_tree(dentry, NULL);
index ef965ab..74b751c 100644 (file)
@@ -84,6 +84,9 @@ struct wimfs_context {
        /* List of lookup table entries in the staging directory */
        struct list_head staging_list;
 
+       /* List of inodes in the mounted image */
+       struct hlist_head *image_inode_list;
+
        /* Name and message queue descriptors for message queues between the filesystem
         * daemon process and the unmount process.  These are used when the filesystem
         * is unmounted and the process running wimlib_mount() (i.e. the `imagex
@@ -289,6 +292,7 @@ static int create_dentry(struct wimfs_context *ctx, const char *path,
        new->d_inode->resolved = 1;
        new->d_inode->ino = ctx->next_ino++;
        dentry_add_child(parent, new);
+       hlist_add_head(&new->d_inode->hlist, ctx->image_inode_list);
        *dentry_ret = new;
        return 0;
 }
@@ -448,7 +452,6 @@ static int extract_resource_to_staging_dir(struct inode *inode,
        int ret;
        int fd;
        struct lookup_table_entry *old_lte, *new_lte;
-       off_t old_size;
        off_t extract_size;
 
        DEBUG("Extracting resource to staging dir: inode %"PRIu64", "
@@ -1266,11 +1269,13 @@ static void wimfs_destroy(void *p)
        if (ret == 0) {
                if (ctx->mount_flags & WIMLIB_MOUNT_FLAG_READWRITE) {
                        if (unmount_flags & WIMLIB_UNMOUNT_FLAG_COMMIT) {
-                               int write_flags;
+                               int write_flags = 0;
                                if (unmount_flags & WIMLIB_UNMOUNT_FLAG_CHECK_INTEGRITY)
-                                       write_flags = WIMLIB_WRITE_FLAG_CHECK_INTEGRITY;
-                               else
-                                       write_flags = 0;
+                                       write_flags |= WIMLIB_WRITE_FLAG_CHECK_INTEGRITY;
+                               if (unmount_flags & WIMLIB_UNMOUNT_FLAG_REBUILD)
+                                       write_flags |= WIMLIB_WRITE_FLAG_REBUILD;
+                               if (unmount_flags & WIMLIB_UNMOUNT_FLAG_RECOMPRESS)
+                                       write_flags |= WIMLIB_WRITE_FLAG_RECOMPRESS;
                                status = rebuild_wim(ctx, write_flags);
                        }
                        ret = delete_staging_dir(ctx);
@@ -1417,8 +1422,6 @@ static int wimfs_link(const char *to, const char *from)
        from_dentry->d_inode = inode;
        inode->link_count++;
 
-       if (inode->lte)
-               inode->lte->refcnt++;
        for (i = 0; i <= inode->num_ads; i++) {
                lte = inode_stream_lte_resolved(inode, i);
                if (lte)
@@ -2160,14 +2163,13 @@ WIMLIBAPI int wimlib_mount_image(WIMStruct *wim, int image, const char *dir,
        }
 
        ret = select_wim_image(wim, image);
-
        if (ret != 0)
                goto out;
 
-       imd = &wim->image_metadata[image - 1];
-
        DEBUG("Selected image %d", image);
 
+       imd = wim_get_current_image_metadata(wim);
+
        if (imd->root_dentry->refcnt != 1) {
                ERROR("Cannot mount image that was just exported with "
                      "wimlib_export_image()");
@@ -2183,7 +2185,7 @@ WIMLIBAPI int wimlib_mount_image(WIMStruct *wim, int image, const char *dir,
        }
 
        if (mount_flags & WIMLIB_MOUNT_FLAG_READWRITE) {
-               ret = lock_wim(wim->fp, wim->filename);
+               ret = lock_wim(wim, wim->fp);
                if (ret != 0)
                        goto out;
        }
@@ -2198,6 +2200,7 @@ WIMLIBAPI int wimlib_mount_image(WIMStruct *wim, int image, const char *dir,
        init_wimfs_context(&ctx);
        ctx.wim = wim;
        ctx.mount_flags = mount_flags;
+       ctx.image_inode_list = &imd->inode_list;
 
        if (mount_flags & WIMLIB_MOUNT_FLAG_STREAM_INTERFACE_WINDOWS)
                ctx.default_lookup_flags = LOOKUP_FLAG_ADS_OK;
@@ -2205,7 +2208,7 @@ WIMLIBAPI int wimlib_mount_image(WIMStruct *wim, int image, const char *dir,
        DEBUG("Unlinking message queues in case they already exist");
        ret = set_message_queue_names(&ctx, dir);
        if (ret != 0)
-               goto out;
+               goto out_unlock;
        unlink_message_queues(&ctx);
 
        DEBUG("Preparing arguments to fuse_main()");
@@ -2291,6 +2294,8 @@ WIMLIBAPI int wimlib_mount_image(WIMStruct *wim, int image, const char *dir,
                ret = WIMLIB_ERR_FUSE;
 out_free_dir_copy:
        FREE(dir_copy);
+out_unlock:
+       wim->wim_locked = 0;
 out_free_message_queue_names:
        free_message_queue_names(&ctx);
 out:
index 58605bc..8b64915 100644 (file)
@@ -636,6 +636,12 @@ typedef int (*wimlib_progress_func_t)(enum wimlib_progress_msg msg_type,
  * discarded.  Ignored for read-only mounts. */
 #define WIMLIB_UNMOUNT_FLAG_COMMIT                     0x00000002
 
+/** See ::WIMLIB_WRITE_FLAG_REBUILD */
+#define WIMLIB_UNMOUNT_FLAG_REBUILD                    0x00000004
+
+/** See ::WIMLIB_WRITE_FLAG_RECOMPRESS */
+#define WIMLIB_UNMOUNT_FLAG_RECOMPRESS                 0x00000008
+
 /******************************
  * WIMLIB_WRITE_FLAG_*        *
  ******************************/
@@ -655,7 +661,7 @@ typedef int (*wimlib_progress_func_t)(enum wimlib_progress_msg msg_type,
 /** Call fsync() when the WIM file is closed */
 #define WIMLIB_WRITE_FLAG_FSYNC                                0x00000008
 
-/** Specifying this flag overrides the default behavior of wimlib_overwrite()
+/* Specifying this flag overrides the default behavior of wimlib_overwrite()
  * after one or more calls to wimlib_delete_image(), which is to rebuild the
  * entire WIM.
  *
@@ -1404,6 +1410,7 @@ extern int wimlib_join(const char **swms, unsigned num_swms,
  *     the same directory as the WIM file that @a wim was originally read from.
  *
  * @return 0 on success; nonzero on error.
+ *
  * @retval ::WIMLIB_ERR_ALREADY_LOCKED
  *     A read-write mount was requested, but an an exclusive advisory lock on
  *     the on-disk WIM file could not be acquired because another thread or
index 4b10dbf..c2457b9 100644 (file)
@@ -317,6 +317,7 @@ struct WIMStruct {
        u8 deletion_occurred : 1;
        u8 all_images_verified : 1;
        u8 full_verification_in_progress : 1;
+       u8 wim_locked : 1;
 };
 
 /* Inline utility functions for WIMStructs. */
@@ -543,9 +544,9 @@ extern int finish_write(WIMStruct *w, int image, int write_flags,
                        wimlib_progress_func_t progress_func);
 
 #if defined(HAVE_SYS_FILE_H) && defined(HAVE_FLOCK)
-extern int lock_wim(FILE *fp, const char *path);
+extern int lock_wim(WIMStruct *w, FILE *fp);
 #else
-static inline int lock_wim(FILE *fp, const char *path)
+static inline int lock_wim(WIMStruct *w, FILE *fp)
 {
        return 0;
 }
index a23aa45..4981400 100644 (file)
@@ -1339,6 +1339,47 @@ static int write_stream_list(struct list_head *stream_list, FILE *out_fp,
                                                &progress);
 }
 
+struct lte_overwrite_prepare_args {
+       WIMStruct *wim;
+       struct list_head *stream_list;
+       off_t end_offset;
+};
+
+static int lte_overwrite_prepare(struct lookup_table_entry *lte, void *arg)
+{
+       struct lte_overwrite_prepare_args *args = arg;
+
+       if (lte->resource_entry.offset +
+           lte->resource_entry.size > args->end_offset)
+       {
+               ERROR("The following resource is after the XML data:");
+               print_lookup_table_entry(lte);
+               return WIMLIB_ERR_RESOURCE_ORDER;
+       }
+
+       lte->out_refcnt = lte->refcnt;
+       memcpy(&lte->output_resource_entry, &lte->resource_entry,
+              sizeof(struct resource_entry));
+       if (!(lte->resource_entry.flags & WIM_RESHDR_FLAG_METADATA)) {
+               wimlib_assert(lte->resource_location != RESOURCE_NONEXISTENT);
+               if (lte->resource_location != RESOURCE_IN_WIM || lte->wim != args->wim)
+                       list_add(&lte->staging_list, args->stream_list);
+       }
+       return 0;
+}
+
+static int wim_find_new_streams(WIMStruct *wim, off_t end_offset,
+                               struct list_head *stream_list)
+{
+       struct lte_overwrite_prepare_args args = {
+               .wim = wim,
+               .stream_list = stream_list,
+               .end_offset = end_offset,
+       };
+
+       return for_lookup_table_entry(wim->lookup_table,
+                                     lte_overwrite_prepare, &args);
+}
 
 static int dentry_find_streams_to_write(struct dentry *dentry,
                                        void *wim)
@@ -1537,22 +1578,24 @@ out:
 }
 
 #if defined(HAVE_SYS_FILE_H) && defined(HAVE_FLOCK)
-int lock_wim(FILE *fp, const char *path)
+int lock_wim(WIMStruct *w, FILE *fp)
 {
        int ret = 0;
-       if (fp) {
+       if (fp && !w->wim_locked) {
                ret = flock(fileno(fp), LOCK_EX | LOCK_NB);
                if (ret != 0) {
                        if (errno == EWOULDBLOCK) {
                                ERROR("`%s' is already being modified or has been "
                                      "mounted read-write\n"
-                                     "        by another process!", path);
+                                     "        by another process!", w->filename);
                                ret = WIMLIB_ERR_ALREADY_LOCKED;
                        } else {
                                WARNING("Failed to lock `%s': %s",
-                                       path, strerror(errno));
+                                       w->filename, strerror(errno));
                                ret = 0;
                        }
+               } else {
+                       w->wim_locked = 1;
                }
        }
        return ret;
@@ -1651,45 +1694,6 @@ out:
        return ret;
 }
 
-static int lte_overwrite_prepare(struct lookup_table_entry *lte,
-                                void *ignore)
-{
-       memcpy(&lte->output_resource_entry, &lte->resource_entry,
-              sizeof(struct resource_entry));
-       lte->out_refcnt = 0;
-       return 0;
-}
-
-static int check_resource_offset(struct lookup_table_entry *lte, void *arg)
-{
-       wimlib_assert(lte->out_refcnt <= lte->refcnt);
-       if (lte->out_refcnt < lte->refcnt) {
-               off_t end_offset = *(u64*)arg;
-               if (lte->resource_entry.offset +
-                   lte->resource_entry.size > end_offset)
-               {
-                       ERROR("The following resource is after the XML data:");
-                       print_lookup_table_entry(lte);
-                       return WIMLIB_ERR_RESOURCE_ORDER;
-               }
-       }
-       return 0;
-}
-
-static int find_new_streams(struct lookup_table_entry *lte, void *arg)
-{
-       if (lte->out_refcnt == lte->refcnt) {
-               /* Newly added stream that is only referenced in the modified
-                * images.  Append it to the list of streams to write. */
-               list_add(&lte->staging_list, (struct list_head*)arg);
-       } else {
-               /* Not a newly added stream.  But set out_refcnt to the full
-                * refcnt so that it's written correctly. */
-               lte->out_refcnt = lte->refcnt;
-       }
-       return 0;
-}
-
 /*
  * Overwrite a WIM, possibly appending streams to it.
  *
@@ -1773,25 +1777,12 @@ static int overwrite_wim_inplace(WIMStruct *w, int write_flags,
                return WIMLIB_ERR_RESOURCE_ORDER;
        }
 
-       DEBUG("Identifying newly added streams");
-       for_lookup_table_entry(w->lookup_table, lte_overwrite_prepare, NULL);
-       INIT_LIST_HEAD(&stream_list);
-       for (int i = modified_image_idx; i < w->hdr.image_count; i++) {
-               DEBUG("Identifiying streams in image %d", i + 1);
-               w->private = &stream_list;
-               for_dentry_in_tree(w->image_metadata[i].root_dentry,
-                                  dentry_find_streams_to_write, w);
-       }
 
        if (w->hdr.integrity.offset)
                old_wim_end = w->hdr.integrity.offset + w->hdr.integrity.size;
        else
                old_wim_end = w->hdr.xml_res_entry.offset + w->hdr.xml_res_entry.size;
 
-       ret = for_lookup_table_entry(w->lookup_table, check_resource_offset,
-                                    &old_wim_end);
-       if (ret != 0)
-               return ret;
 
        if (modified_image_idx == w->hdr.image_count && !w->deletion_occurred) {
                /* If no images have been modified and no images have been
@@ -1801,17 +1792,17 @@ static int overwrite_wim_inplace(WIMStruct *w, int write_flags,
                write_flags |= WIMLIB_WRITE_FLAG_NO_LOOKUP_TABLE |
                               WIMLIB_WRITE_FLAG_CHECKPOINT_AFTER_XML;
        }
-
        INIT_LIST_HEAD(&stream_list);
-       for_lookup_table_entry(w->lookup_table, find_new_streams,
-                              &stream_list);
+       ret = wim_find_new_streams(w, old_wim_end, &stream_list);
+       if (ret != 0)
+               return ret;
 
        ret = open_wim_writable(w, w->filename, false,
                                (write_flags & WIMLIB_WRITE_FLAG_CHECK_INTEGRITY) != 0);
        if (ret != 0)
                return ret;
 
-       ret = lock_wim(w->out_fp, w->filename);
+       ret = lock_wim(w, w->out_fp);
        if (ret != 0) {
                fclose(w->out_fp);
                w->out_fp = NULL;
@@ -1822,6 +1813,7 @@ static int overwrite_wim_inplace(WIMStruct *w, int write_flags,
                ERROR_WITH_ERRNO("Can't seek to end of WIM");
                fclose(w->out_fp);
                w->out_fp = NULL;
+               w->wim_locked = 0;
                return WIMLIB_ERR_WRITE;
        }
 
@@ -1854,6 +1846,7 @@ out_ftruncate:
                        w->filename, old_wim_end);
                truncate(w->filename, old_wim_end);
        }
+       w->wim_locked = 0;
        return ret;
 }
 
@@ -1949,8 +1942,7 @@ WIMLIBAPI int wimlib_overwrite(WIMStruct *w, int write_flags,
                for (i = 0; i < w->hdr.image_count && !w->image_metadata[i].modified; i++)
                        ;
                modified_image_idx = i;
-               for (; i < w->hdr.image_count && w->image_metadata[i].modified &&
-                       !w->image_metadata[i].has_been_mounted_rw; i++)
+               for (; i < w->hdr.image_count && w->image_metadata[i].modified; i++)
                        ;
                if (i == w->hdr.image_count) {
                        ret = overwrite_wim_inplace(w, write_flags, num_threads,