Fix various NTFS capture issues
authorEric Biggers <ebiggers3@gmail.com>
Wed, 29 Aug 2012 21:45:29 +0000 (16:45 -0500)
committerEric Biggers <ebiggers3@gmail.com>
Wed, 29 Aug 2012 21:45:29 +0000 (16:45 -0500)
src/dentry.c
src/lookup_table.c
src/lookup_table.h
src/ntfs-capture.c
src/resource.c
src/sha1.c
src/sha1.h

index fbc62b8..a4f9c43 100644 (file)
@@ -170,13 +170,17 @@ struct ads_entry *dentry_add_ads(struct dentry *dentry, const char *stream_name)
        DEBUG("Add alternate data stream %s:%s",
               dentry->file_name_utf8, stream_name);
 
-       if (dentry->num_ads == 0xffff)
+       if (dentry->num_ads == 0xffff) {
+               ERROR("Too many alternate data streams in one dentry!");
                return NULL;
+       }
        num_ads = dentry->num_ads + 1;
        ads_entries = REALLOC(dentry->ads_entries,
                              num_ads * sizeof(struct ads_entry));
-       if (!ads_entries)
+       if (!ads_entries) {
+               ERROR("Failed to allocate memory for new alternate data stream");
                return NULL;
+       }
        if (ads_entries != dentry->ads_entries) {
                /* We moved the ADS entries.  Adjust the stream lists. */
                for (u16 i = 0; i < dentry->num_ads; i++) {
@@ -1391,10 +1395,7 @@ static u8 *write_dentry(const struct dentry *dentry, u8 *p)
        p = put_u64(p, dentry->creation_time);
        p = put_u64(p, dentry->last_access_time);
        p = put_u64(p, dentry->last_write_time);
-       if (dentry->resolved && dentry->lte)
-               hash = dentry->lte->hash;
-       else
-               hash = dentry->hash;
+       hash = dentry_stream_hash(dentry, 0);
        p = put_bytes(p, SHA1_HASH_SIZE, hash);
        if (dentry->attributes & FILE_ATTRIBUTE_REPARSE_POINT) {
                p = put_zeroes(p, 4);
@@ -1432,10 +1433,7 @@ static u8 *write_dentry(const struct dentry *dentry, u8 *p)
        for (u16 i = 0; i < dentry->num_ads; i++) {
                p = put_u64(p, ads_entry_total_length(&dentry->ads_entries[i]));
                p = put_u64(p, 0); /* Unused */
-               if (dentry->resolved && dentry->ads_entries[i].lte)
-                       hash = dentry->ads_entries[i].lte->hash;
-               else
-                       hash = dentry->ads_entries[i].hash;
+               hash = dentry_stream_hash(dentry, i + 1);
                p = put_bytes(p, SHA1_HASH_SIZE, hash);
                p = put_u16(p, dentry->ads_entries[i].stream_name_len);
                if (dentry->ads_entries[i].stream_name_len) {
index e8f44ea..f4a6775 100644 (file)
@@ -81,9 +81,10 @@ void free_lookup_table_entry(struct lookup_table_entry *lte)
                case RESOURCE_IN_STAGING_FILE:
                case RESOURCE_IN_ATTACHED_BUFFER:
                case RESOURCE_IN_FILE_ON_DISK:
-                       wimlib_assert(&lte->file_on_disk ==
-                                     &lte->staging_file_name ==
-                                     &lte->attached_buffer);
+                       wimlib_assert((&lte->file_on_disk ==
+                                     &lte->staging_file_name)
+                                     && (&lte->file_on_disk ==
+                                     &lte->attached_buffer));
                        FREE(lte->file_on_disk);
                        break;
 #ifdef WITH_NTFS_3G
index afc2117..f0ee25b 100644 (file)
@@ -310,7 +310,7 @@ static inline const u8 *dentry_stream_hash_resolved(const struct dentry *dentry,
        if (lte)
                return lte->hash;
        else
-               return NULL;
+               return zero_hash;
 }
 
 /* 
index eadedac..f16850e 100644 (file)
@@ -133,6 +133,7 @@ static int sd_set_add_sd(struct sd_set *sd_set, const char descriptor[],
        struct wim_security_data *sd;
 
        sha1_buffer((const u8*)descriptor, size, hash);
+
        security_id = lookup_sd(hash, sd_set->root);
        if (security_id >= 0)
                return security_id;
@@ -170,7 +171,7 @@ static int sd_set_add_sd(struct sd_set *sd_set, const char descriptor[],
        sd->total_length += size + sizeof(sd->sizes[0]);
 
        if (sd_set->root)
-               insert_sd_node(sd_set->root, new);
+               insert_sd_node(new, sd_set->root);
        else
                sd_set->root = new;
        return new->security_id;
@@ -192,11 +193,15 @@ static inline ntfschar *attr_record_name(ATTR_RECORD *ar)
  * @ni:  The NTFS inode containing the attribute.
  * @ar:         The ATTR_RECORD describing the attribute.
  * @md:  If successful, the returned SHA1 message digest.
+ * @reparse_tag_ret:   Optional pointer into which the first 4 bytes of the
+ *                             attribute will be written (to get the reparse
+ *                             point ID)
  *
  * Return 0 on success or nonzero on error.
  */
 static int ntfs_attr_sha1sum(ntfs_inode *ni, ATTR_RECORD *ar,
-                            u8 md[SHA1_HASH_SIZE])
+                            u8 md[SHA1_HASH_SIZE],
+                            u32 *reparse_tag_ret)
 {
        s64 pos = 0;
        s64 bytes_remaining;
@@ -214,8 +219,8 @@ static int ntfs_attr_sha1sum(ntfs_inode *ni, ATTR_RECORD *ar,
        bytes_remaining = na->data_size;
        sha1_init(&ctx);
 
-       DEBUG("Calculating SHA1 message digest (%"PRIu64" bytes)",
-                       bytes_remaining);
+       DEBUG2("Calculating SHA1 message digest (%"PRIu64" bytes)",
+              bytes_remaining);
 
        while (bytes_remaining) {
                s64 to_read = min(bytes_remaining, sizeof(buf));
@@ -223,6 +228,8 @@ static int ntfs_attr_sha1sum(ntfs_inode *ni, ATTR_RECORD *ar,
                        ERROR_WITH_ERRNO("Error reading NTFS attribute");
                        return WIMLIB_ERR_NTFS_3G;
                }
+               if (bytes_remaining == na->data_size && reparse_tag_ret)
+                       *reparse_tag_ret = le32_to_cpu(*(u32*)buf);
                sha1_update(&ctx, buf, to_read);
                pos += to_read;
                bytes_remaining -= to_read;
@@ -244,10 +251,10 @@ static int capture_ntfs_streams(struct dentry *dentry, ntfs_inode *ni,
        ntfs_attr_search_ctx *actx;
        u8 attr_hash[SHA1_HASH_SIZE];
        struct ntfs_location *ntfs_loc = NULL;
-       struct lookup_table_entry *lte;
        int ret = 0;
+       struct lookup_table_entry *lte;
 
-       DEBUG("Capturing NTFS data streams from `%s'", path);
+       DEBUG2("Capturing NTFS data streams from `%s'", path);
 
        /* Get context to search the streams of the NTFS file. */
        actx = ntfs_attr_get_search_ctx(ni, NULL);
@@ -263,50 +270,78 @@ static int capture_ntfs_streams(struct dentry *dentry, ntfs_inode *ni,
        {
                char *stream_name_utf8;
                size_t stream_name_utf16_len;
+               u32 reparse_tag;
+               u64 data_size = ntfs_get_attribute_value_length(actx->attr);
 
-               /* Checksum the stream. */
-               ret = ntfs_attr_sha1sum(ni, actx->attr, attr_hash);
-               if (ret != 0)
-                       goto out_put_actx;
-
-               /* Make a lookup table entry for the stream, or use an existing
-                * one if there's already an identical stream. */
-               lte = __lookup_resource(lookup_table, attr_hash);
-               ret = WIMLIB_ERR_NOMEM;
-               if (lte) {
-                       lte->refcnt++;
+               if (data_size == 0) { 
+                       if (errno != 0) {
+                               ERROR_WITH_ERRNO("Failed to get size of attribute of "
+                                                "`%s'", path);
+                               ret = WIMLIB_ERR_NTFS_3G;
+                               goto out_put_actx;
+                       }
+                       /* Empty stream.  No lookup table entry is needed. */
+                       lte = NULL;
                } else {
-                       ntfs_loc = CALLOC(1, sizeof(*ntfs_loc));
-                       if (!ntfs_loc)
+                       if (type == AT_REPARSE_POINT && data_size < 8) {
+                               ERROR("`%s': reparse point buffer too small");
+                               ret = WIMLIB_ERR_NTFS_3G;
                                goto out_put_actx;
-                       ntfs_loc->ntfs_vol_p = ntfs_vol_p;
-                       ntfs_loc->path_utf8 = MALLOC(path_len + 1);
-                       if (!ntfs_loc->path_utf8)
-                               goto out_free_ntfs_loc;
-                       memcpy(ntfs_loc->path_utf8, path, path_len + 1);
-                       ntfs_loc->stream_name_utf16 = MALLOC(actx->attr->name_length * 2);
-                       if (!ntfs_loc->stream_name_utf16)
-                               goto out_free_ntfs_loc;
-                       memcpy(ntfs_loc->stream_name_utf16,
-                              attr_record_name(actx->attr),
-                              actx->attr->name_length * 2);
-
-                       ntfs_loc->stream_name_utf16_num_chars = actx->attr->name_length;
-                       ntfs_loc->is_reparse_point = (type == AT_REPARSE_POINT);
-                       lte = new_lookup_table_entry();
-                       if (!lte)
-                               goto out_free_ntfs_loc;
-                       lte->ntfs_loc = ntfs_loc;
-                       lte->resource_location = RESOURCE_IN_NTFS_VOLUME;
-                       lte->resource_entry.original_size = actx->attr->data_size;
-                       lte->resource_entry.size = actx->attr->data_size;
-                       DEBUG("Add resource for `%s' (size = %zu)",
-                               dentry->file_name_utf8,
-                               lte->resource_entry.original_size);
-                       copy_hash(lte->hash, attr_hash);
-                       lookup_table_insert(lookup_table, lte);
+                       }
+                       /* Checksum the stream. */
+                       ret = ntfs_attr_sha1sum(ni, actx->attr, attr_hash, &reparse_tag);
+                       if (ret != 0)
+                               goto out_put_actx;
+
+                       /* Make a lookup table entry for the stream, or use an existing
+                        * one if there's already an identical stream. */
+                       lte = __lookup_resource(lookup_table, attr_hash);
+                       ret = WIMLIB_ERR_NOMEM;
+                       if (lte) {
+                               lte->refcnt++;
+                       } else {
+                               ntfs_loc = CALLOC(1, sizeof(*ntfs_loc));
+                               if (!ntfs_loc)
+                                       goto out_put_actx;
+                               ntfs_loc->ntfs_vol_p = ntfs_vol_p;
+                               ntfs_loc->path_utf8 = MALLOC(path_len + 1);
+                               if (!ntfs_loc->path_utf8)
+                                       goto out_free_ntfs_loc;
+                               memcpy(ntfs_loc->path_utf8, path, path_len + 1);
+                               ntfs_loc->stream_name_utf16 = MALLOC(actx->attr->name_length * 2);
+                               if (!ntfs_loc->stream_name_utf16)
+                                       goto out_free_ntfs_loc;
+                               memcpy(ntfs_loc->stream_name_utf16,
+                                      attr_record_name(actx->attr),
+                                      actx->attr->name_length * 2);
+
+                               ntfs_loc->stream_name_utf16_num_chars = actx->attr->name_length;
+                               lte = new_lookup_table_entry();
+                               if (!lte)
+                                       goto out_free_ntfs_loc;
+                               lte->ntfs_loc = ntfs_loc;
+                               lte->resource_location = RESOURCE_IN_NTFS_VOLUME;
+                               if (type == AT_REPARSE_POINT) {
+                                       dentry->reparse_tag = reparse_tag;
+                                       ntfs_loc->is_reparse_point = true;
+                                       lte->resource_entry.original_size = data_size - 8;
+                                       lte->resource_entry.size = data_size - 8;
+                               } else {
+                                       ntfs_loc->is_reparse_point = false;
+                                       lte->resource_entry.original_size = data_size;
+                                       lte->resource_entry.size = data_size;
+                               }
+                               ntfs_loc = NULL;
+                               DEBUG("Add resource for `%s' (size = %zu)",
+                                       dentry->file_name_utf8,
+                                       lte->resource_entry.original_size);
+                               copy_hash(lte->hash, attr_hash);
+                               lookup_table_insert(lookup_table, lte);
+                       }
                }
                if (actx->attr->name_length == 0) {
+                       /* Unnamed data stream.  Put the reference to it in the
+                        * dentry. */
                        if (dentry->lte) {
                                ERROR("Found two un-named data streams for "
                                      "`%s'", path);
@@ -315,6 +350,8 @@ static int capture_ntfs_streams(struct dentry *dentry, ntfs_inode *ni,
                        }
                        dentry->lte = lte;
                } else {
+                       /* Named data stream.  Put the reference to it in the
+                        * alternate data stream entries */
                        struct ads_entry *new_ads_entry;
                        size_t stream_name_utf8_len;
                        stream_name_utf8 = utf16_to_utf8((const char*)attr_record_name(actx->attr),
@@ -343,9 +380,9 @@ out_free_ntfs_loc:
 out_put_actx:
        ntfs_attr_put_search_ctx(actx);
        if (ret == 0)
-               DEBUG("Successfully captured NTFS streams from `%s'", path);
+               DEBUG2("Successfully captured NTFS streams from `%s'", path);
        else
-               DEBUG("Failed to capture NTFS streams from `%s", path);
+               ERROR("Failed to capture NTFS streams from `%s", path);
        return ret;
 }
 
@@ -358,6 +395,7 @@ struct readdir_ctx {
        struct sd_set       *sd_set;
        const struct capture_config *config;
        ntfs_volume        **ntfs_vol_p;
+       int                  flags;
 };
 
 static int
@@ -366,7 +404,8 @@ build_dentry_tree_ntfs_recursive(struct dentry **root_p, ntfs_inode *ni,
                                 struct lookup_table *lookup_table,
                                 struct sd_set *sd_set,
                                 const struct capture_config *config,
-                                ntfs_volume **ntfs_vol_p);
+                                ntfs_volume **ntfs_vol_p,
+                                int flags);
 
 static int wim_ntfs_capture_filldir(void *dirent, const ntfschar *name,
                                    const int name_len, const int name_type,
@@ -393,13 +432,10 @@ static int wim_ntfs_capture_filldir(void *dirent, const ntfschar *name,
        if (utf8_name[0] == '.' &&
             (utf8_name[1] == '\0' ||
              (utf8_name[1] == '.' && utf8_name[2] == '\0'))) {
-               DEBUG("Skipping dentry `%s'", utf8_name);
                ret = 0;
                goto out_free_utf8_name;
        }
 
-       DEBUG("Opening inode for `%s'", utf8_name);
-
        ctx = dirent;
 
        ntfs_inode *ni = ntfs_inode_open(ctx->dir_ni->vol, mref);
@@ -414,13 +450,12 @@ static int wim_ntfs_capture_filldir(void *dirent, const ntfschar *name,
        path_len += utf8_name_len;
        ret = build_dentry_tree_ntfs_recursive(&child, ni, ctx->path, path_len,
                                               ctx->lookup_table, ctx->sd_set,
-                                              ctx->config, ctx->ntfs_vol_p);
+                                              ctx->config, ctx->ntfs_vol_p,
+                                              ctx->flags);
 
-       if (child) {
-               DEBUG("Linking dentry `%s' with parent `%s'",
-                     child->file_name_utf8, ctx->parent->file_name_utf8);
+       if (child)
                link_dentry(child, ctx->parent);
-       }
+
        ntfs_inode_close(ni);
 out_free_utf8_name:
        FREE(utf8_name);
@@ -439,7 +474,8 @@ static int build_dentry_tree_ntfs_recursive(struct dentry **root_p,
                                            struct lookup_table *lookup_table,
                                            struct sd_set *sd_set,
                                            const struct capture_config *config,
-                                           ntfs_volume **ntfs_vol_p)
+                                           ntfs_volume **ntfs_vol_p,
+                                           int flags)
 {
        u32 attributes;
        int mrec_flags;
@@ -447,14 +483,25 @@ static int build_dentry_tree_ntfs_recursive(struct dentry **root_p,
        int ret = 0;
        struct dentry *root;
 
+       mrec_flags = ni->mrec->flags;
+       attributes = ntfs_inode_get_attributes(ni);
+
        if (exclude_path(path, config, false)) {
-               DEBUG("Excluding `%s' from capture", path);
+               if (flags & WIMLIB_ADD_IMAGE_FLAG_VERBOSE) {
+                       const char *file_type;
+                       if (attributes & MFT_RECORD_IS_DIRECTORY)
+                               file_type = "directory";
+                       else
+                               file_type = "file";
+                       printf("Excluding %s `%s' from capture\n",
+                              file_type, path);
+               }
+               *root_p = NULL;
                return 0;
        }
 
-       DEBUG("Starting recursive capture at path = `%s'", path);
-       mrec_flags = ni->mrec->flags;
-       attributes = ntfs_inode_get_attributes(ni);
+       if (flags & WIMLIB_ADD_IMAGE_FLAG_VERBOSE)
+               printf("Scanning `%s'\n", path);
 
        root = new_dentry(path_basename(path));
        if (!root)
@@ -469,13 +516,11 @@ static int build_dentry_tree_ntfs_recursive(struct dentry **root_p,
        root->resolved         = true;
 
        if (attributes & FILE_ATTR_REPARSE_POINT) {
-               DEBUG("Reparse point `%s'", path);
                /* Junction point, symbolic link, or other reparse point */
                ret = capture_ntfs_streams(root, ni, path, path_len,
                                           lookup_table, ntfs_vol_p,
                                           AT_REPARSE_POINT);
        } else if (mrec_flags & MFT_RECORD_IS_DIRECTORY) {
-               DEBUG("Directory `%s'", path);
 
                /* Normal directory */
                s64 pos = 0;
@@ -488,6 +533,7 @@ static int build_dentry_tree_ntfs_recursive(struct dentry **root_p,
                        .sd_set       = sd_set,
                        .config       = config,
                        .ntfs_vol_p   = ntfs_vol_p,
+                       .flags        = flags,
                };
                ret = ntfs_readdir(ni, &pos, &ctx, wim_ntfs_capture_filldir);
                if (ret != 0) {
@@ -495,7 +541,6 @@ static int build_dentry_tree_ntfs_recursive(struct dentry **root_p,
                        ret = WIMLIB_ERR_NTFS_3G;
                }
        } else {
-               DEBUG("Normal file `%s'", path);
                /* Normal file */
                ret = capture_ntfs_streams(root, ni, path, path_len,
                                           lookup_table, ntfs_vol_p,
@@ -565,6 +610,7 @@ static int build_dentry_tree_ntfs(struct dentry **root_p,
                                 device);
                return WIMLIB_ERR_NTFS_3G;
        }
+       ntfs_open_secure(vol);
 
        /* We don't want to capture the special NTFS files such as $Bitmap.  Not
         * to be confused with "hidden" or "system" files which are real files
@@ -592,7 +638,7 @@ static int build_dentry_tree_ntfs(struct dentry **root_p,
        path[1] = '\0';
        ret = build_dentry_tree_ntfs_recursive(root_p, root_ni, path, 1,
                                               lookup_table, &sd_set,
-                                              config, ntfs_vol_p);
+                                              config, ntfs_vol_p, flags);
 out_cleanup:
        FREE(path);
        ntfs_inode_close(root_ni);
index 084cae9..fd75a62 100644 (file)
@@ -483,7 +483,13 @@ int read_wim_resource(const struct lookup_table_entry *lte, u8 buf[],
                break;
 #ifdef WITH_NTFS_3G
        case RESOURCE_IN_NTFS_VOLUME:
+               wimlib_assert(lte->ntfs_loc);
                if (lte->attr) {
+                       u64 adjusted_offset;
+                       if (lte->ntfs_loc->is_reparse_point)
+                               adjusted_offset = offset + 8;
+                       else
+                               adjusted_offset = offset;
                        if (ntfs_attr_pread(lte->attr, offset, size, buf) == size) {
                                return 0;
                        } else {
@@ -739,7 +745,7 @@ static int write_wim_resource(struct lookup_table_entry *lte,
        bool raw;
        off_t file_offset;
 #ifdef WITH_NTFS_3G
-       ntfs_inode *ni;
+       ntfs_inode *ni = NULL;
 #endif
 
        /* Original size of the resource */
@@ -798,7 +804,7 @@ static int write_wim_resource(struct lookup_table_entry *lte,
        }
 #ifdef WITH_NTFS_3G
        else if (lte->resource_location == RESOURCE_IN_NTFS_VOLUME
-                  && !lte->attr)
+                 && !lte->attr)
        {
                struct ntfs_location *loc = lte->ntfs_loc;
                wimlib_assert(loc);
@@ -806,15 +812,18 @@ static int write_wim_resource(struct lookup_table_entry *lte,
                if (!ni) {
                        ERROR_WITH_ERRNO("Failed to open inode `%s' in NTFS "
                                         "volume", loc->path_utf8);
+                       ret = WIMLIB_ERR_NTFS_3G;
+                       goto out;
                }
                lte->attr = ntfs_attr_open(ni,
                                           loc->is_reparse_point ? AT_REPARSE_POINT : AT_DATA,
                                           (ntfschar*)loc->stream_name_utf16,
                                           loc->stream_name_utf16_num_chars);
                if (!lte->attr) {
-                       ntfs_inode_close(ni);
                        ERROR_WITH_ERRNO("Failed to open attribute of `%s' in "
                                         "NTFS volume", loc->path_utf8);
+                       ret = WIMLIB_ERR_NTFS_3G;
+                       goto out_fclose;
                }
        }
 #endif
@@ -923,10 +932,13 @@ out_fclose:
                lte->file_on_disk_fp = NULL;
        }
 #ifdef WITH_NTFS_3G
-       else if (lte->resource_location == RESOURCE_IN_NTFS_VOLUME
-                && lte->attr) {
-               ntfs_attr_close(lte->attr);
-               ntfs_inode_close(ni);
+       else if (lte->resource_location == RESOURCE_IN_NTFS_VOLUME) {
+               if (lte->attr) {
+                       ntfs_attr_close(lte->attr);
+                       lte->attr = NULL;
+               } if (ni) {
+                       ntfs_inode_close(ni);
+               }
        }
 #endif
 out:
@@ -953,7 +965,7 @@ static int write_wim_resource_from_buffer(const u8 *buf, u64 buf_size,
        lte.resource_location            = RESOURCE_IN_ATTACHED_BUFFER;
        lte.attached_buffer              = (u8*)buf;
 
-       zero_hash(lte.hash);
+       zero_out_hash(lte.hash);
        ret = write_wim_resource(&lte, out_fp, out_ctype, out_res_entry);
        if (ret != 0)
                return ret;
index 27e0daf..28a7184 100644 (file)
  * Steve Reid's public domain code, or based on Intel's SSSE3 SHA1 code.
  */
 
-#ifndef WITH_LIBCRYPTO
+const u8 zero_hash[SHA1_HASH_SIZE] = {
+       0, 0, 0, 0, 0,
+       0, 0, 0, 0, 0,
+       0, 0, 0, 0, 0,
+       0, 0, 0, 0, 0,
+};
+
 
+#ifndef WITH_LIBCRYPTO
 
 /*  Initialize new context */
 void sha1_init(SHA_CTX* context)
index a0d6335..8573c54 100644 (file)
@@ -9,6 +9,8 @@
 
 #define SHA1_HASH_SIZE 20
 
+extern const u8 zero_hash[SHA1_HASH_SIZE];
+
 static inline void copy_hash(u8 dest[SHA1_HASH_SIZE],
                             const u8 src[SHA1_HASH_SIZE])
 {
@@ -46,7 +48,7 @@ static inline bool is_zero_hash(const u8 hash[SHA1_HASH_SIZE])
        return true;
 }
 
-static inline void zero_hash(u8 hash[SHA1_HASH_SIZE])
+static inline void zero_out_hash(u8 hash[SHA1_HASH_SIZE])
 {
        memset(hash, 0, SHA1_HASH_SIZE);
 }