6 * Copyright (C) 2012 Eric Biggers
8 * This file is part of wimlib, a library for working with WIM files.
10 * wimlib is free software; you can redistribute it and/or modify it under the
11 * terms of the GNU General Public License as published by the Free
12 * Software Foundation; either version 3 of the License, or (at your option)
15 * wimlib is distributed in the hope that it will be useful, but WITHOUT ANY
16 * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
17 * A PARTICULAR PURPOSE. See the GNU General Public License for more
20 * You should have received a copy of the GNU General Public License
21 * along with wimlib; if not, see http://www.gnu.org/licenses/.
24 #include "wimlib_internal.h"
26 #include "timestamp.h"
27 #include "lookup_table.h"
37 /** Private flag: Used to mark that we currently adding the root directory of
39 #define WIMLIB_ADD_IMAGE_FLAG_ROOT 0x80000000
42 * Adds an image (given by its dentry tree) to the image metadata array of a WIM
43 * file, adds an entry to the lookup table for the image metadata, updates the
44 * image count in the header, and selects the new image.
46 * Does not update the XML data.
48 * On failure, WIMLIB_ERR_NOMEM is returned and no changes are made. Otherwise,
49 * 0 is returned and the image metadata array of @w is modified.
51 * @w: The WIMStruct for the WIM file.
52 * @root_dentry: The root of the directory tree for the image.
53 * @sd: The security data for the image.
55 int add_new_dentry_tree(WIMStruct *w, struct dentry *root_dentry,
56 struct wim_security_data *sd)
58 struct lookup_table_entry *metadata_lte;
59 struct image_metadata *imd;
60 struct image_metadata *new_imd;
63 wimlib_assert(root_dentry != NULL);
65 DEBUG("Reallocating image metadata array for image_count = %u",
66 w->hdr.image_count + 1);
67 imd = CALLOC((w->hdr.image_count + 1), sizeof(struct image_metadata));
70 ERROR("Failed to allocate memory for new image metadata array");
74 memcpy(imd, w->image_metadata,
75 w->hdr.image_count * sizeof(struct image_metadata));
77 metadata_lte = new_lookup_table_entry();
81 metadata_lte->resource_entry.flags = WIM_RESHDR_FLAG_METADATA;
82 random_hash(metadata_lte->hash);
83 lookup_table_insert(w->lookup_table, metadata_lte);
85 new_imd = &imd[w->hdr.image_count];
87 new_imd->root_dentry = root_dentry;
88 new_imd->metadata_lte = metadata_lte;
89 new_imd->security_data = sd;
90 new_imd->modified = true;
92 FREE(w->image_metadata);
93 w->image_metadata = imd;
96 /* Change the current image to the new one. There should not be any
97 * ways for this to fail, since the image is valid and the dentry tree
98 * is already in memory. */
99 ret = select_wim_image(w, w->hdr.image_count);
100 wimlib_assert(ret == 0);
105 return WIMLIB_ERR_NOMEM;
111 * Recursively builds a dentry tree from a directory tree on disk, outside the
114 * @root_ret: Place to return a pointer to the root of the dentry tree. Only
115 * modified if successful. NULL if the file or directory was
116 * excluded from capture.
118 * @root_disk_path: The path to the root of the directory tree on disk.
120 * @lookup_table: The lookup table for the WIM file. For each file added to the
121 * dentry tree being built, an entry is added to the lookup table,
122 * unless an identical stream is already in the lookup table.
123 * These lookup table entries that are added point to the path of
126 * @sd: Ignored. (Security data only captured in NTFS mode.)
129 * Configuration for files to be excluded from capture.
131 * @add_flags: Bitwise or of WIMLIB_ADD_IMAGE_FLAG_*
133 * @extra_arg: Ignored. (Only used in NTFS mode.)
135 * @return: 0 on success, nonzero on failure. It is a failure if any of
136 * the files cannot be `stat'ed, or if any of the needed
137 * directories cannot be opened or read. Failure to add the files
138 * to the WIM may still occur later when trying to actually read
139 * the on-disk files during a call to wimlib_write() or
140 * wimlib_overwrite().
142 static int build_dentry_tree(struct dentry **root_ret,
143 const char *root_disk_path,
144 struct lookup_table *lookup_table,
145 struct wim_security_data *sd,
146 const struct capture_config *config,
148 wimlib_progress_func_t progress_func,
151 struct stat root_stbuf;
153 int (*stat_fn)(const char *restrict, struct stat *restrict);
155 const char *filename;
158 if (exclude_path(root_disk_path, config, true)) {
159 if (add_image_flags & WIMLIB_ADD_IMAGE_FLAG_ROOT) {
160 ERROR("Cannot exclude the root directory from capture");
161 return WIMLIB_ERR_INVALID_CAPTURE_CONFIG;
163 if ((add_image_flags & WIMLIB_ADD_IMAGE_FLAG_VERBOSE)
166 union wimlib_progress_info info;
167 info.scan.cur_path = root_disk_path;
168 info.scan.excluded = true;
169 progress_func(WIMLIB_PROGRESS_MSG_SCAN_DENTRY, &info);
175 if ((add_image_flags & WIMLIB_ADD_IMAGE_FLAG_VERBOSE)
178 union wimlib_progress_info info;
179 info.scan.cur_path = root_disk_path;
180 info.scan.excluded = false;
181 progress_func(WIMLIB_PROGRESS_MSG_SCAN_DENTRY, &info);
184 if (add_image_flags & WIMLIB_ADD_IMAGE_FLAG_DEREFERENCE)
189 ret = (*stat_fn)(root_disk_path, &root_stbuf);
191 ERROR_WITH_ERRNO("Failed to stat `%s'", root_disk_path);
192 return WIMLIB_ERR_STAT;
195 if ((add_image_flags & WIMLIB_ADD_IMAGE_FLAG_ROOT) &&
196 !S_ISDIR(root_stbuf.st_mode))
198 /* Do a dereference-stat in case the root is a symbolic link.
199 * This case is allowed, provided that the symbolic link points
201 ret = stat(root_disk_path, &root_stbuf);
203 ERROR_WITH_ERRNO("Failed to stat `%s'", root_disk_path);
204 return WIMLIB_ERR_STAT;
206 if (!S_ISDIR(root_stbuf.st_mode)) {
207 ERROR("`%s' is not a directory", root_disk_path);
208 return WIMLIB_ERR_NOTDIR;
211 if (!S_ISREG(root_stbuf.st_mode) && !S_ISDIR(root_stbuf.st_mode)
212 && !S_ISLNK(root_stbuf.st_mode)) {
213 ERROR("`%s' is not a regular file, directory, or symbolic link.",
215 return WIMLIB_ERR_SPECIAL_FILE;
218 if (add_image_flags & WIMLIB_ADD_IMAGE_FLAG_ROOT)
221 filename = path_basename(root_disk_path);
223 root = new_dentry_with_timeless_inode(filename);
225 return WIMLIB_ERR_NOMEM;
227 inode = root->d_inode;
229 #ifdef HAVE_STAT_NANOSECOND_PRECISION
230 inode->creation_time = timespec_to_wim_timestamp(&root_stbuf.st_mtim);
231 inode->last_write_time = timespec_to_wim_timestamp(&root_stbuf.st_mtim);
232 inode->last_access_time = timespec_to_wim_timestamp(&root_stbuf.st_atim);
234 inode->creation_time = unix_timestamp_to_wim(root_stbuf.st_mtime);
235 inode->last_write_time = unix_timestamp_to_wim(root_stbuf.st_mtime);
236 inode->last_access_time = unix_timestamp_to_wim(root_stbuf.st_atime);
238 if (sizeof(ino_t) >= 8)
239 inode->ino = (u64)root_stbuf.st_ino;
241 inode->ino = (u64)root_stbuf.st_ino |
242 ((u64)root_stbuf.st_dev << ((sizeof(ino_t) * 8) & 63));
244 add_image_flags &= ~WIMLIB_ADD_IMAGE_FLAG_ROOT;
247 if (S_ISREG(root_stbuf.st_mode)) { /* Archiving a regular file */
249 struct lookup_table_entry *lte;
250 u8 hash[SHA1_HASH_SIZE];
252 inode->attributes = FILE_ATTRIBUTE_NORMAL;
254 /* Empty files do not have to have a lookup table entry. */
255 if (root_stbuf.st_size == 0)
258 /* For each regular file, we must check to see if the file is in
259 * the lookup table already; if it is, we increment its refcnt;
260 * otherwise, we create a new lookup table entry and insert it.
263 ret = sha1sum(root_disk_path, hash);
267 lte = __lookup_resource(lookup_table, hash);
270 DEBUG("Add lte reference %u for `%s'", lte->refcnt,
273 char *file_on_disk = STRDUP(root_disk_path);
275 ERROR("Failed to allocate memory for file path");
276 ret = WIMLIB_ERR_NOMEM;
279 lte = new_lookup_table_entry();
282 ret = WIMLIB_ERR_NOMEM;
285 lte->file_on_disk = file_on_disk;
286 lte->resource_location = RESOURCE_IN_FILE_ON_DISK;
287 lte->resource_entry.original_size = root_stbuf.st_size;
288 lte->resource_entry.size = root_stbuf.st_size;
289 copy_hash(lte->hash, hash);
290 lookup_table_insert(lookup_table, lte);
292 root->d_inode->lte = lte;
293 } else if (S_ISDIR(root_stbuf.st_mode)) { /* Archiving a directory */
295 inode->attributes = FILE_ATTRIBUTE_DIRECTORY;
298 struct dirent entry, *result;
299 struct dentry *child;
301 dir = opendir(root_disk_path);
303 ERROR_WITH_ERRNO("Failed to open the directory `%s'",
305 ret = WIMLIB_ERR_OPEN;
309 /* Buffer for names of files in directory. */
310 size_t len = strlen(root_disk_path);
311 char name[len + 1 + FILENAME_MAX + 1];
312 memcpy(name, root_disk_path, len);
315 /* Create a dentry for each entry in the directory on disk, and recurse
316 * to any subdirectories. */
319 ret = readdir_r(dir, &entry, &result);
321 ret = WIMLIB_ERR_READ;
322 ERROR_WITH_ERRNO("Error reading the "
329 if (result->d_name[0] == '.' && (result->d_name[1] == '\0'
330 || (result->d_name[1] == '.' && result->d_name[2] == '\0')))
332 strcpy(name + len + 1, result->d_name);
333 ret = build_dentry_tree(&child, name, lookup_table,
334 NULL, config, add_image_flags,
335 progress_func, NULL);
339 dentry_add_child(root, child);
342 } else { /* Archiving a symbolic link */
343 inode->attributes = FILE_ATTRIBUTE_REPARSE_POINT;
344 inode->reparse_tag = WIM_IO_REPARSE_TAG_SYMLINK;
346 /* The idea here is to call readlink() to get the UNIX target of
347 * the symbolic link, then turn the target into a reparse point
348 * data buffer that contains a relative or absolute symbolic
349 * link (NOT a junction point or *full* path symbolic link with
353 char deref_name_buf[4096];
354 ssize_t deref_name_len;
356 deref_name_len = readlink(root_disk_path, deref_name_buf,
357 sizeof(deref_name_buf) - 1);
358 if (deref_name_len >= 0) {
359 deref_name_buf[deref_name_len] = '\0';
360 DEBUG("Read symlink `%s'", deref_name_buf);
361 ret = inode_set_symlink(root->d_inode, deref_name_buf,
365 * Unfortunately, Windows seems to have the
366 * concept of "file" symbolic links as being
367 * different from "directory" symbolic links...
368 * so FILE_ATTRIBUTE_DIRECTORY needs to be set
369 * on the symbolic link if the *target* of the
370 * symbolic link is a directory.
373 if (stat(root_disk_path, &stbuf) == 0 &&
374 S_ISDIR(stbuf.st_mode))
376 inode->attributes |= FILE_ATTRIBUTE_DIRECTORY;
380 ERROR_WITH_ERRNO("Failed to read target of "
381 "symbolic link `%s'", root_disk_path);
382 ret = WIMLIB_ERR_READLINK;
389 free_dentry_tree(root, lookup_table);
398 COMPRESSION_EXCLUSION_LIST,
402 /* Default capture configuration file when none is specified. */
403 static const char *default_config =
408 "\\System Volume Information\n"
412 "[CompressionExclusionList]\n"
416 "\\WINDOWS\\inf\\*.pnf\n";
418 static void destroy_pattern_list(struct pattern_list *list)
423 static void destroy_capture_config(struct capture_config *config)
425 destroy_pattern_list(&config->exclusion_list);
426 destroy_pattern_list(&config->exclusion_exception);
427 destroy_pattern_list(&config->compression_exclusion_list);
428 destroy_pattern_list(&config->alignment_list);
429 FREE(config->config_str);
430 FREE(config->prefix);
431 memset(config, 0, sizeof(*config));
434 static int pattern_list_add_pattern(struct pattern_list *list,
438 if (list->num_pats >= list->num_allocated_pats) {
439 pats = REALLOC(list->pats,
440 sizeof(list->pats[0]) * (list->num_allocated_pats + 8));
442 return WIMLIB_ERR_NOMEM;
443 list->num_allocated_pats += 8;
446 list->pats[list->num_pats++] = pattern;
450 /* Parses the contents of the image capture configuration file and fills in a
451 * `struct capture_config'. */
452 static int init_capture_config(const char *_config_str, size_t config_len,
453 const char *_prefix, struct capture_config *config)
460 size_t bytes_remaining;
461 enum pattern_type type = NONE;
463 unsigned long line_no = 0;
465 DEBUG("config_len = %zu", config_len);
466 bytes_remaining = config_len;
467 memset(config, 0, sizeof(*config));
468 config_str = MALLOC(config_len);
470 ERROR("Could not duplicate capture config string");
471 return WIMLIB_ERR_NOMEM;
473 prefix = STRDUP(_prefix);
476 return WIMLIB_ERR_NOMEM;
479 memcpy(config_str, _config_str, config_len);
481 config->config_str = config_str;
482 config->prefix = prefix;
483 config->prefix_len = strlen(prefix);
484 while (bytes_remaining) {
487 eol = memchr(p, '\n', bytes_remaining);
489 ERROR("Expected end-of-line in capture config file on "
490 "line %lu", line_no);
491 ret = WIMLIB_ERR_INVALID_CAPTURE_CONFIG;
496 bytes_remaining -= (next_p - p);
500 if (*(eol - 1) == '\r')
504 /* Translate backslash to forward slash */
505 for (char *pp = p; pp != eol; pp++)
509 /* Remove drive letter */
510 if (eol - p > 2 && isalpha(*p) && *(p + 1) == ':')
514 if (strcmp(p, "[ExclusionList]") == 0)
515 type = EXCLUSION_LIST;
516 else if (strcmp(p, "[ExclusionException]") == 0)
517 type = EXCLUSION_EXCEPTION;
518 else if (strcmp(p, "[CompressionExclusionList]") == 0)
519 type = COMPRESSION_EXCLUSION_LIST;
520 else if (strcmp(p, "[AlignmentList]") == 0)
521 type = ALIGNMENT_LIST;
522 else if (p[0] == '[' && strrchr(p, ']')) {
523 ERROR("Unknown capture configuration section `%s'", p);
524 ret = WIMLIB_ERR_INVALID_CAPTURE_CONFIG;
525 } else switch (type) {
527 DEBUG("Adding pattern \"%s\" to exclusion list", p);
528 ret = pattern_list_add_pattern(&config->exclusion_list, p);
530 case EXCLUSION_EXCEPTION:
531 DEBUG("Adding pattern \"%s\" to exclusion exception list", p);
532 ret = pattern_list_add_pattern(&config->exclusion_exception, p);
534 case COMPRESSION_EXCLUSION_LIST:
535 DEBUG("Adding pattern \"%s\" to compression exclusion list", p);
536 ret = pattern_list_add_pattern(&config->compression_exclusion_list, p);
539 DEBUG("Adding pattern \"%s\" to alignment list", p);
540 ret = pattern_list_add_pattern(&config->alignment_list, p);
543 ERROR("Line %lu of capture configuration is not "
544 "in a block (such as [ExclusionList])",
546 ret = WIMLIB_ERR_INVALID_CAPTURE_CONFIG;
554 destroy_capture_config(config);
558 static bool match_pattern(const char *path, const char *path_basename,
559 const struct pattern_list *list)
561 for (size_t i = 0; i < list->num_pats; i++) {
562 const char *pat = list->pats[i];
565 /* Absolute path from root of capture */
568 if (strchr(pat, '/'))
569 /* Relative path from root of capture */
572 /* A file name pattern */
573 string = path_basename;
575 if (fnmatch(pat, string, FNM_PATHNAME
581 DEBUG("`%s' matches the pattern \"%s\"",
589 static void print_pattern_list(const struct pattern_list *list)
591 for (size_t i = 0; i < list->num_pats; i++)
592 printf(" %s\n", list->pats[i]);
595 static void print_capture_config(const struct capture_config *config)
597 if (config->exclusion_list.num_pats) {
598 puts("Files or folders excluded from image capture:");
599 print_pattern_list(&config->exclusion_list);
604 /* Return true if the image capture configuration file indicates we should
605 * exclude the filename @path from capture.
607 * If @exclude_prefix is %true, the part of the path up and including the name
608 * of the directory being captured is not included in the path for matching
609 * purposes. This allows, for example, a pattern like /hiberfil.sys to match a
610 * file /mnt/windows7/hiberfil.sys if we are capturing the /mnt/windows7
613 bool exclude_path(const char *path, const struct capture_config *config,
616 const char *basename = path_basename(path);
617 if (exclude_prefix) {
618 wimlib_assert(strlen(path) >= config->prefix_len);
619 if (memcmp(config->prefix, path, config->prefix_len) == 0
620 && path[config->prefix_len] == '/')
621 path += config->prefix_len;
623 return match_pattern(path, basename, &config->exclusion_list) &&
624 !match_pattern(path, basename, &config->exclusion_exception);
628 WIMLIBAPI int wimlib_add_image(WIMStruct *w, const char *source,
629 const char *name, const char *config_str,
630 size_t config_len, int add_image_flags,
631 wimlib_progress_func_t progress_func)
633 int (*capture_tree)(struct dentry **, const char *,
634 struct lookup_table *,
635 struct wim_security_data *,
636 const struct capture_config *,
637 int, wimlib_progress_func_t, void *);
640 struct dentry *root_dentry = NULL;
641 struct wim_security_data *sd;
642 struct capture_config config;
643 struct inode_table inode_tab;
644 struct hlist_head inode_list;
647 if (add_image_flags & WIMLIB_ADD_IMAGE_FLAG_NTFS) {
649 if (add_image_flags & (WIMLIB_ADD_IMAGE_FLAG_DEREFERENCE)) {
650 ERROR("Cannot dereference files when capturing directly from NTFS");
651 return WIMLIB_ERR_INVALID_PARAM;
653 capture_tree = build_dentry_tree_ntfs;
654 extra_arg = &w->ntfs_vol;
656 ERROR("wimlib was compiled without support for NTFS-3g, so\n"
657 " cannot capture a WIM image directly from a NTFS volume!");
658 return WIMLIB_ERR_UNSUPPORTED;
661 capture_tree = build_dentry_tree;
665 DEBUG("Adding dentry tree from directory or NTFS volume `%s'.", source);
667 if (!name || !*name) {
668 ERROR("Must specify a non-empty string for the image name");
669 return WIMLIB_ERR_INVALID_PARAM;
671 if (!source || !*source) {
672 ERROR("Must specify the name of a directory or NTFS volume");
673 return WIMLIB_ERR_INVALID_PARAM;
676 if (w->hdr.total_parts != 1) {
677 ERROR("Cannot add an image to a split WIM");
678 return WIMLIB_ERR_SPLIT_UNSUPPORTED;
681 if (wimlib_image_name_in_use(w, name)) {
682 ERROR("There is already an image named \"%s\" in `%s'",
684 return WIMLIB_ERR_IMAGE_NAME_COLLISION;
687 DEBUG("Initializing capture configuration");
689 DEBUG("Using default capture configuration");
690 config_str = default_config;
691 config_len = strlen(default_config);
693 ret = init_capture_config(config_str, config_len, source, &config);
696 print_capture_config(&config);
698 DEBUG("Allocating security data");
700 sd = CALLOC(1, sizeof(struct wim_security_data));
702 ret = WIMLIB_ERR_NOMEM;
703 goto out_destroy_config;
705 sd->total_length = 8;
709 union wimlib_progress_info progress;
710 progress.scan.source = source;
711 progress_func(WIMLIB_PROGRESS_MSG_SCAN_BEGIN, &progress);
714 DEBUG("Building dentry tree.");
715 ret = (*capture_tree)(&root_dentry, source, w->lookup_table, sd,
716 &config, add_image_flags | WIMLIB_ADD_IMAGE_FLAG_ROOT,
717 progress_func, extra_arg);
718 destroy_capture_config(&config);
721 ERROR("Failed to build dentry tree for `%s'", source);
722 goto out_free_security_data;
726 union wimlib_progress_info progress;
727 progress.scan.source = source;
728 progress_func(WIMLIB_PROGRESS_MSG_SCAN_END, &progress);
731 DEBUG("Calculating full paths of dentries.");
732 ret = for_dentry_in_tree(root_dentry, calculate_dentry_full_path, NULL);
734 goto out_free_dentry_tree;
736 ret = add_new_dentry_tree(w, root_dentry, sd);
738 goto out_free_dentry_tree;
740 DEBUG("Inserting dentries into inode table");
741 ret = init_inode_table(&inode_tab, 9001);
743 goto out_destroy_imd;
745 for_dentry_in_tree(root_dentry, inode_table_insert, &inode_tab);
747 DEBUG("Cleaning up the hard link groups");
748 ret = fix_inodes(&inode_tab, &inode_list);
749 destroy_inode_table(&inode_tab);
751 goto out_destroy_imd;
753 DEBUG("Assigning hard link group IDs");
754 assign_inode_numbers(&inode_list);
755 w->image_metadata[w->hdr.image_count - 1].inode_list = inode_list;
757 ret = xml_add_image(w, name);
759 goto out_destroy_imd;
761 if (add_image_flags & WIMLIB_ADD_IMAGE_FLAG_BOOT)
762 wimlib_set_boot_idx(w, w->hdr.image_count);
765 destroy_image_metadata(&w->image_metadata[w->hdr.image_count - 1],
767 w->hdr.image_count--;
769 out_free_dentry_tree:
770 free_dentry_tree(root_dentry, w->lookup_table);
771 out_free_security_data:
772 free_security_data(sd);
774 destroy_capture_config(&config);