6 * Copyright (C) 2012, 2013 Biggers
8 * This file is part of wimlib, a library for working with WIM files.
10 * wimlib is free software; you can redistribute it and/or modify it under the
11 * terms of the GNU General Public License as published by the Free
12 * Software Foundation; either version 3 of the License, or (at your option)
15 * wimlib is distributed in the hope that it will be useful, but WITHOUT ANY
16 * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
17 * A PARTICULAR PURPOSE. See the GNU General Public License for more
20 * You should have received a copy of the GNU General Public License
21 * along with wimlib; if not, see http://www.gnu.org/licenses/.
24 #include "wimlib_internal.h"
26 #include "timestamp.h"
27 #include "lookup_table.h"
37 /** Private flag: Used to mark that we currently adding the root directory of
39 #define WIMLIB_ADD_IMAGE_FLAG_ROOT 0x80000000
42 * Adds the dentry tree and security data for a new image to the image metadata
43 * array of the WIMStruct.
45 int add_new_dentry_tree(WIMStruct *w, struct wim_dentry *root_dentry,
46 struct wim_security_data *sd)
48 struct wim_lookup_table_entry *metadata_lte;
49 struct wim_image_metadata *imd;
50 struct wim_image_metadata *new_imd;
52 wimlib_assert(root_dentry != NULL);
54 DEBUG("Reallocating image metadata array for image_count = %u",
55 w->hdr.image_count + 1);
56 imd = CALLOC((w->hdr.image_count + 1), sizeof(struct wim_image_metadata));
59 ERROR("Failed to allocate memory for new image metadata array");
63 memcpy(imd, w->image_metadata,
64 w->hdr.image_count * sizeof(struct wim_image_metadata));
66 metadata_lte = new_lookup_table_entry();
70 metadata_lte->resource_entry.flags = WIM_RESHDR_FLAG_METADATA;
71 random_hash(metadata_lte->hash);
72 lookup_table_insert(w->lookup_table, metadata_lte);
74 new_imd = &imd[w->hdr.image_count];
76 new_imd->root_dentry = root_dentry;
77 new_imd->metadata_lte = metadata_lte;
78 new_imd->security_data = sd;
79 new_imd->modified = 1;
81 FREE(w->image_metadata);
82 w->image_metadata = imd;
88 return WIMLIB_ERR_NOMEM;
94 * build_dentry_tree():
95 * Recursively builds a tree of WIM dentries from an on-disk directory
98 * @root_ret: Place to return a pointer to the root of the dentry tree. Only
99 * modified if successful. Set to NULL if the file or directory was
100 * excluded from capture.
102 * @root_disk_path: The path to the root of the directory tree on disk.
104 * @lookup_table: The lookup table for the WIM file. For each file added to the
105 * dentry tree being built, an entry is added to the lookup table,
106 * unless an identical stream is already in the lookup table.
107 * These lookup table entries that are added point to the path of
110 * @sd: Ignored. (Security data only captured in NTFS mode.)
113 * Configuration for files to be excluded from capture.
115 * @add_flags: Bitwise or of WIMLIB_ADD_IMAGE_FLAG_*
117 * @extra_arg: Ignored. (Only used in NTFS mode.)
119 * @return: 0 on success, nonzero on failure. It is a failure if any of
120 * the files cannot be `stat'ed, or if any of the needed
121 * directories cannot be opened or read. Failure to add the files
122 * to the WIM may still occur later when trying to actually read
123 * the on-disk files during a call to wimlib_write() or
124 * wimlib_overwrite().
126 static int build_dentry_tree(struct wim_dentry **root_ret,
127 const char *root_disk_path,
128 struct wim_lookup_table *lookup_table,
129 struct wim_security_data *sd,
130 const struct capture_config *config,
132 wimlib_progress_func_t progress_func,
135 struct stat root_stbuf;
137 int (*stat_fn)(const char *restrict, struct stat *restrict);
138 struct wim_dentry *root;
139 const char *filename;
140 struct wim_inode *inode;
142 if (exclude_path(root_disk_path, config, true)) {
143 if (add_image_flags & WIMLIB_ADD_IMAGE_FLAG_ROOT) {
144 ERROR("Cannot exclude the root directory from capture");
145 return WIMLIB_ERR_INVALID_CAPTURE_CONFIG;
147 if ((add_image_flags & WIMLIB_ADD_IMAGE_FLAG_VERBOSE)
150 union wimlib_progress_info info;
151 info.scan.cur_path = root_disk_path;
152 info.scan.excluded = true;
153 progress_func(WIMLIB_PROGRESS_MSG_SCAN_DENTRY, &info);
159 if ((add_image_flags & WIMLIB_ADD_IMAGE_FLAG_VERBOSE)
162 union wimlib_progress_info info;
163 info.scan.cur_path = root_disk_path;
164 info.scan.excluded = false;
165 progress_func(WIMLIB_PROGRESS_MSG_SCAN_DENTRY, &info);
168 if (add_image_flags & WIMLIB_ADD_IMAGE_FLAG_DEREFERENCE)
173 ret = (*stat_fn)(root_disk_path, &root_stbuf);
175 ERROR_WITH_ERRNO("Failed to stat `%s'", root_disk_path);
176 return WIMLIB_ERR_STAT;
179 if ((add_image_flags & WIMLIB_ADD_IMAGE_FLAG_ROOT) &&
180 !S_ISDIR(root_stbuf.st_mode))
182 /* Do a dereference-stat in case the root is a symbolic link.
183 * This case is allowed, provided that the symbolic link points
185 ret = stat(root_disk_path, &root_stbuf);
187 ERROR_WITH_ERRNO("Failed to stat `%s'", root_disk_path);
188 return WIMLIB_ERR_STAT;
190 if (!S_ISDIR(root_stbuf.st_mode)) {
191 ERROR("`%s' is not a directory", root_disk_path);
192 return WIMLIB_ERR_NOTDIR;
195 if (!S_ISREG(root_stbuf.st_mode) && !S_ISDIR(root_stbuf.st_mode)
196 && !S_ISLNK(root_stbuf.st_mode)) {
197 ERROR("`%s' is not a regular file, directory, or symbolic link.",
199 return WIMLIB_ERR_SPECIAL_FILE;
202 if (add_image_flags & WIMLIB_ADD_IMAGE_FLAG_ROOT)
205 filename = path_basename(root_disk_path);
207 root = new_dentry_with_timeless_inode(filename);
210 return WIMLIB_ERR_INVALID_UTF8_STRING;
211 else if (errno == ENOMEM)
212 return WIMLIB_ERR_NOMEM;
214 return WIMLIB_ERR_ICONV_NOT_AVAILABLE;
217 inode = root->d_inode;
219 #ifdef HAVE_STAT_NANOSECOND_PRECISION
220 inode->i_creation_time = timespec_to_wim_timestamp(&root_stbuf.st_mtim);
221 inode->i_last_write_time = timespec_to_wim_timestamp(&root_stbuf.st_mtim);
222 inode->i_last_access_time = timespec_to_wim_timestamp(&root_stbuf.st_atim);
224 inode->i_creation_time = unix_timestamp_to_wim(root_stbuf.st_mtime);
225 inode->i_last_write_time = unix_timestamp_to_wim(root_stbuf.st_mtime);
226 inode->i_last_access_time = unix_timestamp_to_wim(root_stbuf.st_atime);
228 if (sizeof(ino_t) >= 8)
229 inode->i_ino = (u64)root_stbuf.st_ino;
231 inode->i_ino = (u64)root_stbuf.st_ino |
232 ((u64)root_stbuf.st_dev << ((sizeof(ino_t) * 8) & 63));
233 inode->i_resolved = 1;
234 if (add_image_flags & WIMLIB_ADD_IMAGE_FLAG_UNIX_DATA) {
235 ret = inode_set_unix_data(inode, root_stbuf.st_uid,
239 UNIX_DATA_ALL | UNIX_DATA_CREATE);
243 add_image_flags &= ~WIMLIB_ADD_IMAGE_FLAG_ROOT;
244 if (S_ISREG(root_stbuf.st_mode)) { /* Archiving a regular file */
246 struct wim_lookup_table_entry *lte;
247 u8 hash[SHA1_HASH_SIZE];
249 inode->i_attributes = FILE_ATTRIBUTE_NORMAL;
251 /* Empty files do not have to have a lookup table entry. */
252 if (root_stbuf.st_size == 0)
255 /* For each regular file, we must check to see if the file is in
256 * the lookup table already; if it is, we increment its refcnt;
257 * otherwise, we create a new lookup table entry and insert it.
260 ret = sha1sum(root_disk_path, hash);
264 lte = __lookup_resource(lookup_table, hash);
267 DEBUG("Add lte reference %u for `%s'", lte->refcnt,
270 char *file_on_disk = STRDUP(root_disk_path);
272 ERROR("Failed to allocate memory for file path");
273 ret = WIMLIB_ERR_NOMEM;
276 lte = new_lookup_table_entry();
279 ret = WIMLIB_ERR_NOMEM;
282 lte->file_on_disk = file_on_disk;
283 lte->resource_location = RESOURCE_IN_FILE_ON_DISK;
284 lte->resource_entry.original_size = root_stbuf.st_size;
285 lte->resource_entry.size = root_stbuf.st_size;
286 copy_hash(lte->hash, hash);
287 lookup_table_insert(lookup_table, lte);
289 root->d_inode->i_lte = lte;
290 } else if (S_ISDIR(root_stbuf.st_mode)) { /* Archiving a directory */
292 inode->i_attributes = FILE_ATTRIBUTE_DIRECTORY;
295 struct dirent entry, *result;
296 struct wim_dentry *child;
298 dir = opendir(root_disk_path);
300 ERROR_WITH_ERRNO("Failed to open the directory `%s'",
302 ret = WIMLIB_ERR_OPEN;
306 /* Buffer for names of files in directory. */
307 size_t len = strlen(root_disk_path);
308 char name[len + 1 + FILENAME_MAX + 1];
309 memcpy(name, root_disk_path, len);
312 /* Create a dentry for each entry in the directory on disk, and recurse
313 * to any subdirectories. */
316 ret = readdir_r(dir, &entry, &result);
318 ret = WIMLIB_ERR_READ;
319 ERROR_WITH_ERRNO("Error reading the "
326 if (result->d_name[0] == '.' && (result->d_name[1] == '\0'
327 || (result->d_name[1] == '.' && result->d_name[2] == '\0')))
329 strcpy(name + len + 1, result->d_name);
330 ret = build_dentry_tree(&child, name, lookup_table,
331 NULL, config, add_image_flags,
332 progress_func, NULL);
336 dentry_add_child(root, child);
339 } else { /* Archiving a symbolic link */
340 inode->i_attributes = FILE_ATTRIBUTE_REPARSE_POINT;
341 inode->i_reparse_tag = WIM_IO_REPARSE_TAG_SYMLINK;
343 /* The idea here is to call readlink() to get the UNIX target of
344 * the symbolic link, then turn the target into a reparse point
345 * data buffer that contains a relative or absolute symbolic
346 * link (NOT a junction point or *full* path symbolic link with
350 char deref_name_buf[4096];
351 ssize_t deref_name_len;
353 deref_name_len = readlink(root_disk_path, deref_name_buf,
354 sizeof(deref_name_buf) - 1);
355 if (deref_name_len >= 0) {
356 deref_name_buf[deref_name_len] = '\0';
357 DEBUG("Read symlink `%s'", deref_name_buf);
358 ret = inode_set_symlink(root->d_inode, deref_name_buf,
362 * Unfortunately, Windows seems to have the
363 * concept of "file" symbolic links as being
364 * different from "directory" symbolic links...
365 * so FILE_ATTRIBUTE_DIRECTORY needs to be set
366 * on the symbolic link if the *target* of the
367 * symbolic link is a directory.
370 if (stat(root_disk_path, &stbuf) == 0 &&
371 S_ISDIR(stbuf.st_mode))
373 inode->i_attributes |= FILE_ATTRIBUTE_DIRECTORY;
377 ERROR_WITH_ERRNO("Failed to read target of "
378 "symbolic link `%s'", root_disk_path);
379 ret = WIMLIB_ERR_READLINK;
386 free_dentry_tree(root, lookup_table);
395 COMPRESSION_EXCLUSION_LIST,
399 #define COMPAT_DEFAULT_CONFIG
401 /* Default capture configuration file when none is specified. */
402 static const char *default_config =
403 #ifdef COMPAT_DEFAULT_CONFIG /* XXX: This policy is being moved to library
404 users. The next ABI-incompatible library
405 version will default to the empty string here. */
410 "\\System Volume Information\n"
414 "[CompressionExclusionList]\n"
418 "\\WINDOWS\\inf\\*.pnf\n";
423 static void destroy_pattern_list(struct pattern_list *list)
428 static void destroy_capture_config(struct capture_config *config)
430 destroy_pattern_list(&config->exclusion_list);
431 destroy_pattern_list(&config->exclusion_exception);
432 destroy_pattern_list(&config->compression_exclusion_list);
433 destroy_pattern_list(&config->alignment_list);
434 FREE(config->config_str);
435 FREE(config->prefix);
436 memset(config, 0, sizeof(*config));
439 static int pattern_list_add_pattern(struct pattern_list *list,
443 if (list->num_pats >= list->num_allocated_pats) {
444 pats = REALLOC(list->pats,
445 sizeof(list->pats[0]) * (list->num_allocated_pats + 8));
447 return WIMLIB_ERR_NOMEM;
448 list->num_allocated_pats += 8;
451 list->pats[list->num_pats++] = pattern;
455 /* Parses the contents of the image capture configuration file and fills in a
456 * `struct capture_config'. */
457 static int init_capture_config(const char *_config_str, size_t config_len,
458 const char *_prefix, struct capture_config *config)
465 size_t bytes_remaining;
466 enum pattern_type type = NONE;
468 unsigned long line_no = 0;
470 DEBUG("config_len = %zu", config_len);
471 bytes_remaining = config_len;
472 memset(config, 0, sizeof(*config));
473 config_str = MALLOC(config_len);
475 ERROR("Could not duplicate capture config string");
476 return WIMLIB_ERR_NOMEM;
478 prefix = STRDUP(_prefix);
481 return WIMLIB_ERR_NOMEM;
484 memcpy(config_str, _config_str, config_len);
486 config->config_str = config_str;
487 config->prefix = prefix;
488 config->prefix_len = strlen(prefix);
489 while (bytes_remaining) {
492 eol = memchr(p, '\n', bytes_remaining);
494 ERROR("Expected end-of-line in capture config file on "
495 "line %lu", line_no);
496 ret = WIMLIB_ERR_INVALID_CAPTURE_CONFIG;
501 bytes_remaining -= (next_p - p);
505 if (*(eol - 1) == '\r')
509 /* Translate backslash to forward slash */
510 for (char *pp = p; pp != eol; pp++)
514 /* Remove drive letter */
515 if (eol - p > 2 && isalpha(*p) && *(p + 1) == ':')
519 if (strcmp(p, "[ExclusionList]") == 0)
520 type = EXCLUSION_LIST;
521 else if (strcmp(p, "[ExclusionException]") == 0)
522 type = EXCLUSION_EXCEPTION;
523 else if (strcmp(p, "[CompressionExclusionList]") == 0)
524 type = COMPRESSION_EXCLUSION_LIST;
525 else if (strcmp(p, "[AlignmentList]") == 0)
526 type = ALIGNMENT_LIST;
527 else if (p[0] == '[' && strrchr(p, ']')) {
528 ERROR("Unknown capture configuration section `%s'", p);
529 ret = WIMLIB_ERR_INVALID_CAPTURE_CONFIG;
530 } else switch (type) {
532 DEBUG("Adding pattern \"%s\" to exclusion list", p);
533 ret = pattern_list_add_pattern(&config->exclusion_list, p);
535 case EXCLUSION_EXCEPTION:
536 DEBUG("Adding pattern \"%s\" to exclusion exception list", p);
537 ret = pattern_list_add_pattern(&config->exclusion_exception, p);
539 case COMPRESSION_EXCLUSION_LIST:
540 DEBUG("Adding pattern \"%s\" to compression exclusion list", p);
541 ret = pattern_list_add_pattern(&config->compression_exclusion_list, p);
544 DEBUG("Adding pattern \"%s\" to alignment list", p);
545 ret = pattern_list_add_pattern(&config->alignment_list, p);
548 ERROR("Line %lu of capture configuration is not "
549 "in a block (such as [ExclusionList])",
551 ret = WIMLIB_ERR_INVALID_CAPTURE_CONFIG;
559 destroy_capture_config(config);
563 static bool match_pattern(const char *path, const char *path_basename,
564 const struct pattern_list *list)
566 for (size_t i = 0; i < list->num_pats; i++) {
567 const char *pat = list->pats[i];
570 /* Absolute path from root of capture */
573 if (strchr(pat, '/'))
574 /* Relative path from root of capture */
577 /* A file name pattern */
578 string = path_basename;
580 if (fnmatch(pat, string, FNM_PATHNAME
586 DEBUG("`%s' matches the pattern \"%s\"",
594 /* Return true if the image capture configuration file indicates we should
595 * exclude the filename @path from capture.
597 * If @exclude_prefix is %true, the part of the path up and including the name
598 * of the directory being captured is not included in the path for matching
599 * purposes. This allows, for example, a pattern like /hiberfil.sys to match a
600 * file /mnt/windows7/hiberfil.sys if we are capturing the /mnt/windows7
603 bool exclude_path(const char *path, const struct capture_config *config,
606 const char *basename = path_basename(path);
607 if (exclude_prefix) {
608 wimlib_assert(strlen(path) >= config->prefix_len);
609 if (memcmp(config->prefix, path, config->prefix_len) == 0
610 && path[config->prefix_len] == '/')
611 path += config->prefix_len;
613 return match_pattern(path, basename, &config->exclusion_list) &&
614 !match_pattern(path, basename, &config->exclusion_exception);
618 WIMLIBAPI int wimlib_add_image(WIMStruct *w, const char *source,
619 const char *name, const char *config_str,
620 size_t config_len, int add_image_flags,
621 wimlib_progress_func_t progress_func)
623 int (*capture_tree)(struct wim_dentry **, const char *,
624 struct wim_lookup_table *,
625 struct wim_security_data *,
626 const struct capture_config *,
627 int, wimlib_progress_func_t, void *);
630 struct wim_dentry *root_dentry = NULL;
631 struct wim_security_data *sd;
632 struct capture_config config;
633 struct wim_image_metadata *imd;
636 if (add_image_flags & WIMLIB_ADD_IMAGE_FLAG_NTFS) {
638 if (add_image_flags & WIMLIB_ADD_IMAGE_FLAG_DEREFERENCE) {
639 ERROR("Cannot dereference files when capturing directly from NTFS");
640 return WIMLIB_ERR_INVALID_PARAM;
642 if (add_image_flags & WIMLIB_ADD_IMAGE_FLAG_UNIX_DATA) {
643 ERROR("Capturing UNIX owner and mode not supported "
644 "when capturing directly from NTFS");
645 return WIMLIB_ERR_INVALID_PARAM;
647 capture_tree = build_dentry_tree_ntfs;
648 extra_arg = &w->ntfs_vol;
650 ERROR("wimlib was compiled without support for NTFS-3g, so\n"
651 " cannot capture a WIM image directly from a NTFS volume!");
652 return WIMLIB_ERR_UNSUPPORTED;
655 capture_tree = build_dentry_tree;
659 DEBUG("Adding dentry tree from directory or NTFS volume `%s'.", source);
661 if (!name || !*name) {
662 ERROR("Must specify a non-empty string for the image name");
663 return WIMLIB_ERR_INVALID_PARAM;
665 if (!source || !*source) {
666 ERROR("Must specify the name of a directory or NTFS volume");
667 return WIMLIB_ERR_INVALID_PARAM;
670 if (w->hdr.total_parts != 1) {
671 ERROR("Cannot add an image to a split WIM");
672 return WIMLIB_ERR_SPLIT_UNSUPPORTED;
675 if (wimlib_image_name_in_use(w, name)) {
676 ERROR("There is already an image named \"%s\" in `%s'",
678 return WIMLIB_ERR_IMAGE_NAME_COLLISION;
681 DEBUG("Initializing capture configuration");
683 DEBUG("Using default capture configuration");
684 config_str = default_config;
685 config_len = strlen(default_config);
687 ret = init_capture_config(config_str, config_len, source, &config);
691 DEBUG("Allocating security data");
693 sd = CALLOC(1, sizeof(struct wim_security_data));
695 ret = WIMLIB_ERR_NOMEM;
696 goto out_destroy_config;
698 sd->total_length = 8;
702 union wimlib_progress_info progress;
703 progress.scan.source = source;
704 progress_func(WIMLIB_PROGRESS_MSG_SCAN_BEGIN, &progress);
707 DEBUG("Building dentry tree.");
708 ret = (*capture_tree)(&root_dentry, source, w->lookup_table, sd,
709 &config, add_image_flags | WIMLIB_ADD_IMAGE_FLAG_ROOT,
710 progress_func, extra_arg);
711 destroy_capture_config(&config);
714 ERROR("Failed to build dentry tree for `%s'", source);
715 goto out_free_security_data;
719 union wimlib_progress_info progress;
720 progress.scan.source = source;
721 progress_func(WIMLIB_PROGRESS_MSG_SCAN_END, &progress);
724 DEBUG("Calculating full paths of dentries.");
725 ret = for_dentry_in_tree(root_dentry, calculate_dentry_full_path, NULL);
727 goto out_free_dentry_tree;
729 ret = add_new_dentry_tree(w, root_dentry, sd);
731 goto out_free_dentry_tree;
733 imd = &w->image_metadata[w->hdr.image_count - 1];
735 ret = dentry_tree_fix_inodes(root_dentry, &imd->inode_list);
737 goto out_destroy_imd;
739 DEBUG("Assigning hard link group IDs");
740 assign_inode_numbers(&imd->inode_list);
742 ret = xml_add_image(w, name);
744 goto out_destroy_imd;
746 if (add_image_flags & WIMLIB_ADD_IMAGE_FLAG_BOOT)
747 wimlib_set_boot_idx(w, w->hdr.image_count);
750 destroy_image_metadata(&w->image_metadata[w->hdr.image_count - 1],
752 w->hdr.image_count--;
754 out_free_dentry_tree:
755 free_dentry_tree(root_dentry, w->lookup_table);
756 out_free_security_data:
757 free_security_data(sd);
759 destroy_capture_config(&config);