Update progress functions
[wimlib] / src / unix_capture.c
1 /*
2  * unix_capture.c:  Capture a directory tree on UNIX.
3  */
4
5 /*
6  * Copyright (C) 2012, 2013, 2014 Eric Biggers
7  *
8  * This file is part of wimlib, a library for working with WIM files.
9  *
10  * wimlib is free software; you can redistribute it and/or modify it under the
11  * terms of the GNU General Public License as published by the Free
12  * Software Foundation; either version 3 of the License, or (at your option)
13  * any later version.
14  *
15  * wimlib is distributed in the hope that it will be useful, but WITHOUT ANY
16  * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
17  * A PARTICULAR PURPOSE. See the GNU General Public License for more
18  * details.
19  *
20  * You should have received a copy of the GNU General Public License
21  * along with wimlib; if not, see http://www.gnu.org/licenses/.
22  */
23
24 #ifndef __WIN32__
25
26 #ifdef HAVE_CONFIG_H
27 #  include "config.h"
28 #endif
29
30 #include <dirent.h>
31 #include <errno.h>
32 #include <fcntl.h>
33 #include <limits.h> /* for PATH_MAX */
34 #include <sys/stat.h>
35 #include <unistd.h>
36
37 #include "wimlib/capture.h"
38 #include "wimlib/dentry.h"
39 #include "wimlib/error.h"
40 #include "wimlib/lookup_table.h"
41 #include "wimlib/reparse.h"
42 #include "wimlib/timestamp.h"
43
44 #ifdef HAVE_FDOPENDIR
45 #  define my_fdopendir(dirfd_p) fdopendir(*(dirfd_p))
46 #else
47 static DIR *
48 my_fdopendir(int *dirfd_p)
49 {
50         DIR *dir = NULL;
51         int old_pwd;
52
53         old_pwd = open(".", O_RDONLY);
54         if (old_pwd >= 0) {
55                 if (!fchdir(*dirfd_p)) {
56                         dir = opendir(".");
57                         if (dir) {
58                                 close(*dirfd_p);
59                                 *dirfd_p = dirfd(dir);
60                         }
61                         fchdir(old_pwd);
62                 }
63                 close(old_pwd);
64         }
65         return dir;
66 }
67 #endif
68
69 #ifdef HAVE_OPENAT
70 #  define my_openat(full_path, dirfd, relpath, flags) \
71                 openat((dirfd), (relpath), (flags))
72 #else
73 #  define my_openat(full_path, dirfd, relpath, flags) \
74                 open((full_path), (flags))
75 #endif
76
77 #ifdef HAVE_READLINKAT
78 #  define my_readlinkat(full_path, dirfd, relpath, buf, bufsize) \
79                 readlinkat((dirfd), (relpath), (buf), (bufsize))
80 #else
81 #  define my_readlinkat(full_path, dirfd, relpath, buf, bufsize) \
82                 readlink((full_path), (buf), (bufsize))
83 #endif
84
85 #ifdef HAVE_FSTATAT
86 #  define my_fstatat(full_path, dirfd, relpath, stbuf, flags)   \
87         fstatat((dirfd), (relpath), (stbuf), (flags))
88 #else
89 #  define my_fstatat(full_path, dirfd, relpath, stbuf, flags)   \
90         ((flags) & AT_SYMLINK_NOFOLLOW) ? \
91                 lstat((full_path), (stbuf)) : \
92                 stat((full_path), (stbuf))
93 #endif
94
95 #ifndef AT_FDCWD
96 #  define AT_FDCWD      -100
97 #endif
98
99 #ifndef AT_SYMLINK_NOFOLLOW
100 #  define AT_SYMLINK_NOFOLLOW   0x100
101 #endif
102
103 static int
104 unix_scan_regular_file(const char *path, u64 size, struct wim_inode *inode,
105                        struct list_head *unhashed_streams)
106 {
107         struct wim_lookup_table_entry *lte;
108         char *file_on_disk;
109
110         inode->i_attributes = FILE_ATTRIBUTE_NORMAL;
111
112         /* Empty files do not have to have a lookup table entry. */
113         if (!size)
114                 return 0;
115
116         file_on_disk = STRDUP(path);
117         if (!file_on_disk)
118                 return WIMLIB_ERR_NOMEM;
119         lte = new_lookup_table_entry();
120         if (!lte) {
121                 FREE(file_on_disk);
122                 return WIMLIB_ERR_NOMEM;
123         }
124         lte->file_on_disk = file_on_disk;
125         lte->resource_location = RESOURCE_IN_FILE_ON_DISK;
126         lte->size = size;
127         add_unhashed_stream(lte, inode, 0, unhashed_streams);
128         inode->i_lte = lte;
129         return 0;
130 }
131
132 static int
133 unix_build_dentry_tree_recursive(struct wim_dentry **tree_ret,
134                                  char *path, size_t path_len,
135                                  int dirfd, const char *relpath,
136                                  struct add_image_params *params);
137
138 static int
139 unix_scan_directory(struct wim_dentry *dir_dentry,
140                     char *full_path, size_t full_path_len,
141                     int parent_dirfd, const char *dir_relpath,
142                     struct add_image_params *params)
143 {
144
145         int dirfd;
146         DIR *dir;
147         int ret;
148
149         dirfd = my_openat(full_path, parent_dirfd, dir_relpath, O_RDONLY);
150         if (dirfd < 0) {
151                 ERROR_WITH_ERRNO("\"%s\": Can't open directory", full_path);
152                 return WIMLIB_ERR_OPENDIR;
153         }
154
155         dir_dentry->d_inode->i_attributes = FILE_ATTRIBUTE_DIRECTORY;
156         dir = my_fdopendir(&dirfd);
157         if (!dir) {
158                 ERROR_WITH_ERRNO("\"%s\": Can't open directory", full_path);
159                 close(dirfd);
160                 return WIMLIB_ERR_OPENDIR;
161         }
162
163         ret = 0;
164         for (;;) {
165                 struct dirent *entry;
166                 struct wim_dentry *child;
167                 size_t name_len;
168
169                 errno = 0;
170                 entry = readdir(dir);
171                 if (!entry) {
172                         if (errno) {
173                                 ret = WIMLIB_ERR_READ;
174                                 ERROR_WITH_ERRNO("\"%s\": Error reading directory",
175                                                  full_path);
176                         }
177                         break;
178                 }
179
180                 if (entry->d_name[0] == '.' &&
181                     (entry->d_name[1] == '\0' ||
182                      (entry->d_name[1] == '.' && entry->d_name[2] == '\0')))
183                         continue;
184
185                 full_path[full_path_len] = '/';
186                 name_len = strlen(entry->d_name);
187                 memcpy(&full_path[full_path_len + 1], entry->d_name, name_len + 1);
188                 ret = unix_build_dentry_tree_recursive(&child,
189                                                        full_path,
190                                                        full_path_len + 1 + name_len,
191                                                        dirfd,
192                                                        &full_path[full_path_len + 1],
193                                                        params);
194                 full_path[full_path_len] = '\0';
195                 if (ret)
196                         break;
197                 if (child)
198                         dentry_add_child(dir_dentry, child);
199         }
200         closedir(dir);
201         return ret;
202 }
203
204 /* Given an absolute symbolic link target @dest (UNIX-style, beginning
205  * with '/'), determine whether it points into the directory specified by
206  * @ino and @dev.  If so, return the target modified to be "absolute"
207  * relative to this directory.  Otherwise, return NULL.  */
208 static char *
209 unix_fixup_abslink(char *dest, u64 ino, u64 dev)
210 {
211         char *p = dest;
212
213         do {
214                 char save;
215                 struct stat stbuf;
216                 int ret;
217
218                 /* Skip non-slashes.  */
219                 while (*p && *p != '/')
220                         p++;
221
222                 /* Skip slashes.  */
223                 while (*p && *p == '/')
224                         p++;
225
226                 /* Get inode and device for this prefix.  */
227                 save = *p;
228                 *p = '\0';
229                 ret = stat(dest, &stbuf);
230                 *p = save;
231
232                 if (ret) {
233                         /* stat() failed.  Assume the link points outside the
234                          * directory tree being captured.  */
235                         break;
236                 }
237
238                 if (stbuf.st_ino == ino && stbuf.st_dev == dev) {
239                         /* Link points inside directory tree being captured.
240                          * Return abbreviated path.  */
241                         *--p = '/';
242                         while (p > dest && *(p - 1) == '/')
243                                 p--;
244                         return p;
245                 }
246         } while (*p);
247
248         /* Link does not point inside directory tree being captured.  */
249         return NULL;
250 }
251
252 static int
253 unix_scan_symlink(struct wim_dentry **root_p, const char *full_path,
254                   int dirfd, const char *relpath,
255                   struct wim_inode *inode, struct add_image_params *params)
256 {
257         char deref_name_buf[4096];
258         ssize_t deref_name_len;
259         char *dest;
260         int ret;
261
262         inode->i_attributes = FILE_ATTRIBUTE_REPARSE_POINT;
263         inode->i_reparse_tag = WIM_IO_REPARSE_TAG_SYMLINK;
264
265         /* The idea here is to call readlink() to get the UNIX target of the
266          * symbolic link, then turn the target into a reparse point data buffer
267          * that contains a relative or absolute symbolic link. */
268         deref_name_len = my_readlinkat(full_path, dirfd, relpath,
269                                        deref_name_buf, sizeof(deref_name_buf) - 1);
270         if (deref_name_len < 0) {
271                 ERROR_WITH_ERRNO("\"%s\": Can't read target of symbolic link",
272                                  full_path);
273                 return WIMLIB_ERR_READLINK;
274         }
275
276         dest = deref_name_buf;
277
278         dest[deref_name_len] = '\0';
279
280         if ((params->add_flags & WIMLIB_ADD_FLAG_RPFIX) &&
281              dest[0] == '/')
282         {
283                 dest = unix_fixup_abslink(dest,
284                                           params->capture_root_ino,
285                                           params->capture_root_dev);
286                 if (!dest) {
287                         /* RPFIX (reparse point fixup) mode:  Ignore
288                          * absolute symbolic link that points out of the
289                          * tree to be captured.  */
290                         free_dentry(*root_p);
291                         *root_p = NULL;
292                         params->progress.scan.cur_path = full_path;
293                         params->progress.scan.symlink_target = deref_name_buf;
294                         return do_capture_progress(params,
295                                                    WIMLIB_SCAN_DENTRY_EXCLUDED_SYMLINK,
296                                                    NULL);
297                 }
298                 inode->i_not_rpfixed = 0;
299         }
300         ret = wim_inode_set_symlink(inode, dest, params->lookup_table);
301         if (ret)
302                 return ret;
303
304         /* Unfortunately, Windows seems to have the concept of "file" symbolic
305          * links as being different from "directory" symbolic links...  so
306          * FILE_ATTRIBUTE_DIRECTORY needs to be set on the symbolic link if the
307          * *target* of the symbolic link is a directory.  */
308         struct stat stbuf;
309         if (my_fstatat(full_path, dirfd, relpath, &stbuf, 0) == 0 &&
310             S_ISDIR(stbuf.st_mode))
311                 inode->i_attributes |= FILE_ATTRIBUTE_DIRECTORY;
312         return 0;
313 }
314
315 static int
316 unix_build_dentry_tree_recursive(struct wim_dentry **tree_ret,
317                                  char *full_path, size_t full_path_len,
318                                  int dirfd, const char *relpath,
319                                  struct add_image_params *params)
320 {
321         struct wim_dentry *tree = NULL;
322         struct wim_inode *inode = NULL;
323         int ret;
324         struct stat stbuf;
325         int stat_flags;
326
327         if (should_exclude_path(full_path + params->capture_root_nchars,
328                                 full_path_len - params->capture_root_nchars,
329                                 params->config))
330         {
331                 ret = 0;
332                 goto out_progress;
333         }
334
335         if (params->add_flags & (WIMLIB_ADD_FLAG_DEREFERENCE |
336                                  WIMLIB_ADD_FLAG_ROOT))
337                 stat_flags = 0;
338         else
339                 stat_flags = AT_SYMLINK_NOFOLLOW;
340
341         ret = my_fstatat(full_path, dirfd, relpath, &stbuf, stat_flags);
342
343         if (ret) {
344                 ERROR_WITH_ERRNO("\"%s\": Can't read metadata", full_path);
345                 ret = WIMLIB_ERR_STAT;
346                 goto out;
347         }
348
349         if (unlikely(!S_ISREG(stbuf.st_mode) &&
350                      !S_ISDIR(stbuf.st_mode) &&
351                      !S_ISLNK(stbuf.st_mode)))
352         {
353                 if (params->add_flags & WIMLIB_ADD_FLAG_NO_UNSUPPORTED_EXCLUDE)
354                 {
355                         ERROR("\"%s\": File type is unsupported", full_path);
356                         ret = WIMLIB_ERR_UNSUPPORTED_FILE;
357                         goto out;
358                 }
359                 params->progress.scan.cur_path = full_path;
360                 ret = do_capture_progress(params, WIMLIB_SCAN_DENTRY_UNSUPPORTED, NULL);
361                 goto out;
362         }
363
364         ret = inode_table_new_dentry(params->inode_table, relpath,
365                                      stbuf.st_ino, stbuf.st_dev,
366                                      S_ISDIR(stbuf.st_mode), &tree);
367         if (ret)
368                 goto out;
369
370         inode = tree->d_inode;
371
372         if (inode->i_nlink > 1) {
373                 /* Already seen this inode?  */
374                 ret = 0;
375                 goto out_progress;
376         }
377
378 #ifdef HAVE_STAT_NANOSECOND_PRECISION
379         inode->i_creation_time = timespec_to_wim_timestamp(stbuf.st_mtim);
380         inode->i_last_write_time = timespec_to_wim_timestamp(stbuf.st_mtim);
381         inode->i_last_access_time = timespec_to_wim_timestamp(stbuf.st_atim);
382 #else
383         inode->i_creation_time = unix_timestamp_to_wim(stbuf.st_mtime);
384         inode->i_last_write_time = unix_timestamp_to_wim(stbuf.st_mtime);
385         inode->i_last_access_time = unix_timestamp_to_wim(stbuf.st_atime);
386 #endif
387         inode->i_resolved = 1;
388         if (params->add_flags & WIMLIB_ADD_FLAG_UNIX_DATA) {
389                 inode->i_unix_data.uid = stbuf.st_uid;
390                 inode->i_unix_data.gid = stbuf.st_gid;
391                 inode->i_unix_data.mode = stbuf.st_mode;
392         }
393
394         if (params->add_flags & WIMLIB_ADD_FLAG_ROOT) {
395                 params->capture_root_ino = stbuf.st_ino;
396                 params->capture_root_dev = stbuf.st_dev;
397                 params->add_flags &= ~WIMLIB_ADD_FLAG_ROOT;
398         }
399
400         if (S_ISREG(stbuf.st_mode)) {
401                 ret = unix_scan_regular_file(full_path, stbuf.st_size,
402                                              inode, params->unhashed_streams);
403         } else if (S_ISDIR(stbuf.st_mode)) {
404                 ret = unix_scan_directory(tree, full_path, full_path_len,
405                                           dirfd, relpath, params);
406         } else {
407                 ret = unix_scan_symlink(&tree, full_path, dirfd, relpath,
408                                         inode, params);
409                 if (!tree)
410                         goto out;
411         }
412
413         if (ret)
414                 goto out;
415
416 out_progress:
417         params->progress.scan.cur_path = full_path;
418         if (likely(tree))
419                 ret = do_capture_progress(params, WIMLIB_SCAN_DENTRY_OK, inode);
420         else
421                 ret = do_capture_progress(params, WIMLIB_SCAN_DENTRY_EXCLUDED, NULL);
422 out:
423         if (likely(ret == 0))
424                 *tree_ret = tree;
425         else
426                 free_dentry_tree(tree, params->lookup_table);
427         return ret;
428 }
429
430 /*
431  * unix_build_dentry_tree():
432  *      Builds a tree of WIM dentries from an on-disk directory tree (UNIX
433  *      version; no NTFS-specific data is captured).
434  *
435  * @root_ret:   Place to return a pointer to the root of the dentry tree.  Only
436  *              modified if successful.  Set to NULL if the file or directory was
437  *              excluded from capture.
438  *
439  * @root_disk_path:  The path to the root of the directory tree on disk.
440  *
441  * @params:     See doc for `struct add_image_params'.
442  *
443  * @return:     0 on success, nonzero on failure.  It is a failure if any of
444  *              the files cannot be `stat'ed, or if any of the needed
445  *              directories cannot be opened or read.  Failure to add the files
446  *              to the WIM may still occur later when trying to actually read
447  *              the on-disk files during a call to wimlib_write() or
448  *              wimlib_overwrite().
449  */
450 int
451 unix_build_dentry_tree(struct wim_dentry **root_ret,
452                        const char *root_disk_path,
453                        struct add_image_params *params)
454 {
455         size_t path_len;
456         size_t path_bufsz;
457         char *path_buf;
458         int ret;
459
460         path_len = strlen(root_disk_path);
461         path_bufsz = min(32790, PATH_MAX + 1);
462
463         if (path_len >= path_bufsz)
464                 return WIMLIB_ERR_INVALID_PARAM;
465
466         path_buf = MALLOC(path_bufsz);
467         if (!path_buf)
468                 return WIMLIB_ERR_NOMEM;
469         memcpy(path_buf, root_disk_path, path_len + 1);
470
471         params->capture_root_nchars = path_len;
472
473         ret = unix_build_dentry_tree_recursive(root_ret, path_buf, path_len,
474                                                AT_FDCWD, path_buf, params);
475         FREE(path_buf);
476         return ret;
477 }
478
479 #endif /* !__WIN32__ */