unix_capture.c: add error messages for invalid UTF-8
[wimlib] / src / unix_capture.c
1 /*
2  * unix_capture.c:  Capture a directory tree on UNIX.
3  */
4
5 /*
6  * Copyright (C) 2012-2016 Eric Biggers
7  *
8  * This file is free software; you can redistribute it and/or modify it under
9  * the terms of the GNU Lesser General Public License as published by the Free
10  * Software Foundation; either version 3 of the License, or (at your option) any
11  * later version.
12  *
13  * This file is distributed in the hope that it will be useful, but WITHOUT
14  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
15  * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
16  * details.
17  *
18  * You should have received a copy of the GNU Lesser General Public License
19  * along with this file; if not, see http://www.gnu.org/licenses/.
20  */
21
22 #ifndef __WIN32__
23
24 #ifdef HAVE_CONFIG_H
25 #  include "config.h"
26 #endif
27
28 #include <dirent.h>
29 #include <errno.h>
30 #include <fcntl.h>
31 #include <limits.h> /* for PATH_MAX */
32 #include <sys/stat.h>
33 #include <unistd.h>
34
35 #include "wimlib/blob_table.h"
36 #include "wimlib/dentry.h"
37 #include "wimlib/error.h"
38 #include "wimlib/reparse.h"
39 #include "wimlib/scan.h"
40 #include "wimlib/timestamp.h"
41 #include "wimlib/unix_data.h"
42
43 #ifdef HAVE_FDOPENDIR
44 #  define my_fdopendir(dirfd_p) fdopendir(*(dirfd_p))
45 #else
46 static DIR *
47 my_fdopendir(int *dirfd_p)
48 {
49         DIR *dir = NULL;
50         int old_pwd;
51
52         old_pwd = open(".", O_RDONLY);
53         if (old_pwd >= 0) {
54                 if (!fchdir(*dirfd_p)) {
55                         dir = opendir(".");
56                         if (dir) {
57                                 close(*dirfd_p);
58                                 *dirfd_p = dirfd(dir);
59                         }
60                         fchdir(old_pwd);
61                 }
62                 close(old_pwd);
63         }
64         return dir;
65 }
66 #endif
67
68 #ifdef HAVE_OPENAT
69 #  define my_openat(full_path, dirfd, relpath, flags) \
70                 openat((dirfd), (relpath), (flags))
71 #else
72 #  define my_openat(full_path, dirfd, relpath, flags) \
73                 open((full_path), (flags))
74 #endif
75
76 #ifdef HAVE_READLINKAT
77 #  define my_readlinkat(full_path, dirfd, relpath, buf, bufsize) \
78                 readlinkat((dirfd), (relpath), (buf), (bufsize))
79 #else
80 #  define my_readlinkat(full_path, dirfd, relpath, buf, bufsize) \
81                 readlink((full_path), (buf), (bufsize))
82 #endif
83
84 #ifdef HAVE_FSTATAT
85 #  define my_fstatat(full_path, dirfd, relpath, stbuf, flags)   \
86         fstatat((dirfd), (relpath), (stbuf), (flags))
87 #else
88 #  define my_fstatat(full_path, dirfd, relpath, stbuf, flags)   \
89         ((flags) & AT_SYMLINK_NOFOLLOW) ? \
90                 lstat((full_path), (stbuf)) : \
91                 stat((full_path), (stbuf))
92 #endif
93
94 #ifndef AT_FDCWD
95 #  define AT_FDCWD      -100
96 #endif
97
98 #ifndef AT_SYMLINK_NOFOLLOW
99 #  define AT_SYMLINK_NOFOLLOW   0x100
100 #endif
101
102 static int
103 unix_scan_regular_file(const char *path, u64 size, struct wim_inode *inode,
104                        struct list_head *unhashed_blobs)
105 {
106         struct blob_descriptor *blob = NULL;
107         struct wim_inode_stream *strm;
108
109         inode->i_attributes = FILE_ATTRIBUTE_NORMAL;
110
111         if (size) {
112                 blob = new_blob_descriptor();
113                 if (unlikely(!blob))
114                         goto err_nomem;
115                 blob->file_on_disk = STRDUP(path);
116                 if (unlikely(!blob->file_on_disk))
117                         goto err_nomem;
118                 blob->blob_location = BLOB_IN_FILE_ON_DISK;
119                 blob->size = size;
120                 blob->file_inode = inode;
121         }
122
123         strm = inode_add_stream(inode, STREAM_TYPE_DATA, NO_STREAM_NAME, blob);
124         if (unlikely(!strm))
125                 goto err_nomem;
126
127         prepare_unhashed_blob(blob, inode, strm->stream_id, unhashed_blobs);
128         return 0;
129
130 err_nomem:
131         free_blob_descriptor(blob);
132         return WIMLIB_ERR_NOMEM;
133 }
134
135 static int
136 unix_build_dentry_tree_recursive(struct wim_dentry **tree_ret,
137                                  char *path, size_t path_len,
138                                  int dirfd, const char *relpath,
139                                  struct scan_params *params);
140
141 static int
142 unix_scan_directory(struct wim_dentry *dir_dentry,
143                     char *full_path, size_t full_path_len,
144                     int parent_dirfd, const char *dir_relpath,
145                     struct scan_params *params)
146 {
147
148         int dirfd;
149         DIR *dir;
150         int ret;
151
152         dirfd = my_openat(full_path, parent_dirfd, dir_relpath, O_RDONLY);
153         if (dirfd < 0) {
154                 ERROR_WITH_ERRNO("\"%s\": Can't open directory", full_path);
155                 return WIMLIB_ERR_OPENDIR;
156         }
157
158         dir_dentry->d_inode->i_attributes = FILE_ATTRIBUTE_DIRECTORY;
159         dir = my_fdopendir(&dirfd);
160         if (!dir) {
161                 ERROR_WITH_ERRNO("\"%s\": Can't open directory", full_path);
162                 close(dirfd);
163                 return WIMLIB_ERR_OPENDIR;
164         }
165
166         ret = 0;
167         for (;;) {
168                 struct dirent *entry;
169                 struct wim_dentry *child;
170                 size_t name_len;
171
172                 errno = 0;
173                 entry = readdir(dir);
174                 if (!entry) {
175                         if (errno) {
176                                 ret = WIMLIB_ERR_READ;
177                                 ERROR_WITH_ERRNO("\"%s\": Error reading directory",
178                                                  full_path);
179                         }
180                         break;
181                 }
182
183                 name_len = strlen(entry->d_name);
184
185                 if (should_ignore_filename(entry->d_name, name_len))
186                         continue;
187
188                 full_path[full_path_len] = '/';
189                 memcpy(&full_path[full_path_len + 1], entry->d_name, name_len + 1);
190                 ret = unix_build_dentry_tree_recursive(&child,
191                                                        full_path,
192                                                        full_path_len + 1 + name_len,
193                                                        dirfd,
194                                                        &full_path[full_path_len + 1],
195                                                        params);
196                 full_path[full_path_len] = '\0';
197                 if (ret)
198                         break;
199                 attach_scanned_tree(dir_dentry, child, params->blob_table);
200         }
201         closedir(dir);
202         return ret;
203 }
204
205 /*
206  * Given an absolute symbolic link target (UNIX-style, beginning with '/'),
207  * determine whether it points into the directory identified by @ino and @dev.
208  * If yes, return the suffix of @target which is relative to this directory, but
209  * retaining leading slashes.  If no, return @target.
210  *
211  * Here are some examples, assuming that the @ino/@dev directory is "/home/e":
212  *
213  *      Original target         New target
214  *      ---------------         ----------
215  *      /home/e/test            /test
216  *      /home/e/test/           /test/
217  *      //home//e//test//       //test//
218  *      /home/e                                         (empty string)
219  *      /home/e/                /
220  *      /usr/lib                /usr/lib                (external link)
221  *
222  * Because of the possibility of other links into the @ino/@dev directory and/or
223  * multiple path separators, we can't simply do a string comparison; instead we
224  * need to stat() each ancestor directory.
225  *
226  * If the link points directly to the @ino/@dev directory with no trailing
227  * slashes, then the new target will be an empty string.  This is not a valid
228  * UNIX symlink target, but we store this in the archive anyway since the target
229  * is intended to be de-relativized when the link is extracted.
230  */
231 static char *
232 unix_relativize_link_target(char *target, u64 ino, u64 dev)
233 {
234         char *p = target;
235
236         do {
237                 char save;
238                 struct stat stbuf;
239                 int ret;
240
241                 /* Skip slashes (guaranteed to be at least one here)  */
242                 do {
243                         p++;
244                 } while (*p == '/');
245
246                 /* End of string?  */
247                 if (!*p)
248                         break;
249
250                 /* Skip non-slashes (guaranteed to be at least one here)  */
251                 do {
252                         p++;
253                 } while (*p && *p != '/');
254
255                 /* Get the inode and device numbers for this prefix.  */
256                 save = *p;
257                 *p = '\0';
258                 ret = stat(target, &stbuf);
259                 *p = save;
260
261                 if (ret) {
262                         /* stat() failed.  Assume the link points outside the
263                          * directory tree being captured.  */
264                         break;
265                 }
266
267                 if (stbuf.st_ino == ino && stbuf.st_dev == dev) {
268                         /* Link points inside directory tree being captured.
269                          * Return abbreviated path.  */
270                         return p;
271                 }
272         } while (*p);
273
274         /* Link does not point inside directory tree being captured.  */
275         return target;
276 }
277
278 static noinline_for_stack int
279 unix_scan_symlink(const char *full_path, int dirfd, const char *relpath,
280                   struct wim_inode *inode, struct scan_params *params)
281 {
282         char orig_target[REPARSE_POINT_MAX_SIZE];
283         char *target = orig_target;
284         int ret;
285
286         /* Read the UNIX symbolic link target.  */
287         ret = my_readlinkat(full_path, dirfd, relpath, target,
288                             sizeof(orig_target));
289         if (unlikely(ret < 0)) {
290                 ERROR_WITH_ERRNO("\"%s\": Can't read target of symbolic link",
291                                  full_path);
292                 return WIMLIB_ERR_READLINK;
293         }
294         if (unlikely(ret >= sizeof(orig_target))) {
295                 ERROR("\"%s\": target of symbolic link is too long", full_path);
296                 return WIMLIB_ERR_READLINK;
297         }
298         target[ret] = '\0';
299
300         /* If the link is absolute and reparse point fixups are enabled, then
301          * change it to be "absolute" relative to the tree being captured.  */
302         if (target[0] == '/' && (params->add_flags & WIMLIB_ADD_FLAG_RPFIX)) {
303                 int status = WIMLIB_SCAN_DENTRY_NOT_FIXED_SYMLINK;
304
305                 params->progress.scan.cur_path = full_path;
306                 params->progress.scan.symlink_target = target;
307
308                 target = unix_relativize_link_target(target,
309                                                      params->capture_root_ino,
310                                                      params->capture_root_dev);
311                 if (target != orig_target) {
312                         /* Link target was fixed.  */
313                         inode->i_rp_flags &= ~WIM_RP_FLAG_NOT_FIXED;
314                         status = WIMLIB_SCAN_DENTRY_FIXED_SYMLINK;
315                 }
316                 ret = do_scan_progress(params, status, NULL);
317                 if (ret)
318                         return ret;
319         }
320
321         /* Translate the UNIX symlink target into a Windows reparse point.  */
322         ret = wim_inode_set_symlink(inode, target, params->blob_table);
323         if (unlikely(ret)) {
324                 if (ret == WIMLIB_ERR_INVALID_UTF8_STRING) {
325                         ERROR("\"%s\": target of symbolic link is not valid "
326                               "UTF-8.  This is not supported.", full_path);
327                 }
328                 return ret;
329         }
330
331         /* On Windows, a reparse point can be set on both directory and
332          * non-directory files.  Usually, a link that is intended to point to a
333          * (non-)directory is stored as a reparse point on a (non-)directory
334          * file.  Replicate this behavior by examining the target file.  */
335         struct stat stbuf;
336         if (my_fstatat(full_path, dirfd, relpath, &stbuf, 0) == 0 &&
337             S_ISDIR(stbuf.st_mode))
338                 inode->i_attributes |= FILE_ATTRIBUTE_DIRECTORY;
339         return 0;
340 }
341
342 static int
343 unix_build_dentry_tree_recursive(struct wim_dentry **tree_ret,
344                                  char *full_path, size_t full_path_len,
345                                  int dirfd, const char *relpath,
346                                  struct scan_params *params)
347 {
348         struct wim_dentry *tree = NULL;
349         struct wim_inode *inode = NULL;
350         int ret;
351         struct stat stbuf;
352         int stat_flags;
353
354         ret = try_exclude(full_path, params);
355         if (unlikely(ret < 0)) /* Excluded? */
356                 goto out_progress;
357         if (unlikely(ret > 0)) /* Error? */
358                 goto out;
359
360         if (params->add_flags & (WIMLIB_ADD_FLAG_DEREFERENCE |
361                                  WIMLIB_ADD_FLAG_ROOT))
362                 stat_flags = 0;
363         else
364                 stat_flags = AT_SYMLINK_NOFOLLOW;
365
366         ret = my_fstatat(full_path, dirfd, relpath, &stbuf, stat_flags);
367
368         if (ret) {
369                 ERROR_WITH_ERRNO("\"%s\": Can't read metadata", full_path);
370                 ret = WIMLIB_ERR_STAT;
371                 goto out;
372         }
373
374         if (!(params->add_flags & WIMLIB_ADD_FLAG_UNIX_DATA)) {
375                 if (unlikely(!S_ISREG(stbuf.st_mode) &&
376                              !S_ISDIR(stbuf.st_mode) &&
377                              !S_ISLNK(stbuf.st_mode)))
378                 {
379                         if (params->add_flags &
380                             WIMLIB_ADD_FLAG_NO_UNSUPPORTED_EXCLUDE)
381                         {
382                                 ERROR("\"%s\": File type is unsupported",
383                                       full_path);
384                                 ret = WIMLIB_ERR_UNSUPPORTED_FILE;
385                                 goto out;
386                         }
387                         params->progress.scan.cur_path = full_path;
388                         ret = do_scan_progress(params,
389                                                WIMLIB_SCAN_DENTRY_UNSUPPORTED,
390                                                NULL);
391                         goto out;
392                 }
393         }
394
395         ret = inode_table_new_dentry(params->inode_table, relpath,
396                                      stbuf.st_ino, stbuf.st_dev, false, &tree);
397         if (unlikely(ret)) {
398                 if (ret == WIMLIB_ERR_INVALID_UTF8_STRING) {
399                         ERROR("\"%s\": filename is not valid UTF-8.  "
400                               "This is not supported.", full_path);
401                 }
402                 goto out;
403         }
404
405         inode = tree->d_inode;
406
407         /* Already seen this inode?  */
408         if (inode->i_nlink > 1)
409                 goto out_progress;
410
411 #ifdef HAVE_STAT_NANOSECOND_PRECISION
412         inode->i_creation_time = timespec_to_wim_timestamp(&stbuf.st_mtim);
413         inode->i_last_write_time = timespec_to_wim_timestamp(&stbuf.st_mtim);
414         inode->i_last_access_time = timespec_to_wim_timestamp(&stbuf.st_atim);
415 #else
416         inode->i_creation_time = time_t_to_wim_timestamp(stbuf.st_mtime);
417         inode->i_last_write_time = time_t_to_wim_timestamp(stbuf.st_mtime);
418         inode->i_last_access_time = time_t_to_wim_timestamp(stbuf.st_atime);
419 #endif
420         if (params->add_flags & WIMLIB_ADD_FLAG_UNIX_DATA) {
421                 struct wimlib_unix_data unix_data;
422
423                 unix_data.uid = stbuf.st_uid;
424                 unix_data.gid = stbuf.st_gid;
425                 unix_data.mode = stbuf.st_mode;
426                 unix_data.rdev = stbuf.st_rdev;
427                 if (!inode_set_unix_data(inode, &unix_data, UNIX_DATA_ALL)) {
428                         ret = WIMLIB_ERR_NOMEM;
429                         goto out;
430                 }
431         }
432
433         if (params->add_flags & WIMLIB_ADD_FLAG_ROOT) {
434                 params->capture_root_ino = stbuf.st_ino;
435                 params->capture_root_dev = stbuf.st_dev;
436                 params->add_flags &= ~WIMLIB_ADD_FLAG_ROOT;
437         }
438
439         if (S_ISREG(stbuf.st_mode)) {
440                 ret = unix_scan_regular_file(full_path, stbuf.st_size,
441                                              inode, params->unhashed_blobs);
442         } else if (S_ISDIR(stbuf.st_mode)) {
443                 ret = unix_scan_directory(tree, full_path, full_path_len,
444                                           dirfd, relpath, params);
445         } else if (S_ISLNK(stbuf.st_mode)) {
446                 ret = unix_scan_symlink(full_path, dirfd, relpath,
447                                         inode, params);
448         }
449
450         if (ret)
451                 goto out;
452
453 out_progress:
454         params->progress.scan.cur_path = full_path;
455         if (likely(tree))
456                 ret = do_scan_progress(params, WIMLIB_SCAN_DENTRY_OK, inode);
457         else
458                 ret = do_scan_progress(params, WIMLIB_SCAN_DENTRY_EXCLUDED, NULL);
459 out:
460         if (unlikely(ret)) {
461                 free_dentry_tree(tree, params->blob_table);
462                 tree = NULL;
463                 ret = report_scan_error(params, ret, full_path);
464         }
465         *tree_ret = tree;
466         return ret;
467 }
468
469 /*
470  * unix_build_dentry_tree():
471  *      Builds a tree of WIM dentries from an on-disk directory tree (UNIX
472  *      version; no NTFS-specific data is captured).
473  *
474  * @root_ret:   Place to return a pointer to the root of the dentry tree.  Set
475  *              to NULL if the file or directory was excluded from capture.
476  *
477  * @root_disk_path:  The path to the root of the directory tree on disk.
478  *
479  * @params:     See doc for `struct scan_params'.
480  *
481  * @return:     0 on success, nonzero on failure.  It is a failure if any of
482  *              the files cannot be `stat'ed, or if any of the needed
483  *              directories cannot be opened or read.  Failure to add the files
484  *              to the WIM may still occur later when trying to actually read
485  *              the on-disk files during a call to wimlib_write() or
486  *              wimlib_overwrite().
487  */
488 int
489 unix_build_dentry_tree(struct wim_dentry **root_ret,
490                        const char *root_disk_path, struct scan_params *params)
491 {
492         size_t path_len;
493         size_t path_bufsz;
494         char *path_buf;
495         int ret;
496
497         path_len = strlen(root_disk_path);
498         path_bufsz = min(32790, PATH_MAX + 1);
499
500         if (path_len >= path_bufsz)
501                 return WIMLIB_ERR_INVALID_PARAM;
502
503         path_buf = MALLOC(path_bufsz);
504         if (!path_buf)
505                 return WIMLIB_ERR_NOMEM;
506         memcpy(path_buf, root_disk_path, path_len + 1);
507
508         params->capture_root_nchars = path_len;
509
510         ret = unix_build_dentry_tree_recursive(root_ret, path_buf, path_len,
511                                                AT_FDCWD, path_buf, params);
512         FREE(path_buf);
513         return ret;
514 }
515
516 #endif /* !__WIN32__ */