]> wimlib.net Git - wimlib/blob - src/extract.c
Recognize tagged metadata items and use for UNIX data
[wimlib] / src / extract.c
1 /*
2  * extract.c
3  *
4  * Support for extracting WIM images, or files or directories contained in a WIM
5  * image.
6  */
7
8 /*
9  * Copyright (C) 2012, 2013, 2014 Eric Biggers
10  *
11  * This file is part of wimlib, a library for working with WIM files.
12  *
13  * wimlib is free software; you can redistribute it and/or modify it under the
14  * terms of the GNU General Public License as published by the Free
15  * Software Foundation; either version 3 of the License, or (at your option)
16  * any later version.
17  *
18  * wimlib is distributed in the hope that it will be useful, but WITHOUT ANY
19  * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
20  * A PARTICULAR PURPOSE. See the GNU General Public License for more
21  * details.
22  *
23  * You should have received a copy of the GNU General Public License
24  * along with wimlib; if not, see http://www.gnu.org/licenses/.
25  */
26
27 /*
28  * This file provides the API functions wimlib_extract_image(),
29  * wimlib_extract_image_from_pipe(), wimlib_extract_paths(), and
30  * wimlib_extract_pathlist().  Internally, all end up calling
31  * do_wimlib_extract_paths() and extract_trees().
32  *
33  * Although wimlib supports multiple extraction modes/backends (NTFS-3g, UNIX,
34  * Win32), this file does not itself have code to extract files or directories
35  * to any specific target; instead, it handles generic functionality and relies
36  * on lower-level callback functions declared in `struct apply_operations' to do
37  * the actual extraction.
38  */
39
40 #ifdef HAVE_CONFIG_H
41 #  include "config.h"
42 #endif
43
44 #include "wimlib/apply.h"
45 #include "wimlib/dentry.h"
46 #include "wimlib/encoding.h"
47 #include "wimlib/endianness.h"
48 #include "wimlib/error.h"
49 #include "wimlib/lookup_table.h"
50 #include "wimlib/metadata.h"
51 #include "wimlib/pathlist.h"
52 #include "wimlib/paths.h"
53 #include "wimlib/reparse.h"
54 #include "wimlib/resource.h"
55 #include "wimlib/security.h"
56 #include "wimlib/unix_data.h"
57 #ifdef __WIN32__
58 #  include "wimlib/win32.h" /* for realpath() equivalent */
59 #endif
60 #include "wimlib/xml.h"
61 #include "wimlib/wildcard.h"
62 #include "wimlib/wim.h"
63
64 #include <errno.h>
65 #include <fcntl.h>
66 #include <stdlib.h>
67 #include <sys/stat.h>
68 #include <unistd.h>
69
70 #define WIMLIB_EXTRACT_FLAG_MULTI_IMAGE 0x80000000
71 #define WIMLIB_EXTRACT_FLAG_FROM_PIPE   0x40000000
72 #define WIMLIB_EXTRACT_FLAG_IMAGEMODE   0x20000000
73
74 /* Keep in sync with wimlib.h  */
75 #define WIMLIB_EXTRACT_MASK_PUBLIC                              \
76         (WIMLIB_EXTRACT_FLAG_NTFS                       |       \
77          WIMLIB_EXTRACT_FLAG_UNIX_DATA                  |       \
78          WIMLIB_EXTRACT_FLAG_NO_ACLS                    |       \
79          WIMLIB_EXTRACT_FLAG_STRICT_ACLS                |       \
80          WIMLIB_EXTRACT_FLAG_RPFIX                      |       \
81          WIMLIB_EXTRACT_FLAG_NORPFIX                    |       \
82          WIMLIB_EXTRACT_FLAG_TO_STDOUT                  |       \
83          WIMLIB_EXTRACT_FLAG_REPLACE_INVALID_FILENAMES  |       \
84          WIMLIB_EXTRACT_FLAG_ALL_CASE_CONFLICTS         |       \
85          WIMLIB_EXTRACT_FLAG_STRICT_TIMESTAMPS          |       \
86          WIMLIB_EXTRACT_FLAG_STRICT_SHORT_NAMES         |       \
87          WIMLIB_EXTRACT_FLAG_STRICT_SYMLINKS            |       \
88          WIMLIB_EXTRACT_FLAG_GLOB_PATHS                 |       \
89          WIMLIB_EXTRACT_FLAG_STRICT_GLOB                |       \
90          WIMLIB_EXTRACT_FLAG_NO_ATTRIBUTES              |       \
91          WIMLIB_EXTRACT_FLAG_NO_PRESERVE_DIR_STRUCTURE  |       \
92          WIMLIB_EXTRACT_FLAG_WIMBOOT)
93
94 /* Check whether the extraction of a dentry should be skipped completely.  */
95 static bool
96 dentry_is_supported(struct wim_dentry *dentry,
97                     const struct wim_features *supported_features)
98 {
99         struct wim_inode *inode = dentry->d_inode;
100
101         if (inode->i_attributes & FILE_ATTRIBUTE_REPARSE_POINT) {
102                 return supported_features->reparse_points ||
103                         (inode_is_symlink(inode) &&
104                          supported_features->symlink_reparse_points);
105         }
106         if (inode->i_attributes & FILE_ATTRIBUTE_ENCRYPTED) {
107                 if (inode->i_attributes & FILE_ATTRIBUTE_DIRECTORY)
108                         return supported_features->encrypted_directories != 0;
109                 else
110                         return supported_features->encrypted_files != 0;
111         }
112         return true;
113 }
114
115
116 #define PWM_ALLOW_WIM_HDR 0x00001
117 #define PWM_SILENT_EOF    0x00002
118
119 /* Read the header from a stream in a pipable WIM.  */
120 static int
121 read_pwm_stream_header(WIMStruct *pwm, struct wim_lookup_table_entry *lte,
122                        struct wim_resource_spec *rspec,
123                        int flags, struct wim_header_disk *hdr_ret)
124 {
125         union {
126                 struct pwm_stream_hdr stream_hdr;
127                 struct wim_header_disk pwm_hdr;
128         } buf;
129         struct wim_reshdr reshdr;
130         int ret;
131
132         ret = full_read(&pwm->in_fd, &buf.stream_hdr, sizeof(buf.stream_hdr));
133         if (ret)
134                 goto read_error;
135
136         if ((flags & PWM_ALLOW_WIM_HDR) && buf.stream_hdr.magic == PWM_MAGIC) {
137                 BUILD_BUG_ON(sizeof(buf.pwm_hdr) < sizeof(buf.stream_hdr));
138                 ret = full_read(&pwm->in_fd, &buf.stream_hdr + 1,
139                                 sizeof(buf.pwm_hdr) - sizeof(buf.stream_hdr));
140
141                 if (ret)
142                         goto read_error;
143                 lte->resource_location = RESOURCE_NONEXISTENT;
144                 memcpy(hdr_ret, &buf.pwm_hdr, sizeof(buf.pwm_hdr));
145                 return 0;
146         }
147
148         if (le64_to_cpu(buf.stream_hdr.magic) != PWM_STREAM_MAGIC) {
149                 ERROR("Data read on pipe is invalid (expected stream header).");
150                 return WIMLIB_ERR_INVALID_PIPABLE_WIM;
151         }
152
153         copy_hash(lte->hash, buf.stream_hdr.hash);
154
155         reshdr.size_in_wim = 0;
156         reshdr.flags = le32_to_cpu(buf.stream_hdr.flags);
157         reshdr.offset_in_wim = pwm->in_fd.offset;
158         reshdr.uncompressed_size = le64_to_cpu(buf.stream_hdr.uncompressed_size);
159         wim_res_hdr_to_spec(&reshdr, pwm, rspec);
160         lte_bind_wim_resource_spec(lte, rspec);
161         lte->flags = rspec->flags;
162         lte->size = rspec->uncompressed_size;
163         lte->offset_in_res = 0;
164         return 0;
165
166 read_error:
167         if (ret != WIMLIB_ERR_UNEXPECTED_END_OF_FILE || !(flags & PWM_SILENT_EOF))
168                 ERROR_WITH_ERRNO("Error reading pipable WIM from pipe");
169         return ret;
170 }
171
172 static int
173 load_streams_from_pipe(struct apply_ctx *ctx,
174                        const struct read_stream_list_callbacks *cbs)
175 {
176         struct wim_lookup_table_entry *found_lte = NULL;
177         struct wim_resource_spec *rspec = NULL;
178         struct wim_lookup_table *lookup_table;
179         int ret;
180
181         ret = WIMLIB_ERR_NOMEM;
182         found_lte = new_lookup_table_entry();
183         if (!found_lte)
184                 goto out;
185
186         rspec = MALLOC(sizeof(struct wim_resource_spec));
187         if (!rspec)
188                 goto out;
189
190         lookup_table = ctx->wim->lookup_table;
191         memcpy(ctx->progress.extract.guid, ctx->wim->hdr.guid, WIM_GUID_LEN);
192         ctx->progress.extract.part_number = ctx->wim->hdr.part_number;
193         ctx->progress.extract.total_parts = ctx->wim->hdr.total_parts;
194         ret = extract_progress(ctx, WIMLIB_PROGRESS_MSG_EXTRACT_SPWM_PART_BEGIN);
195         if (ret)
196                 goto out;
197
198         while (ctx->num_streams_remaining) {
199                 struct wim_header_disk pwm_hdr;
200                 struct wim_lookup_table_entry *needed_lte;
201
202                 if (found_lte->resource_location != RESOURCE_NONEXISTENT)
203                         lte_unbind_wim_resource_spec(found_lte);
204                 ret = read_pwm_stream_header(ctx->wim, found_lte, rspec,
205                                              PWM_ALLOW_WIM_HDR, &pwm_hdr);
206                 if (ret)
207                         goto out;
208
209                 if ((found_lte->resource_location != RESOURCE_NONEXISTENT)
210                     && !(found_lte->flags & WIM_RESHDR_FLAG_METADATA)
211                     && (needed_lte = lookup_stream(lookup_table, found_lte->hash))
212                     && (needed_lte->out_refcnt))
213                 {
214                         needed_lte->offset_in_res = found_lte->offset_in_res;
215                         needed_lte->flags = found_lte->flags;
216                         needed_lte->size = found_lte->size;
217
218                         lte_unbind_wim_resource_spec(found_lte);
219                         lte_bind_wim_resource_spec(needed_lte, rspec);
220
221                         ret = (*cbs->begin_stream)(needed_lte, 0,
222                                                    cbs->begin_stream_ctx);
223                         if (ret) {
224                                 lte_unbind_wim_resource_spec(needed_lte);
225                                 goto out;
226                         }
227
228                         ret = extract_stream(needed_lte, needed_lte->size,
229                                              cbs->consume_chunk,
230                                              cbs->consume_chunk_ctx);
231
232                         ret = (*cbs->end_stream)(needed_lte, ret,
233                                                  cbs->end_stream_ctx);
234                         lte_unbind_wim_resource_spec(needed_lte);
235                         if (ret)
236                                 goto out;
237                         ctx->num_streams_remaining--;
238                 } else if (found_lte->resource_location != RESOURCE_NONEXISTENT) {
239                         ret = skip_wim_stream(found_lte);
240                         if (ret)
241                                 goto out;
242                 } else {
243                         u16 part_number = le16_to_cpu(pwm_hdr.part_number);
244                         u16 total_parts = le16_to_cpu(pwm_hdr.total_parts);
245
246                         if (part_number != ctx->progress.extract.part_number ||
247                             total_parts != ctx->progress.extract.total_parts ||
248                             memcmp(pwm_hdr.guid, ctx->progress.extract.guid,
249                                    WIM_GUID_LEN))
250                         {
251                                 ctx->progress.extract.part_number = part_number;
252                                 ctx->progress.extract.total_parts = total_parts;
253                                 memcpy(ctx->progress.extract.guid,
254                                        pwm_hdr.guid, WIM_GUID_LEN);
255                                 ret = extract_progress(ctx,
256                                                        WIMLIB_PROGRESS_MSG_EXTRACT_SPWM_PART_BEGIN);
257                                 if (ret)
258                                         goto out;
259                         }
260                 }
261         }
262         ret = 0;
263 out:
264         if (found_lte->resource_location != RESOURCE_IN_WIM)
265                 FREE(rspec);
266         free_lookup_table_entry(found_lte);
267         return ret;
268 }
269
270 static int
271 begin_extract_stream_with_progress(struct wim_lookup_table_entry *lte,
272                                    u32 flags, void *_ctx)
273 {
274         struct apply_ctx *ctx = _ctx;
275
276         ctx->cur_stream = lte;
277
278         return (*ctx->saved_cbs->begin_stream)(lte, flags,
279                                                ctx->saved_cbs->begin_stream_ctx);
280 }
281
282 static int
283 consume_chunk_with_progress(const void *chunk, size_t size, void *_ctx)
284 {
285         struct apply_ctx *ctx = _ctx;
286         union wimlib_progress_info *progress = &ctx->progress;
287         int ret;
288
289         if (likely(ctx->supported_features.hard_links)) {
290                 progress->extract.completed_bytes +=
291                         (u64)size * ctx->cur_stream->out_refcnt;
292         } else {
293                 const struct stream_owner *owners = stream_owners(ctx->cur_stream);
294                 for (u32 i = 0; i < ctx->cur_stream->out_refcnt; i++) {
295                         const struct wim_inode *inode = owners[i].inode;
296                         const struct wim_dentry *dentry;
297
298                         list_for_each_entry(dentry,
299                                             &inode->i_extraction_aliases,
300                                             d_extraction_alias_node)
301                         {
302                                 progress->extract.completed_bytes += size;
303                         }
304                 }
305         }
306         if (progress->extract.completed_bytes >= ctx->next_progress) {
307
308                 ret = extract_progress(ctx, WIMLIB_PROGRESS_MSG_EXTRACT_STREAMS);
309                 if (ret)
310                         return ret;
311
312                 if (progress->extract.completed_bytes >=
313                     progress->extract.total_bytes)
314                 {
315                         ctx->next_progress = UINT64_MAX;
316                 } else {
317                         ctx->next_progress += progress->extract.total_bytes / 128;
318                         if (ctx->next_progress > progress->extract.total_bytes)
319                                 ctx->next_progress = progress->extract.total_bytes;
320                 }
321         }
322         return (*ctx->saved_cbs->consume_chunk)(chunk, size,
323                                                 ctx->saved_cbs->consume_chunk_ctx);
324 }
325
326 /*
327  * Read the list of single-instance streams to extract and feed their data into
328  * the specified callback functions.
329  *
330  * This handles checksumming each stream.
331  *
332  * This also handles sending WIMLIB_PROGRESS_MSG_EXTRACT_STREAMS.
333  *
334  * This also works if the WIM is being read from a pipe, whereas attempting to
335  * read streams directly (e.g. with read_full_stream_into_buf()) will not.
336  */
337 int
338 extract_stream_list(struct apply_ctx *ctx,
339                     const struct read_stream_list_callbacks *cbs)
340 {
341         struct read_stream_list_callbacks wrapper_cbs = {
342                 .begin_stream      = begin_extract_stream_with_progress,
343                 .begin_stream_ctx  = ctx,
344                 .consume_chunk     = consume_chunk_with_progress,
345                 .consume_chunk_ctx = ctx,
346                 .end_stream        = cbs->end_stream,
347                 .end_stream_ctx    = cbs->end_stream_ctx,
348         };
349         if (ctx->progfunc) {
350                 ctx->saved_cbs = cbs;
351                 cbs = &wrapper_cbs;
352         }
353         if (ctx->extract_flags & WIMLIB_EXTRACT_FLAG_FROM_PIPE) {
354                 return load_streams_from_pipe(ctx, cbs);
355         } else {
356                 return read_stream_list(&ctx->stream_list,
357                                         offsetof(struct wim_lookup_table_entry,
358                                                  extraction_list),
359                                         cbs, VERIFY_STREAM_HASHES);
360         }
361 }
362
363 /* Extract a WIM dentry to standard output.
364  *
365  * This obviously doesn't make sense in all cases.  We return an error if the
366  * dentry does not correspond to a regular file.  Otherwise we extract the
367  * unnamed data stream only.  */
368 static int
369 extract_dentry_to_stdout(struct wim_dentry *dentry,
370                          const struct wim_lookup_table *lookup_table)
371 {
372         struct wim_inode *inode = dentry->d_inode;
373         struct wim_lookup_table_entry *lte;
374         struct filedes _stdout;
375
376         if (inode->i_attributes & (FILE_ATTRIBUTE_REPARSE_POINT |
377                                    FILE_ATTRIBUTE_DIRECTORY))
378         {
379                 ERROR("\"%"TS"\" is not a regular file and therefore cannot be "
380                       "extracted to standard output", dentry_full_path(dentry));
381                 return WIMLIB_ERR_NOT_A_REGULAR_FILE;
382         }
383
384         lte = inode_unnamed_lte(inode, lookup_table);
385         if (!lte) {
386                 const u8 *hash = inode_unnamed_stream_hash(inode);
387                 if (!is_zero_hash(hash))
388                         return stream_not_found_error(inode, hash);
389                 return 0;
390         }
391
392         filedes_init(&_stdout, STDOUT_FILENO);
393         return extract_full_stream_to_fd(lte, &_stdout);
394 }
395
396 static int
397 extract_dentries_to_stdout(struct wim_dentry **dentries, size_t num_dentries,
398                            const struct wim_lookup_table *lookup_table)
399 {
400         for (size_t i = 0; i < num_dentries; i++) {
401                 int ret = extract_dentry_to_stdout(dentries[i], lookup_table);
402                 if (ret)
403                         return ret;
404         }
405         return 0;
406 }
407
408 /**********************************************************************/
409
410 /*
411  * Removes duplicate dentries from the array.
412  *
413  * Returns the new number of dentries, packed at the front of the array.
414  */
415 static size_t
416 remove_duplicate_trees(struct wim_dentry **trees, size_t num_trees)
417 {
418         size_t i, j = 0;
419         for (i = 0; i < num_trees; i++) {
420                 if (!trees[i]->tmp_flag) {
421                         /* Found distinct dentry.  */
422                         trees[i]->tmp_flag = 1;
423                         trees[j++] = trees[i];
424                 }
425         }
426         for (i = 0; i < j; i++)
427                 trees[i]->tmp_flag = 0;
428         return j;
429 }
430
431 /*
432  * Remove dentries that are descendants of other dentries in the array.
433  *
434  * Returns the new number of dentries, packed at the front of the array.
435  */
436 static size_t
437 remove_contained_trees(struct wim_dentry **trees, size_t num_trees)
438 {
439         size_t i, j = 0;
440         for (i = 0; i < num_trees; i++)
441                 trees[i]->tmp_flag = 1;
442         for (i = 0; i < num_trees; i++) {
443                 struct wim_dentry *d = trees[i];
444                 while (!dentry_is_root(d)) {
445                         d = d->parent;
446                         if (d->tmp_flag)
447                                 goto tree_contained;
448                 }
449                 trees[j++] = trees[i];
450                 continue;
451
452         tree_contained:
453                 trees[i]->tmp_flag = 0;
454         }
455
456         for (i = 0; i < j; i++)
457                 trees[i]->tmp_flag = 0;
458         return j;
459 }
460
461 static int
462 dentry_append_to_list(struct wim_dentry *dentry, void *_dentry_list)
463 {
464         struct list_head *dentry_list = _dentry_list;
465         list_add_tail(&dentry->d_extraction_list_node, dentry_list);
466         return 0;
467 }
468
469 static void
470 dentry_reset_extraction_list_node(struct wim_dentry *dentry)
471 {
472         dentry->d_extraction_list_node = (struct list_head){NULL, NULL};
473 }
474
475 static int
476 dentry_delete_from_list(struct wim_dentry *dentry, void *_ignore)
477 {
478         list_del(&dentry->d_extraction_list_node);
479         dentry_reset_extraction_list_node(dentry);
480         return 0;
481 }
482
483 /*
484  * Build the preliminary list of dentries to be extracted.
485  *
486  * The list maintains the invariant that if d1 and d2 are in the list and d1 is
487  * an ancestor of d2, then d1 appears before d2 in the list.
488  */
489 static void
490 build_dentry_list(struct list_head *dentry_list, struct wim_dentry **trees,
491                   size_t num_trees, bool add_ancestors)
492 {
493         INIT_LIST_HEAD(dentry_list);
494
495         /* Add the trees recursively.  */
496         for (size_t i = 0; i < num_trees; i++)
497                 for_dentry_in_tree(trees[i], dentry_append_to_list, dentry_list);
498
499         /* If requested, add ancestors of the trees.  */
500         if (add_ancestors) {
501                 for (size_t i = 0; i < num_trees; i++) {
502                         struct wim_dentry *dentry = trees[i];
503                         struct wim_dentry *ancestor;
504                         struct list_head *place_after;
505
506                         if (dentry_is_root(dentry))
507                                 continue;
508
509                         place_after = dentry_list;
510                         ancestor = dentry;
511                         do {
512                                 ancestor = ancestor->parent;
513                                 if (will_extract_dentry(ancestor)) {
514                                         place_after = &ancestor->d_extraction_list_node;
515                                         break;
516                                 }
517                         } while (!dentry_is_root(ancestor));
518
519                         ancestor = dentry;
520                         do {
521                                 ancestor = ancestor->parent;
522                                 if (will_extract_dentry(ancestor))
523                                         break;
524                                 list_add(&ancestor->d_extraction_list_node, place_after);
525                         } while (!dentry_is_root(ancestor));
526                 }
527         }
528 }
529
530 static void
531 destroy_dentry_list(struct list_head *dentry_list)
532 {
533         struct wim_dentry *dentry, *tmp;
534         struct wim_inode *inode;
535
536         list_for_each_entry_safe(dentry, tmp, dentry_list, d_extraction_list_node) {
537                 inode = dentry->d_inode;
538                 dentry_reset_extraction_list_node(dentry);
539                 inode->i_visited = 0;
540                 if ((void *)dentry->d_extraction_name != (void *)dentry->file_name)
541                         FREE(dentry->d_extraction_name);
542                 dentry->d_extraction_name = NULL;
543                 dentry->d_extraction_name_nchars = 0;
544         }
545 }
546
547 static void
548 destroy_stream_list(struct list_head *stream_list)
549 {
550         struct wim_lookup_table_entry *lte;
551
552         list_for_each_entry(lte, stream_list, extraction_list)
553                 if (lte->out_refcnt > ARRAY_LEN(lte->inline_stream_owners))
554                         FREE(lte->stream_owners);
555 }
556
557 #ifdef __WIN32__
558 static const utf16lechar replacement_char = cpu_to_le16(0xfffd);
559 #else
560 static const utf16lechar replacement_char = cpu_to_le16('?');
561 #endif
562
563 static bool
564 file_name_valid(utf16lechar *name, size_t num_chars, bool fix)
565 {
566         size_t i;
567
568         if (num_chars == 0)
569                 return true;
570         for (i = 0; i < num_chars; i++) {
571                 switch (name[i]) {
572         #ifdef __WIN32__
573                 case cpu_to_le16('\\'):
574                 case cpu_to_le16(':'):
575                 case cpu_to_le16('*'):
576                 case cpu_to_le16('?'):
577                 case cpu_to_le16('"'):
578                 case cpu_to_le16('<'):
579                 case cpu_to_le16('>'):
580                 case cpu_to_le16('|'):
581         #endif
582                 case cpu_to_le16('/'):
583                 case cpu_to_le16('\0'):
584                         if (fix)
585                                 name[i] = replacement_char;
586                         else
587                                 return false;
588                 }
589         }
590
591 #ifdef __WIN32__
592         if (name[num_chars - 1] == cpu_to_le16(' ') ||
593             name[num_chars - 1] == cpu_to_le16('.'))
594         {
595                 if (fix)
596                         name[num_chars - 1] = replacement_char;
597                 else
598                         return false;
599         }
600 #endif
601         return true;
602 }
603
604 static int
605 dentry_calculate_extraction_name(struct wim_dentry *dentry,
606                                  struct apply_ctx *ctx)
607 {
608         int ret;
609
610         if (!dentry_is_supported(dentry, &ctx->supported_features))
611                 goto skip_dentry;
612
613         if (dentry_is_root(dentry))
614                 return 0;
615
616         if (ctx->extract_flags & WIMLIB_EXTRACT_FLAG_NTFS) {
617                 dentry->d_extraction_name = dentry->file_name;
618                 dentry->d_extraction_name_nchars = dentry->file_name_nbytes /
619                                                    sizeof(utf16lechar);
620                 return 0;
621         }
622
623         if (!ctx->supported_features.case_sensitive_filenames) {
624                 struct wim_dentry *other;
625                 list_for_each_entry(other, &dentry->d_ci_conflict_list,
626                                     d_ci_conflict_list)
627                 {
628                         if (will_extract_dentry(other)) {
629                                 if (ctx->extract_flags &
630                                     WIMLIB_EXTRACT_FLAG_ALL_CASE_CONFLICTS) {
631                                         WARNING("\"%"TS"\" has the same "
632                                                 "case-insensitive name as "
633                                                 "\"%"TS"\"; extracting "
634                                                 "dummy name instead",
635                                                 dentry_full_path(dentry),
636                                                 dentry_full_path(other));
637                                         goto out_replace;
638                                 } else {
639                                         WARNING("Not extracting \"%"TS"\": "
640                                                 "has same case-insensitive "
641                                                 "name as \"%"TS"\"",
642                                                 dentry_full_path(dentry),
643                                                 dentry_full_path(other));
644                                         goto skip_dentry;
645                                 }
646                         }
647                 }
648         }
649
650         if (file_name_valid(dentry->file_name, dentry->file_name_nbytes / 2, false)) {
651                 ret = utf16le_get_tstr(dentry->file_name,
652                                        dentry->file_name_nbytes,
653                                        (const tchar **)&dentry->d_extraction_name,
654                                        &dentry->d_extraction_name_nchars);
655                 dentry->d_extraction_name_nchars /= sizeof(tchar);
656                 return ret;
657         } else {
658                 if (ctx->extract_flags & WIMLIB_EXTRACT_FLAG_REPLACE_INVALID_FILENAMES)
659                 {
660                         WARNING("\"%"TS"\" has an invalid filename "
661                                 "that is not supported on this platform; "
662                                 "extracting dummy name instead",
663                                 dentry_full_path(dentry));
664                         goto out_replace;
665                 } else {
666                         WARNING("Not extracting \"%"TS"\": has an invalid filename "
667                                 "that is not supported on this platform",
668                                 dentry_full_path(dentry));
669                         goto skip_dentry;
670                 }
671         }
672
673 out_replace:
674         {
675                 utf16lechar utf16_name_copy[dentry->file_name_nbytes / 2];
676
677                 memcpy(utf16_name_copy, dentry->file_name, dentry->file_name_nbytes);
678                 file_name_valid(utf16_name_copy, dentry->file_name_nbytes / 2, true);
679
680                 const tchar *tchar_name;
681                 size_t tchar_nchars;
682
683                 ret = utf16le_get_tstr(utf16_name_copy,
684                                        dentry->file_name_nbytes,
685                                        &tchar_name, &tchar_nchars);
686                 if (ret)
687                         return ret;
688
689                 tchar_nchars /= sizeof(tchar);
690
691                 size_t fixed_name_num_chars = tchar_nchars;
692                 tchar fixed_name[tchar_nchars + 50];
693
694                 tmemcpy(fixed_name, tchar_name, tchar_nchars);
695                 fixed_name_num_chars += tsprintf(fixed_name + tchar_nchars,
696                                                  T(" (invalid filename #%lu)"),
697                                                  ++ctx->invalid_sequence);
698
699                 utf16le_put_tstr(tchar_name);
700
701                 dentry->d_extraction_name = memdup(fixed_name,
702                                                    2 * fixed_name_num_chars + 2);
703                 if (!dentry->d_extraction_name)
704                         return WIMLIB_ERR_NOMEM;
705                 dentry->d_extraction_name_nchars = fixed_name_num_chars;
706         }
707         return 0;
708
709 skip_dentry:
710         for_dentry_in_tree(dentry, dentry_delete_from_list, NULL);
711         return 0;
712 }
713
714 /*
715  * Calculate the actual filename component at which each WIM dentry will be
716  * extracted, with special handling for dentries that are unsupported by the
717  * extraction backend or have invalid names.
718  *
719  * ctx->supported_features must be filled in.
720  *
721  * Possible error codes: WIMLIB_ERR_NOMEM, WIMLIB_ERR_INVALID_UTF16_STRING
722  */
723 static int
724 dentry_list_calculate_extraction_names(struct list_head *dentry_list,
725                                        struct apply_ctx *ctx)
726 {
727         struct list_head *prev, *cur;
728
729         /* Can't use list_for_each_entry() because a call to
730          * dentry_calculate_extraction_name() may delete the current dentry and
731          * its children from the list.  */
732
733         prev = dentry_list;
734         for (;;) {
735                 struct wim_dentry *dentry;
736                 int ret;
737
738                 cur = prev->next;
739                 if (cur == dentry_list)
740                         break;
741
742                 dentry = list_entry(cur, struct wim_dentry, d_extraction_list_node);
743
744                 ret = dentry_calculate_extraction_name(dentry, ctx);
745                 if (ret)
746                         return ret;
747
748                 if (prev->next == cur)
749                         prev = cur;
750                 else
751                         ; /* Current dentry and its children (which follow in
752                              the list) were deleted.  prev stays the same.  */
753         }
754         return 0;
755 }
756
757 static int
758 dentry_resolve_streams(struct wim_dentry *dentry, int extract_flags,
759                        struct wim_lookup_table *lookup_table)
760 {
761         struct wim_inode *inode = dentry->d_inode;
762         struct wim_lookup_table_entry *lte;
763         int ret;
764         bool force = false;
765
766         /* Special case:  when extracting from a pipe, the WIM lookup table is
767          * initially empty, so "resolving" an inode's streams is initially not
768          * possible.  However, we still need to keep track of which streams,
769          * identified by SHA1 message digests, need to be extracted, so we
770          * "resolve" the inode's streams anyway by allocating new entries.  */
771         if (extract_flags & WIMLIB_EXTRACT_FLAG_FROM_PIPE)
772                 force = true;
773         ret = inode_resolve_streams(inode, lookup_table, force);
774         if (ret)
775                 return ret;
776         for (u32 i = 0; i <= inode->i_num_ads; i++) {
777                 lte = inode_stream_lte_resolved(inode, i);
778                 if (lte)
779                         lte->out_refcnt = 0;
780         }
781         return 0;
782 }
783
784 /*
785  * For each dentry to be extracted, resolve all streams in the corresponding
786  * inode and set 'out_refcnt' in each to 0.
787  *
788  * Possible error codes: WIMLIB_ERR_RESOURCE_NOT_FOUND, WIMLIB_ERR_NOMEM.
789  */
790 static int
791 dentry_list_resolve_streams(struct list_head *dentry_list,
792                             struct apply_ctx *ctx)
793 {
794         struct wim_dentry *dentry;
795         int ret;
796
797         list_for_each_entry(dentry, dentry_list, d_extraction_list_node) {
798                 ret = dentry_resolve_streams(dentry,
799                                              ctx->extract_flags,
800                                              ctx->wim->lookup_table);
801                 if (ret)
802                         return ret;
803         }
804         return 0;
805 }
806
807 static int
808 ref_stream(struct wim_lookup_table_entry *lte, u32 stream_idx,
809            struct wim_dentry *dentry, struct apply_ctx *ctx)
810 {
811         struct wim_inode *inode = dentry->d_inode;
812         struct stream_owner *stream_owners;
813
814         if (!lte)
815                 return 0;
816
817         /* Tally the size only for each extraction of the stream (not hard
818          * links).  */
819         if (inode->i_visited && ctx->supported_features.hard_links)
820                 return 0;
821
822         ctx->progress.extract.total_bytes += lte->size;
823         ctx->progress.extract.num_streams++;
824
825         if (inode->i_visited)
826                 return 0;
827
828         /* Add stream to the dentry_list only one time, even if it's going
829          * to be extracted to multiple inodes.  */
830         if (lte->out_refcnt == 0) {
831                 list_add_tail(&lte->extraction_list, &ctx->stream_list);
832                 ctx->num_streams_remaining++;
833         }
834
835         /* If inode not yet been visited, append it to the stream_owners array.  */
836         if (lte->out_refcnt < ARRAY_LEN(lte->inline_stream_owners)) {
837                 stream_owners = lte->inline_stream_owners;
838         } else {
839                 struct stream_owner *prev_stream_owners;
840                 size_t alloc_stream_owners;
841
842                 if (lte->out_refcnt == ARRAY_LEN(lte->inline_stream_owners)) {
843                         prev_stream_owners = NULL;
844                         alloc_stream_owners = ARRAY_LEN(lte->inline_stream_owners);
845                 } else {
846                         prev_stream_owners = lte->stream_owners;
847                         alloc_stream_owners = lte->alloc_stream_owners;
848                 }
849
850                 if (lte->out_refcnt == alloc_stream_owners) {
851                         alloc_stream_owners *= 2;
852                         stream_owners = REALLOC(prev_stream_owners,
853                                                alloc_stream_owners *
854                                                 sizeof(stream_owners[0]));
855                         if (!stream_owners)
856                                 return WIMLIB_ERR_NOMEM;
857                         if (!prev_stream_owners) {
858                                 memcpy(stream_owners,
859                                        lte->inline_stream_owners,
860                                        sizeof(lte->inline_stream_owners));
861                         }
862                         lte->stream_owners = stream_owners;
863                         lte->alloc_stream_owners = alloc_stream_owners;
864                 }
865                 stream_owners = lte->stream_owners;
866         }
867         stream_owners[lte->out_refcnt].inode = inode;
868         if (stream_idx == 0) {
869                 stream_owners[lte->out_refcnt].stream_name = NULL;
870         } else {
871                 stream_owners[lte->out_refcnt].stream_name =
872                         inode->i_ads_entries[stream_idx - 1].stream_name;
873         }
874         lte->out_refcnt++;
875         return 0;
876 }
877
878 static int
879 dentry_ref_streams(struct wim_dentry *dentry, struct apply_ctx *ctx)
880 {
881         struct wim_inode *inode = dentry->d_inode;
882         int ret;
883
884         /* The unnamed data stream will always be extracted, except in an
885          * unlikely case.  */
886         if (!inode_is_encrypted_directory(inode)) {
887                 u16 stream_idx;
888                 struct wim_lookup_table_entry *stream;
889
890                 stream = inode_unnamed_stream_resolved(inode, &stream_idx);
891                 ret = ref_stream(stream, stream_idx, dentry, ctx);
892                 if (ret)
893                         return ret;
894         }
895
896         /* Named data streams will be extracted only if supported in the current
897          * extraction mode and volume, and to avoid complications, if not doing
898          * a linked extraction.  */
899         if (ctx->supported_features.named_data_streams) {
900                 for (u16 i = 0; i < inode->i_num_ads; i++) {
901                         if (!ads_entry_is_named_stream(&inode->i_ads_entries[i]))
902                                 continue;
903                         ret = ref_stream(inode->i_ads_entries[i].lte, i + 1,
904                                          dentry, ctx);
905                         if (ret)
906                                 return ret;
907                 }
908         }
909         inode->i_visited = 1;
910         return 0;
911 }
912
913 /*
914  * For each dentry to be extracted, iterate through the data streams of the
915  * corresponding inode.  For each such stream that is not to be ignored due to
916  * the supported features or extraction flags, add it to the list of streams to
917  * be extracted (ctx->stream_list) if not already done so.
918  *
919  * Also builds a mapping from each stream to the inodes referencing it.
920  *
921  * This also initializes the extract progress info with byte and stream
922  * information.
923  *
924  * ctx->supported_features must be filled in.
925  *
926  * Possible error codes: WIMLIB_ERR_NOMEM.
927  */
928 static int
929 dentry_list_ref_streams(struct list_head *dentry_list, struct apply_ctx *ctx)
930 {
931         struct wim_dentry *dentry;
932         int ret;
933
934         list_for_each_entry(dentry, dentry_list, d_extraction_list_node) {
935                 ret = dentry_ref_streams(dentry, ctx);
936                 if (ret)
937                         return ret;
938         }
939         list_for_each_entry(dentry, dentry_list, d_extraction_list_node)
940                 dentry->d_inode->i_visited = 0;
941         return 0;
942 }
943
944 static void
945 dentry_list_build_inode_alias_lists(struct list_head *dentry_list)
946 {
947         struct wim_dentry *dentry;
948         struct wim_inode *inode;
949
950         list_for_each_entry(dentry, dentry_list, d_extraction_list_node) {
951                 inode = dentry->d_inode;
952                 if (!inode->i_visited)
953                         INIT_LIST_HEAD(&inode->i_extraction_aliases);
954                 list_add_tail(&dentry->d_extraction_alias_node,
955                               &inode->i_extraction_aliases);
956                 inode->i_visited = 1;
957         }
958         list_for_each_entry(dentry, dentry_list, d_extraction_list_node)
959                 dentry->d_inode->i_visited = 0;
960 }
961
962 static void
963 inode_tally_features(const struct wim_inode *inode,
964                      struct wim_features *features)
965 {
966         if (inode->i_attributes & FILE_ATTRIBUTE_ARCHIVE)
967                 features->archive_files++;
968         if (inode->i_attributes & FILE_ATTRIBUTE_HIDDEN)
969                 features->hidden_files++;
970         if (inode->i_attributes & FILE_ATTRIBUTE_SYSTEM)
971                 features->system_files++;
972         if (inode->i_attributes & FILE_ATTRIBUTE_COMPRESSED)
973                 features->compressed_files++;
974         if (inode->i_attributes & FILE_ATTRIBUTE_ENCRYPTED) {
975                 if (inode->i_attributes & FILE_ATTRIBUTE_DIRECTORY)
976                         features->encrypted_directories++;
977                 else
978                         features->encrypted_files++;
979         }
980         if (inode->i_attributes & FILE_ATTRIBUTE_NOT_CONTENT_INDEXED)
981                 features->not_context_indexed_files++;
982         if (inode->i_attributes & FILE_ATTRIBUTE_SPARSE_FILE)
983                 features->sparse_files++;
984         if (inode_has_named_stream(inode))
985                 features->named_data_streams++;
986         if (inode->i_attributes & FILE_ATTRIBUTE_REPARSE_POINT) {
987                 features->reparse_points++;
988                 if (inode_is_symlink(inode))
989                         features->symlink_reparse_points++;
990                 else
991                         features->other_reparse_points++;
992         }
993         if (inode->i_security_id != -1)
994                 features->security_descriptors++;
995         if (inode_has_unix_data(inode))
996                 features->unix_data++;
997 }
998
999 /* Tally features necessary to extract a dentry and the corresponding inode.  */
1000 static void
1001 dentry_tally_features(struct wim_dentry *dentry, struct wim_features *features)
1002 {
1003         struct wim_inode *inode = dentry->d_inode;
1004
1005         if (dentry_has_short_name(dentry))
1006                 features->short_names++;
1007
1008         if (inode->i_visited) {
1009                 features->hard_links++;
1010         } else {
1011                 inode_tally_features(inode, features);
1012                 inode->i_visited = 1;
1013         }
1014 }
1015
1016 /* Tally the features necessary to extract the specified dentries.  */
1017 static void
1018 dentry_list_get_features(struct list_head *dentry_list,
1019                          struct wim_features *features)
1020 {
1021         struct wim_dentry *dentry;
1022
1023         list_for_each_entry(dentry, dentry_list, d_extraction_list_node)
1024                 dentry_tally_features(dentry, features);
1025
1026         list_for_each_entry(dentry, dentry_list, d_extraction_list_node)
1027                 dentry->d_inode->i_visited = 0;
1028 }
1029
1030 static int
1031 do_feature_check(const struct wim_features *required_features,
1032                  const struct wim_features *supported_features,
1033                  int extract_flags)
1034 {
1035         /* File attributes.  */
1036         if (!(extract_flags & WIMLIB_EXTRACT_FLAG_NO_ATTRIBUTES)) {
1037                 /* Note: Don't bother the user about FILE_ATTRIBUTE_ARCHIVE.
1038                  * We're an archive program, so theoretically we can do what we
1039                  * want with it.  */
1040
1041                 if (required_features->hidden_files &&
1042                     !supported_features->hidden_files)
1043                         WARNING("Ignoring FILE_ATTRIBUTE_HIDDEN of %lu files",
1044                                 required_features->hidden_files);
1045
1046                 if (required_features->system_files &&
1047                     !supported_features->system_files)
1048                         WARNING("Ignoring FILE_ATTRIBUTE_SYSTEM of %lu files",
1049                                 required_features->system_files);
1050
1051                 if (required_features->compressed_files &&
1052                     !supported_features->compressed_files)
1053                         WARNING("Ignoring FILE_ATTRIBUTE_COMPRESSED of %lu files",
1054                                 required_features->compressed_files);
1055
1056                 if (required_features->not_context_indexed_files &&
1057                     !supported_features->not_context_indexed_files)
1058                         WARNING("Ignoring FILE_ATTRIBUTE_NOT_CONTENT_INDEXED of %lu files",
1059                                 required_features->not_context_indexed_files);
1060
1061                 if (required_features->sparse_files &&
1062                     !supported_features->sparse_files)
1063                         WARNING("Ignoring FILE_ATTRIBUTE_SPARSE_FILE of %lu files",
1064                                 required_features->sparse_files);
1065
1066                 if (required_features->encrypted_directories &&
1067                     !supported_features->encrypted_directories)
1068                         WARNING("Ignoring FILE_ATTRIBUTE_ENCRYPTED of %lu directories",
1069                                 required_features->encrypted_directories);
1070         }
1071
1072         /* Encrypted files.  */
1073         if (required_features->encrypted_files &&
1074             !supported_features->encrypted_files)
1075                 WARNING("Ignoring %lu encrypted files",
1076                         required_features->encrypted_files);
1077
1078         /* Named data streams.  */
1079         if (required_features->named_data_streams &&
1080             (!supported_features->named_data_streams))
1081                 WARNING("Ignoring named data streams of %lu files",
1082                         required_features->named_data_streams);
1083
1084         /* Hard links.  */
1085         if (required_features->hard_links && !supported_features->hard_links)
1086                 WARNING("Extracting %lu hard links as independent files",
1087                         required_features->hard_links);
1088
1089         /* Symbolic links and reparse points.  */
1090         if ((extract_flags & WIMLIB_EXTRACT_FLAG_STRICT_SYMLINKS) &&
1091             required_features->symlink_reparse_points &&
1092             !supported_features->symlink_reparse_points &&
1093             !supported_features->reparse_points)
1094         {
1095                 ERROR("Extraction backend does not support symbolic links!");
1096                 return WIMLIB_ERR_UNSUPPORTED;
1097         }
1098         if (required_features->reparse_points &&
1099             !supported_features->reparse_points)
1100         {
1101                 if (supported_features->symlink_reparse_points) {
1102                         if (required_features->other_reparse_points) {
1103                                 WARNING("Ignoring %lu non-symlink/junction "
1104                                         "reparse point files",
1105                                         required_features->other_reparse_points);
1106                         }
1107                 } else {
1108                         WARNING("Ignoring %lu reparse point files",
1109                                 required_features->reparse_points);
1110                 }
1111         }
1112
1113         /* Security descriptors.  */
1114         if (((extract_flags & (WIMLIB_EXTRACT_FLAG_STRICT_ACLS |
1115                                WIMLIB_EXTRACT_FLAG_UNIX_DATA))
1116              == WIMLIB_EXTRACT_FLAG_STRICT_ACLS) &&
1117             required_features->security_descriptors &&
1118             !supported_features->security_descriptors)
1119         {
1120                 ERROR("Extraction backend does not support security descriptors!");
1121                 return WIMLIB_ERR_UNSUPPORTED;
1122         }
1123         if (!(extract_flags & WIMLIB_EXTRACT_FLAG_NO_ACLS) &&
1124             required_features->security_descriptors &&
1125             !supported_features->security_descriptors)
1126                 WARNING("Ignoring Windows NT security descriptors of %lu files",
1127                         required_features->security_descriptors);
1128
1129         /* UNIX data.  */
1130         if ((extract_flags & WIMLIB_EXTRACT_FLAG_UNIX_DATA) &&
1131             required_features->unix_data && !supported_features->unix_data)
1132         {
1133                 ERROR("Extraction backend does not support UNIX data!");
1134                 return WIMLIB_ERR_UNSUPPORTED;
1135         }
1136
1137         /* DOS Names.  */
1138         if (required_features->short_names &&
1139             !supported_features->short_names)
1140         {
1141                 if (extract_flags & WIMLIB_EXTRACT_FLAG_STRICT_SHORT_NAMES) {
1142                         ERROR("Extraction backend does not support DOS names!");
1143                         return WIMLIB_ERR_UNSUPPORTED;
1144                 }
1145                 WARNING("Ignoring DOS names of %lu files",
1146                         required_features->short_names);
1147         }
1148
1149         /* Timestamps.  */
1150         if ((extract_flags & WIMLIB_EXTRACT_FLAG_STRICT_TIMESTAMPS) &&
1151             !supported_features->timestamps)
1152         {
1153                 ERROR("Extraction backend does not support timestamps!");
1154                 return WIMLIB_ERR_UNSUPPORTED;
1155         }
1156
1157         return 0;
1158 }
1159
1160 static const struct apply_operations *
1161 select_apply_operations(int extract_flags)
1162 {
1163 #ifdef WITH_NTFS_3G
1164         if (extract_flags & WIMLIB_EXTRACT_FLAG_NTFS)
1165                 return &ntfs_3g_apply_ops;
1166 #endif
1167 #ifdef __WIN32__
1168         return &win32_apply_ops;
1169 #else
1170         return &unix_apply_ops;
1171 #endif
1172 }
1173
1174 static int
1175 extract_trees(WIMStruct *wim, struct wim_dentry **trees, size_t num_trees,
1176               const tchar *target, int extract_flags)
1177 {
1178         const struct apply_operations *ops;
1179         struct apply_ctx *ctx;
1180         int ret;
1181         LIST_HEAD(dentry_list);
1182
1183         if (extract_flags & WIMLIB_EXTRACT_FLAG_TO_STDOUT) {
1184                 ret = extract_dentries_to_stdout(trees, num_trees,
1185                                                  wim->lookup_table);
1186                 goto out;
1187         }
1188
1189         num_trees = remove_duplicate_trees(trees, num_trees);
1190         num_trees = remove_contained_trees(trees, num_trees);
1191
1192         ops = select_apply_operations(extract_flags);
1193
1194         if (num_trees > 1 && ops->single_tree_only) {
1195                 ERROR("Extracting multiple directory trees "
1196                       "at once is not supported in %s extraction mode!",
1197                       ops->name);
1198                 ret = WIMLIB_ERR_UNSUPPORTED;
1199                 goto out;
1200         }
1201
1202         ctx = CALLOC(1, ops->context_size);
1203         if (!ctx) {
1204                 ret = WIMLIB_ERR_NOMEM;
1205                 goto out;
1206         }
1207
1208         ctx->wim = wim;
1209         ctx->target = target;
1210         ctx->target_nchars = tstrlen(target);
1211         ctx->extract_flags = extract_flags;
1212         if (ctx->wim->progfunc) {
1213                 ctx->progfunc = ctx->wim->progfunc;
1214                 ctx->progctx = ctx->wim->progctx;
1215                 ctx->progress.extract.image = wim->current_image;
1216                 ctx->progress.extract.extract_flags = (extract_flags &
1217                                                        WIMLIB_EXTRACT_MASK_PUBLIC);
1218                 ctx->progress.extract.wimfile_name = wim->filename;
1219                 ctx->progress.extract.image_name = wimlib_get_image_name(wim,
1220                                                                          wim->current_image);
1221                 ctx->progress.extract.target = target;
1222         }
1223         INIT_LIST_HEAD(&ctx->stream_list);
1224
1225         ret = (*ops->get_supported_features)(target, &ctx->supported_features);
1226         if (ret)
1227                 goto out_cleanup;
1228
1229         build_dentry_list(&dentry_list, trees, num_trees,
1230                           !(extract_flags &
1231                             WIMLIB_EXTRACT_FLAG_NO_PRESERVE_DIR_STRUCTURE));
1232
1233         dentry_list_get_features(&dentry_list, &ctx->required_features);
1234
1235         ret = do_feature_check(&ctx->required_features, &ctx->supported_features,
1236                                ctx->extract_flags);
1237         if (ret)
1238                 goto out_cleanup;
1239
1240         ret = dentry_list_calculate_extraction_names(&dentry_list, ctx);
1241         if (ret)
1242                 goto out_cleanup;
1243
1244         ret = dentry_list_resolve_streams(&dentry_list, ctx);
1245         if (ret)
1246                 goto out_cleanup;
1247
1248         ret = dentry_list_ref_streams(&dentry_list, ctx);
1249         if (ret)
1250                 goto out_cleanup;
1251
1252         dentry_list_build_inode_alias_lists(&dentry_list);
1253
1254         if (extract_flags & WIMLIB_EXTRACT_FLAG_FROM_PIPE) {
1255                 /* When extracting from a pipe, the number of bytes of data to
1256                  * extract can't be determined in the normal way (examining the
1257                  * lookup table), since at this point all we have is a set of
1258                  * SHA1 message digests of streams that need to be extracted.
1259                  * However, we can get a reasonably accurate estimate by taking
1260                  * <TOTALBYTES> from the corresponding <IMAGE> in the WIM XML
1261                  * data.  This does assume that a full image is being extracted,
1262                  * but currently there is no API for doing otherwise.  (Also,
1263                  * subtract <HARDLINKBYTES> from this if hard links are
1264                  * supported by the extraction mode.)  */
1265                 ctx->progress.extract.total_bytes =
1266                         wim_info_get_image_total_bytes(wim->wim_info,
1267                                                        wim->current_image);
1268                 if (ctx->supported_features.hard_links) {
1269                         ctx->progress.extract.total_bytes -=
1270                                 wim_info_get_image_hard_link_bytes(wim->wim_info,
1271                                                                    wim->current_image);
1272                 }
1273         }
1274
1275         ret = extract_progress(ctx,
1276                                ((extract_flags & WIMLIB_EXTRACT_FLAG_IMAGEMODE) ?
1277                                        WIMLIB_PROGRESS_MSG_EXTRACT_IMAGE_BEGIN :
1278                                        WIMLIB_PROGRESS_MSG_EXTRACT_TREE_BEGIN));
1279         if (ret)
1280                 goto out_cleanup;
1281
1282         ret = (*ops->extract)(&dentry_list, ctx);
1283         if (ret)
1284                 goto out_cleanup;
1285
1286         if (ctx->progress.extract.completed_bytes <
1287             ctx->progress.extract.total_bytes)
1288         {
1289                 ctx->progress.extract.completed_bytes =
1290                         ctx->progress.extract.total_bytes;
1291                 ret = extract_progress(ctx, WIMLIB_PROGRESS_MSG_EXTRACT_STREAMS);
1292                 if (ret)
1293                         goto out_cleanup;
1294         }
1295
1296         ret = extract_progress(ctx,
1297                                ((extract_flags & WIMLIB_EXTRACT_FLAG_IMAGEMODE) ?
1298                                        WIMLIB_PROGRESS_MSG_EXTRACT_IMAGE_END :
1299                                        WIMLIB_PROGRESS_MSG_EXTRACT_TREE_END));
1300 out_cleanup:
1301         destroy_stream_list(&ctx->stream_list);
1302         destroy_dentry_list(&dentry_list);
1303         FREE(ctx);
1304 out:
1305         return ret;
1306 }
1307
1308 static int
1309 mkdir_if_needed(const tchar *target)
1310 {
1311         struct stat stbuf;
1312         if (tstat(target, &stbuf)) {
1313                 if (errno == ENOENT) {
1314                         if (tmkdir(target, 0755)) {
1315                                 ERROR_WITH_ERRNO("Failed to create directory "
1316                                                  "\"%"TS"\"", target);
1317                                 return WIMLIB_ERR_MKDIR;
1318                         }
1319                 } else {
1320                         ERROR_WITH_ERRNO("Failed to stat \"%"TS"\"", target);
1321                         return WIMLIB_ERR_STAT;
1322                 }
1323         } else if (!S_ISDIR(stbuf.st_mode)) {
1324                 ERROR("\"%"TS"\" is not a directory", target);
1325                 return WIMLIB_ERR_NOTDIR;
1326         }
1327         return 0;
1328 }
1329
1330 /* Make sure the extraction flags make sense, and update them if needed.  */
1331 static int
1332 check_extract_flags(const WIMStruct *wim, int *extract_flags_p)
1333 {
1334         int extract_flags = *extract_flags_p;
1335
1336         /* Check for invalid flag combinations  */
1337
1338         if ((extract_flags &
1339              (WIMLIB_EXTRACT_FLAG_NO_ACLS |
1340               WIMLIB_EXTRACT_FLAG_STRICT_ACLS)) == (WIMLIB_EXTRACT_FLAG_NO_ACLS |
1341                                                     WIMLIB_EXTRACT_FLAG_STRICT_ACLS))
1342                 return WIMLIB_ERR_INVALID_PARAM;
1343
1344         if ((extract_flags &
1345              (WIMLIB_EXTRACT_FLAG_RPFIX |
1346               WIMLIB_EXTRACT_FLAG_NORPFIX)) == (WIMLIB_EXTRACT_FLAG_RPFIX |
1347                                                 WIMLIB_EXTRACT_FLAG_NORPFIX))
1348                 return WIMLIB_ERR_INVALID_PARAM;
1349
1350 #ifndef WITH_NTFS_3G
1351         if (extract_flags & WIMLIB_EXTRACT_FLAG_NTFS) {
1352                 ERROR("wimlib was compiled without support for NTFS-3g, so\n"
1353                       "        it cannot apply a WIM image directly to an NTFS volume.");
1354                 return WIMLIB_ERR_UNSUPPORTED;
1355         }
1356 #endif
1357
1358 #ifndef __WIN32__
1359         if (extract_flags & WIMLIB_EXTRACT_FLAG_WIMBOOT) {
1360                 ERROR("WIMBoot extraction is only supported on Windows!");
1361                 return WIMLIB_ERR_UNSUPPORTED;
1362         }
1363 #endif
1364
1365         if ((extract_flags & (WIMLIB_EXTRACT_FLAG_RPFIX |
1366                               WIMLIB_EXTRACT_FLAG_NORPFIX |
1367                               WIMLIB_EXTRACT_FLAG_IMAGEMODE)) ==
1368                                         WIMLIB_EXTRACT_FLAG_IMAGEMODE)
1369         {
1370                 /* For full-image extraction, do reparse point fixups by default
1371                  * if the WIM header says they are enabled.  */
1372                 if (wim->hdr.flags & WIM_HDR_FLAG_RP_FIX)
1373                         extract_flags |= WIMLIB_EXTRACT_FLAG_RPFIX;
1374         }
1375
1376         *extract_flags_p = extract_flags;
1377         return 0;
1378 }
1379
1380 static u32
1381 get_wildcard_flags(int extract_flags)
1382 {
1383         u32 wildcard_flags = 0;
1384
1385         if (extract_flags & WIMLIB_EXTRACT_FLAG_STRICT_GLOB)
1386                 wildcard_flags |= WILDCARD_FLAG_ERROR_IF_NO_MATCH;
1387         else
1388                 wildcard_flags |= WILDCARD_FLAG_WARN_IF_NO_MATCH;
1389
1390         if (default_ignore_case)
1391                 wildcard_flags |= WILDCARD_FLAG_CASE_INSENSITIVE;
1392
1393         return wildcard_flags;
1394 }
1395
1396 struct append_dentry_ctx {
1397         struct wim_dentry **dentries;
1398         size_t num_dentries;
1399         size_t num_alloc_dentries;
1400 };
1401
1402 static int
1403 append_dentry_cb(struct wim_dentry *dentry, void *_ctx)
1404 {
1405         struct append_dentry_ctx *ctx = _ctx;
1406
1407         if (ctx->num_dentries == ctx->num_alloc_dentries) {
1408                 struct wim_dentry **new_dentries;
1409                 size_t new_length;
1410
1411                 new_length = max(ctx->num_alloc_dentries + 8,
1412                                  ctx->num_alloc_dentries * 3 / 2);
1413                 new_dentries = REALLOC(ctx->dentries,
1414                                        new_length * sizeof(ctx->dentries[0]));
1415                 if (new_dentries == NULL)
1416                         return WIMLIB_ERR_NOMEM;
1417                 ctx->dentries = new_dentries;
1418                 ctx->num_alloc_dentries = new_length;
1419         }
1420         ctx->dentries[ctx->num_dentries++] = dentry;
1421         return 0;
1422 }
1423
1424 static int
1425 do_wimlib_extract_paths(WIMStruct *wim, int image, const tchar *target,
1426                         const tchar * const *paths, size_t num_paths,
1427                         int extract_flags)
1428 {
1429         int ret;
1430         struct wim_dentry **trees;
1431         size_t num_trees;
1432
1433         if (wim == NULL || target == NULL || target[0] == T('\0') ||
1434             (num_paths != 0 && paths == NULL))
1435                 return WIMLIB_ERR_INVALID_PARAM;
1436
1437         ret = check_extract_flags(wim, &extract_flags);
1438         if (ret)
1439                 return ret;
1440
1441         ret = select_wim_image(wim, image);
1442         if (ret)
1443                 return ret;
1444
1445         ret = wim_checksum_unhashed_streams(wim);
1446         if (ret)
1447                 return ret;
1448
1449         if ((extract_flags & (WIMLIB_EXTRACT_FLAG_NTFS |
1450                               WIMLIB_EXTRACT_FLAG_NO_PRESERVE_DIR_STRUCTURE)) ==
1451             (WIMLIB_EXTRACT_FLAG_NO_PRESERVE_DIR_STRUCTURE))
1452         {
1453                 ret = mkdir_if_needed(target);
1454                 if (ret)
1455                         return ret;
1456         }
1457
1458         if (extract_flags & WIMLIB_EXTRACT_FLAG_GLOB_PATHS) {
1459
1460                 struct append_dentry_ctx append_dentry_ctx = {
1461                         .dentries = NULL,
1462                         .num_dentries = 0,
1463                         .num_alloc_dentries = 0,
1464                 };
1465
1466                 u32 wildcard_flags = get_wildcard_flags(extract_flags);
1467
1468                 for (size_t i = 0; i < num_paths; i++) {
1469                         tchar *path = canonicalize_wim_path(paths[i]);
1470                         if (path == NULL) {
1471                                 ret = WIMLIB_ERR_NOMEM;
1472                                 trees = append_dentry_ctx.dentries;
1473                                 goto out_free_trees;
1474                         }
1475                         ret = expand_wildcard(wim, path,
1476                                               append_dentry_cb,
1477                                               &append_dentry_ctx,
1478                                               wildcard_flags);
1479                         FREE(path);
1480                         if (ret) {
1481                                 trees = append_dentry_ctx.dentries;
1482                                 goto out_free_trees;
1483                         }
1484                 }
1485                 trees = append_dentry_ctx.dentries;
1486                 num_trees = append_dentry_ctx.num_dentries;
1487         } else {
1488                 trees = MALLOC(num_paths * sizeof(trees[0]));
1489                 if (trees == NULL)
1490                         return WIMLIB_ERR_NOMEM;
1491
1492                 for (size_t i = 0; i < num_paths; i++) {
1493
1494                         tchar *path = canonicalize_wim_path(paths[i]);
1495                         if (path == NULL) {
1496                                 ret = WIMLIB_ERR_NOMEM;
1497                                 goto out_free_trees;
1498                         }
1499
1500                         trees[i] = get_dentry(wim, path,
1501                                               WIMLIB_CASE_PLATFORM_DEFAULT);
1502                         FREE(path);
1503                         if (trees[i] == NULL) {
1504                                   ERROR("Path \"%"TS"\" does not exist "
1505                                         "in WIM image %d",
1506                                         paths[i], wim->current_image);
1507                                   ret = WIMLIB_ERR_PATH_DOES_NOT_EXIST;
1508                                   goto out_free_trees;
1509                         }
1510                 }
1511                 num_trees = num_paths;
1512         }
1513
1514         if (num_trees == 0) {
1515                 ret = 0;
1516                 goto out_free_trees;
1517         }
1518
1519         ret = extract_trees(wim, trees, num_trees, target, extract_flags);
1520 out_free_trees:
1521         FREE(trees);
1522         return ret;
1523 }
1524
1525 static int
1526 extract_single_image(WIMStruct *wim, int image,
1527                      const tchar *target, int extract_flags)
1528 {
1529         const tchar *path = WIMLIB_WIM_ROOT_PATH;
1530         extract_flags |= WIMLIB_EXTRACT_FLAG_IMAGEMODE;
1531         return do_wimlib_extract_paths(wim, image, target, &path, 1, extract_flags);
1532 }
1533
1534 static const tchar * const filename_forbidden_chars =
1535 T(
1536 #ifdef __WIN32__
1537 "<>:\"/\\|?*"
1538 #else
1539 "/"
1540 #endif
1541 );
1542
1543 /* This function checks if it is okay to use a WIM image's name as a directory
1544  * name.  */
1545 static bool
1546 image_name_ok_as_dir(const tchar *image_name)
1547 {
1548         return image_name && *image_name &&
1549                 !tstrpbrk(image_name, filename_forbidden_chars) &&
1550                 tstrcmp(image_name, T(".")) &&
1551                 tstrcmp(image_name, T(".."));
1552 }
1553
1554 /* Extracts all images from the WIM to the directory @target, with the images
1555  * placed in subdirectories named by their image names. */
1556 static int
1557 extract_all_images(WIMStruct *wim, const tchar *target, int extract_flags)
1558 {
1559         size_t image_name_max_len = max(xml_get_max_image_name_len(wim), 20);
1560         size_t output_path_len = tstrlen(target);
1561         tchar buf[output_path_len + 1 + image_name_max_len + 1];
1562         int ret;
1563         int image;
1564         const tchar *image_name;
1565
1566         extract_flags |= WIMLIB_EXTRACT_FLAG_MULTI_IMAGE;
1567
1568         if (extract_flags & WIMLIB_EXTRACT_FLAG_NTFS) {
1569                 ERROR("Cannot extract multiple images in NTFS extraction mode.");
1570                 return WIMLIB_ERR_INVALID_PARAM;
1571         }
1572
1573         ret = mkdir_if_needed(target);
1574         if (ret)
1575                 return ret;
1576         tmemcpy(buf, target, output_path_len);
1577         buf[output_path_len] = OS_PREFERRED_PATH_SEPARATOR;
1578         for (image = 1; image <= wim->hdr.image_count; image++) {
1579                 image_name = wimlib_get_image_name(wim, image);
1580                 if (image_name_ok_as_dir(image_name)) {
1581                         tstrcpy(buf + output_path_len + 1, image_name);
1582                 } else {
1583                         /* Image name is empty or contains forbidden characters.
1584                          * Use image number instead. */
1585                         tsprintf(buf + output_path_len + 1, T("%d"), image);
1586                 }
1587                 ret = extract_single_image(wim, image, buf, extract_flags);
1588                 if (ret)
1589                         return ret;
1590         }
1591         return 0;
1592 }
1593
1594 static int
1595 do_wimlib_extract_image(WIMStruct *wim, int image, const tchar *target,
1596                         int extract_flags)
1597 {
1598         if (extract_flags & (WIMLIB_EXTRACT_FLAG_NO_PRESERVE_DIR_STRUCTURE |
1599                              WIMLIB_EXTRACT_FLAG_TO_STDOUT |
1600                              WIMLIB_EXTRACT_FLAG_GLOB_PATHS))
1601                 return WIMLIB_ERR_INVALID_PARAM;
1602
1603         if (image == WIMLIB_ALL_IMAGES)
1604                 return extract_all_images(wim, target, extract_flags);
1605         else
1606                 return extract_single_image(wim, image, target, extract_flags);
1607 }
1608
1609
1610 /****************************************************************************
1611  *                          Extraction API                                  *
1612  ****************************************************************************/
1613
1614 WIMLIBAPI int
1615 wimlib_extract_paths(WIMStruct *wim, int image, const tchar *target,
1616                      const tchar * const *paths, size_t num_paths,
1617                      int extract_flags)
1618 {
1619         if (extract_flags & ~WIMLIB_EXTRACT_MASK_PUBLIC)
1620                 return WIMLIB_ERR_INVALID_PARAM;
1621
1622         return do_wimlib_extract_paths(wim, image, target, paths, num_paths,
1623                                        extract_flags);
1624 }
1625
1626 WIMLIBAPI int
1627 wimlib_extract_pathlist(WIMStruct *wim, int image, const tchar *target,
1628                         const tchar *path_list_file, int extract_flags)
1629 {
1630         int ret;
1631         tchar **paths;
1632         size_t num_paths;
1633         void *mem;
1634
1635         ret = read_path_list_file(path_list_file, &paths, &num_paths, &mem);
1636         if (ret) {
1637                 ERROR("Failed to read path list file \"%"TS"\"",
1638                       path_list_file);
1639                 return ret;
1640         }
1641
1642         ret = wimlib_extract_paths(wim, image, target,
1643                                    (const tchar * const *)paths, num_paths,
1644                                    extract_flags);
1645         FREE(paths);
1646         FREE(mem);
1647         return ret;
1648 }
1649
1650 WIMLIBAPI int
1651 wimlib_extract_image_from_pipe_with_progress(int pipe_fd,
1652                                              const tchar *image_num_or_name,
1653                                              const tchar *target,
1654                                              int extract_flags,
1655                                              wimlib_progress_func_t progfunc,
1656                                              void *progctx)
1657 {
1658         int ret;
1659         WIMStruct *pwm;
1660         struct filedes *in_fd;
1661         int image;
1662         unsigned i;
1663
1664         if (extract_flags & ~WIMLIB_EXTRACT_MASK_PUBLIC)
1665                 return WIMLIB_ERR_INVALID_PARAM;
1666
1667         /* Read the WIM header from the pipe and get a WIMStruct to represent
1668          * the pipable WIM.  Caveats:  Unlike getting a WIMStruct with
1669          * wimlib_open_wim(), getting a WIMStruct in this way will result in
1670          * an empty lookup table, no XML data read, and no filename set.  */
1671         ret = open_wim_as_WIMStruct(&pipe_fd, WIMLIB_OPEN_FLAG_FROM_PIPE, &pwm,
1672                                     progfunc, progctx);
1673         if (ret)
1674                 return ret;
1675
1676         /* Sanity check to make sure this is a pipable WIM.  */
1677         if (pwm->hdr.magic != PWM_MAGIC) {
1678                 ERROR("The WIM being read from file descriptor %d "
1679                       "is not pipable!", pipe_fd);
1680                 ret = WIMLIB_ERR_NOT_PIPABLE;
1681                 goto out_wimlib_free;
1682         }
1683
1684         /* Sanity check to make sure the first part of a pipable split WIM is
1685          * sent over the pipe first.  */
1686         if (pwm->hdr.part_number != 1) {
1687                 ERROR("The first part of the split WIM must be "
1688                       "sent over the pipe first.");
1689                 ret = WIMLIB_ERR_INVALID_PIPABLE_WIM;
1690                 goto out_wimlib_free;
1691         }
1692
1693         in_fd = &pwm->in_fd;
1694         wimlib_assert(in_fd->offset == WIM_HEADER_DISK_SIZE);
1695
1696         /* As mentioned, the WIMStruct we created from the pipe does not have
1697          * XML data yet.  Fix this by reading the extra copy of the XML data
1698          * that directly follows the header in pipable WIMs.  (Note: see
1699          * write_pipable_wim() for more details about the format of pipable
1700          * WIMs.)  */
1701         {
1702                 struct wim_lookup_table_entry xml_lte;
1703                 struct wim_resource_spec xml_rspec;
1704                 ret = read_pwm_stream_header(pwm, &xml_lte, &xml_rspec, 0, NULL);
1705                 if (ret)
1706                         goto out_wimlib_free;
1707
1708                 if (!(xml_lte.flags & WIM_RESHDR_FLAG_METADATA))
1709                 {
1710                         ERROR("Expected XML data, but found non-metadata "
1711                               "stream.");
1712                         ret = WIMLIB_ERR_INVALID_PIPABLE_WIM;
1713                         goto out_wimlib_free;
1714                 }
1715
1716                 wim_res_spec_to_hdr(&xml_rspec, &pwm->hdr.xml_data_reshdr);
1717
1718                 ret = read_wim_xml_data(pwm);
1719                 if (ret)
1720                         goto out_wimlib_free;
1721
1722                 if (wim_info_get_num_images(pwm->wim_info) != pwm->hdr.image_count) {
1723                         ERROR("Image count in XML data is not the same as in WIM header.");
1724                         ret = WIMLIB_ERR_IMAGE_COUNT;
1725                         goto out_wimlib_free;
1726                 }
1727         }
1728
1729         /* Get image index (this may use the XML data that was just read to
1730          * resolve an image name).  */
1731         if (image_num_or_name) {
1732                 image = wimlib_resolve_image(pwm, image_num_or_name);
1733                 if (image == WIMLIB_NO_IMAGE) {
1734                         ERROR("\"%"TS"\" is not a valid image in the pipable WIM!",
1735                               image_num_or_name);
1736                         ret = WIMLIB_ERR_INVALID_IMAGE;
1737                         goto out_wimlib_free;
1738                 } else if (image == WIMLIB_ALL_IMAGES) {
1739                         ERROR("Applying all images from a pipe is not supported!");
1740                         ret = WIMLIB_ERR_INVALID_IMAGE;
1741                         goto out_wimlib_free;
1742                 }
1743         } else {
1744                 if (pwm->hdr.image_count != 1) {
1745                         ERROR("No image was specified, but the pipable WIM "
1746                               "did not contain exactly 1 image");
1747                         ret = WIMLIB_ERR_INVALID_IMAGE;
1748                         goto out_wimlib_free;
1749                 }
1750                 image = 1;
1751         }
1752
1753         /* Load the needed metadata resource.  */
1754         for (i = 1; i <= pwm->hdr.image_count; i++) {
1755                 struct wim_lookup_table_entry *metadata_lte;
1756                 struct wim_image_metadata *imd;
1757                 struct wim_resource_spec *metadata_rspec;
1758
1759                 metadata_lte = new_lookup_table_entry();
1760                 if (metadata_lte == NULL) {
1761                         ret = WIMLIB_ERR_NOMEM;
1762                         goto out_wimlib_free;
1763                 }
1764                 metadata_rspec = MALLOC(sizeof(struct wim_resource_spec));
1765                 if (metadata_rspec == NULL) {
1766                         ret = WIMLIB_ERR_NOMEM;
1767                         free_lookup_table_entry(metadata_lte);
1768                         goto out_wimlib_free;
1769                 }
1770
1771                 ret = read_pwm_stream_header(pwm, metadata_lte, metadata_rspec, 0, NULL);
1772                 imd = pwm->image_metadata[i - 1];
1773                 imd->metadata_lte = metadata_lte;
1774                 if (ret) {
1775                         FREE(metadata_rspec);
1776                         goto out_wimlib_free;
1777                 }
1778
1779                 if (!(metadata_lte->flags & WIM_RESHDR_FLAG_METADATA)) {
1780                         ERROR("Expected metadata resource, but found "
1781                               "non-metadata stream.");
1782                         ret = WIMLIB_ERR_INVALID_PIPABLE_WIM;
1783                         goto out_wimlib_free;
1784                 }
1785
1786                 if (i == image) {
1787                         /* Metadata resource is for the image being extracted.
1788                          * Parse it and save the metadata in memory.  */
1789                         ret = read_metadata_resource(pwm, imd);
1790                         if (ret)
1791                                 goto out_wimlib_free;
1792                         imd->modified = 1;
1793                 } else {
1794                         /* Metadata resource is not for the image being
1795                          * extracted.  Skip over it.  */
1796                         ret = skip_wim_stream(metadata_lte);
1797                         if (ret)
1798                                 goto out_wimlib_free;
1799                 }
1800         }
1801         /* Extract the image.  */
1802         extract_flags |= WIMLIB_EXTRACT_FLAG_FROM_PIPE;
1803         ret = do_wimlib_extract_image(pwm, image, target, extract_flags);
1804         /* Clean up and return.  */
1805 out_wimlib_free:
1806         wimlib_free(pwm);
1807         return ret;
1808 }
1809
1810
1811 WIMLIBAPI int
1812 wimlib_extract_image_from_pipe(int pipe_fd, const tchar *image_num_or_name,
1813                                const tchar *target, int extract_flags)
1814 {
1815         return wimlib_extract_image_from_pipe_with_progress(pipe_fd,
1816                                                             image_num_or_name,
1817                                                             target,
1818                                                             extract_flags,
1819                                                             NULL,
1820                                                             NULL);
1821 }
1822
1823 WIMLIBAPI int
1824 wimlib_extract_image(WIMStruct *wim, int image, const tchar *target,
1825                      int extract_flags)
1826 {
1827         if (extract_flags & ~WIMLIB_EXTRACT_MASK_PUBLIC)
1828                 return WIMLIB_ERR_INVALID_PARAM;
1829         return do_wimlib_extract_image(wim, image, target, extract_flags);
1830 }