]> wimlib.net Git - wimlib/blob - src/extract.c
struct wim_dentry: Rename 'parent' to 'd_parent'
[wimlib] / src / extract.c
1 /*
2  * extract.c
3  *
4  * Support for extracting WIM images, or files or directories contained in a WIM
5  * image.
6  */
7
8 /*
9  * Copyright (C) 2012, 2013, 2014 Eric Biggers
10  *
11  * This file is part of wimlib, a library for working with WIM files.
12  *
13  * wimlib is free software; you can redistribute it and/or modify it under the
14  * terms of the GNU General Public License as published by the Free
15  * Software Foundation; either version 3 of the License, or (at your option)
16  * any later version.
17  *
18  * wimlib is distributed in the hope that it will be useful, but WITHOUT ANY
19  * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
20  * A PARTICULAR PURPOSE. See the GNU General Public License for more
21  * details.
22  *
23  * You should have received a copy of the GNU General Public License
24  * along with wimlib; if not, see http://www.gnu.org/licenses/.
25  */
26
27 /*
28  * This file provides the API functions wimlib_extract_image(),
29  * wimlib_extract_image_from_pipe(), wimlib_extract_paths(), and
30  * wimlib_extract_pathlist().  Internally, all end up calling
31  * do_wimlib_extract_paths() and extract_trees().
32  *
33  * Although wimlib supports multiple extraction modes/backends (NTFS-3g, UNIX,
34  * Win32), this file does not itself have code to extract files or directories
35  * to any specific target; instead, it handles generic functionality and relies
36  * on lower-level callback functions declared in `struct apply_operations' to do
37  * the actual extraction.
38  */
39
40 #ifdef HAVE_CONFIG_H
41 #  include "config.h"
42 #endif
43
44 #include "wimlib/apply.h"
45 #include "wimlib/dentry.h"
46 #include "wimlib/encoding.h"
47 #include "wimlib/endianness.h"
48 #include "wimlib/error.h"
49 #include "wimlib/lookup_table.h"
50 #include "wimlib/metadata.h"
51 #include "wimlib/pathlist.h"
52 #include "wimlib/paths.h"
53 #include "wimlib/reparse.h"
54 #include "wimlib/resource.h"
55 #include "wimlib/security.h"
56 #include "wimlib/unix_data.h"
57 #ifdef __WIN32__
58 #  include "wimlib/win32.h" /* for realpath() equivalent */
59 #endif
60 #include "wimlib/xml.h"
61 #include "wimlib/wildcard.h"
62 #include "wimlib/wim.h"
63
64 #include <errno.h>
65 #include <fcntl.h>
66 #include <stdlib.h>
67 #include <sys/stat.h>
68 #include <unistd.h>
69
70 #define WIMLIB_EXTRACT_FLAG_MULTI_IMAGE 0x80000000
71 #define WIMLIB_EXTRACT_FLAG_FROM_PIPE   0x40000000
72 #define WIMLIB_EXTRACT_FLAG_IMAGEMODE   0x20000000
73
74 /* Keep in sync with wimlib.h  */
75 #define WIMLIB_EXTRACT_MASK_PUBLIC                              \
76         (WIMLIB_EXTRACT_FLAG_NTFS                       |       \
77          WIMLIB_EXTRACT_FLAG_UNIX_DATA                  |       \
78          WIMLIB_EXTRACT_FLAG_NO_ACLS                    |       \
79          WIMLIB_EXTRACT_FLAG_STRICT_ACLS                |       \
80          WIMLIB_EXTRACT_FLAG_RPFIX                      |       \
81          WIMLIB_EXTRACT_FLAG_NORPFIX                    |       \
82          WIMLIB_EXTRACT_FLAG_TO_STDOUT                  |       \
83          WIMLIB_EXTRACT_FLAG_REPLACE_INVALID_FILENAMES  |       \
84          WIMLIB_EXTRACT_FLAG_ALL_CASE_CONFLICTS         |       \
85          WIMLIB_EXTRACT_FLAG_STRICT_TIMESTAMPS          |       \
86          WIMLIB_EXTRACT_FLAG_STRICT_SHORT_NAMES         |       \
87          WIMLIB_EXTRACT_FLAG_STRICT_SYMLINKS            |       \
88          WIMLIB_EXTRACT_FLAG_GLOB_PATHS                 |       \
89          WIMLIB_EXTRACT_FLAG_STRICT_GLOB                |       \
90          WIMLIB_EXTRACT_FLAG_NO_ATTRIBUTES              |       \
91          WIMLIB_EXTRACT_FLAG_NO_PRESERVE_DIR_STRUCTURE  |       \
92          WIMLIB_EXTRACT_FLAG_WIMBOOT)
93
94 /* Check whether the extraction of a dentry should be skipped completely.  */
95 static bool
96 dentry_is_supported(struct wim_dentry *dentry,
97                     const struct wim_features *supported_features)
98 {
99         struct wim_inode *inode = dentry->d_inode;
100
101         if (inode->i_attributes & FILE_ATTRIBUTE_REPARSE_POINT) {
102                 return supported_features->reparse_points ||
103                         (inode_is_symlink(inode) &&
104                          supported_features->symlink_reparse_points);
105         }
106         if (inode->i_attributes & FILE_ATTRIBUTE_ENCRYPTED) {
107                 if (inode->i_attributes & FILE_ATTRIBUTE_DIRECTORY)
108                         return supported_features->encrypted_directories != 0;
109                 else
110                         return supported_features->encrypted_files != 0;
111         }
112         return true;
113 }
114
115
116 #define PWM_ALLOW_WIM_HDR 0x00001
117 #define PWM_SILENT_EOF    0x00002
118
119 /* Read the header from a stream in a pipable WIM.  */
120 static int
121 read_pwm_stream_header(WIMStruct *pwm, struct wim_lookup_table_entry *lte,
122                        struct wim_resource_spec *rspec,
123                        int flags, struct wim_header_disk *hdr_ret)
124 {
125         union {
126                 struct pwm_stream_hdr stream_hdr;
127                 struct wim_header_disk pwm_hdr;
128         } buf;
129         struct wim_reshdr reshdr;
130         int ret;
131
132         ret = full_read(&pwm->in_fd, &buf.stream_hdr, sizeof(buf.stream_hdr));
133         if (ret)
134                 goto read_error;
135
136         if ((flags & PWM_ALLOW_WIM_HDR) && buf.stream_hdr.magic == PWM_MAGIC) {
137                 BUILD_BUG_ON(sizeof(buf.pwm_hdr) < sizeof(buf.stream_hdr));
138                 ret = full_read(&pwm->in_fd, &buf.stream_hdr + 1,
139                                 sizeof(buf.pwm_hdr) - sizeof(buf.stream_hdr));
140
141                 if (ret)
142                         goto read_error;
143                 lte->resource_location = RESOURCE_NONEXISTENT;
144                 memcpy(hdr_ret, &buf.pwm_hdr, sizeof(buf.pwm_hdr));
145                 return 0;
146         }
147
148         if (le64_to_cpu(buf.stream_hdr.magic) != PWM_STREAM_MAGIC) {
149                 ERROR("Data read on pipe is invalid (expected stream header).");
150                 return WIMLIB_ERR_INVALID_PIPABLE_WIM;
151         }
152
153         copy_hash(lte->hash, buf.stream_hdr.hash);
154
155         reshdr.size_in_wim = 0;
156         reshdr.flags = le32_to_cpu(buf.stream_hdr.flags);
157         reshdr.offset_in_wim = pwm->in_fd.offset;
158         reshdr.uncompressed_size = le64_to_cpu(buf.stream_hdr.uncompressed_size);
159         wim_res_hdr_to_spec(&reshdr, pwm, rspec);
160         lte_bind_wim_resource_spec(lte, rspec);
161         lte->flags = rspec->flags;
162         lte->size = rspec->uncompressed_size;
163         lte->offset_in_res = 0;
164         return 0;
165
166 read_error:
167         if (ret != WIMLIB_ERR_UNEXPECTED_END_OF_FILE || !(flags & PWM_SILENT_EOF))
168                 ERROR_WITH_ERRNO("Error reading pipable WIM from pipe");
169         return ret;
170 }
171
172 static int
173 load_streams_from_pipe(struct apply_ctx *ctx,
174                        const struct read_stream_list_callbacks *cbs)
175 {
176         struct wim_lookup_table_entry *found_lte = NULL;
177         struct wim_resource_spec *rspec = NULL;
178         struct wim_lookup_table *lookup_table;
179         int ret;
180
181         ret = WIMLIB_ERR_NOMEM;
182         found_lte = new_lookup_table_entry();
183         if (!found_lte)
184                 goto out;
185
186         rspec = MALLOC(sizeof(struct wim_resource_spec));
187         if (!rspec)
188                 goto out;
189
190         lookup_table = ctx->wim->lookup_table;
191         memcpy(ctx->progress.extract.guid, ctx->wim->hdr.guid, WIM_GUID_LEN);
192         ctx->progress.extract.part_number = ctx->wim->hdr.part_number;
193         ctx->progress.extract.total_parts = ctx->wim->hdr.total_parts;
194         ret = extract_progress(ctx, WIMLIB_PROGRESS_MSG_EXTRACT_SPWM_PART_BEGIN);
195         if (ret)
196                 goto out;
197
198         while (ctx->num_streams_remaining) {
199                 struct wim_header_disk pwm_hdr;
200                 struct wim_lookup_table_entry *needed_lte;
201
202                 if (found_lte->resource_location != RESOURCE_NONEXISTENT)
203                         lte_unbind_wim_resource_spec(found_lte);
204                 ret = read_pwm_stream_header(ctx->wim, found_lte, rspec,
205                                              PWM_ALLOW_WIM_HDR, &pwm_hdr);
206                 if (ret)
207                         goto out;
208
209                 if ((found_lte->resource_location != RESOURCE_NONEXISTENT)
210                     && !(found_lte->flags & WIM_RESHDR_FLAG_METADATA)
211                     && (needed_lte = lookup_stream(lookup_table, found_lte->hash))
212                     && (needed_lte->out_refcnt))
213                 {
214                         needed_lte->offset_in_res = found_lte->offset_in_res;
215                         needed_lte->flags = found_lte->flags;
216                         needed_lte->size = found_lte->size;
217
218                         lte_unbind_wim_resource_spec(found_lte);
219                         lte_bind_wim_resource_spec(needed_lte, rspec);
220
221                         ret = (*cbs->begin_stream)(needed_lte, 0,
222                                                    cbs->begin_stream_ctx);
223                         if (ret) {
224                                 lte_unbind_wim_resource_spec(needed_lte);
225                                 goto out;
226                         }
227
228                         ret = extract_stream(needed_lte, needed_lte->size,
229                                              cbs->consume_chunk,
230                                              cbs->consume_chunk_ctx);
231
232                         ret = (*cbs->end_stream)(needed_lte, ret,
233                                                  cbs->end_stream_ctx);
234                         lte_unbind_wim_resource_spec(needed_lte);
235                         if (ret)
236                                 goto out;
237                         ctx->num_streams_remaining--;
238                 } else if (found_lte->resource_location != RESOURCE_NONEXISTENT) {
239                         ret = skip_wim_stream(found_lte);
240                         if (ret)
241                                 goto out;
242                 } else {
243                         u16 part_number = le16_to_cpu(pwm_hdr.part_number);
244                         u16 total_parts = le16_to_cpu(pwm_hdr.total_parts);
245
246                         if (part_number != ctx->progress.extract.part_number ||
247                             total_parts != ctx->progress.extract.total_parts ||
248                             memcmp(pwm_hdr.guid, ctx->progress.extract.guid,
249                                    WIM_GUID_LEN))
250                         {
251                                 ctx->progress.extract.part_number = part_number;
252                                 ctx->progress.extract.total_parts = total_parts;
253                                 memcpy(ctx->progress.extract.guid,
254                                        pwm_hdr.guid, WIM_GUID_LEN);
255                                 ret = extract_progress(ctx,
256                                                        WIMLIB_PROGRESS_MSG_EXTRACT_SPWM_PART_BEGIN);
257                                 if (ret)
258                                         goto out;
259                         }
260                 }
261         }
262         ret = 0;
263 out:
264         if (found_lte->resource_location != RESOURCE_IN_WIM)
265                 FREE(rspec);
266         free_lookup_table_entry(found_lte);
267         return ret;
268 }
269
270 static int
271 begin_extract_stream_with_progress(struct wim_lookup_table_entry *lte,
272                                    u32 flags, void *_ctx)
273 {
274         struct apply_ctx *ctx = _ctx;
275
276         ctx->cur_stream = lte;
277
278         return (*ctx->saved_cbs->begin_stream)(lte, flags,
279                                                ctx->saved_cbs->begin_stream_ctx);
280 }
281
282 static int
283 consume_chunk_with_progress(const void *chunk, size_t size, void *_ctx)
284 {
285         struct apply_ctx *ctx = _ctx;
286         union wimlib_progress_info *progress = &ctx->progress;
287         int ret;
288
289         if (likely(ctx->supported_features.hard_links)) {
290                 progress->extract.completed_bytes +=
291                         (u64)size * ctx->cur_stream->out_refcnt;
292         } else {
293                 const struct stream_owner *owners = stream_owners(ctx->cur_stream);
294                 for (u32 i = 0; i < ctx->cur_stream->out_refcnt; i++) {
295                         const struct wim_inode *inode = owners[i].inode;
296                         const struct wim_dentry *dentry;
297
298                         list_for_each_entry(dentry,
299                                             &inode->i_extraction_aliases,
300                                             d_extraction_alias_node)
301                         {
302                                 progress->extract.completed_bytes += size;
303                         }
304                 }
305         }
306         if (progress->extract.completed_bytes >= ctx->next_progress) {
307
308                 ret = extract_progress(ctx, WIMLIB_PROGRESS_MSG_EXTRACT_STREAMS);
309                 if (ret)
310                         return ret;
311
312                 if (progress->extract.completed_bytes >=
313                     progress->extract.total_bytes)
314                 {
315                         ctx->next_progress = UINT64_MAX;
316                 } else {
317                         ctx->next_progress += progress->extract.total_bytes / 128;
318                         if (ctx->next_progress > progress->extract.total_bytes)
319                                 ctx->next_progress = progress->extract.total_bytes;
320                 }
321         }
322         return (*ctx->saved_cbs->consume_chunk)(chunk, size,
323                                                 ctx->saved_cbs->consume_chunk_ctx);
324 }
325
326 /*
327  * Read the list of single-instance streams to extract and feed their data into
328  * the specified callback functions.
329  *
330  * This handles checksumming each stream.
331  *
332  * This also handles sending WIMLIB_PROGRESS_MSG_EXTRACT_STREAMS.
333  *
334  * This also works if the WIM is being read from a pipe, whereas attempting to
335  * read streams directly (e.g. with read_full_stream_into_buf()) will not.
336  */
337 int
338 extract_stream_list(struct apply_ctx *ctx,
339                     const struct read_stream_list_callbacks *cbs)
340 {
341         struct read_stream_list_callbacks wrapper_cbs = {
342                 .begin_stream      = begin_extract_stream_with_progress,
343                 .begin_stream_ctx  = ctx,
344                 .consume_chunk     = consume_chunk_with_progress,
345                 .consume_chunk_ctx = ctx,
346                 .end_stream        = cbs->end_stream,
347                 .end_stream_ctx    = cbs->end_stream_ctx,
348         };
349         if (ctx->progfunc) {
350                 ctx->saved_cbs = cbs;
351                 cbs = &wrapper_cbs;
352         }
353         if (ctx->extract_flags & WIMLIB_EXTRACT_FLAG_FROM_PIPE) {
354                 return load_streams_from_pipe(ctx, cbs);
355         } else {
356                 return read_stream_list(&ctx->stream_list,
357                                         offsetof(struct wim_lookup_table_entry,
358                                                  extraction_list),
359                                         cbs, VERIFY_STREAM_HASHES);
360         }
361 }
362
363 /* Extract a WIM dentry to standard output.
364  *
365  * This obviously doesn't make sense in all cases.  We return an error if the
366  * dentry does not correspond to a regular file.  Otherwise we extract the
367  * unnamed data stream only.  */
368 static int
369 extract_dentry_to_stdout(struct wim_dentry *dentry,
370                          const struct wim_lookup_table *lookup_table)
371 {
372         struct wim_inode *inode = dentry->d_inode;
373         struct wim_lookup_table_entry *lte;
374         struct filedes _stdout;
375
376         if (inode->i_attributes & (FILE_ATTRIBUTE_REPARSE_POINT |
377                                    FILE_ATTRIBUTE_DIRECTORY))
378         {
379                 ERROR("\"%"TS"\" is not a regular file and therefore cannot be "
380                       "extracted to standard output", dentry_full_path(dentry));
381                 return WIMLIB_ERR_NOT_A_REGULAR_FILE;
382         }
383
384         lte = inode_unnamed_lte(inode, lookup_table);
385         if (!lte) {
386                 const u8 *hash = inode_unnamed_stream_hash(inode);
387                 if (!is_zero_hash(hash))
388                         return stream_not_found_error(inode, hash);
389                 return 0;
390         }
391
392         filedes_init(&_stdout, STDOUT_FILENO);
393         return extract_full_stream_to_fd(lte, &_stdout);
394 }
395
396 static int
397 extract_dentries_to_stdout(struct wim_dentry **dentries, size_t num_dentries,
398                            const struct wim_lookup_table *lookup_table)
399 {
400         for (size_t i = 0; i < num_dentries; i++) {
401                 int ret = extract_dentry_to_stdout(dentries[i], lookup_table);
402                 if (ret)
403                         return ret;
404         }
405         return 0;
406 }
407
408 /**********************************************************************/
409
410 /*
411  * Removes duplicate dentries from the array.
412  *
413  * Returns the new number of dentries, packed at the front of the array.
414  */
415 static size_t
416 remove_duplicate_trees(struct wim_dentry **trees, size_t num_trees)
417 {
418         size_t i, j = 0;
419         for (i = 0; i < num_trees; i++) {
420                 if (!trees[i]->tmp_flag) {
421                         /* Found distinct dentry.  */
422                         trees[i]->tmp_flag = 1;
423                         trees[j++] = trees[i];
424                 }
425         }
426         for (i = 0; i < j; i++)
427                 trees[i]->tmp_flag = 0;
428         return j;
429 }
430
431 /*
432  * Remove dentries that are descendants of other dentries in the array.
433  *
434  * Returns the new number of dentries, packed at the front of the array.
435  */
436 static size_t
437 remove_contained_trees(struct wim_dentry **trees, size_t num_trees)
438 {
439         size_t i, j = 0;
440         for (i = 0; i < num_trees; i++)
441                 trees[i]->tmp_flag = 1;
442         for (i = 0; i < num_trees; i++) {
443                 struct wim_dentry *d = trees[i];
444                 while (!dentry_is_root(d)) {
445                         d = d->d_parent;
446                         if (d->tmp_flag)
447                                 goto tree_contained;
448                 }
449                 trees[j++] = trees[i];
450                 continue;
451
452         tree_contained:
453                 trees[i]->tmp_flag = 0;
454         }
455
456         for (i = 0; i < j; i++)
457                 trees[i]->tmp_flag = 0;
458         return j;
459 }
460
461 static int
462 dentry_append_to_list(struct wim_dentry *dentry, void *_dentry_list)
463 {
464         struct list_head *dentry_list = _dentry_list;
465         list_add_tail(&dentry->d_extraction_list_node, dentry_list);
466         return 0;
467 }
468
469 static void
470 dentry_reset_extraction_list_node(struct wim_dentry *dentry)
471 {
472         dentry->d_extraction_list_node = (struct list_head){NULL, NULL};
473 }
474
475 static int
476 dentry_delete_from_list(struct wim_dentry *dentry, void *_ignore)
477 {
478         list_del(&dentry->d_extraction_list_node);
479         dentry_reset_extraction_list_node(dentry);
480         return 0;
481 }
482
483 /*
484  * Build the preliminary list of dentries to be extracted.
485  *
486  * The list maintains the invariant that if d1 and d2 are in the list and d1 is
487  * an ancestor of d2, then d1 appears before d2 in the list.
488  */
489 static void
490 build_dentry_list(struct list_head *dentry_list, struct wim_dentry **trees,
491                   size_t num_trees, bool add_ancestors)
492 {
493         INIT_LIST_HEAD(dentry_list);
494
495         /* Add the trees recursively.  */
496         for (size_t i = 0; i < num_trees; i++)
497                 for_dentry_in_tree(trees[i], dentry_append_to_list, dentry_list);
498
499         /* If requested, add ancestors of the trees.  */
500         if (add_ancestors) {
501                 for (size_t i = 0; i < num_trees; i++) {
502                         struct wim_dentry *dentry = trees[i];
503                         struct wim_dentry *ancestor;
504                         struct list_head *place_after;
505
506                         if (dentry_is_root(dentry))
507                                 continue;
508
509                         place_after = dentry_list;
510                         ancestor = dentry;
511                         do {
512                                 ancestor = ancestor->d_parent;
513                                 if (will_extract_dentry(ancestor)) {
514                                         place_after = &ancestor->d_extraction_list_node;
515                                         break;
516                                 }
517                         } while (!dentry_is_root(ancestor));
518
519                         ancestor = dentry;
520                         do {
521                                 ancestor = ancestor->d_parent;
522                                 if (will_extract_dentry(ancestor))
523                                         break;
524                                 list_add(&ancestor->d_extraction_list_node, place_after);
525                         } while (!dentry_is_root(ancestor));
526                 }
527         }
528 }
529
530 static void
531 destroy_dentry_list(struct list_head *dentry_list)
532 {
533         struct wim_dentry *dentry, *tmp;
534         struct wim_inode *inode;
535
536         list_for_each_entry_safe(dentry, tmp, dentry_list, d_extraction_list_node) {
537                 inode = dentry->d_inode;
538                 dentry_reset_extraction_list_node(dentry);
539                 inode->i_visited = 0;
540                 if ((void *)dentry->d_extraction_name != (void *)dentry->file_name)
541                         FREE(dentry->d_extraction_name);
542                 dentry->d_extraction_name = NULL;
543                 dentry->d_extraction_name_nchars = 0;
544         }
545 }
546
547 static void
548 destroy_stream_list(struct list_head *stream_list)
549 {
550         struct wim_lookup_table_entry *lte;
551
552         list_for_each_entry(lte, stream_list, extraction_list)
553                 if (lte->out_refcnt > ARRAY_LEN(lte->inline_stream_owners))
554                         FREE(lte->stream_owners);
555 }
556
557 #ifdef __WIN32__
558 static const utf16lechar replacement_char = cpu_to_le16(0xfffd);
559 #else
560 static const utf16lechar replacement_char = cpu_to_le16('?');
561 #endif
562
563 static bool
564 file_name_valid(utf16lechar *name, size_t num_chars, bool fix)
565 {
566         size_t i;
567
568         if (num_chars == 0)
569                 return true;
570         for (i = 0; i < num_chars; i++) {
571                 switch (name[i]) {
572         #ifdef __WIN32__
573                 case cpu_to_le16('\\'):
574                 case cpu_to_le16(':'):
575                 case cpu_to_le16('*'):
576                 case cpu_to_le16('?'):
577                 case cpu_to_le16('"'):
578                 case cpu_to_le16('<'):
579                 case cpu_to_le16('>'):
580                 case cpu_to_le16('|'):
581         #endif
582                 case cpu_to_le16('/'):
583                 case cpu_to_le16('\0'):
584                         if (fix)
585                                 name[i] = replacement_char;
586                         else
587                                 return false;
588                 }
589         }
590
591 #ifdef __WIN32__
592         if (name[num_chars - 1] == cpu_to_le16(' ') ||
593             name[num_chars - 1] == cpu_to_le16('.'))
594         {
595                 if (fix)
596                         name[num_chars - 1] = replacement_char;
597                 else
598                         return false;
599         }
600 #endif
601         return true;
602 }
603
604 static int
605 dentry_calculate_extraction_name(struct wim_dentry *dentry,
606                                  struct apply_ctx *ctx)
607 {
608         int ret;
609
610         if (!dentry_is_supported(dentry, &ctx->supported_features))
611                 goto skip_dentry;
612
613         if (dentry_is_root(dentry))
614                 return 0;
615
616         if (ctx->extract_flags & WIMLIB_EXTRACT_FLAG_NTFS) {
617                 dentry->d_extraction_name = dentry->file_name;
618                 dentry->d_extraction_name_nchars = dentry->file_name_nbytes /
619                                                    sizeof(utf16lechar);
620                 return 0;
621         }
622
623         if (!ctx->supported_features.case_sensitive_filenames) {
624                 struct wim_dentry *other;
625                 list_for_each_entry(other, &dentry->d_ci_conflict_list,
626                                     d_ci_conflict_list)
627                 {
628                         if (will_extract_dentry(other)) {
629                                 if (ctx->extract_flags &
630                                     WIMLIB_EXTRACT_FLAG_ALL_CASE_CONFLICTS) {
631                                         WARNING("\"%"TS"\" has the same "
632                                                 "case-insensitive name as "
633                                                 "\"%"TS"\"; extracting "
634                                                 "dummy name instead",
635                                                 dentry_full_path(dentry),
636                                                 dentry_full_path(other));
637                                         goto out_replace;
638                                 } else {
639                                         WARNING("Not extracting \"%"TS"\": "
640                                                 "has same case-insensitive "
641                                                 "name as \"%"TS"\"",
642                                                 dentry_full_path(dentry),
643                                                 dentry_full_path(other));
644                                         goto skip_dentry;
645                                 }
646                         }
647                 }
648         }
649
650         if (file_name_valid(dentry->file_name, dentry->file_name_nbytes / 2, false)) {
651                 ret = utf16le_get_tstr(dentry->file_name,
652                                        dentry->file_name_nbytes,
653                                        (const tchar **)&dentry->d_extraction_name,
654                                        &dentry->d_extraction_name_nchars);
655                 dentry->d_extraction_name_nchars /= sizeof(tchar);
656                 return ret;
657         } else {
658                 if (ctx->extract_flags & WIMLIB_EXTRACT_FLAG_REPLACE_INVALID_FILENAMES)
659                 {
660                         WARNING("\"%"TS"\" has an invalid filename "
661                                 "that is not supported on this platform; "
662                                 "extracting dummy name instead",
663                                 dentry_full_path(dentry));
664                         goto out_replace;
665                 } else {
666                         WARNING("Not extracting \"%"TS"\": has an invalid filename "
667                                 "that is not supported on this platform",
668                                 dentry_full_path(dentry));
669                         goto skip_dentry;
670                 }
671         }
672
673 out_replace:
674         {
675                 utf16lechar utf16_name_copy[dentry->file_name_nbytes / 2];
676
677                 memcpy(utf16_name_copy, dentry->file_name, dentry->file_name_nbytes);
678                 file_name_valid(utf16_name_copy, dentry->file_name_nbytes / 2, true);
679
680                 const tchar *tchar_name;
681                 size_t tchar_nchars;
682
683                 ret = utf16le_get_tstr(utf16_name_copy,
684                                        dentry->file_name_nbytes,
685                                        &tchar_name, &tchar_nchars);
686                 if (ret)
687                         return ret;
688
689                 tchar_nchars /= sizeof(tchar);
690
691                 size_t fixed_name_num_chars = tchar_nchars;
692                 tchar fixed_name[tchar_nchars + 50];
693
694                 tmemcpy(fixed_name, tchar_name, tchar_nchars);
695                 fixed_name_num_chars += tsprintf(fixed_name + tchar_nchars,
696                                                  T(" (invalid filename #%lu)"),
697                                                  ++ctx->invalid_sequence);
698
699                 utf16le_put_tstr(tchar_name);
700
701                 dentry->d_extraction_name = memdup(fixed_name,
702                                                    2 * fixed_name_num_chars + 2);
703                 if (!dentry->d_extraction_name)
704                         return WIMLIB_ERR_NOMEM;
705                 dentry->d_extraction_name_nchars = fixed_name_num_chars;
706         }
707         return 0;
708
709 skip_dentry:
710         for_dentry_in_tree(dentry, dentry_delete_from_list, NULL);
711         return 0;
712 }
713
714 /*
715  * Calculate the actual filename component at which each WIM dentry will be
716  * extracted, with special handling for dentries that are unsupported by the
717  * extraction backend or have invalid names.
718  *
719  * ctx->supported_features must be filled in.
720  *
721  * Possible error codes: WIMLIB_ERR_NOMEM, WIMLIB_ERR_INVALID_UTF16_STRING
722  */
723 static int
724 dentry_list_calculate_extraction_names(struct list_head *dentry_list,
725                                        struct apply_ctx *ctx)
726 {
727         struct list_head *prev, *cur;
728
729         /* Can't use list_for_each_entry() because a call to
730          * dentry_calculate_extraction_name() may delete the current dentry and
731          * its children from the list.  */
732
733         prev = dentry_list;
734         for (;;) {
735                 struct wim_dentry *dentry;
736                 int ret;
737
738                 cur = prev->next;
739                 if (cur == dentry_list)
740                         break;
741
742                 dentry = list_entry(cur, struct wim_dentry, d_extraction_list_node);
743
744                 ret = dentry_calculate_extraction_name(dentry, ctx);
745                 if (ret)
746                         return ret;
747
748                 if (prev->next == cur)
749                         prev = cur;
750                 else
751                         ; /* Current dentry and its children (which follow in
752                              the list) were deleted.  prev stays the same.  */
753         }
754         return 0;
755 }
756
757 static int
758 dentry_resolve_streams(struct wim_dentry *dentry, int extract_flags,
759                        struct wim_lookup_table *lookup_table)
760 {
761         struct wim_inode *inode = dentry->d_inode;
762         struct wim_lookup_table_entry *lte;
763         int ret;
764         bool force = false;
765
766         /* Special case:  when extracting from a pipe, the WIM lookup table is
767          * initially empty, so "resolving" an inode's streams is initially not
768          * possible.  However, we still need to keep track of which streams,
769          * identified by SHA1 message digests, need to be extracted, so we
770          * "resolve" the inode's streams anyway by allocating new entries.  */
771         if (extract_flags & WIMLIB_EXTRACT_FLAG_FROM_PIPE)
772                 force = true;
773         ret = inode_resolve_streams(inode, lookup_table, force);
774         if (ret)
775                 return ret;
776         for (u32 i = 0; i <= inode->i_num_ads; i++) {
777                 lte = inode_stream_lte_resolved(inode, i);
778                 if (lte)
779                         lte->out_refcnt = 0;
780         }
781         return 0;
782 }
783
784 /*
785  * For each dentry to be extracted, resolve all streams in the corresponding
786  * inode and set 'out_refcnt' in each to 0.
787  *
788  * Possible error codes: WIMLIB_ERR_RESOURCE_NOT_FOUND, WIMLIB_ERR_NOMEM.
789  */
790 static int
791 dentry_list_resolve_streams(struct list_head *dentry_list,
792                             struct apply_ctx *ctx)
793 {
794         struct wim_dentry *dentry;
795         int ret;
796
797         list_for_each_entry(dentry, dentry_list, d_extraction_list_node) {
798                 ret = dentry_resolve_streams(dentry,
799                                              ctx->extract_flags,
800                                              ctx->wim->lookup_table);
801                 if (ret)
802                         return ret;
803         }
804         return 0;
805 }
806
807 static int
808 ref_stream(struct wim_lookup_table_entry *lte, u32 stream_idx,
809            struct wim_dentry *dentry, struct apply_ctx *ctx)
810 {
811         struct wim_inode *inode = dentry->d_inode;
812         struct stream_owner *stream_owners;
813
814         if (!lte)
815                 return 0;
816
817         /* Tally the size only for each extraction of the stream (not hard
818          * links).  */
819         if (inode->i_visited && ctx->supported_features.hard_links)
820                 return 0;
821
822         ctx->progress.extract.total_bytes += lte->size;
823         ctx->progress.extract.num_streams++;
824
825         if (inode->i_visited)
826                 return 0;
827
828         /* Add stream to the dentry_list only one time, even if it's going
829          * to be extracted to multiple inodes.  */
830         if (lte->out_refcnt == 0) {
831                 list_add_tail(&lte->extraction_list, &ctx->stream_list);
832                 ctx->num_streams_remaining++;
833         }
834
835         /* If inode not yet been visited, append it to the stream_owners array.  */
836         if (lte->out_refcnt < ARRAY_LEN(lte->inline_stream_owners)) {
837                 stream_owners = lte->inline_stream_owners;
838         } else {
839                 struct stream_owner *prev_stream_owners;
840                 size_t alloc_stream_owners;
841
842                 if (lte->out_refcnt == ARRAY_LEN(lte->inline_stream_owners)) {
843                         prev_stream_owners = NULL;
844                         alloc_stream_owners = ARRAY_LEN(lte->inline_stream_owners);
845                 } else {
846                         prev_stream_owners = lte->stream_owners;
847                         alloc_stream_owners = lte->alloc_stream_owners;
848                 }
849
850                 if (lte->out_refcnt == alloc_stream_owners) {
851                         alloc_stream_owners *= 2;
852                         stream_owners = REALLOC(prev_stream_owners,
853                                                alloc_stream_owners *
854                                                 sizeof(stream_owners[0]));
855                         if (!stream_owners)
856                                 return WIMLIB_ERR_NOMEM;
857                         if (!prev_stream_owners) {
858                                 memcpy(stream_owners,
859                                        lte->inline_stream_owners,
860                                        sizeof(lte->inline_stream_owners));
861                         }
862                         lte->stream_owners = stream_owners;
863                         lte->alloc_stream_owners = alloc_stream_owners;
864                 }
865                 stream_owners = lte->stream_owners;
866         }
867         stream_owners[lte->out_refcnt].inode = inode;
868         if (stream_idx == 0) {
869                 stream_owners[lte->out_refcnt].stream_name = NULL;
870         } else {
871                 stream_owners[lte->out_refcnt].stream_name =
872                         inode->i_ads_entries[stream_idx - 1].stream_name;
873         }
874         lte->out_refcnt++;
875         return 0;
876 }
877
878 static int
879 dentry_ref_streams(struct wim_dentry *dentry, struct apply_ctx *ctx)
880 {
881         struct wim_inode *inode = dentry->d_inode;
882         int ret;
883
884         /* The unnamed data stream will always be extracted, except in an
885          * unlikely case.  */
886         if (!inode_is_encrypted_directory(inode)) {
887                 u16 stream_idx;
888                 struct wim_lookup_table_entry *stream;
889
890                 stream = inode_unnamed_stream_resolved(inode, &stream_idx);
891                 ret = ref_stream(stream, stream_idx, dentry, ctx);
892                 if (ret)
893                         return ret;
894         }
895
896         /* Named data streams will be extracted only if supported in the current
897          * extraction mode and volume, and to avoid complications, if not doing
898          * a linked extraction.  */
899         if (ctx->supported_features.named_data_streams) {
900                 for (u16 i = 0; i < inode->i_num_ads; i++) {
901                         if (!ads_entry_is_named_stream(&inode->i_ads_entries[i]))
902                                 continue;
903                         ret = ref_stream(inode->i_ads_entries[i].lte, i + 1,
904                                          dentry, ctx);
905                         if (ret)
906                                 return ret;
907                 }
908         }
909         inode->i_visited = 1;
910         return 0;
911 }
912
913 /*
914  * For each dentry to be extracted, iterate through the data streams of the
915  * corresponding inode.  For each such stream that is not to be ignored due to
916  * the supported features or extraction flags, add it to the list of streams to
917  * be extracted (ctx->stream_list) if not already done so.
918  *
919  * Also builds a mapping from each stream to the inodes referencing it.
920  *
921  * This also initializes the extract progress info with byte and stream
922  * information.
923  *
924  * ctx->supported_features must be filled in.
925  *
926  * Possible error codes: WIMLIB_ERR_NOMEM.
927  */
928 static int
929 dentry_list_ref_streams(struct list_head *dentry_list, struct apply_ctx *ctx)
930 {
931         struct wim_dentry *dentry;
932         int ret;
933
934         list_for_each_entry(dentry, dentry_list, d_extraction_list_node) {
935                 ret = dentry_ref_streams(dentry, ctx);
936                 if (ret)
937                         return ret;
938         }
939         list_for_each_entry(dentry, dentry_list, d_extraction_list_node)
940                 dentry->d_inode->i_visited = 0;
941         return 0;
942 }
943
944 static void
945 dentry_list_build_inode_alias_lists(struct list_head *dentry_list)
946 {
947         struct wim_dentry *dentry;
948         struct wim_inode *inode;
949
950         list_for_each_entry(dentry, dentry_list, d_extraction_list_node) {
951                 inode = dentry->d_inode;
952                 if (!inode->i_visited)
953                         INIT_LIST_HEAD(&inode->i_extraction_aliases);
954                 list_add_tail(&dentry->d_extraction_alias_node,
955                               &inode->i_extraction_aliases);
956                 inode->i_visited = 1;
957         }
958         list_for_each_entry(dentry, dentry_list, d_extraction_list_node)
959                 dentry->d_inode->i_visited = 0;
960 }
961
962 static void
963 inode_tally_features(const struct wim_inode *inode,
964                      struct wim_features *features)
965 {
966         if (inode->i_attributes & FILE_ATTRIBUTE_ARCHIVE)
967                 features->archive_files++;
968         if (inode->i_attributes & FILE_ATTRIBUTE_HIDDEN)
969                 features->hidden_files++;
970         if (inode->i_attributes & FILE_ATTRIBUTE_SYSTEM)
971                 features->system_files++;
972         if (inode->i_attributes & FILE_ATTRIBUTE_COMPRESSED)
973                 features->compressed_files++;
974         if (inode->i_attributes & FILE_ATTRIBUTE_ENCRYPTED) {
975                 if (inode->i_attributes & FILE_ATTRIBUTE_DIRECTORY)
976                         features->encrypted_directories++;
977                 else
978                         features->encrypted_files++;
979         }
980         if (inode->i_attributes & FILE_ATTRIBUTE_NOT_CONTENT_INDEXED)
981                 features->not_context_indexed_files++;
982         if (inode->i_attributes & FILE_ATTRIBUTE_SPARSE_FILE)
983                 features->sparse_files++;
984         if (inode_has_named_stream(inode))
985                 features->named_data_streams++;
986         if (inode->i_attributes & FILE_ATTRIBUTE_REPARSE_POINT) {
987                 features->reparse_points++;
988                 if (inode_is_symlink(inode))
989                         features->symlink_reparse_points++;
990                 else
991                         features->other_reparse_points++;
992         }
993         if (inode->i_security_id != -1)
994                 features->security_descriptors++;
995         if (inode_has_unix_data(inode))
996                 features->unix_data++;
997 }
998
999 /* Tally features necessary to extract a dentry and the corresponding inode.  */
1000 static void
1001 dentry_tally_features(struct wim_dentry *dentry, struct wim_features *features)
1002 {
1003         struct wim_inode *inode = dentry->d_inode;
1004
1005         if (dentry_has_short_name(dentry))
1006                 features->short_names++;
1007
1008         if (inode->i_visited) {
1009                 features->hard_links++;
1010         } else {
1011                 inode_tally_features(inode, features);
1012                 inode->i_visited = 1;
1013         }
1014 }
1015
1016 /* Tally the features necessary to extract the specified dentries.  */
1017 static void
1018 dentry_list_get_features(struct list_head *dentry_list,
1019                          struct wim_features *features)
1020 {
1021         struct wim_dentry *dentry;
1022
1023         list_for_each_entry(dentry, dentry_list, d_extraction_list_node)
1024                 dentry_tally_features(dentry, features);
1025
1026         list_for_each_entry(dentry, dentry_list, d_extraction_list_node)
1027                 dentry->d_inode->i_visited = 0;
1028 }
1029
1030 static int
1031 do_feature_check(const struct wim_features *required_features,
1032                  const struct wim_features *supported_features,
1033                  int extract_flags)
1034 {
1035         /* File attributes.  */
1036         if (!(extract_flags & WIMLIB_EXTRACT_FLAG_NO_ATTRIBUTES)) {
1037                 /* Note: Don't bother the user about FILE_ATTRIBUTE_ARCHIVE.
1038                  * We're an archive program, so theoretically we can do what we
1039                  * want with it.  */
1040
1041                 if (required_features->hidden_files &&
1042                     !supported_features->hidden_files)
1043                         WARNING("Ignoring FILE_ATTRIBUTE_HIDDEN of %lu files",
1044                                 required_features->hidden_files);
1045
1046                 if (required_features->system_files &&
1047                     !supported_features->system_files)
1048                         WARNING("Ignoring FILE_ATTRIBUTE_SYSTEM of %lu files",
1049                                 required_features->system_files);
1050
1051                 if (required_features->compressed_files &&
1052                     !supported_features->compressed_files)
1053                         WARNING("Ignoring FILE_ATTRIBUTE_COMPRESSED of %lu files",
1054                                 required_features->compressed_files);
1055
1056                 if (required_features->not_context_indexed_files &&
1057                     !supported_features->not_context_indexed_files)
1058                         WARNING("Ignoring FILE_ATTRIBUTE_NOT_CONTENT_INDEXED of %lu files",
1059                                 required_features->not_context_indexed_files);
1060
1061                 if (required_features->sparse_files &&
1062                     !supported_features->sparse_files)
1063                         WARNING("Ignoring FILE_ATTRIBUTE_SPARSE_FILE of %lu files",
1064                                 required_features->sparse_files);
1065
1066                 if (required_features->encrypted_directories &&
1067                     !supported_features->encrypted_directories)
1068                         WARNING("Ignoring FILE_ATTRIBUTE_ENCRYPTED of %lu directories",
1069                                 required_features->encrypted_directories);
1070         }
1071
1072         /* Encrypted files.  */
1073         if (required_features->encrypted_files &&
1074             !supported_features->encrypted_files)
1075                 WARNING("Ignoring %lu encrypted files",
1076                         required_features->encrypted_files);
1077
1078         /* Named data streams.  */
1079         if (required_features->named_data_streams &&
1080             (!supported_features->named_data_streams))
1081                 WARNING("Ignoring named data streams of %lu files",
1082                         required_features->named_data_streams);
1083
1084         /* Hard links.  */
1085         if (required_features->hard_links && !supported_features->hard_links)
1086                 WARNING("Extracting %lu hard links as independent files",
1087                         required_features->hard_links);
1088
1089         /* Symbolic links and reparse points.  */
1090         if ((extract_flags & WIMLIB_EXTRACT_FLAG_STRICT_SYMLINKS) &&
1091             required_features->symlink_reparse_points &&
1092             !supported_features->symlink_reparse_points &&
1093             !supported_features->reparse_points)
1094         {
1095                 ERROR("Extraction backend does not support symbolic links!");
1096                 return WIMLIB_ERR_UNSUPPORTED;
1097         }
1098         if (required_features->reparse_points &&
1099             !supported_features->reparse_points)
1100         {
1101                 if (supported_features->symlink_reparse_points) {
1102                         if (required_features->other_reparse_points) {
1103                                 WARNING("Ignoring %lu non-symlink/junction "
1104                                         "reparse point files",
1105                                         required_features->other_reparse_points);
1106                         }
1107                 } else {
1108                         WARNING("Ignoring %lu reparse point files",
1109                                 required_features->reparse_points);
1110                 }
1111         }
1112
1113         /* Security descriptors.  */
1114         if (((extract_flags & (WIMLIB_EXTRACT_FLAG_STRICT_ACLS |
1115                                WIMLIB_EXTRACT_FLAG_UNIX_DATA))
1116              == WIMLIB_EXTRACT_FLAG_STRICT_ACLS) &&
1117             required_features->security_descriptors &&
1118             !supported_features->security_descriptors)
1119         {
1120                 ERROR("Extraction backend does not support security descriptors!");
1121                 return WIMLIB_ERR_UNSUPPORTED;
1122         }
1123         if (!(extract_flags & WIMLIB_EXTRACT_FLAG_NO_ACLS) &&
1124             required_features->security_descriptors &&
1125             !supported_features->security_descriptors)
1126                 WARNING("Ignoring Windows NT security descriptors of %lu files",
1127                         required_features->security_descriptors);
1128
1129         /* UNIX data.  */
1130         if ((extract_flags & WIMLIB_EXTRACT_FLAG_UNIX_DATA) &&
1131             required_features->unix_data && !supported_features->unix_data)
1132         {
1133                 ERROR("Extraction backend does not support UNIX data!");
1134                 return WIMLIB_ERR_UNSUPPORTED;
1135         }
1136
1137         if (required_features->unix_data &&
1138             !(extract_flags & WIMLIB_EXTRACT_FLAG_UNIX_DATA))
1139         {
1140                 WARNING("Ignoring UNIX metadata of %lu files",
1141                         required_features->unix_data);
1142         }
1143
1144         /* DOS Names.  */
1145         if (required_features->short_names &&
1146             !supported_features->short_names)
1147         {
1148                 if (extract_flags & WIMLIB_EXTRACT_FLAG_STRICT_SHORT_NAMES) {
1149                         ERROR("Extraction backend does not support DOS names!");
1150                         return WIMLIB_ERR_UNSUPPORTED;
1151                 }
1152                 WARNING("Ignoring DOS names of %lu files",
1153                         required_features->short_names);
1154         }
1155
1156         /* Timestamps.  */
1157         if ((extract_flags & WIMLIB_EXTRACT_FLAG_STRICT_TIMESTAMPS) &&
1158             !supported_features->timestamps)
1159         {
1160                 ERROR("Extraction backend does not support timestamps!");
1161                 return WIMLIB_ERR_UNSUPPORTED;
1162         }
1163
1164         return 0;
1165 }
1166
1167 static const struct apply_operations *
1168 select_apply_operations(int extract_flags)
1169 {
1170 #ifdef WITH_NTFS_3G
1171         if (extract_flags & WIMLIB_EXTRACT_FLAG_NTFS)
1172                 return &ntfs_3g_apply_ops;
1173 #endif
1174 #ifdef __WIN32__
1175         return &win32_apply_ops;
1176 #else
1177         return &unix_apply_ops;
1178 #endif
1179 }
1180
1181 static int
1182 extract_trees(WIMStruct *wim, struct wim_dentry **trees, size_t num_trees,
1183               const tchar *target, int extract_flags)
1184 {
1185         const struct apply_operations *ops;
1186         struct apply_ctx *ctx;
1187         int ret;
1188         LIST_HEAD(dentry_list);
1189
1190         if (extract_flags & WIMLIB_EXTRACT_FLAG_TO_STDOUT) {
1191                 ret = extract_dentries_to_stdout(trees, num_trees,
1192                                                  wim->lookup_table);
1193                 goto out;
1194         }
1195
1196         num_trees = remove_duplicate_trees(trees, num_trees);
1197         num_trees = remove_contained_trees(trees, num_trees);
1198
1199         ops = select_apply_operations(extract_flags);
1200
1201         if (num_trees > 1 && ops->single_tree_only) {
1202                 ERROR("Extracting multiple directory trees "
1203                       "at once is not supported in %s extraction mode!",
1204                       ops->name);
1205                 ret = WIMLIB_ERR_UNSUPPORTED;
1206                 goto out;
1207         }
1208
1209         ctx = CALLOC(1, ops->context_size);
1210         if (!ctx) {
1211                 ret = WIMLIB_ERR_NOMEM;
1212                 goto out;
1213         }
1214
1215         ctx->wim = wim;
1216         ctx->target = target;
1217         ctx->target_nchars = tstrlen(target);
1218         ctx->extract_flags = extract_flags;
1219         if (ctx->wim->progfunc) {
1220                 ctx->progfunc = ctx->wim->progfunc;
1221                 ctx->progctx = ctx->wim->progctx;
1222                 ctx->progress.extract.image = wim->current_image;
1223                 ctx->progress.extract.extract_flags = (extract_flags &
1224                                                        WIMLIB_EXTRACT_MASK_PUBLIC);
1225                 ctx->progress.extract.wimfile_name = wim->filename;
1226                 ctx->progress.extract.image_name = wimlib_get_image_name(wim,
1227                                                                          wim->current_image);
1228                 ctx->progress.extract.target = target;
1229         }
1230         INIT_LIST_HEAD(&ctx->stream_list);
1231
1232         ret = (*ops->get_supported_features)(target, &ctx->supported_features);
1233         if (ret)
1234                 goto out_cleanup;
1235
1236         build_dentry_list(&dentry_list, trees, num_trees,
1237                           !(extract_flags &
1238                             WIMLIB_EXTRACT_FLAG_NO_PRESERVE_DIR_STRUCTURE));
1239
1240         dentry_list_get_features(&dentry_list, &ctx->required_features);
1241
1242         ret = do_feature_check(&ctx->required_features, &ctx->supported_features,
1243                                ctx->extract_flags);
1244         if (ret)
1245                 goto out_cleanup;
1246
1247         ret = dentry_list_calculate_extraction_names(&dentry_list, ctx);
1248         if (ret)
1249                 goto out_cleanup;
1250
1251         ret = dentry_list_resolve_streams(&dentry_list, ctx);
1252         if (ret)
1253                 goto out_cleanup;
1254
1255         ret = dentry_list_ref_streams(&dentry_list, ctx);
1256         if (ret)
1257                 goto out_cleanup;
1258
1259         dentry_list_build_inode_alias_lists(&dentry_list);
1260
1261         if (extract_flags & WIMLIB_EXTRACT_FLAG_FROM_PIPE) {
1262                 /* When extracting from a pipe, the number of bytes of data to
1263                  * extract can't be determined in the normal way (examining the
1264                  * lookup table), since at this point all we have is a set of
1265                  * SHA1 message digests of streams that need to be extracted.
1266                  * However, we can get a reasonably accurate estimate by taking
1267                  * <TOTALBYTES> from the corresponding <IMAGE> in the WIM XML
1268                  * data.  This does assume that a full image is being extracted,
1269                  * but currently there is no API for doing otherwise.  (Also,
1270                  * subtract <HARDLINKBYTES> from this if hard links are
1271                  * supported by the extraction mode.)  */
1272                 ctx->progress.extract.total_bytes =
1273                         wim_info_get_image_total_bytes(wim->wim_info,
1274                                                        wim->current_image);
1275                 if (ctx->supported_features.hard_links) {
1276                         ctx->progress.extract.total_bytes -=
1277                                 wim_info_get_image_hard_link_bytes(wim->wim_info,
1278                                                                    wim->current_image);
1279                 }
1280         }
1281
1282         ret = extract_progress(ctx,
1283                                ((extract_flags & WIMLIB_EXTRACT_FLAG_IMAGEMODE) ?
1284                                        WIMLIB_PROGRESS_MSG_EXTRACT_IMAGE_BEGIN :
1285                                        WIMLIB_PROGRESS_MSG_EXTRACT_TREE_BEGIN));
1286         if (ret)
1287                 goto out_cleanup;
1288
1289         ret = (*ops->extract)(&dentry_list, ctx);
1290         if (ret)
1291                 goto out_cleanup;
1292
1293         if (ctx->progress.extract.completed_bytes <
1294             ctx->progress.extract.total_bytes)
1295         {
1296                 ctx->progress.extract.completed_bytes =
1297                         ctx->progress.extract.total_bytes;
1298                 ret = extract_progress(ctx, WIMLIB_PROGRESS_MSG_EXTRACT_STREAMS);
1299                 if (ret)
1300                         goto out_cleanup;
1301         }
1302
1303         ret = extract_progress(ctx,
1304                                ((extract_flags & WIMLIB_EXTRACT_FLAG_IMAGEMODE) ?
1305                                        WIMLIB_PROGRESS_MSG_EXTRACT_IMAGE_END :
1306                                        WIMLIB_PROGRESS_MSG_EXTRACT_TREE_END));
1307 out_cleanup:
1308         destroy_stream_list(&ctx->stream_list);
1309         destroy_dentry_list(&dentry_list);
1310         FREE(ctx);
1311 out:
1312         return ret;
1313 }
1314
1315 static int
1316 mkdir_if_needed(const tchar *target)
1317 {
1318         struct stat stbuf;
1319         if (tstat(target, &stbuf)) {
1320                 if (errno == ENOENT) {
1321                         if (tmkdir(target, 0755)) {
1322                                 ERROR_WITH_ERRNO("Failed to create directory "
1323                                                  "\"%"TS"\"", target);
1324                                 return WIMLIB_ERR_MKDIR;
1325                         }
1326                 } else {
1327                         ERROR_WITH_ERRNO("Failed to stat \"%"TS"\"", target);
1328                         return WIMLIB_ERR_STAT;
1329                 }
1330         } else if (!S_ISDIR(stbuf.st_mode)) {
1331                 ERROR("\"%"TS"\" is not a directory", target);
1332                 return WIMLIB_ERR_NOTDIR;
1333         }
1334         return 0;
1335 }
1336
1337 /* Make sure the extraction flags make sense, and update them if needed.  */
1338 static int
1339 check_extract_flags(const WIMStruct *wim, int *extract_flags_p)
1340 {
1341         int extract_flags = *extract_flags_p;
1342
1343         /* Check for invalid flag combinations  */
1344
1345         if ((extract_flags &
1346              (WIMLIB_EXTRACT_FLAG_NO_ACLS |
1347               WIMLIB_EXTRACT_FLAG_STRICT_ACLS)) == (WIMLIB_EXTRACT_FLAG_NO_ACLS |
1348                                                     WIMLIB_EXTRACT_FLAG_STRICT_ACLS))
1349                 return WIMLIB_ERR_INVALID_PARAM;
1350
1351         if ((extract_flags &
1352              (WIMLIB_EXTRACT_FLAG_RPFIX |
1353               WIMLIB_EXTRACT_FLAG_NORPFIX)) == (WIMLIB_EXTRACT_FLAG_RPFIX |
1354                                                 WIMLIB_EXTRACT_FLAG_NORPFIX))
1355                 return WIMLIB_ERR_INVALID_PARAM;
1356
1357 #ifndef WITH_NTFS_3G
1358         if (extract_flags & WIMLIB_EXTRACT_FLAG_NTFS) {
1359                 ERROR("wimlib was compiled without support for NTFS-3g, so\n"
1360                       "        it cannot apply a WIM image directly to an NTFS volume.");
1361                 return WIMLIB_ERR_UNSUPPORTED;
1362         }
1363 #endif
1364
1365 #ifndef __WIN32__
1366         if (extract_flags & WIMLIB_EXTRACT_FLAG_WIMBOOT) {
1367                 ERROR("WIMBoot extraction is only supported on Windows!");
1368                 return WIMLIB_ERR_UNSUPPORTED;
1369         }
1370 #endif
1371
1372         if ((extract_flags & (WIMLIB_EXTRACT_FLAG_RPFIX |
1373                               WIMLIB_EXTRACT_FLAG_NORPFIX |
1374                               WIMLIB_EXTRACT_FLAG_IMAGEMODE)) ==
1375                                         WIMLIB_EXTRACT_FLAG_IMAGEMODE)
1376         {
1377                 /* For full-image extraction, do reparse point fixups by default
1378                  * if the WIM header says they are enabled.  */
1379                 if (wim->hdr.flags & WIM_HDR_FLAG_RP_FIX)
1380                         extract_flags |= WIMLIB_EXTRACT_FLAG_RPFIX;
1381         }
1382
1383         *extract_flags_p = extract_flags;
1384         return 0;
1385 }
1386
1387 static u32
1388 get_wildcard_flags(int extract_flags)
1389 {
1390         u32 wildcard_flags = 0;
1391
1392         if (extract_flags & WIMLIB_EXTRACT_FLAG_STRICT_GLOB)
1393                 wildcard_flags |= WILDCARD_FLAG_ERROR_IF_NO_MATCH;
1394         else
1395                 wildcard_flags |= WILDCARD_FLAG_WARN_IF_NO_MATCH;
1396
1397         if (default_ignore_case)
1398                 wildcard_flags |= WILDCARD_FLAG_CASE_INSENSITIVE;
1399
1400         return wildcard_flags;
1401 }
1402
1403 struct append_dentry_ctx {
1404         struct wim_dentry **dentries;
1405         size_t num_dentries;
1406         size_t num_alloc_dentries;
1407 };
1408
1409 static int
1410 append_dentry_cb(struct wim_dentry *dentry, void *_ctx)
1411 {
1412         struct append_dentry_ctx *ctx = _ctx;
1413
1414         if (ctx->num_dentries == ctx->num_alloc_dentries) {
1415                 struct wim_dentry **new_dentries;
1416                 size_t new_length;
1417
1418                 new_length = max(ctx->num_alloc_dentries + 8,
1419                                  ctx->num_alloc_dentries * 3 / 2);
1420                 new_dentries = REALLOC(ctx->dentries,
1421                                        new_length * sizeof(ctx->dentries[0]));
1422                 if (new_dentries == NULL)
1423                         return WIMLIB_ERR_NOMEM;
1424                 ctx->dentries = new_dentries;
1425                 ctx->num_alloc_dentries = new_length;
1426         }
1427         ctx->dentries[ctx->num_dentries++] = dentry;
1428         return 0;
1429 }
1430
1431 static int
1432 do_wimlib_extract_paths(WIMStruct *wim, int image, const tchar *target,
1433                         const tchar * const *paths, size_t num_paths,
1434                         int extract_flags)
1435 {
1436         int ret;
1437         struct wim_dentry **trees;
1438         size_t num_trees;
1439
1440         if (wim == NULL || target == NULL || target[0] == T('\0') ||
1441             (num_paths != 0 && paths == NULL))
1442                 return WIMLIB_ERR_INVALID_PARAM;
1443
1444         ret = check_extract_flags(wim, &extract_flags);
1445         if (ret)
1446                 return ret;
1447
1448         ret = select_wim_image(wim, image);
1449         if (ret)
1450                 return ret;
1451
1452         ret = wim_checksum_unhashed_streams(wim);
1453         if (ret)
1454                 return ret;
1455
1456         if ((extract_flags & (WIMLIB_EXTRACT_FLAG_NTFS |
1457                               WIMLIB_EXTRACT_FLAG_NO_PRESERVE_DIR_STRUCTURE)) ==
1458             (WIMLIB_EXTRACT_FLAG_NO_PRESERVE_DIR_STRUCTURE))
1459         {
1460                 ret = mkdir_if_needed(target);
1461                 if (ret)
1462                         return ret;
1463         }
1464
1465         if (extract_flags & WIMLIB_EXTRACT_FLAG_GLOB_PATHS) {
1466
1467                 struct append_dentry_ctx append_dentry_ctx = {
1468                         .dentries = NULL,
1469                         .num_dentries = 0,
1470                         .num_alloc_dentries = 0,
1471                 };
1472
1473                 u32 wildcard_flags = get_wildcard_flags(extract_flags);
1474
1475                 for (size_t i = 0; i < num_paths; i++) {
1476                         tchar *path = canonicalize_wim_path(paths[i]);
1477                         if (path == NULL) {
1478                                 ret = WIMLIB_ERR_NOMEM;
1479                                 trees = append_dentry_ctx.dentries;
1480                                 goto out_free_trees;
1481                         }
1482                         ret = expand_wildcard(wim, path,
1483                                               append_dentry_cb,
1484                                               &append_dentry_ctx,
1485                                               wildcard_flags);
1486                         FREE(path);
1487                         if (ret) {
1488                                 trees = append_dentry_ctx.dentries;
1489                                 goto out_free_trees;
1490                         }
1491                 }
1492                 trees = append_dentry_ctx.dentries;
1493                 num_trees = append_dentry_ctx.num_dentries;
1494         } else {
1495                 trees = MALLOC(num_paths * sizeof(trees[0]));
1496                 if (trees == NULL)
1497                         return WIMLIB_ERR_NOMEM;
1498
1499                 for (size_t i = 0; i < num_paths; i++) {
1500
1501                         tchar *path = canonicalize_wim_path(paths[i]);
1502                         if (path == NULL) {
1503                                 ret = WIMLIB_ERR_NOMEM;
1504                                 goto out_free_trees;
1505                         }
1506
1507                         trees[i] = get_dentry(wim, path,
1508                                               WIMLIB_CASE_PLATFORM_DEFAULT);
1509                         FREE(path);
1510                         if (trees[i] == NULL) {
1511                                   ERROR("Path \"%"TS"\" does not exist "
1512                                         "in WIM image %d",
1513                                         paths[i], wim->current_image);
1514                                   ret = WIMLIB_ERR_PATH_DOES_NOT_EXIST;
1515                                   goto out_free_trees;
1516                         }
1517                 }
1518                 num_trees = num_paths;
1519         }
1520
1521         if (num_trees == 0) {
1522                 ret = 0;
1523                 goto out_free_trees;
1524         }
1525
1526         ret = extract_trees(wim, trees, num_trees, target, extract_flags);
1527 out_free_trees:
1528         FREE(trees);
1529         return ret;
1530 }
1531
1532 static int
1533 extract_single_image(WIMStruct *wim, int image,
1534                      const tchar *target, int extract_flags)
1535 {
1536         const tchar *path = WIMLIB_WIM_ROOT_PATH;
1537         extract_flags |= WIMLIB_EXTRACT_FLAG_IMAGEMODE;
1538         return do_wimlib_extract_paths(wim, image, target, &path, 1, extract_flags);
1539 }
1540
1541 static const tchar * const filename_forbidden_chars =
1542 T(
1543 #ifdef __WIN32__
1544 "<>:\"/\\|?*"
1545 #else
1546 "/"
1547 #endif
1548 );
1549
1550 /* This function checks if it is okay to use a WIM image's name as a directory
1551  * name.  */
1552 static bool
1553 image_name_ok_as_dir(const tchar *image_name)
1554 {
1555         return image_name && *image_name &&
1556                 !tstrpbrk(image_name, filename_forbidden_chars) &&
1557                 tstrcmp(image_name, T(".")) &&
1558                 tstrcmp(image_name, T(".."));
1559 }
1560
1561 /* Extracts all images from the WIM to the directory @target, with the images
1562  * placed in subdirectories named by their image names. */
1563 static int
1564 extract_all_images(WIMStruct *wim, const tchar *target, int extract_flags)
1565 {
1566         size_t image_name_max_len = max(xml_get_max_image_name_len(wim), 20);
1567         size_t output_path_len = tstrlen(target);
1568         tchar buf[output_path_len + 1 + image_name_max_len + 1];
1569         int ret;
1570         int image;
1571         const tchar *image_name;
1572
1573         extract_flags |= WIMLIB_EXTRACT_FLAG_MULTI_IMAGE;
1574
1575         if (extract_flags & WIMLIB_EXTRACT_FLAG_NTFS) {
1576                 ERROR("Cannot extract multiple images in NTFS extraction mode.");
1577                 return WIMLIB_ERR_INVALID_PARAM;
1578         }
1579
1580         ret = mkdir_if_needed(target);
1581         if (ret)
1582                 return ret;
1583         tmemcpy(buf, target, output_path_len);
1584         buf[output_path_len] = OS_PREFERRED_PATH_SEPARATOR;
1585         for (image = 1; image <= wim->hdr.image_count; image++) {
1586                 image_name = wimlib_get_image_name(wim, image);
1587                 if (image_name_ok_as_dir(image_name)) {
1588                         tstrcpy(buf + output_path_len + 1, image_name);
1589                 } else {
1590                         /* Image name is empty or contains forbidden characters.
1591                          * Use image number instead. */
1592                         tsprintf(buf + output_path_len + 1, T("%d"), image);
1593                 }
1594                 ret = extract_single_image(wim, image, buf, extract_flags);
1595                 if (ret)
1596                         return ret;
1597         }
1598         return 0;
1599 }
1600
1601 static int
1602 do_wimlib_extract_image(WIMStruct *wim, int image, const tchar *target,
1603                         int extract_flags)
1604 {
1605         if (extract_flags & (WIMLIB_EXTRACT_FLAG_NO_PRESERVE_DIR_STRUCTURE |
1606                              WIMLIB_EXTRACT_FLAG_TO_STDOUT |
1607                              WIMLIB_EXTRACT_FLAG_GLOB_PATHS))
1608                 return WIMLIB_ERR_INVALID_PARAM;
1609
1610         if (image == WIMLIB_ALL_IMAGES)
1611                 return extract_all_images(wim, target, extract_flags);
1612         else
1613                 return extract_single_image(wim, image, target, extract_flags);
1614 }
1615
1616
1617 /****************************************************************************
1618  *                          Extraction API                                  *
1619  ****************************************************************************/
1620
1621 WIMLIBAPI int
1622 wimlib_extract_paths(WIMStruct *wim, int image, const tchar *target,
1623                      const tchar * const *paths, size_t num_paths,
1624                      int extract_flags)
1625 {
1626         if (extract_flags & ~WIMLIB_EXTRACT_MASK_PUBLIC)
1627                 return WIMLIB_ERR_INVALID_PARAM;
1628
1629         return do_wimlib_extract_paths(wim, image, target, paths, num_paths,
1630                                        extract_flags);
1631 }
1632
1633 WIMLIBAPI int
1634 wimlib_extract_pathlist(WIMStruct *wim, int image, const tchar *target,
1635                         const tchar *path_list_file, int extract_flags)
1636 {
1637         int ret;
1638         tchar **paths;
1639         size_t num_paths;
1640         void *mem;
1641
1642         ret = read_path_list_file(path_list_file, &paths, &num_paths, &mem);
1643         if (ret) {
1644                 ERROR("Failed to read path list file \"%"TS"\"",
1645                       path_list_file);
1646                 return ret;
1647         }
1648
1649         ret = wimlib_extract_paths(wim, image, target,
1650                                    (const tchar * const *)paths, num_paths,
1651                                    extract_flags);
1652         FREE(paths);
1653         FREE(mem);
1654         return ret;
1655 }
1656
1657 WIMLIBAPI int
1658 wimlib_extract_image_from_pipe_with_progress(int pipe_fd,
1659                                              const tchar *image_num_or_name,
1660                                              const tchar *target,
1661                                              int extract_flags,
1662                                              wimlib_progress_func_t progfunc,
1663                                              void *progctx)
1664 {
1665         int ret;
1666         WIMStruct *pwm;
1667         struct filedes *in_fd;
1668         int image;
1669         unsigned i;
1670
1671         if (extract_flags & ~WIMLIB_EXTRACT_MASK_PUBLIC)
1672                 return WIMLIB_ERR_INVALID_PARAM;
1673
1674         /* Read the WIM header from the pipe and get a WIMStruct to represent
1675          * the pipable WIM.  Caveats:  Unlike getting a WIMStruct with
1676          * wimlib_open_wim(), getting a WIMStruct in this way will result in
1677          * an empty lookup table, no XML data read, and no filename set.  */
1678         ret = open_wim_as_WIMStruct(&pipe_fd, WIMLIB_OPEN_FLAG_FROM_PIPE, &pwm,
1679                                     progfunc, progctx);
1680         if (ret)
1681                 return ret;
1682
1683         /* Sanity check to make sure this is a pipable WIM.  */
1684         if (pwm->hdr.magic != PWM_MAGIC) {
1685                 ERROR("The WIM being read from file descriptor %d "
1686                       "is not pipable!", pipe_fd);
1687                 ret = WIMLIB_ERR_NOT_PIPABLE;
1688                 goto out_wimlib_free;
1689         }
1690
1691         /* Sanity check to make sure the first part of a pipable split WIM is
1692          * sent over the pipe first.  */
1693         if (pwm->hdr.part_number != 1) {
1694                 ERROR("The first part of the split WIM must be "
1695                       "sent over the pipe first.");
1696                 ret = WIMLIB_ERR_INVALID_PIPABLE_WIM;
1697                 goto out_wimlib_free;
1698         }
1699
1700         in_fd = &pwm->in_fd;
1701         wimlib_assert(in_fd->offset == WIM_HEADER_DISK_SIZE);
1702
1703         /* As mentioned, the WIMStruct we created from the pipe does not have
1704          * XML data yet.  Fix this by reading the extra copy of the XML data
1705          * that directly follows the header in pipable WIMs.  (Note: see
1706          * write_pipable_wim() for more details about the format of pipable
1707          * WIMs.)  */
1708         {
1709                 struct wim_lookup_table_entry xml_lte;
1710                 struct wim_resource_spec xml_rspec;
1711                 ret = read_pwm_stream_header(pwm, &xml_lte, &xml_rspec, 0, NULL);
1712                 if (ret)
1713                         goto out_wimlib_free;
1714
1715                 if (!(xml_lte.flags & WIM_RESHDR_FLAG_METADATA))
1716                 {
1717                         ERROR("Expected XML data, but found non-metadata "
1718                               "stream.");
1719                         ret = WIMLIB_ERR_INVALID_PIPABLE_WIM;
1720                         goto out_wimlib_free;
1721                 }
1722
1723                 wim_res_spec_to_hdr(&xml_rspec, &pwm->hdr.xml_data_reshdr);
1724
1725                 ret = read_wim_xml_data(pwm);
1726                 if (ret)
1727                         goto out_wimlib_free;
1728
1729                 if (wim_info_get_num_images(pwm->wim_info) != pwm->hdr.image_count) {
1730                         ERROR("Image count in XML data is not the same as in WIM header.");
1731                         ret = WIMLIB_ERR_IMAGE_COUNT;
1732                         goto out_wimlib_free;
1733                 }
1734         }
1735
1736         /* Get image index (this may use the XML data that was just read to
1737          * resolve an image name).  */
1738         if (image_num_or_name) {
1739                 image = wimlib_resolve_image(pwm, image_num_or_name);
1740                 if (image == WIMLIB_NO_IMAGE) {
1741                         ERROR("\"%"TS"\" is not a valid image in the pipable WIM!",
1742                               image_num_or_name);
1743                         ret = WIMLIB_ERR_INVALID_IMAGE;
1744                         goto out_wimlib_free;
1745                 } else if (image == WIMLIB_ALL_IMAGES) {
1746                         ERROR("Applying all images from a pipe is not supported!");
1747                         ret = WIMLIB_ERR_INVALID_IMAGE;
1748                         goto out_wimlib_free;
1749                 }
1750         } else {
1751                 if (pwm->hdr.image_count != 1) {
1752                         ERROR("No image was specified, but the pipable WIM "
1753                               "did not contain exactly 1 image");
1754                         ret = WIMLIB_ERR_INVALID_IMAGE;
1755                         goto out_wimlib_free;
1756                 }
1757                 image = 1;
1758         }
1759
1760         /* Load the needed metadata resource.  */
1761         for (i = 1; i <= pwm->hdr.image_count; i++) {
1762                 struct wim_lookup_table_entry *metadata_lte;
1763                 struct wim_image_metadata *imd;
1764                 struct wim_resource_spec *metadata_rspec;
1765
1766                 metadata_lte = new_lookup_table_entry();
1767                 if (metadata_lte == NULL) {
1768                         ret = WIMLIB_ERR_NOMEM;
1769                         goto out_wimlib_free;
1770                 }
1771                 metadata_rspec = MALLOC(sizeof(struct wim_resource_spec));
1772                 if (metadata_rspec == NULL) {
1773                         ret = WIMLIB_ERR_NOMEM;
1774                         free_lookup_table_entry(metadata_lte);
1775                         goto out_wimlib_free;
1776                 }
1777
1778                 ret = read_pwm_stream_header(pwm, metadata_lte, metadata_rspec, 0, NULL);
1779                 imd = pwm->image_metadata[i - 1];
1780                 imd->metadata_lte = metadata_lte;
1781                 if (ret) {
1782                         FREE(metadata_rspec);
1783                         goto out_wimlib_free;
1784                 }
1785
1786                 if (!(metadata_lte->flags & WIM_RESHDR_FLAG_METADATA)) {
1787                         ERROR("Expected metadata resource, but found "
1788                               "non-metadata stream.");
1789                         ret = WIMLIB_ERR_INVALID_PIPABLE_WIM;
1790                         goto out_wimlib_free;
1791                 }
1792
1793                 if (i == image) {
1794                         /* Metadata resource is for the image being extracted.
1795                          * Parse it and save the metadata in memory.  */
1796                         ret = read_metadata_resource(pwm, imd);
1797                         if (ret)
1798                                 goto out_wimlib_free;
1799                         imd->modified = 1;
1800                 } else {
1801                         /* Metadata resource is not for the image being
1802                          * extracted.  Skip over it.  */
1803                         ret = skip_wim_stream(metadata_lte);
1804                         if (ret)
1805                                 goto out_wimlib_free;
1806                 }
1807         }
1808         /* Extract the image.  */
1809         extract_flags |= WIMLIB_EXTRACT_FLAG_FROM_PIPE;
1810         ret = do_wimlib_extract_image(pwm, image, target, extract_flags);
1811         /* Clean up and return.  */
1812 out_wimlib_free:
1813         wimlib_free(pwm);
1814         return ret;
1815 }
1816
1817
1818 WIMLIBAPI int
1819 wimlib_extract_image_from_pipe(int pipe_fd, const tchar *image_num_or_name,
1820                                const tchar *target, int extract_flags)
1821 {
1822         return wimlib_extract_image_from_pipe_with_progress(pipe_fd,
1823                                                             image_num_or_name,
1824                                                             target,
1825                                                             extract_flags,
1826                                                             NULL,
1827                                                             NULL);
1828 }
1829
1830 WIMLIBAPI int
1831 wimlib_extract_image(WIMStruct *wim, int image, const tchar *target,
1832                      int extract_flags)
1833 {
1834         if (extract_flags & ~WIMLIB_EXTRACT_MASK_PUBLIC)
1835                 return WIMLIB_ERR_INVALID_PARAM;
1836         return do_wimlib_extract_image(wim, image, target, extract_flags);
1837 }