/** Compressed resources in the WIM use XPRESS compression. */
WIMLIB_COMPRESSION_TYPE_XPRESS = 2,
- /** TODO */
+ /** Compressed resources in the WIM use LZMS compression. Currently,
+ * wimlib has a decompressor for this format but not a compressor. LZMS
+ * compression is only compatible with wimlib v1.6.0 and later and with
+ * WIMGAPI Windows 8 and later (and some restrictions apply on the
+ * latter). */
WIMLIB_COMPRESSION_TYPE_LZMS = 3,
};
uint32_t reserved[9];
};
-/** Information about a unique resource in the WIM file.
- */
+/** Information about a unique stream in the WIM file. (A stream is the same
+ * thing as a "resource", except in the case of packed resources.) */
struct wimlib_resource_entry {
- /** Uncompressed size of the resource in bytes. */
+ /** Uncompressed size of the stream in bytes. */
uint64_t uncompressed_size;
- /** Compressed size of the resource in bytes. This will be the same as
- * @p uncompressed_size if the resource is uncompressed. */
+ /** Compressed size of the stream in bytes. This will be the same as @p
+ * uncompressed_size if the stream is uncompressed. Or, if @p
+ * is_packed_streams is 1, this will be 0. */
uint64_t compressed_size;
- /** Offset, in bytes, of this resource from the start of the WIM file.
+ /** Offset, in bytes, of this stream from the start of the WIM file. Or
+ * if @p packed is 1, then this is actually the offset at which this
+ * stream begins in the uncompressed contents of the packed resource.
*/
uint64_t offset;
- /** SHA1 message digest of the resource's uncompressed contents. */
+ /** SHA1 message digest of the stream's uncompressed contents. */
uint8_t sha1_hash[20];
- /** Which part number of the split WIM this resource is in. This should
+ /** Which part number of the split WIM this stream is in. This should
* be the same as the part number provided by wimlib_get_wim_info(). */
uint32_t part_number;
- /** Number of times this resource is referenced over all WIM images. */
+ /** Number of times this stream is referenced over all WIM images. */
uint32_t reference_count;
- /** 1 if this resource is compressed. */
+ /** 1 if this stream is compressed. */
uint32_t is_compressed : 1;
- /** 1 if this resource is a metadata resource rather than a file
- * resource. */
+ /** 1 if this stream is a metadata resource rather than a file resource.
+ * */
uint32_t is_metadata : 1;
uint32_t is_free : 1;
uint32_t is_spanned : 1;
- /** 1 if this resource was not found in the lookup table of the
+ /** 1 if this stream was not found in the lookup table of the
* ::WIMStruct. This normally implies a missing call to
* wimlib_reference_resource_files() or wimlib_reference_resources().
- */
+ * */
uint32_t is_missing : 1;
- uint32_t is_packed_streams : 1;
+ /** 1 if this stream is located in a packed resource which may contain
+ * other streams (all compressed together) as well. */
+ uint32_t packed : 1;
uint32_t reserved_flags : 26;
+ /** If @p packed is 1, then this will specify the offset of the packed
+ * resource in the WIM. */
uint64_t raw_resource_offset_in_wim;
- uint64_t raw_resource_uncompressed_size;
+
+ /** If @p is_packed_streams is 1, then this will specify the compressed
+ * size of the packed resource in the WIM. */
uint64_t raw_resource_compressed_size;
- uint64_t reserved[1];
+ uint64_t reserved[2];
};
/** A stream of a file in the WIM. */
* chunk of the WIM does not match the corresponding message digest given
* in the integrity table.
* @retval ::WIMLIB_ERR_INVALID_CHUNK_SIZE
- * Resources in @p wim_file are compressed, but the chunk size is not 32768.
+ * Resources in @p wim_file are compressed, but the chunk size was invalid
+ * for the WIM's compression format.
* @retval ::WIMLIB_ERR_INVALID_COMPRESSION_TYPE
* The header of @p wim_file says that resources in the WIM are compressed,
* but the header flag indicating LZX or XPRESS compression is not set.
*
* <b>WARNING: Changing the compression chunk size to any value other than the
* default of 32768 bytes eliminates compatibility with Microsoft's software,
- * except when increasing the XPRESS chunk size before Windows 8.</b>
+ * except when increasing the XPRESS chunk size before Windows 8. Chunk sizes
+ * other than 32768 are also incompatible with wimlib v1.5.3 and earlier.</b>
*
* @param wim
* ::WIMStruct for a WIM.
* library clients looking to make use of wimlib's compression code for another
* purpose.
*
- * As of wimlib v1.5.4, this function can be used with @p chunk_size greater
+ * As of wimlib v1.6.0, this function can be used with @p chunk_size greater
* than 32768 bytes and is only limited by available memory. However, the
* XPRESS format itself still caps match offsets to 65535, so if a larger chunk
* size is chosen, then the matching will effectively occur in a sliding window
wimlib_warning_with_errno(const tchar *format, ...)
_format_attribute(printf, 1, 2) _cold_attribute;
# define ERROR(format, ...) wimlib_error(T(format), ## __VA_ARGS__)
-# define ERROR_WITH_ERRNO(format, ...) wimlib_error_with_errno(T(format), ## __VA_ARGS__)
+# define ERROR_WITH_ERRNO(format, ...) wimlib_error_with_errno(T(format), ## __VA_ARGS__)
# define WARNING(format, ...) wimlib_warning(T(format), ## __VA_ARGS__)
# define WARNING_WITH_ERRNO(format, ...) wimlib_warning_with_errno(T(format), ## __VA_ARGS__)
extern bool wimlib_print_errors;
wimlib_debug(const tchar *file, int line, const char *func,
const tchar *format, ...);
# define DEBUG(format, ...) \
- wimlib_debug(T(__FILE__), __LINE__, __func__, T(format), ## __VA_ARGS__)
+ wimlib_debug(T(__FILE__), __LINE__, __func__, T(format), ## __VA_ARGS__)
#else
# define DEBUG(format, ...) dummy_tprintf(T(format), ## __VA_ARGS__)
/* Version number used for WIMs that allow multiple streams packed into one
* resource (WIM_RESHDR_FLAG_PACKED_STREAMS). New as of Windows 8 WIMGAPI; used
- * for the Windows 8 web downloader, but yet properly documented by Microsoft.
- */
+ * for the Windows 8 web downloader, but yet not yet properly documented by
+ * Microsoft. */
#define WIM_VERSION_PACKED_STREAMS 0xe00
/* Note: there is another WIM version from Vista pre-releases, but it is not
* supported by wimlib. */
-/* WIM magic characters, translated to a single 64-bit little endian number. */
-#define WIM_MAGIC \
- cpu_to_le64(((u64)'M' << 0) | \
- ((u64)'S' << 8) | \
- ((u64)'W' << 16) | \
- ((u64)'I' << 24) | \
- ((u64)'M' << 32) | \
- ((u64)'\0' << 40) | \
- ((u64)'\0' << 48) | \
- ((u64)'\0' << 54))
-
-/* wimlib pipable WIM magic characters, translated to a single 64-bit little
- * endian number. */
-#define PWM_MAGIC \
- cpu_to_le64(((u64)'W' << 0) | \
- ((u64)'L' << 8) | \
- ((u64)'P' << 16) | \
- ((u64)'W' << 24) | \
- ((u64)'M' << 32) | \
- ((u64)'\0' << 40) | \
- ((u64)'\0' << 48) | \
- ((u64)'\0' << 54))
+/* WIM magic characters, translated to a single 64-bit number. */
+#define WIM_MAGIC \
+ (((u64)'M' << 0) | \
+ ((u64)'S' << 8) | \
+ ((u64)'W' << 16) | \
+ ((u64)'I' << 24) | \
+ ((u64)'M' << 32) | \
+ ((u64)'\0' << 40) | \
+ ((u64)'\0' << 48) | \
+ ((u64)'\0' << 54))
+
+/* wimlib pipable WIM magic characters, translated to a single 64-bit number.
+ * */
+#define PWM_MAGIC \
+ (((u64)'W' << 0) | \
+ ((u64)'L' << 8) | \
+ ((u64)'P' << 16) | \
+ ((u64)'W' << 24) | \
+ ((u64)'M' << 32) | \
+ ((u64)'\0' << 40) | \
+ ((u64)'\0' << 48) | \
+ ((u64)'\0' << 54))
/* On-disk format of the WIM header. */
struct wim_header_disk {
- /* +0x00: Magic characters "MSWIM\0\0\0" */
+ /* +0x00: Magic characters "MSWIM\0\0\0". */
le64 magic;
/* +0x08: Size of the WIM header, in bytes; WIM_HEADER_DISK_SIZE
- * expected (currently the only supported value). */
+ * expected (currently the only supported value). */
u32 hdr_size;
- /* +0x0c: Version of the WIM file
- * TODO */
+ /* +0x0c: Version of the WIM file. Recognized values are the
+ * WIM_VERSION_* constants from above. */
u32 wim_version;
- /* +0x10: Flags for the WIM file (WIM_HDR_FLAG_*) */
+ /* +0x10: Flags for the WIM file (WIM_HDR_FLAG_*). */
u32 wim_flags;
- /* +0x14: Chunk size for compressed resources in the WIM, or 0 if the
- * WIM is uncompressed. */
+ /* +0x14: Uncompressed chunk size for compressed resources in the WIM
+ * other than packed resources, or 0 if the WIM is uncompressed. */
u32 chunk_size;
/* +0x18: Globally unique identifier for the WIM file. Basically a
* is not split. */
u16 total_parts;
- /* +0x2c: Number of images in the WIM. */
+ /* +0x2c: Number of images in the WIM. */
u32 image_count;
- /* +0x30: Location and size of the WIM's lookup table. */
+ /* +0x30: Location and size of the WIM's lookup table. */
struct wim_reshdr_disk lookup_table_reshdr;
- /* +0x48: Location and size of the WIM's XML data. */
+ /* +0x48: Location and size of the WIM's XML data. */
struct wim_reshdr_disk xml_data_reshdr;
/* +0x60: Location and size of metadata resource for the bootable image
- * of the WIM, or all zeroes if no image is bootable. */
+ * of the WIM, or all zeroes if no image is bootable. */
struct wim_reshdr_disk boot_metadata_reshdr;
/* +0x78: 1-based index of the bootable image of the WIM, or 0 if no
- * image is bootable. */
+ * image is bootable. */
u32 boot_idx;
/* +0x7c: Location and size of the WIM's integrity table, or all zeroes
* if the WIM has no integrity table.
*
- * Note the integrity_table_reshdr here is 4-byte aligned even though
- * it would ordinarily be 8-byte aligned--- hence, the _packed_attribute
- * on the `struct wim_header_disk' is essential. */
+ * Note the integrity_table_reshdr here is 4-byte aligned even though it
+ * would ordinarily be 8-byte aligned--- hence, the _packed_attribute on
+ * this structure is essential. */
struct wim_reshdr_disk integrity_table_reshdr;
- /* +0x94: Unused bytes. */
+ /* +0x94: Unused bytes. */
u8 unused[60];
/* +0xd0 (208) */
} _packed_attribute;
-/* Header at the very beginning of the WIM file. This is the in-memory
- * representation and does not include all fields; see `struct wim_header_disk'
- * for the on-disk structure. */
+/* In-memory representation of a WIM header. See `struct wim_header_disk' for
+ * field descriptions. */
struct wim_header {
-
- /* Magic characters: either WIM_MAGIC or PWM_MAGIC. */
- le64 magic;
-
- /* Version of the WIM file */
+ u64 magic;
u32 wim_version;
-
- /* Bitwise OR of one or more of the WIM_HDR_FLAG_* defined below. */
u32 flags;
-
- /* Compressed resource chunk size */
u32 chunk_size;
-
- /* A unique identifier for the WIM file. */
u8 guid[WIM_GID_LEN];
-
- /* Part number of the WIM file in a spanned set. */
u16 part_number;
-
- /* Total number of parts in a spanned set. */
u16 total_parts;
-
- /* Number of images in the WIM file. */
u32 image_count;
-
- /* Location, size, and flags of the lookup table of the WIM. */
struct wim_reshdr lookup_table_reshdr;
-
- /* Location, size, and flags for the XML data of the WIM. */
struct wim_reshdr xml_data_reshdr;
-
- /* Location, size, and flags for the boot metadata. This means the
- * metadata resource for the image specified by boot_idx below. Should
- * be zeroed out if boot_idx is 0. */
struct wim_reshdr boot_metadata_reshdr;
-
- /* The index of the bootable image in the WIM file. If 0, there are no
- * bootable images available. */
u32 boot_idx;
-
- /* The location of the optional integrity table used to verify the
- * integrity WIM. Zeroed out if there is no integrity table.*/
struct wim_reshdr integrity_table_reshdr;
};
*/
#define list_for_each_entry(pos, head, member) \
for (pos = list_entry((head)->next, typeof(*pos), member); \
- &pos->member != (head); \
+ &pos->member != (head); \
pos = list_entry(pos->member.next, typeof(*pos), member))
/**
#define list_for_each_entry_safe(pos, n, head, member) \
for (pos = list_entry((head)->next, typeof(*pos), member), \
n = list_entry(pos->member.next, typeof(*pos), member); \
- &pos->member != (head); \
+ &pos->member != (head); \
pos = n, n = list_entry(n->member.next, typeof(*n), member))
/*
* @head: the head for your list.
* @member: the name of the hlist_node within the struct.
*/
-#define hlist_for_each_entry_safe(tpos, pos, n, head, member) \
+#define hlist_for_each_entry_safe(tpos, pos, n, head, member) \
for (pos = (head)->first; \
- pos && ({ n = pos->next; 1; }) && \
+ pos && ({ n = pos->next; 1; }) && \
({ tpos = hlist_entry(pos, typeof(*tpos), member); 1;}); \
pos = n)
* `struct wim_resource_spec' pointed to by @rspec. @offset_in_res
* identifies the offset at which this particular stream begins in the
* uncompressed data of the resource; this is normally 0, but in general
- * a WIM resource may contain multiple streams. */
+ * a WIM resource may be "packed" and potentially contain multiple
+ * streams. */
RESOURCE_IN_WIM,
/* The stream is located in the external file named by @file_on_disk.
- * On Windows, @file_on_disk may actually specify a named data stream.
- */
+ * On Windows, @file_on_disk may actually specify a named data stream
+ * (file path, then colon, then name of the stream). */
RESOURCE_IN_FILE_ON_DISK,
/* The stream is directly attached in the in-memory buffer pointed to by
};
-/*
- * An entry in the lookup table in the WIM file.
- *
- * It is used to find data streams for files in the WIM.
+/* Specification for a stream, which may be the contents of a file (unnamed data
+ * stream), a named data stream, reparse point data, or a WIM metadata resource.
*
- * Metadata resources and reparse point data buffers will also have lookup table
- * entries associated with the data.
- *
- * The lookup_table_entry for a given dentry or alternate stream entry in the
- * WIM is found using the SHA1 message digest field.
- */
+ * One instance of this structure is created for each entry in the WIM's lookup
+ * table, hence the name of the struct. Each of these entries contains the SHA1
+ * message digest of a stream and the location of the stream data in the WIM
+ * file (size, location, flags). The in-memory lookup table is a map from SHA1
+ * message digests to stream locations. */
struct wim_lookup_table_entry {
- /* List of lookup table entries in this hash bucket */
+ /* List node for a hash bucket of the lookup table. */
struct hlist_node hash_list;
- /* Uncompressed size of the stream. */
+ /* Uncompressed size of this stream. */
u64 size;
/* Stream flags (WIM_RESHDR_FLAG_*). */
u32 flags : 8;
- /* One of the `enum resource_location' values documented above. */
- u32 resource_location : 5;
+ /* One of the `enum resource_location' values documented above. */
+ u32 resource_location : 4;
/* 1 if this stream has not had a SHA1 message digest calculated for it
- * yet */
+ * yet. */
u32 unhashed : 1;
- /* Temoorary files used for writing; set as documented for
+ /* Temoorary fields used when writing streams; set as documented for
* prepare_stream_list_for_write(). */
u32 unique_size : 1;
u32 will_be_in_output_wim : 1;
/* Set to 1 when a metadata entry has its checksum changed; in such
- * cases the hash is no longer valid to verify the data if the metadata
- * resource is read again. */
+ * cases the hash cannot be used to verify the data if the metadata
+ * resource is read again. (This could be avoided if we used separate
+ * fields for input/output checksum, but most stream entries wouldn't
+ * need this.) */
u32 dont_check_metadata_hash : 1;
union {
};
/* Number of times this lookup table entry is referenced by dentries in
- * the WIM. */
+ * the WIM. When a WIM's lookup table is read, this field is
+ * initialized from a corresponding entry; while it should be correct,
+ * in general it may not be. wim_recalculate_refcnts() recalculates the
+ * reference counts for all streams and is run before doing any
+ * deletions. */
u32 refcnt;
- /* Actual reference count to this stream (only used while verifying an
- * image). */
- u32 real_refcnt;
-
- /* When a WIM file is written, out_refcnt starts at 0 and is incremented
- * whenever the stream pointed to by this lookup table entry needs to be
- * written. The stream only need to be written when out_refcnt is
- * nonzero, since otherwise it is not referenced by any dentries. */
+ /* When a WIM file is written, this is set to the number of references
+ * (by dentries) to this stream in the output WIM file.
+ *
+ * During extraction, this is set to the number of times the stream must
+ * be extracted.
+ *
+ * During image export, this is set to the number of references of this
+ * stream that originated from the source WIM. */
u32 out_refcnt;
#ifdef WITH_FUSE
- /* Number of times this stream has been opened (used only during
- * mounting) */
+ /* Number of times this stream has been opened; used only during
+ * mounting. */
u16 num_opened_fds;
#endif
- /* Pointers to somewhere where the stream is actually located. See the
- * comments for the @resource_location field above. */
+ /* Specification of where this stream is actually located. Which member
+ * is valid is determined by the @resource_location field. */
union {
struct {
struct wim_resource_spec *rspec;
};
/* Links together streams that share the same underlying WIM resource.
- * The head is wim_resource_spec.stream_list. */
+ * The head is the `stream_list' member of `struct wim_resource_spec'.
+ */
struct list_head rspec_node;
- /* This field is used for the special hardlink or symlink image
- * extraction mode. In these mode, all identical files are linked
- * together, and @extracted_file will be set to the filename of the
- * first extracted file containing this stream. */
+ /* This field is used during the hardlink and symlink image extraction
+ * modes. In these modes, all identical files are linked together, and
+ * @extracted_file will be set to the filename of the first extracted
+ * file containing this stream. */
tchar *extracted_file;
/* Temporary fields */
union {
- /* Used temporarily during WIM file writing */
+ /* Fields used temporarily during WIM file writing. */
struct {
union {
+ /* List node used for stream size table. */
struct hlist_node hash_list_2;
+
+ /* Metadata for the underlying packed resource
+ * in the WIM being written (only valid if
+ * WIM_RESHDR_FLAG_PACKED_STREAMS set in
+ * out_reshdr.flags). */
struct {
u64 out_res_offset_in_wim;
u64 out_res_size_in_wim;
- u64 out_res_uncompressed_size;
};
};
+
/* Links streams being written to the WIM. */
struct list_head write_streams_list;
+ /* Metadata for this stream in the WIM being written.
+ */
struct wim_reshdr out_reshdr;
};
/* Used temporarily during extraction */
union {
- /* out_refcnt tracks number of slots filled */
- struct wim_dentry *inline_lte_dentries[8];
+ /* Dentries to extract that reference this stream.
+ * out_refcnt tracks the number of slots filled. */
+ struct wim_dentry *inline_lte_dentries[7];
struct {
struct wim_dentry **lte_dentries;
- unsigned long alloc_lte_dentries;
+ size_t alloc_lte_dentries;
};
};
+
+ /* Actual reference count to this stream (only used while
+ * verifying an image). */
+ u32 real_refcnt;
};
- /* Temporary list fields */
+ /* Temporary list fields. */
union {
/* Links streams for writing lookup table. */
struct list_head lookup_table_list;
extern void
lookup_table_insert(struct wim_lookup_table *table, struct wim_lookup_table_entry *lte);
-/* Unlinks a lookup table entry from the table; does not free it. */
+/* Unlinks a lookup table entry from the table; does not free it. */
static inline void
lookup_table_unlink(struct wim_lookup_table *table, struct wim_lookup_table_entry *lte)
{
lte_unbind_wim_resource_spec(struct wim_lookup_table_entry *lte)
{
list_del(<e->rspec_node);
- lte->rspec = NULL;
lte->resource_location = RESOURCE_NONEXISTENT;
}
#define LZMS_NUM_LITERAL_SYMS 256
#define LZMS_NUM_LEN_SYMS 54
#define LZMS_NUM_DELTA_POWER_SYMS 8
-#define LZMS_MAX_NUM_OFFSET_SYMS 799
+#define LZMS_MAX_NUM_OFFSET_SYMS 799
#define LZMS_MAX_NUM_SYMS 799
#define LZMS_MAX_CODEWORD_LEN 15
#define LZX_ALIGNEDCODE_NUM_SYMBOLS 8
#define LZX_ALIGNEDCODE_TABLEBITS 7
-#define LZX_ALIGNEDCODE_ELEMENT_SIZE 3
+#define LZX_ALIGNEDCODE_ELEMENT_SIZE 3
/* Maximum allowed length of Huffman codewords. */
#define LZX_MAX_MAIN_CODEWORD_LEN 16
u64 offset_in_wim;
/* The size of this resource in the WIM file. For compressed resources
- * this is the compressed size. */
+ * this is the compressed size, including overhead such as the chunk
+ * table. */
u64 size_in_wim;
/* The number of bytes of uncompressed data this resource decompresses
/* The list of streams this resource contains. */
struct list_head stream_list;
- /* Flags for this resource (WIM_RESHDR_FLAG_*) */
+ /* Flags for this resource (WIM_RESHDR_FLAG_*). */
u32 flags : 8;
- /* This flag will be set if the WIM is pipable. In such cases, the
- * resource will be in a slightly different format if it is compressed.
- * This is a wimlib extension. */
+ /* [wimlib extension] This flag will be set if the WIM is pipable. In
+ * such cases, the resource will be in a slightly different format if it
+ * is compressed. */
u32 is_pipable : 1;
/* Temporary flag. */
le64 uncompressed_size;
} _packed_attribute;
-/* In-memory version of a WIM resource header. */
+/* In-memory version of a WIM resource header (`struct wim_reshdr_disk'). */
struct wim_reshdr {
u64 size_in_wim : 56;
u64 flags : 8;
u64 uncompressed_size;
};
-/* Flags for the `flags' field of WIM resource headers. */
+/* Flags for the `flags' field of WIM resource headers (`struct wim_reshdr').
+ */
/* Unknown meaning; may be intended to indicate spaces in the WIM that are free
* to overwrite. Currently ignored by wimlib. */
le64 res_usize;
/* Number of bytes each compressed chunk decompresses into, except
- * possibly the last which decompresses into the remainder. */
+ * possibly the last which decompresses into the remainder. This
+ * overrides the chunk size specified by the WIM header. */
le32 chunk_size;
/* Compression format used for compressed chunks:
* 0 = None
* 1 = LZX
* 2 = XPRESS
- * 3 = LZMS */
+ * 3 = LZMS
+ *
+ * This overrides the compression type specified by the WIM header. */
le32 compression_format;
/* This header is directly followed by a table of compressed sizes of
- * the chunks. */
+ * the chunks (4 bytes per entry). */
} _packed_attribute;
/* wimlib internal flags used when writing resources. */
extern int
skip_wim_stream(struct wim_lookup_table_entry *lte);
-extern int
-read_stream_prefix(const struct wim_lookup_table_entry *lte,
- u64 size, consume_data_callback_t cb,
- void *cb_ctx);
-
+/*
+ * Type of callback function for beginning to read a stream.
+ *
+ * @lte:
+ * Stream that is about to be read.
+ *
+ * @is_partial_res:
+ * Set to true if the stream is just one of several being read from a
+ * single pack and therefore would be extra expensive to read
+ * independently.
+ *
+ * @ctx:
+ * User-provided context.
+ *
+ * Must return 0 on success, a positive error code on failure, or the special
+ * value BEGIN_STREAM_STATUS_SKIP_STREAM to indicate that the stream should not
+ * be read, and read_stream_list() should continue on to the next stream
+ * (without calling @consume_chunk or @end_stream).
+ */
typedef int (*read_stream_list_begin_stream_t)(struct wim_lookup_table_entry *lte,
bool is_partial_res,
void *ctx);
+
+#define BEGIN_STREAM_STATUS_SKIP_STREAM -1
+
+/*
+ * Type of callback function for finishing reading a stream.
+ *
+ * @lte:
+ * Stream that has been fully read, or stream that started being read but
+ * could not be fully read due to a read error.
+ *
+ * @status:
+ * 0 if reading the stream was successful; otherwise a nonzero error code
+ * that specifies the return status.
+ *
+ * @ctx:
+ * User-provided context.
+ */
typedef int (*read_stream_list_end_stream_t)(struct wim_lookup_table_entry *lte,
int status,
void *ctx);
-#define VERIFY_STREAM_HASHES 0x1
-#define COMPUTE_MISSING_STREAM_HASHES 0x2
-#define STREAM_LIST_ALREADY_SORTED 0x4
-#define BEGIN_STREAM_STATUS_SKIP_STREAM -1
/* Callback functions and contexts for read_stream_list(). */
struct read_stream_list_callbacks {
/* Called when a chunk of data has been read. */
consume_data_callback_t consume_chunk;
- /* Called when a stream has been fully read. */
+ /* Called when a stream has been fully read. A successful call to
+ * @begin_stream will always be matched by a call to @end_stream. */
read_stream_list_end_stream_t end_stream;
/* Parameter passed to @begin_stream. */
void *end_stream_ctx;
};
+/* Flags for read_stream_list() */
+#define VERIFY_STREAM_HASHES 0x1
+#define COMPUTE_MISSING_STREAM_HASHES 0x2
+#define STREAM_LIST_ALREADY_SORTED 0x4
+
extern int
read_stream_list(struct list_head *stream_list,
size_t list_head_offset,
read_win32_file_prefix(const struct wim_lookup_table_entry *lte,
u64 size,
consume_data_callback_t cb,
- u32 in_chunk_size,
- void *ctx_or_buf,
- int _ignored_flags);
+ void *cb_ctx);
extern int
read_win32_encrypted_file_prefix(const struct wim_lookup_table_entry *lte,
u64 size,
consume_data_callback_t cb,
- u32 in_chunk_size,
- void *ctx_or_buf,
- int _ignored_flags);
+ void *cb_ctx);
extern int
extern unsigned
win32_get_number_of_processors(void);
+extern u64
+win32_get_avail_memory(void);
+
extern tchar *
realpath(const tchar *path, tchar *resolved_path);
//#define ENABLE_XPRESS_DEBUG
#ifdef ENABLE_XPRESS_DEBUG
# define XPRESS_DEBUG DEBUG
-# define XPRESS_ASSERT wimlib_assert
+# define XPRESS_ASSERT wimlib_assert
#else
# define XPRESS_DEBUG(format, ...)
# define XPRESS_ASSERT(...)
{
tprintf(T("Uncompressed size = %"PRIu64" bytes\n"),
resource->uncompressed_size);
- if (resource->is_packed_streams) {
- tprintf(T("Raw uncompressed size = %"PRIu64" bytes\n"),
- resource->raw_resource_uncompressed_size);
-
+ if (resource->packed) {
tprintf(T("Raw compressed size = %"PRIu64" bytes\n"),
resource->raw_resource_compressed_size);
resource->offset);
}
-
tprintf(T("Part Number = %u\n"), resource->part_number);
tprintf(T("Reference Count = %u\n"), resource->reference_count);
tprintf(T("WIM_RESHDR_FLAG_FREE "));
if (resource->is_spanned)
tprintf(T("WIM_RESHDR_FLAG_SPANNED "));
- if (resource->is_packed_streams)
+ if (resource->packed)
tprintf(T("WIM_RESHDR_FLAG_PACKED_STREAMS "));
tputchar(T('\n'));
tputchar(T('\n'));
* replacement function in win32.c. */
if (fnmatch(pat, string, FNM_PATHNAME | FNM_NOESCAPE
#ifdef FNM_CASEFOLD
- | FNM_CASEFOLD
+ | FNM_CASEFOLD
#endif
) == 0)
{
* @num_syms: The number of symbols in the alphabet.
*
* @max_codeword_len: The maximum allowed length of a codeword in the code.
- * Note that if the code being created runs up against
- * this restriction, the code ultimately created will be
- * suboptimal, although there are some advantages for
- * limiting the length of the codewords.
+ * Note that if the code being created runs up against
+ * this restriction, the code ultimately created will be
+ * suboptimal, although there are some advantages for
+ * limiting the length of the codewords.
*
* @freq_tab: An array of length @num_syms that contains the frequencies
- * of each symbol in the uncompressed data.
+ * of each symbol in the uncompressed data.
*
* @lens: An array of length @num_syms into which the lengths of the
- * codewords for each symbol will be written.
+ * codewords for each symbol will be written.
*
* @codewords: An array of @num_syms short integers into which the
- * codewords for each symbol will be written. The first
- * lens[i] bits of codewords[i] will contain the codeword
- * for symbol i.
+ * codewords for each symbol will be written. The first
+ * lens[i] bits of codewords[i] will contain the codeword
+ * for symbol i.
*/
void
make_canonical_huffman_code(unsigned num_syms,
* though).
*
* @decode_table: The array in which to create the fast huffman decoding
- * table. It must have a length of at least
- * (2**table_bits) + 2 * num_syms to guarantee
- * that there is enough space. Also must be 16-byte
- * aligned (at least when USE_SSE2_FILL gets defined).
+ * table. It must have a length of at least
+ * (2**table_bits) + 2 * num_syms to guarantee
+ * that there is enough space. Also must be 16-byte
+ * aligned (at least when USE_SSE2_FILL gets defined).
*
- * @num_syms: Number of symbols in the alphabet, including symbols
+ * @num_syms: Number of symbols in the alphabet, including symbols
* that do not appear in this particular input chunk.
*
* @table_bits: Any symbols with a code length of table_bits or less can
- * be decoded in one lookup of the table. 2**table_bits
- * must be greater than or equal to @num_syms if there are
- * any Huffman codes longer than @table_bits.
+ * be decoded in one lookup of the table. 2**table_bits
+ * must be greater than or equal to @num_syms if there are
+ * any Huffman codes longer than @table_bits.
*
* @lens: An array of length @num_syms, indexable by symbol, that
- * gives the length of the Huffman codeword for that
- * symbol. Because the Huffman tree is in canonical form,
- * it can be reconstructed by only knowing the length of
- * the codeword for each symbol. It is assumed, but not
- * checked, that every length is less than
- * @max_codeword_len.
+ * gives the length of the Huffman codeword for that
+ * symbol. Because the Huffman tree is in canonical form,
+ * it can be reconstructed by only knowing the length of
+ * the codeword for each symbol. It is assumed, but not
+ * checked, that every length is less than
+ * @max_codeword_len.
*
* @max_codeword_len: The longest codeword length allowed in the compression
- * format.
+ * format.
*
* Returns 0 on success; returns -1 if the length values do not correspond to a
* valid Huffman tree.
return 0;
}
- if (buf.stream_hdr.magic != PWM_STREAM_MAGIC) {
+ if (le64_to_cpu(buf.stream_hdr.magic) != PWM_STREAM_MAGIC) {
ERROR("Data read on pipe is invalid (expected stream header).");
return WIMLIB_ERR_INVALID_PIPABLE_WIM;
}
&& (needed_lte = lookup_resource(lookup_table, found_lte->hash))
&& (needed_lte->out_refcnt))
{
- char *tmpfile_name = NULL;
+ tchar *tmpfile_name = NULL;
struct wim_lookup_table_entry *lte_override;
struct wim_lookup_table_entry tmpfile_lte;
for (bytes_remaining = count;
bytes_remaining != 0;
bytes_remaining -= bytes_read, buf += bytes_read,
- offset += bytes_read)
+ offset += bytes_read)
{
bytes_read = pread(fd->fd, buf, bytes_remaining, offset);
if (unlikely(bytes_read <= 0)) {
for (bytes_remaining = count;
bytes_remaining != 0;
bytes_remaining -= bytes_written, buf += bytes_written,
- offset += bytes_written)
+ offset += bytes_written)
{
bytes_written = pwrite(fd->fd, buf, bytes_remaining, offset);
if (unlikely(bytes_written < 0)) {
return 0;
}
+#if 0
/* Wrapper around writev() that checks for errors and keep retrying until all
* requested bytes have been written.
*
fd->offset += total_bytes_written;
return 0;
}
+#endif
ssize_t
raw_pread(struct filedes *fd, void *buf, size_t count, off_t offset)
if (ret)
goto read_error;
- if (disk_hdr.magic != WIM_MAGIC) {
- if (disk_hdr.magic == PWM_MAGIC) {
+ hdr->magic = le64_to_cpu(disk_hdr.magic);
+
+ if (hdr->magic != WIM_MAGIC) {
+ if (hdr->magic == PWM_MAGIC) {
/* Pipable WIM: Use header at end instead, unless
* actually reading from a pipe. */
if (!in_fd->is_pipe) {
return WIMLIB_ERR_NOT_A_WIM_FILE;
}
}
- hdr->magic = disk_hdr.magic;
if (le32_to_cpu(disk_hdr.hdr_size) != sizeof(struct wim_header_disk)) {
ERROR("\"%"TS"\": Header size is invalid (%u bytes)",
((hdr->magic == PWM_MAGIC) ? "pipable " : ""),
offset);
- disk_hdr.magic = hdr->magic;
+ disk_hdr.magic = cpu_to_le64(hdr->magic);
disk_hdr.hdr_size = cpu_to_le32(sizeof(struct wim_header_disk));
disk_hdr.wim_version = cpu_to_le32(hdr->wim_version);
disk_hdr.wim_flags = cpu_to_le32(hdr->flags);
const char *name;
};
struct hdr_flag hdr_flags[] = {
- {WIM_HDR_FLAG_RESERVED, "RESERVED"},
+ {WIM_HDR_FLAG_RESERVED, "RESERVED"},
{WIM_HDR_FLAG_COMPRESSION, "COMPRESSION"},
{WIM_HDR_FLAG_READONLY, "READONLY"},
{WIM_HDR_FLAG_SPANNED, "SPANNED"},
* correspond to a spanned set.
*
* @wim:
- * Part 1 of the set.
+ * Part 1 of the set.
*
* @additional_swms:
- * All parts of the set other than part 1.
+ * All parts of the set other than part 1.
*
* @num_additional_swms:
- * Number of WIMStructs in @additional_swms. Or, the total number of parts
- * in the set minus 1.
+ * Number of WIMStructs in @additional_swms. Or, the total number of parts
+ * in the set minus 1.
*
* @return:
- * 0 on success; WIMLIB_ERR_SPLIT_INVALID if the set is not valid.
+ * 0 on success; WIMLIB_ERR_SPLIT_INVALID if the set is not valid.
*/
static int
verify_swm_set(WIMStruct *wim, WIMStruct **additional_swms,
if (v)
return v;
- return cmp_u64(lte1->rspec->offset_in_wim + lte1->offset_in_res,
- lte2->rspec->offset_in_wim + lte2->offset_in_res);
+ if (lte1->rspec->offset_in_wim != lte2->rspec->offset_in_wim)
+ return cmp_u64(lte1->rspec->offset_in_wim,
+ lte2->rspec->offset_in_wim);
+
+ return cmp_u64(lte1->offset_in_res, lte2->offset_in_res);
case RESOURCE_IN_FILE_ON_DISK:
#ifdef WITH_FUSE
goto invalid;
/* Verify that each stream in the resource has a valid offset and size,
- * and that no streams overlap. */
+ * and that no streams overlap, and that the streams were added in order
+ * of increasing offset. */
cur_offset = 0;
list_for_each_entry(lte, &rspec->stream_list, rspec_node) {
if (lte->offset_in_res + lte->size < lte->size ||
if (reshdr.flags & WIM_RESHDR_FLAG_PACKED_STREAMS) {
cur_rspec->size_in_wim = 0;
cur_rspec->uncompressed_size = 0;
- cur_rspec->flags = WIM_RESHDR_FLAG_PACKED_STREAMS;
+ cur_rspec->offset_in_wim = 0;
}
if (prev_entry) {
wentry->offset = lte->rspec->offset_in_wim;
}
wentry->raw_resource_offset_in_wim = lte->rspec->offset_in_wim;
- wentry->raw_resource_uncompressed_size = lte->rspec->uncompressed_size;
+ /*wentry->raw_resource_uncompressed_size = lte->rspec->uncompressed_size;*/
wentry->raw_resource_compressed_size = lte->rspec->size_in_wim;
}
copy_hash(wentry->sha1_hash, lte->hash);
wentry->is_metadata = (lte->flags & WIM_RESHDR_FLAG_METADATA) != 0;
wentry->is_free = (lte->flags & WIM_RESHDR_FLAG_FREE) != 0;
wentry->is_spanned = (lte->flags & WIM_RESHDR_FLAG_SPANNED) != 0;
- wentry->is_packed_streams = (lte->flags & WIM_RESHDR_FLAG_PACKED_STREAMS) != 0;
+ wentry->packed = (lte->flags & WIM_RESHDR_FLAG_PACKED_STREAMS) != 0;
}
struct iterate_lte_context {
* @window: The window of uncompressed data.
* @bytes_remaining: The number of bytes remaining in the window.
* @strstart: The index of the start of the string in the window that
- * we are trying to find a match for.
+ * we are trying to find a match for.
* @prev_tab: The array of prev pointers for the hash table.
* @cur_match: The index of the head of the hash chain for matches
- * having the hash value of the string beginning
- * at index @strstart.
+ * having the hash value of the string beginning
+ * at index @strstart.
* @prev_len: The length of the match that was found for the string
- * beginning at (@strstart - 1).
+ * beginning at (@strstart - 1).
* @match_start_ret: A location into which the index of the start of the
- * match will be returned.
+ * match will be returned.
* @params: Parameters that affect how long the search will proceed
- * before going with the best that has been found
- * so far.
+ * before going with the best that has been found
+ * so far.
* @min_start_pos: If the chain reaches a match starting before this
* position (including the end-of-chain 0), the search will
* be terminated.
* @record_literal: Consumer for literals.
* @record_ctx: Context passed to @record_match and @record_literal.
* @params: Structure that contains parameters that affect how the
- * analysis proceeds (mainly how good the matches
- * have to be).
+ * analysis proceeds (mainly how good the matches
+ * have to be).
* @prev_tab: Temporary space containing least @window_size elements.
*/
void
* code length values from the input.
*
* @istream: The bit stream for the input. It is positioned on the beginning
- * of the pretree for the code length values.
+ * of the pretree for the code length values.
* @lens: An array that contains the length values from the previous time
- * the code lengths for this Huffman tree were read, or all
- * 0's if this is the first time.
+ * the code lengths for this Huffman tree were read, or all
+ * 0's if this is the first time.
* @num_lens: Number of length values to decode and return.
*
*/
*
* @istream: The input bitstream.
* @block_size_ret: A pointer to an int into which the size of the block,
- * in bytes, will be returned.
+ * in bytes, will be returned.
* @block_type_ret: A pointer to an int into which the type of the block
- * (LZX_BLOCKTYPE_*) will be returned.
+ * (LZX_BLOCKTYPE_*) will be returned.
* @tables: A pointer to a lzx_tables structure in which the
- * main tree, the length tree, and possibly the
- * aligned offset tree will be constructed.
+ * main tree, the length tree, and possibly the
+ * aligned offset tree will be constructed.
* @queue: A pointer to the least-recently-used queue into which
- * R0, R1, and R2 will be written (only for uncompressed
- * blocks, which contain this information in the header)
+ * R0, R1, and R2 will be written (only for uncompressed
+ * blocks, which contain this information in the header)
*/
static int
lzx_read_block_header(struct input_bitstream *istream,
* tree.
*
* @block_type: The type of the block (LZX_BLOCKTYPE_ALIGNED or
- * LZX_BLOCKTYPE_VERBATIM)
+ * LZX_BLOCKTYPE_VERBATIM)
*
* @bytes_remaining: The amount of uncompressed data remaining to be
- * uncompressed in this block. It is an error if the match
- * is longer than this number.
+ * uncompressed in this block. It is an error if the match
+ * is longer than this number.
*
* @window: A pointer to the window into which the uncompressed
- * data is being written.
+ * data is being written.
*
* @window_pos: The current byte offset in the window.
*
*
* Returns the length of the match, or a negative number on error. The possible
* error cases are:
- * - Match would exceed the amount of data remaining to be uncompressed.
- * - Match refers to data before the window.
- * - The input bitstream ended unexpectedly.
+ * - Match would exceed the amount of data remaining to be uncompressed.
+ * - Match refers to data before the window.
+ * - The input bitstream ended unexpectedly.
*/
static int
lzx_decode_match(unsigned main_element, int block_type,
* been read.
*
* @block_type: The type of the block (LZX_BLOCKTYPE_VERBATIM or
- * LZX_BLOCKTYPE_ALIGNED)
+ * LZX_BLOCKTYPE_ALIGNED)
* @block_size: The size of the block, in bytes.
* @num_main_syms: Number of symbols in the main alphabet.
* @window: Pointer to the decompression window.
* @window_pos: The current position in the window. Will be 0 for the first
- * block.
+ * block.
* @tables: The Huffman decoding tables for the block (main, length, and
- * aligned offset, the latter only for LZX_BLOCKTYPE_ALIGNED)
+ * aligned offset, the latter only for LZX_BLOCKTYPE_ALIGNED)
* @queue: The least-recently-used queue for match offsets.
* @istream: The input bitstream for the compressed literals.
*/
case RESOURCE_IN_WIM:
if (read_partial_wim_stream_into_buf(fd->f_lte, size,
offset, buf))
- ret = -errno;
+ ret = errno ? -errno : -EIO;
else
ret = size;
break;
int ret;
u8 buf[BUFFER_SIZE];
- ni = ntfs_pathname_to_inode(vol, NULL, loc->path);
+ ni = ntfs_pathname_to_inode(vol, NULL, loc->path);
if (!ni) {
ERROR_WITH_ERRNO("Can't find NTFS inode for \"%"TS"\"", loc->path);
ret = WIMLIB_ERR_NTFS_3G;
/* Read raw data from a file descriptor at the specified offset, feeding the
* data it in chunks into the specified callback function. */
static int
-read_raw_file_data(struct filedes *in_fd, u64 size,
- consume_data_callback_t cb, void *cb_ctx, u64 offset)
+read_raw_file_data(struct filedes *in_fd, u64 offset, u64 size,
+ consume_data_callback_t cb, void *cb_ctx)
{
u8 buf[BUFFER_SIZE];
size_t bytes_to_read;
cb, cb_ctx);
} else {
return read_raw_file_data(&rspec->wim->in_fd,
+ rspec->offset_in_wim + offset,
size,
cb,
- cb_ctx,
- rspec->offset_in_wim + offset);
+ cb_ctx);
}
}
return WIMLIB_ERR_OPEN;
}
filedes_init(&fd, raw_fd);
- ret = read_raw_file_data(&fd, size, cb, cb_ctx, 0);
+ ret = read_raw_file_data(&fd, 0, size, cb, cb_ctx);
filedes_close(&fd);
return ret;
}
* reasons, depending on the stream location), or if @cb returned nonzero in
* which case that error code will be returned.
*/
-int
+static int
read_stream_prefix(const struct wim_lookup_table_entry *lte, u64 size,
consume_data_callback_t cb, void *cb_ctx)
{
* @cbs
* Callback functions to accept the stream data.
* @flags
+ * Bitwise OR of zero or more of the following flags:
*
+ * VERIFY_STREAM_HASHES:
+ * For all streams being read that have already had SHA1 message
+ * digests computed, calculate the SHA1 message digest of the read
+ * data and compare it with the previously computed value. If they
+ * do not match, return WIMLIB_ERR_INVALID_RESOURCE_HASH.
+ *
+ * COMPUTE_MISSING_STREAM_HASHES
+ * For all streams being read that have not yet had their SHA1
+ * message digests computed, calculate and save their SHA1 message
+ * digests.
+ *
+ * STREAM_LIST_ALREADY_SORTED
+ * @stream_list is already sorted in sequential order for reading.
*
* Returns 0 on success; a nonzero error code on failure. Failure can occur due
* to an error reading the data or due to an error status being returned by any
extract_stream(struct wim_lookup_table_entry *lte, u64 size,
consume_data_callback_t extract_chunk, void *extract_chunk_arg)
{
+ wimlib_assert(size <= lte->size);
if (size == lte->size) {
/* Do SHA1. */
struct read_stream_list_callbacks cbs = {
* specification. */
void
wim_res_hdr_to_spec(const struct wim_reshdr *reshdr, WIMStruct *wim,
- struct wim_resource_spec *spec)
+ struct wim_resource_spec *rspec)
{
- spec->wim = wim;
- spec->offset_in_wim = reshdr->offset_in_wim;
- spec->size_in_wim = reshdr->size_in_wim;
- spec->uncompressed_size = reshdr->uncompressed_size;
- INIT_LIST_HEAD(&spec->stream_list);
- spec->flags = reshdr->flags;
- spec->is_pipable = wim_is_pipable(wim);
+ rspec->wim = wim;
+ rspec->offset_in_wim = reshdr->offset_in_wim;
+ rspec->size_in_wim = reshdr->size_in_wim;
+ rspec->uncompressed_size = reshdr->uncompressed_size;
+ INIT_LIST_HEAD(&rspec->stream_list);
+ rspec->flags = reshdr->flags;
+ rspec->is_pipable = wim_is_pipable(wim);
}
/* Convert a stand-alone resource specification to a WIM resource header. */
* Reads the security data from the metadata resource of a WIM image.
*
* @metadata_resource: An array that contains the uncompressed metadata
- * resource for the WIM image.
+ * resource for the WIM image.
* @metadata_resource_len: The length of @metadata_resource. It must be at
* least 8 bytes.
* @sd_ret: A pointer to a pointer to a wim_security_data structure that
- * will be filled in with a pointer to a new wim_security_data
- * structure containing the security data on success.
+ * will be filled in with a pointer to a new wim_security_data
+ * structure containing the security data on success.
*
* Note: There is no `offset' argument because the security data is located at
* the beginning of the metadata resource.
*
* Return values:
* WIMLIB_ERR_SUCCESS (0)
- * WIMLIB_ERR_INVALID_METADATA_RESOURCE
- * WIMLIB_ERR_NOMEM
+ * WIMLIB_ERR_INVALID_METADATA_RESOURCE
+ * WIMLIB_ERR_NOMEM
*/
int
read_wim_security_data(const u8 metadata_resource[], size_t metadata_resource_len,
/*
* unix_build_dentry_tree():
- * Builds a tree of WIM dentries from an on-disk directory tree (UNIX
- * version; no NTFS-specific data is captured).
+ * Builds a tree of WIM dentries from an on-disk directory tree (UNIX
+ * version; no NTFS-specific data is captured).
*
* @root_ret: Place to return a pointer to the root of the dentry tree. Only
* modified if successful. Set to NULL if the file or directory was
if (path_len >= path_bufsz)
return WIMLIB_ERR_INVALID_PARAM;
- path_buf = MALLOC(path_bufsz);
+ path_buf = MALLOC(path_bufsz);
if (!path_buf)
return WIMLIB_ERR_NOMEM;
memcpy(path_buf, root_disk_path, path_len + 1);
/* Attach or overlay a branch onto the WIM image.
*
* @root_p:
- * Pointer to the root of the WIM image, or pointer to NULL if it has not
- * been created yet.
+ * Pointer to the root of the WIM image, or pointer to NULL if it has not
+ * been created yet.
* @branch
- * Branch to add.
+ * Branch to add.
* @target_path:
- * Path in the WIM image to add the branch, with leading and trailing
- * slashes stripped.
+ * Path in the WIM image to add the branch, with leading and trailing
+ * slashes stripped.
*/
static int
attach_branch(struct wim_dentry **root_p, struct wim_dentry *branch,
static u32
wim_default_chunk_size(int ctype)
{
- return 32768;
+ switch (ctype) {
+ case WIMLIB_COMPRESSION_TYPE_LZMS:
+ return 131072;
+ default:
+ return 32768;
+ }
}
/*
"32768, 65536, 131072, ..., 2097152.");
break;
case WIMLIB_COMPRESSION_TYPE_LZMS:
- ERROR("Valid chunk sizes for LZMS are 131072.");
+ ERROR("Valid chunk sizes for LZMS are "
+ "32768, 65536, 131072, ..., 67108864.");
break;
}
return WIMLIB_ERR_INVALID_CHUNK_SIZE;
}
- if (chunk_size != 32768) {
+ if (chunk_size != 32768 &&
+ wim->out_compression_type != WIMLIB_COMPRESSION_TYPE_LZMS)
+ {
WARNING ("Changing the compression chunk size to any value other than\n"
" the default of 32768 bytes eliminates compatibility with\n"
" Microsoft's software!");
read_win32_file_prefix(const struct wim_lookup_table_entry *lte,
u64 size,
consume_data_callback_t cb,
- u32 in_chunk_size,
- void *ctx_or_buf,
- int _ignored_flags)
+ void *cb_ctx)
{
int ret = 0;
- void *out_buf;
- bool out_buf_malloced;
u64 bytes_remaining;
+ u8 buf[BUFFER_SIZE];
HANDLE hFile = win32_open_existing_file(lte->file_on_disk,
FILE_READ_DATA);
return WIMLIB_ERR_OPEN;
}
- out_buf_malloced = false;
- if (cb) {
- if (in_chunk_size <= STACK_MAX) {
- out_buf = alloca(in_chunk_size);
- } else {
- out_buf = MALLOC(in_chunk_size);
- if (out_buf == NULL) {
- ret = WIMLIB_ERR_NOMEM;
- goto out_close_handle;
- }
- out_buf_malloced = true;
- }
- } else {
- out_buf = ctx_or_buf;
- }
-
bytes_remaining = size;
while (bytes_remaining) {
DWORD bytesToRead, bytesRead;
- bytesToRead = min(in_chunk_size, bytes_remaining);
- if (!ReadFile(hFile, out_buf, bytesToRead, &bytesRead, NULL) ||
+ bytesToRead = min(sizeof(buf), bytes_remaining);
+ if (!ReadFile(hFile, buf, bytesToRead, &bytesRead, NULL) ||
bytesRead != bytesToRead)
{
set_errno_from_GetLastError();
break;
}
bytes_remaining -= bytesRead;
- if (cb) {
- ret = (*cb)(out_buf, bytesRead, ctx_or_buf);
- if (ret)
- break;
- } else {
- out_buf += bytesRead;
- }
+ ret = (*cb)(buf, bytesRead, cb_ctx);
+ if (ret)
+ break;
}
- if (out_buf_malloced)
- FREE(out_buf);
out_close_handle:
CloseHandle(hFile);
return ret;
struct win32_encrypted_read_ctx {
consume_data_callback_t read_prefix_cb;
- void *read_prefix_ctx_or_buf;
+ void *read_prefix_ctx;
int wimlib_err_code;
- void *buf;
- size_t buf_filled;
u64 bytes_remaining;
- u32 in_chunk_size;
};
static DWORD WINAPI
-win32_encrypted_export_cb(unsigned char *_data, void *_ctx, unsigned long len)
+win32_encrypted_export_cb(unsigned char *data, void *_ctx, unsigned long len)
{
- const void *data = _data;
struct win32_encrypted_read_ctx *ctx = _ctx;
- u32 in_chunk_size = ctx->in_chunk_size;
int ret;
+ size_t bytes_to_consume = min(len, ctx->bytes_remaining);
- DEBUG("len = %lu", len);
- if (ctx->read_prefix_cb) {
- /* The length of the buffer passed to the ReadEncryptedFileRaw()
- * export callback is undocumented, so we assume it may be of
- * arbitrary size. */
- size_t bytes_to_buffer = min(ctx->bytes_remaining - ctx->buf_filled,
- len);
- while (bytes_to_buffer) {
- size_t bytes_to_copy_to_buf =
- min(bytes_to_buffer, in_chunk_size - ctx->buf_filled);
-
- memcpy(ctx->buf + ctx->buf_filled, data,
- bytes_to_copy_to_buf);
- ctx->buf_filled += bytes_to_copy_to_buf;
- data += bytes_to_copy_to_buf;
- bytes_to_buffer -= bytes_to_copy_to_buf;
-
- if (ctx->buf_filled == in_chunk_size ||
- ctx->buf_filled == ctx->bytes_remaining)
- {
- ret = (*ctx->read_prefix_cb)(ctx->buf,
- ctx->buf_filled,
- ctx->read_prefix_ctx_or_buf);
- if (ret) {
- ctx->wimlib_err_code = ret;
- /* Shouldn't matter what error code is returned
- * here, as long as it isn't ERROR_SUCCESS. */
- return ERROR_READ_FAULT;
- }
- ctx->bytes_remaining -= ctx->buf_filled;
- ctx->buf_filled = 0;
- }
- }
- } else {
- size_t len_to_copy = min(len, ctx->bytes_remaining);
- ctx->read_prefix_ctx_or_buf = mempcpy(ctx->read_prefix_ctx_or_buf,
- data,
- len_to_copy);
- ctx->bytes_remaining -= len_to_copy;
+ ret = (*ctx->read_prefix_cb)(data, bytes_to_consume, ctx->read_prefix_ctx);
+ if (ret) {
+ ctx->wimlib_err_code = ret;
+ /* Shouldn't matter what error code is returned here, as long as
+ * it isn't ERROR_SUCCESS. */
+ return ERROR_READ_FAULT;
}
+ ctx->bytes_remaining -= bytes_to_consume;
return ERROR_SUCCESS;
}
read_win32_encrypted_file_prefix(const struct wim_lookup_table_entry *lte,
u64 size,
consume_data_callback_t cb,
- u32 in_chunk_size,
- void *ctx_or_buf,
- int _ignored_flags)
+ void *cb_ctx)
{
struct win32_encrypted_read_ctx export_ctx;
DWORD err;
size, lte->file_on_disk);
export_ctx.read_prefix_cb = cb;
- export_ctx.read_prefix_ctx_or_buf = ctx_or_buf;
+ export_ctx.read_prefix_ctx = cb_ctx;
export_ctx.wimlib_err_code = 0;
- if (cb) {
- export_ctx.buf = MALLOC(in_chunk_size);
- if (!export_ctx.buf)
- return WIMLIB_ERR_NOMEM;
- } else {
- export_ctx.buf = NULL;
- }
- export_ctx.buf_filled = 0;
export_ctx.bytes_remaining = size;
err = OpenEncryptedFileRaw(lte->file_on_disk, 0, &file_ctx);
set_errno_from_win32_error(err);
ERROR_WITH_ERRNO("Failed to open encrypted file \"%ls\" "
"for raw read", lte->file_on_disk);
- ret = WIMLIB_ERR_OPEN;
- goto out_free_buf;
+ return WIMLIB_ERR_OPEN;
}
err = ReadEncryptedFileRaw(win32_encrypted_export_cb,
&export_ctx, file_ctx);
ret = 0;
}
CloseEncryptedFileRaw(file_ctx);
-out_free_buf:
- FREE(export_ctx.buf);
return ret;
}
return -1;
}
-/* Use the Win32 API to get the number of processors */
+/* Use the Win32 API to get the number of processors. */
unsigned
win32_get_number_of_processors(void)
{
return sysinfo.dwNumberOfProcessors;
}
+/* Use the Win32 API to get the amount of available memory. */
+u64
+win32_get_avail_memory(void)
+{
+ MEMORYSTATUSEX status = {
+ .dwLength = sizeof(status),
+ };
+ GlobalMemoryStatusEx(&status);
+ return status.ullTotalPhys;
+}
+
/* Replacement for POSIX-2008 realpath(). Warning: partial functionality only
* (resolved_path must be NULL). Also I highly doubt that GetFullPathName
* really does the right thing under all circumstances. */
return do_pread_or_pwrite(fd, (void*)buf, count, offset, true);
}
+#if 0
/* Dumb Windows implementation of writev(). It writes the vectors one at a
* time. */
ssize_t writev(int fd, const struct iovec *iov, int iovcnt)
}
return total_bytes_written;
}
+#endif
int
win32_get_file_and_vol_ids(const wchar_t *path, u64 *ino_ret, u64 *dev_ret)
#include "wimlib/lookup_table.h"
#include "wimlib/metadata.h"
#include "wimlib/resource.h"
+#include "wimlib/win32.h"
#include "wimlib/write.h"
#include "wimlib/xml.h"
lte->out_res_offset_in_wim = rspec->offset_in_wim;
lte->out_res_size_in_wim = rspec->size_in_wim;
- lte->out_res_uncompressed_size = rspec->uncompressed_size;
+ /*lte->out_res_uncompressed_size = rspec->uncompressed_size;*/
} else {
wimlib_assert(!(lte->flags & WIM_RESHDR_FLAG_PACKED_STREAMS));
u32 reshdr_flags;
int ret;
- stream_hdr.magic = PWM_STREAM_MAGIC;
+ stream_hdr.magic = cpu_to_le64(PWM_STREAM_MAGIC);
stream_hdr.uncompressed_size = cpu_to_le64(lte->size);
if (additional_reshdr_flags & PWM_RESHDR_FLAG_UNHASHED) {
zero_out_hash(stream_hdr.hash);
};
static u64
-get_chunk_entry_size(u64 res_size)
+get_chunk_entry_size(u64 res_size, int write_resource_flags)
{
- if (res_size <= UINT32_MAX)
+ if (res_size <= UINT32_MAX ||
+ (write_resource_flags & WIM_RESHDR_FLAG_PACKED_STREAMS))
return 4;
else
return 8;
* prove to be needed. At this point, we just use @chunk_csizes
* for a buffer of 0's because the actual compressed chunk sizes
* are unknown. */
- reserve_size = expected_num_chunk_entries * get_chunk_entry_size(res_expected_size);
+ reserve_size = expected_num_chunk_entries *
+ get_chunk_entry_size(res_expected_size,
+ ctx->write_resource_flags);
if (ctx->write_resource_flags & WIMLIB_WRITE_RESOURCE_FLAG_PACK_STREAMS)
reserve_size += sizeof(struct alt_chunk_table_header_disk);
memset(ctx->chunk_csizes, 0, reserve_size);
if (!(ctx->write_resource_flags & WIMLIB_WRITE_RESOURCE_FLAG_PACK_STREAMS))
actual_num_chunk_entries--;
- chunk_entry_size = get_chunk_entry_size(res_actual_size);
+ chunk_entry_size = get_chunk_entry_size(res_actual_size,
+ ctx->write_resource_flags);
typedef le64 __attribute__((may_alias)) aliased_le64_t;
typedef le32 __attribute__((may_alias)) aliased_le32_t;
lte->out_reshdr.offset_in_wim = offset_in_res;
lte->out_res_offset_in_wim = reshdr.offset_in_wim;
lte->out_res_size_in_wim = reshdr.size_in_wim;
- lte->out_res_uncompressed_size = reshdr.uncompressed_size;
+ /*lte->out_res_uncompressed_size = reshdr.uncompressed_size;*/
offset_in_res += lte->size;
}
wimlib_assert(offset_in_res == reshdr.uncompressed_size);
if (lte2->out_reshdr.flags & WIM_RESHDR_FLAG_PACKED_STREAMS)
return -1;
}
- return cmp_u64(lte1->out_reshdr.offset_in_wim,
+ return cmp_u64(lte1->out_reshdr.offset_in_wim,
lte2->out_reshdr.offset_in_wim);
}
return false;
if (wim->chunk_size != wim->out_chunk_size)
return false;
+ } else {
+ if (write_flags & WIMLIB_WRITE_FLAG_NO_PACK_STREAMS)
+ return false;
}
return true;
for (size_t i = 0; i < num_specs; i++) {
int rc = xml_write_string(writer, specs[i].name,
*(const tchar * const *)
- (struct_with_strings + specs[i].offset));
+ (struct_with_strings + specs[i].offset));
if (rc)
return rc;
}
*
* Also, a caveat--- according to Microsoft's documentation for XPRESS,
*
- * "Some implementation of the decompression algorithm expect an extra
- * symbol to mark the end of the data. Specifically, some implementations
- * fail during decompression if the Huffman symbol 256 is not found after
- * the actual data."
+ * "Some implementation of the decompression algorithm expect an extra
+ * symbol to mark the end of the data. Specifically, some implementations
+ * fail during decompression if the Huffman symbol 256 is not found after
+ * the actual data."
*
* This is the case for the implementation in WIMGAPI. However, wimlib's
* decompressor in this file currently does not care if this extra symbol is
git://wimlib.net / wimlib / commitdiff