X-Git-Url: https://wimlib.net/git/?p=wimlib;a=blobdiff_plain;f=src%2Flzx-decompress.c;h=2405b9493ad9710f0476af9e2745fe8c59490b2f;hp=6e4098fffa1c218aae813ec30f558683be4a5df2;hb=439beeac4b861e7a95169d7d4397acddb440e73a;hpb=40beb80283a2df7af88c8359ca41adb814585e9a diff --git a/src/lzx-decompress.c b/src/lzx-decompress.c index 6e4098ff..2405b949 100644 --- a/src/lzx-decompress.c +++ b/src/lzx-decompress.c @@ -7,7 +7,7 @@ */ /* - * Copyright (C) 2012 Eric Biggers + * Copyright (C) 2012, 2013 Eric Biggers * * This file is part of wimlib, a library for working with WIM files. * @@ -28,31 +28,28 @@ /* * LZX is a LZ77 and Huffman-code based compression format that has many * similarities to the DEFLATE format used in zlib. The compression ratio is as - * good or better than DEFLATE. However, in WIM files only up to 32768 bytes of - * data can ever compressed be in the same LZX block, so a .tar.gz file could - * potentially be smaller than a WIM file that uses LZX compression because it - * can use a larger LZ77 window size. + * good or better than DEFLATE. * * Some notes on the LZX compression format as used in Windows Imaging (WIM) * files: * * A compressed WIM resource consists of a table of chunk offsets followed by * the compressed chunks themselves. All compressed chunks except possibly the - * last decompress to WIM_CHUNK_SIZE (= 32768) bytes. This is quite similar to - * the cabinet (.cab) file format, but they are not the same. According to the - * cabinet format documentation, the LZX block size is independent from the - * CFDATA blocks, and a LZX block may span several CFDATA blocks. However, in - * WIMs, LZX blocks do not appear to ever span multiple WIM chunks. Note that - * this means any WIM chunk may be decompressed or compressed independently from - * any other chunk, which is convenient. + * last decompress to a fixed number of bytes, by default 32768. This is quite + * similar to the cabinet (.cab) file format, but they are not the same. + * According to the cabinet format documentation, the LZX block size is + * independent from the CFDATA blocks, and a LZX block may span several CFDATA + * blocks. However, in WIMs, LZX blocks do not appear to ever span multiple WIM + * chunks. Note that this means any WIM chunk may be decompressed or compressed + * independently from any other chunk, which allows random access. * * A LZX compressed WIM chunk contains one or more LZX blocks of the aligned, * verbatim, or uncompressed block types. For aligned and verbatim blocks, the * size of the block in uncompressed bytes is specified by a bit following the 3 * bits that specify the block type, possibly followed by an additional 16 bits. - * '1' means to use the default block size (equal to 32768, the size of a WIM - * chunk--- and this seems to only be valid for the first LZX block in a WIM - * chunk), while '0' means that the block size is provided by the next 16 bits. + * '1' means to use the default block size (equal to 32768, the default size of + * a WIM chunk), while '0' means that the block size is provided by the next 16 + * bits. * * The cabinet format, as documented, allows for the possibility that a * compressed CFDATA chunk is up to 6144 bytes larger than the data it @@ -85,13 +82,13 @@ * defined in the specification. * * The LZX document states that aligned offset blocks have their aligned offset - * huffman tree AFTER the main and length trees. The implementation suggests + * Huffman tree AFTER the main and length trees. The implementation suggests * that the aligned offset tree is BEFORE the main and length trees. * * The LZX document decoding algorithm states that, in an aligned offset block, * if an extra_bits value is 1, 2 or 3, then that number of bits should be read * and the result added to the match offset. This is correct for 1 and 2, but - * not 3, where just a huffman symbol (using the aligned tree) should be read. + * not 3, where just a Huffman symbol (using the aligned tree) should be read. * * Regarding the E8 preprocessing, the LZX document states 'No translation may * be performed on the last 6 bytes of the input block'. This is correct. @@ -100,77 +97,98 @@ * would cause the next four bytes to be modified, at least one of which would * be in the last 6 bytes, which is not allowed according to the spec. * - * The specification states that the huffman trees must always contain at least + * The specification states that the Huffman trees must always contain at least * one element. However, many CAB files contain blocks where the length tree is * completely empty (because there are no matches), and this is expected to * succeed. */ -#include "util.h" -#include "lzx.h" -#include "decompress.h" +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include "wimlib.h" +#include "wimlib/decompressor_ops.h" +#include "wimlib/decompress_common.h" +#include "wimlib/lzx.h" +#include "wimlib/util.h" + #include /* Huffman decoding tables and maps from symbols to code lengths. */ struct lzx_tables { - u16 maintree_decode_table[(1 << LZX_MAINTREE_TABLEBITS) + - (LZX_MAINTREE_NUM_SYMBOLS * 2)]; - u8 maintree_lens[LZX_MAINTREE_NUM_SYMBOLS]; + u16 maintree_decode_table[(1 << LZX_MAINCODE_TABLEBITS) + + (LZX_MAINCODE_MAX_NUM_SYMBOLS * 2)] + _aligned_attribute(DECODE_TABLE_ALIGNMENT); + u8 maintree_lens[LZX_MAINCODE_MAX_NUM_SYMBOLS]; - u16 lentree_decode_table[(1 << LZX_LENTREE_TABLEBITS) + - (LZX_LENTREE_NUM_SYMBOLS * 2)]; - u8 lentree_lens[LZX_LENTREE_NUM_SYMBOLS]; + u16 lentree_decode_table[(1 << LZX_LENCODE_TABLEBITS) + + (LZX_LENCODE_NUM_SYMBOLS * 2)] + _aligned_attribute(DECODE_TABLE_ALIGNMENT); + u8 lentree_lens[LZX_LENCODE_NUM_SYMBOLS]; - u16 alignedtree_decode_table[(1 << LZX_ALIGNEDTREE_TABLEBITS) + - (LZX_ALIGNEDTREE_NUM_SYMBOLS * 2)]; - u8 alignedtree_lens[LZX_ALIGNEDTREE_NUM_SYMBOLS]; -}; + u16 alignedtree_decode_table[(1 << LZX_ALIGNEDCODE_TABLEBITS) + + (LZX_ALIGNEDCODE_NUM_SYMBOLS * 2)] + _aligned_attribute(DECODE_TABLE_ALIGNMENT); + u8 alignedtree_lens[LZX_ALIGNEDCODE_NUM_SYMBOLS]; +} _aligned_attribute(DECODE_TABLE_ALIGNMENT); +struct lzx_decompressor { + u32 max_window_size; + unsigned num_main_syms; + struct lzx_tables tables; +}; /* * Reads a Huffman-encoded symbol using the pre-tree. */ -static inline int read_huffsym_using_pretree(struct input_bitstream *istream, - const u16 pretree_decode_table[], - const u8 pretree_lens[], unsigned *n) +static inline int +read_huffsym_using_pretree(struct input_bitstream *istream, + const u16 pretree_decode_table[], + const u8 pretree_lens[], unsigned *n) { return read_huffsym(istream, pretree_decode_table, pretree_lens, - LZX_PRETREE_NUM_SYMBOLS, LZX_PRETREE_TABLEBITS, n, - LZX_MAX_CODEWORD_LEN); + LZX_PRECODE_NUM_SYMBOLS, LZX_PRECODE_TABLEBITS, n, + LZX_MAX_PRE_CODEWORD_LEN); } /* Reads a Huffman-encoded symbol using the main tree. */ -static inline int read_huffsym_using_maintree(struct input_bitstream *istream, - const struct lzx_tables *tables, - unsigned *n) +static inline int +read_huffsym_using_maintree(struct input_bitstream *istream, + const struct lzx_tables *tables, + unsigned *n, + unsigned num_main_syms) { return read_huffsym(istream, tables->maintree_decode_table, - tables->maintree_lens, LZX_MAINTREE_NUM_SYMBOLS, - LZX_MAINTREE_TABLEBITS, n, LZX_MAX_CODEWORD_LEN); + tables->maintree_lens, num_main_syms, + LZX_MAINCODE_TABLEBITS, n, LZX_MAX_MAIN_CODEWORD_LEN); } /* Reads a Huffman-encoded symbol using the length tree. */ -static inline int read_huffsym_using_lentree(struct input_bitstream *istream, - const struct lzx_tables *tables, - unsigned *n) +static inline int +read_huffsym_using_lentree(struct input_bitstream *istream, + const struct lzx_tables *tables, + unsigned *n) { return read_huffsym(istream, tables->lentree_decode_table, - tables->lentree_lens, LZX_LENTREE_NUM_SYMBOLS, - LZX_LENTREE_TABLEBITS, n, LZX_MAX_CODEWORD_LEN); + tables->lentree_lens, LZX_LENCODE_NUM_SYMBOLS, + LZX_LENCODE_TABLEBITS, n, LZX_MAX_LEN_CODEWORD_LEN); } /* Reads a Huffman-encoded symbol using the aligned offset tree. */ -static inline int read_huffsym_using_alignedtree(struct input_bitstream *istream, - const struct lzx_tables *tables, - unsigned *n) +static inline int +read_huffsym_using_alignedtree(struct input_bitstream *istream, + const struct lzx_tables *tables, + unsigned *n) { return read_huffsym(istream, tables->alignedtree_decode_table, tables->alignedtree_lens, - LZX_ALIGNEDTREE_NUM_SYMBOLS, - LZX_ALIGNEDTREE_TABLEBITS, n, 8); + LZX_ALIGNEDCODE_NUM_SYMBOLS, + LZX_ALIGNEDCODE_TABLEBITS, n, + LZX_MAX_ALIGNED_CODEWORD_LEN); } /* @@ -178,41 +196,43 @@ static inline int read_huffsym_using_alignedtree(struct input_bitstream *istream * code length values from the input. * * @istream: The bit stream for the input. It is positioned on the beginning - * of the pretree for the code length values. + * of the pretree for the code length values. * @lens: An array that contains the length values from the previous time - * the code lengths for this Huffman tree were read, or all - * 0's if this is the first time. + * the code lengths for this Huffman tree were read, or all + * 0's if this is the first time. * @num_lens: Number of length values to decode and return. * */ -static int lzx_read_code_lens(struct input_bitstream *istream, u8 lens[], - unsigned num_lens) +static int +lzx_read_code_lens(struct input_bitstream *istream, u8 lens[], + unsigned num_lens) { /* Declare the decoding table and length table for the pretree. */ - u16 pretree_decode_table[(1 << LZX_PRETREE_TABLEBITS) + - (LZX_PRETREE_NUM_SYMBOLS * 2)]; - u8 pretree_lens[LZX_PRETREE_NUM_SYMBOLS]; + u16 pretree_decode_table[(1 << LZX_PRECODE_TABLEBITS) + + (LZX_PRECODE_NUM_SYMBOLS * 2)] + _aligned_attribute(DECODE_TABLE_ALIGNMENT); + u8 pretree_lens[LZX_PRECODE_NUM_SYMBOLS]; unsigned i; - unsigned len; + u32 len; int ret; /* Read the code lengths of the pretree codes. There are 20 lengths of * 4 bits each. */ - for (i = 0; i < LZX_PRETREE_NUM_SYMBOLS; i++) { - ret = bitstream_read_bits(istream, LZX_PRETREE_ELEMENT_SIZE, + for (i = 0; i < LZX_PRECODE_NUM_SYMBOLS; i++) { + ret = bitstream_read_bits(istream, LZX_PRECODE_ELEMENT_SIZE, &len); - if (ret != 0) + if (ret) return ret; pretree_lens[i] = len; } /* Make the decoding table for the pretree. */ ret = make_huffman_decode_table(pretree_decode_table, - LZX_PRETREE_NUM_SYMBOLS, - LZX_PRETREE_TABLEBITS, + LZX_PRECODE_NUM_SYMBOLS, + LZX_PRECODE_TABLEBITS, pretree_lens, - LZX_MAX_CODEWORD_LEN); - if (ret != 0) + LZX_MAX_PRE_CODEWORD_LEN); + if (ret) return ret; /* Pointer past the last length value that needs to be filled in. */ @@ -227,19 +247,19 @@ static int lzx_read_code_lens(struct input_bitstream *istream, u8 lens[], * the next lengths are all equal to the next symbol in the * input. */ unsigned tree_code; - unsigned num_zeroes; + u32 num_zeroes; unsigned code; - unsigned num_same; - char value; + u32 num_same; + signed char value; ret = read_huffsym_using_pretree(istream, pretree_decode_table, - pretree_lens, &tree_code); - if (ret != 0) + pretree_lens, &tree_code); + if (ret) return ret; switch (tree_code) { case 17: /* Run of 0's */ ret = bitstream_read_bits(istream, 4, &num_zeroes); - if (ret != 0) + if (ret) return ret; num_zeroes += 4; while (num_zeroes--) { @@ -250,7 +270,7 @@ static int lzx_read_code_lens(struct input_bitstream *istream, u8 lens[], break; case 18: /* Longer run of 0's */ ret = bitstream_read_bits(istream, 5, &num_zeroes); - if (ret != 0) + if (ret) return ret; num_zeroes += 20; while (num_zeroes--) { @@ -261,16 +281,16 @@ static int lzx_read_code_lens(struct input_bitstream *istream, u8 lens[], break; case 19: /* Run of identical lengths */ ret = bitstream_read_bits(istream, 1, &num_same); - if (ret != 0) + if (ret) return ret; num_same += 4; - ret = read_huffsym_using_pretree(istream, - pretree_decode_table, - pretree_lens, &code); - if (ret != 0) + pretree_decode_table, + pretree_lens, + &code); + if (ret) return ret; - value = (char)*lens - (char)code; + value = (signed char)*lens - (signed char)code; if (value < 0) value += 17; while (num_same--) { @@ -280,7 +300,7 @@ static int lzx_read_code_lens(struct input_bitstream *istream, u8 lens[], } break; default: /* Difference from old length. */ - value = (char)*lens - (char)tree_code; + value = (signed char)*lens - (signed char)tree_code; if (value < 0) value += 17; *lens = value; @@ -296,52 +316,64 @@ static int lzx_read_code_lens(struct input_bitstream *istream, u8 lens[], * * @istream: The input bitstream. * @block_size_ret: A pointer to an int into which the size of the block, - * in bytes, will be returned. + * in bytes, will be returned. * @block_type_ret: A pointer to an int into which the type of the block - * (LZX_BLOCKTYPE_*) will be returned. + * (LZX_BLOCKTYPE_*) will be returned. * @tables: A pointer to a lzx_tables structure in which the - * main tree, the length tree, and possibly the - * aligned offset tree will be constructed. + * main tree, the length tree, and possibly the + * aligned offset tree will be constructed. * @queue: A pointer to the least-recently-used queue into which - * R0, R1, and R2 will be written (only for uncompressed - * blocks, which contain this information in the header) + * R0, R1, and R2 will be written (only for uncompressed + * blocks, which contain this information in the header) */ -static int lzx_read_block_header(struct input_bitstream *istream, - unsigned *block_size_ret, - unsigned *block_type_ret, - struct lzx_tables *tables, - struct lru_queue *queue) +static int +lzx_read_block_header(struct input_bitstream *istream, + unsigned num_main_syms, + unsigned max_window_size, + unsigned *block_size_ret, + unsigned *block_type_ret, + struct lzx_tables *tables, + struct lzx_lru_queue *queue) { int ret; - int block_type; + unsigned block_type; unsigned block_size; - int s; - int i; - unsigned len; - int32_t R[3]; ret = bitstream_ensure_bits(istream, 4); - if (ret != 0) { - ERROR("LZX input stream overrun"); + if (ret) return ret; - } /* The first three bits tell us what kind of block it is, and are one * of the LZX_BLOCKTYPE_* values. */ block_type = bitstream_read_bits_nocheck(istream, 3); - /* The next bit indicates whether the block size is the default (32768), - * indicated by a 1 bit, or whether the block size is given by the next - * 16 bits, indicated by a 0 bit. */ - s = bitstream_read_bits_nocheck(istream, 1); - - if (s == 1) { - block_size = 1 << 15; + /* Read the block size. This mirrors the behavior + * lzx_write_compressed_block() in lzx-compress.c; see that for more + * details. */ + if (bitstream_read_bits_nocheck(istream, 1)) { + block_size = LZX_DEFAULT_BLOCK_SIZE; } else { - ret = bitstream_read_bits(istream, 16, &block_size); - if (ret != 0) + u32 tmp; + block_size = 0; + + ret = bitstream_read_bits(istream, 8, &tmp); + if (ret) return ret; - block_size = le16_to_cpu(block_size); + block_size |= tmp; + + ret = bitstream_read_bits(istream, 8, &tmp); + if (ret) + return ret; + block_size <<= 8; + block_size |= tmp; + + if (max_window_size >= 65536) { + ret = bitstream_read_bits(istream, 8, &tmp); + if (ret) + return ret; + block_size <<= 8; + block_size |= tmp; + } } switch (block_type) { @@ -349,24 +381,26 @@ static int lzx_read_block_header(struct input_bitstream *istream, /* Read the path lengths for the elements of the aligned tree, * then build it. */ - for (i = 0; i < LZX_ALIGNEDTREE_NUM_SYMBOLS; i++) { + for (unsigned i = 0; i < LZX_ALIGNEDCODE_NUM_SYMBOLS; i++) { + u32 len; + ret = bitstream_read_bits(istream, - LZX_ALIGNEDTREE_ELEMENT_SIZE, + LZX_ALIGNEDCODE_ELEMENT_SIZE, &len); - if (ret != 0) + if (ret) return ret; tables->alignedtree_lens[i] = len; } LZX_DEBUG("Building the aligned tree."); ret = make_huffman_decode_table(tables->alignedtree_decode_table, - LZX_ALIGNEDTREE_NUM_SYMBOLS, - LZX_ALIGNEDTREE_TABLEBITS, + LZX_ALIGNEDCODE_NUM_SYMBOLS, + LZX_ALIGNEDCODE_TABLEBITS, tables->alignedtree_lens, - 8); - if (ret != 0) { - ERROR("lzx_decompress(): Failed to make the decode " - "table for the aligned offset tree"); + LZX_MAX_ALIGNED_CODEWORD_LEN); + if (ret) { + LZX_DEBUG("Failed to make the decode table for the " + "aligned offset tree"); return ret; } @@ -382,10 +416,9 @@ static int lzx_read_block_header(struct input_bitstream *istream, * tree. */ ret = lzx_read_code_lens(istream, tables->maintree_lens, LZX_NUM_CHARS); - if (ret != 0) { - ERROR("lzx_decompress(): Failed to read the code " - "lengths for the first 256 elements of the " - "main tree"); + if (ret) { + LZX_DEBUG("Failed to read the code lengths for the " + "first 256 elements of the main tree"); return ret; } @@ -393,14 +426,13 @@ static int lzx_read_block_header(struct input_bitstream *istream, * tree. */ LZX_DEBUG("Reading path lengths for remaining elements of " "main tree (%d elements).", - LZX_MAINTREE_NUM_SYMBOLS - LZX_NUM_CHARS); + num_main_syms - LZX_NUM_CHARS); ret = lzx_read_code_lens(istream, tables->maintree_lens + LZX_NUM_CHARS, - LZX_MAINTREE_NUM_SYMBOLS - LZX_NUM_CHARS); - if (ret != 0) { - ERROR("lzx_decompress(): Failed to read the path " - "lengths for the remaining elements of the main " - "tree"); + num_main_syms - LZX_NUM_CHARS); + if (ret) { + LZX_DEBUG("Failed to read the path lengths for the " + "remaining elements of the main tree"); return ret; } @@ -408,54 +440,74 @@ static int lzx_read_block_header(struct input_bitstream *istream, "table for the main tree."); ret = make_huffman_decode_table(tables->maintree_decode_table, - LZX_MAINTREE_NUM_SYMBOLS, - LZX_MAINTREE_TABLEBITS, + num_main_syms, + LZX_MAINCODE_TABLEBITS, tables->maintree_lens, - LZX_MAX_CODEWORD_LEN); - if (ret != 0) { - ERROR("lzx_decompress(): Failed to make the decode " - "table for the main tree"); + LZX_MAX_MAIN_CODEWORD_LEN); + if (ret) { + LZX_DEBUG("Failed to make the decode " + "table for the main tree"); return ret; } LZX_DEBUG("Reading path lengths for the length tree."); ret = lzx_read_code_lens(istream, tables->lentree_lens, - LZX_LENTREE_NUM_SYMBOLS); - if (ret != 0) { - ERROR("lzx_decompress(): Failed to read the path " - "lengths for the length tree"); + LZX_LENCODE_NUM_SYMBOLS); + if (ret) { + LZX_DEBUG("Failed to read the path " + "lengths for the length tree"); return ret; } LZX_DEBUG("Building the length tree."); ret = make_huffman_decode_table(tables->lentree_decode_table, - LZX_LENTREE_NUM_SYMBOLS, - LZX_LENTREE_TABLEBITS, + LZX_LENCODE_NUM_SYMBOLS, + LZX_LENCODE_TABLEBITS, tables->lentree_lens, - LZX_MAX_CODEWORD_LEN); - if (ret != 0) { - ERROR("lzx_decompress(): Failed to build the length " - "Huffman tree"); + LZX_MAX_LEN_CODEWORD_LEN); + if (ret) { + LZX_DEBUG("Failed to build the length Huffman tree"); return ret; } - + /* The bitstream of compressed literals and matches for this + * block directly follows and will be read in + * lzx_decompress_block(). */ break; - case LZX_BLOCKTYPE_UNCOMPRESSED: LZX_DEBUG("Found uncompressed block."); - ret = align_input_bitstream(istream, true); - if (ret != 0) - return ret; - ret = bitstream_read_bytes(istream, sizeof(R), R); - if (ret != 0) - return ret; - queue->R0 = le32_to_cpu(R[0]); - queue->R1 = le32_to_cpu(R[1]); - queue->R2 = le32_to_cpu(R[2]); + /* Before reading the three LRU match offsets from the + * uncompressed block header, the stream needs to be aligned on + * a 16-bit boundary. But, unexpectedly, if the stream is + * *already* aligned, the correct thing to do is to throw away + * the next 16 bits. */ + if (istream->bitsleft == 0) { + if (istream->data_bytes_left < 14) { + LZX_DEBUG("Insufficient length in " + "uncompressed block"); + return -1; + } + istream->data += 2; + istream->data_bytes_left -= 2; + } else { + if (istream->data_bytes_left < 12) { + LZX_DEBUG("Insufficient length in " + "uncompressed block"); + return -1; + } + istream->bitsleft = 0; + istream->bitbuf = 0; + } + queue->R[0] = le32_to_cpu(*(le32*)(istream->data + 0)); + queue->R[1] = le32_to_cpu(*(le32*)(istream->data + 4)); + queue->R[2] = le32_to_cpu(*(le32*)(istream->data + 8)); + istream->data += 12; + istream->data_bytes_left -= 12; + /* The uncompressed data of this block directly follows and will + * be read in lzx_decompress(). */ break; default: - LZX_DEBUG("Found invalid block."); - return 1; + LZX_DEBUG("Found invalid block"); + return -1; } *block_type_ret = block_type; *block_size_ret = block_size; @@ -463,35 +515,47 @@ static int lzx_read_block_header(struct input_bitstream *istream, } /* - * Decodes a compressed literal match value. It refers to some match_offset to - * a point earlier in the window, and some match_len, for which the data is to - * be copied to the current position in the window. + * Decodes a compressed match from a block of LZX-compressed data. A match + * refers to some match_offset to a point earlier in the window as well as some + * match_len, for which the data is to be copied to the current position in the + * window. * * @main_element: The start of the match data, as decoded using the main - * tree. - * @block_type: The type of the block (LZX_BLOCKTYPE_ALIGNED or + * tree. + * + * @block_type: The type of the block (LZX_BLOCKTYPE_ALIGNED or * LZX_BLOCKTYPE_VERBATIM) + * * @bytes_remaining: The amount of uncompressed data remaining to be - * uncompressed. It is an error if the match - * is longer than @bytes_remaining. - * @window: A pointer to the window into which the uncompressed - * data is being written. - * @window_pos: The current position in the window. - * @tables: Contains the Huffman tables for the block (main, - * length, and also aligned offset only for - * LZX_BLOCKTYPE_ALIGNED) - * @queue: The least-recently used queue for match offsets. - * @istream: The input bitstream. - * - * Returns the length of the match, or -1 on error (match would exceed - * the amount of data needing to be uncompressed, or match refers to data before - * the window, or the input bitstream ended unexpectedly). + * uncompressed in this block. It is an error if the match + * is longer than this number. + * + * @window: A pointer to the window into which the uncompressed + * data is being written. + * + * @window_pos: The current byte offset in the window. + * + * @tables: The Huffman decoding tables for this LZX block (main + * code, length code, and for LZX_BLOCKTYPE_ALIGNED blocks, + * also the aligned offset code). + * + * @queue: The least-recently used queue for match offsets. + * + * @istream: The input bitstream. + * + * Returns the length of the match, or a negative number on error. The possible + * error cases are: + * - Match would exceed the amount of data remaining to be uncompressed. + * - Match refers to data before the window. + * - The input bitstream ended unexpectedly. */ -static int lzx_decode_match(int main_element, int block_type, - int bytes_remaining, u8 *window, int window_pos, - const struct lzx_tables *tables, - struct lru_queue *queue, - struct input_bitstream *istream) +static int +lzx_decode_match(unsigned main_element, int block_type, + unsigned bytes_remaining, u8 *window, + unsigned window_pos, + const struct lzx_tables *tables, + struct lzx_lru_queue *queue, + struct input_bitstream *istream) { unsigned length_header; unsigned position_slot; @@ -499,10 +563,10 @@ static int lzx_decode_match(int main_element, int block_type, unsigned match_offset; unsigned additional_len; unsigned num_extra_bits; - unsigned verbatim_bits; - unsigned aligned_bits; + u32 verbatim_bits; + u32 aligned_bits; + unsigned i; int ret; - int i; u8 *match_dest; u8 *match_src; @@ -516,15 +580,16 @@ static int lzx_decode_match(int main_element, int block_type, position_slot = main_element >> 3; /* If the length_header is less than LZX_NUM_PRIMARY_LENS (= 7), it - * gives the match length as the offset from LZX_MIN_MATCH. Otherwise, - * the length is given by an additional symbol encoded using the length - * tree, offset by 9 (LZX_MIN_MATCH + LZX_NUM_PRIMARY_LENS) */ - match_len = LZX_MIN_MATCH + length_header; + * gives the match length as the offset from LZX_MIN_MATCH_LEN. + * Otherwise, the length is given by an additional symbol encoded using + * the length tree, offset by 9 (LZX_MIN_MATCH_LEN + + * LZX_NUM_PRIMARY_LENS) */ + match_len = LZX_MIN_MATCH_LEN + length_header; if (length_header == LZX_NUM_PRIMARY_LENS) { ret = read_huffsym_using_lentree(istream, tables, - &additional_len); - if (ret != 0) - return -1; + &additional_len); + if (ret) + return ret; match_len += additional_len; } @@ -534,18 +599,18 @@ static int lzx_decode_match(int main_element, int block_type, * queue. */ switch (position_slot) { case 0: - match_offset = queue->R0; + match_offset = queue->R[0]; break; case 1: - match_offset = queue->R1; - swap(queue->R0, queue->R1); + match_offset = queue->R[1]; + swap(queue->R[0], queue->R[1]); break; case 2: /* The queue doesn't work quite the same as a real LRU queue, * since using the R2 offset doesn't bump the R1 offset down to * R2. */ - match_offset = queue->R2; - swap(queue->R0, queue->R2); + match_offset = queue->R[2]; + swap(queue->R[0], queue->R[2]); break; default: /* Otherwise, the offset was not encoded as one the offsets in @@ -554,7 +619,7 @@ static int lzx_decode_match(int main_element, int block_type, * decode the match offset. */ /* Look up the number of extra bits that need to be read. */ - num_extra_bits = lzx_extra_bits[position_slot]; + num_extra_bits = lzx_get_num_extra_bits(position_slot); /* For aligned blocks, if there are at least 3 extra bits, the * actual number of extra bits is 3 less, and they encode a @@ -571,16 +636,16 @@ static int lzx_decode_match(int main_element, int block_type, * num_extra_bits == 3, the assignment to verbatim_bits * will just set it to 0. ) */ ret = bitstream_read_bits(istream, num_extra_bits - 3, - &verbatim_bits); - if (ret != 0) - return -1; + &verbatim_bits); + if (ret) + return ret; verbatim_bits <<= 3; ret = read_huffsym_using_alignedtree(istream, tables, &aligned_bits); - if (ret != 0) - return -1; + if (ret) + return ret; } else { /* For non-aligned blocks, or for aligned blocks with * less than 3 extra bits, the extra bits are added @@ -588,42 +653,44 @@ static int lzx_decode_match(int main_element, int block_type, * the alignment is taken to be 0. */ ret = bitstream_read_bits(istream, num_extra_bits, &verbatim_bits); - if (ret != 0) - return -1; + if (ret) + return ret; aligned_bits = 0; } /* Calculate the match offset. */ - match_offset = lzx_position_base[position_slot] + verbatim_bits + - aligned_bits - 2; + match_offset = lzx_position_base[position_slot] + + verbatim_bits + aligned_bits - LZX_OFFSET_OFFSET; /* Update the LRU queue. */ - queue->R2 = queue->R1; - queue->R1 = queue->R0; - queue->R0 = match_offset; + queue->R[2] = queue->R[1]; + queue->R[1] = queue->R[0]; + queue->R[0] = match_offset; break; } /* Verify that the match is in the bounds of the part of the window * currently in use, then copy the source of the match to the current * position. */ - match_dest = window + window_pos; - match_src = match_dest - match_offset; if (match_len > bytes_remaining) { - ERROR("lzx_decode_match(): Match of length %d bytes overflows " - "uncompressed block size", match_len); + LZX_DEBUG("Match of length %u bytes overflows " + "uncompressed block size", match_len); return -1; } - if (match_src < window) { - ERROR("lzx_decode_match(): Match of length %d bytes references " - "data before window (match_offset = %d, window_pos = %d)", - match_len, match_offset, window_pos); + if (match_offset > window_pos) { + LZX_DEBUG("Match of length %u bytes references " + "data before window (match_offset = %u, " + "window_pos = %u)", + match_len, match_offset, window_pos); return -1; } + match_dest = window + window_pos; + match_src = match_dest - match_offset; + #if 0 printf("Match: src %u, dst %u, len %u\n", match_src - window, match_dest - window, @@ -643,161 +710,180 @@ static int lzx_decode_match(int main_element, int block_type, return match_len; } - +static void +undo_call_insn_translation(u32 *call_insn_target, s32 input_pos, + s32 file_size) +{ + s32 abs_offset; + s32 rel_offset; + + abs_offset = le32_to_cpu(*call_insn_target); + if (abs_offset >= -input_pos && abs_offset < file_size) { + if (abs_offset >= 0) { + /* "good translation" */ + rel_offset = abs_offset - input_pos; + } else { + /* "compensating translation" */ + rel_offset = abs_offset + file_size; + } + *call_insn_target = cpu_to_le32(rel_offset); + } +} /* Undo the 'E8' preprocessing, where the targets of x86 CALL instructions were - * changed from relative offsets to absolute offsets. This type of - * preprocessing can be used on any binary data even if it is not actually - * machine code. It seems to always be used in WIM files, even though there is - * no bit to indicate that it actually is used, unlike in the LZX compressed - * format as used in other file formats, where a bit is reserved for that - * purpose. */ -static void undo_call_insn_preprocessing(u8 uncompressed_data[], - unsigned uncompressed_data_len) + * changed from relative offsets to absolute offsets. + * + * Note that this call instruction preprocessing can and will be used on any + * data even if it is not actually x86 machine code. In fact, this type of + * preprocessing appears to always be used in LZX-compressed resources in WIM + * files; there is no bit to indicate whether it is used or not, unlike in the + * LZX compressed format as used in cabinet files, where a bit is reserved for + * that purpose. + * + * Call instruction preprocessing is disabled in the last 6 bytes of the + * uncompressed data, which really means the 5-byte call instruction cannot + * start in the last 10 bytes of the uncompressed data. This is one of the + * errors in the LZX documentation. + * + * Call instruction preprocessing does not appear to be disabled after the + * 32768th chunk of a WIM stream, which is apparently is yet another difference + * from the LZX compression used in cabinet files. + * + * Call instruction processing is supposed to take the file size as a parameter, + * as it is used in calculating the translated jump targets. But in WIM files, + * this file size is always the same (LZX_WIM_MAGIC_FILESIZE == 12000000).*/ +static void +undo_call_insn_preprocessing(u8 *uncompressed_data, s32 uncompressed_size) { - int i = 0; - int file_size = LZX_MAGIC_FILESIZE; - int32_t abs_offset; - int32_t rel_offset; - - /* Not enabled in the last 6 bytes, which means the 5-byte call - * instruction cannot start in the last *10* bytes. */ - while (i < uncompressed_data_len - 10) { - if (uncompressed_data[i] != 0xe8) { - i++; - continue; - } - abs_offset = le32_to_cpu(*(int32_t*)(uncompressed_data + i + 1)); - - if (abs_offset >= -i && abs_offset < file_size) { - if (abs_offset >= 0) { - /* "good translation" */ - rel_offset = abs_offset - i; - } else { - /* "compensating translation" */ - rel_offset = abs_offset + file_size; - } - *(int32_t*)(uncompressed_data + i + 1) = - cpu_to_le32(rel_offset); + for (s32 i = 0; i < uncompressed_size - 10; i++) { + if (uncompressed_data[i] == 0xe8) { + undo_call_insn_translation((u32*)&uncompressed_data[i + 1], + i, + LZX_WIM_MAGIC_FILESIZE); + i += 4; } - i += 5; } } /* - * Decompresses a compressed block of data from which the header has already + * Decompresses a LZX-compressed block of data from which the header has already * been read. * * @block_type: The type of the block (LZX_BLOCKTYPE_VERBATIM or - * LZX_BLOCKTYPE_ALIGNED) + * LZX_BLOCKTYPE_ALIGNED) * @block_size: The size of the block, in bytes. + * @num_main_syms: Number of symbols in the main alphabet. * @window: Pointer to the decompression window. * @window_pos: The current position in the window. Will be 0 for the first - * block. + * block. * @tables: The Huffman decoding tables for the block (main, length, and - * aligned offset, the latter only for LZX_BLOCKTYPE_ALIGNED) + * aligned offset, the latter only for LZX_BLOCKTYPE_ALIGNED) * @queue: The least-recently-used queue for match offsets. * @istream: The input bitstream for the compressed literals. */ -static int lzx_decompress_block(int block_type, int block_size, u8 *window, - int window_pos, - const struct lzx_tables *tables, - struct lru_queue *queue, - struct input_bitstream *istream) +static int +lzx_decompress_block(int block_type, unsigned block_size, + unsigned num_main_syms, + u8 *window, + unsigned window_pos, + const struct lzx_tables *tables, + struct lzx_lru_queue *queue, + struct input_bitstream *istream) { - unsigned bytes_remaining; unsigned main_element; - int match_len; + unsigned end; int ret; + int match_len; - bytes_remaining = block_size; - while (bytes_remaining > 0) { - + end = window_pos + block_size; + while (window_pos < end) { ret = read_huffsym_using_maintree(istream, tables, - &main_element); - if (ret != 0) + &main_element, + num_main_syms); + if (ret) return ret; if (main_element < LZX_NUM_CHARS) { /* literal: 0 to LZX_NUM_CHARS - 1 */ - window[window_pos + block_size - bytes_remaining] = - main_element; - bytes_remaining--; + window[window_pos++] = main_element; } else { - /* match: LZX_NUM_CHARS to LZX_MAINTREE_NUM_SYMBOLS - 1 */ + /* match: LZX_NUM_CHARS to num_main_syms - 1 */ match_len = lzx_decode_match(main_element, - block_type, bytes_remaining, window, - block_size + window_pos - - bytes_remaining, - tables, queue, istream); - if (match_len == -1) - return 1; - - bytes_remaining -= match_len; + block_type, + end - window_pos, + window, + window_pos, + tables, + queue, + istream); + if (match_len < 0) + return match_len; + window_pos += match_len; } } return 0; } -/* - * Decompresses a block of LZX-compressed data using a window size of 32768. - * - * @compressed_data: A pointer to the compressed data. - * @compressed_len: The length of the compressed data, in bytes. - * @uncompressed_data: A pointer to the buffer into which to write the - * uncompressed data. - * @uncompressed_len: The length of the uncompressed data. - * - * Return non-zero on failure. - */ -int lzx_decompress(const void *compressed_data, unsigned compressed_len, - void *uncompressed_data, unsigned uncompressed_len) +static int +lzx_decompress(const void *compressed_data, size_t compressed_size, + void *uncompressed_data, size_t uncompressed_size, + void *_ctx) { - struct lzx_tables tables; + struct lzx_decompressor *ctx = _ctx; struct input_bitstream istream; - struct lru_queue queue; - unsigned bytes_remaining; + struct lzx_lru_queue queue; + unsigned window_pos; unsigned block_size; unsigned block_type; int ret; + bool e8_preprocessing_done; + + LZX_DEBUG("compressed_data = %p, compressed_size = %zu, " + "uncompressed_data = %p, uncompressed_size = %zu, " + "max_window_size=%u).", + compressed_data, compressed_size, + uncompressed_data, uncompressed_size, + ctx->max_window_size); + + if (uncompressed_size > ctx->max_window_size) { + LZX_DEBUG("Uncompressed size of %zu exceeds " + "window size of %u!", + uncompressed_size, ctx->max_window_size); + return -1; + } - LZX_DEBUG("lzx_decompress (compressed_data = %p, compressed_len = %d, " - "uncompressed_data = %p, uncompressed_len = %d).", - compressed_data, compressed_len, - uncompressed_data, uncompressed_len); - - wimlib_assert(uncompressed_len <= 32768); - - memset(tables.maintree_lens, 0, sizeof(tables.maintree_lens)); - memset(tables.lentree_lens, 0, sizeof(tables.lentree_lens)); - queue.R0 = 1; - queue.R1 = 1; - queue.R2 = 1; - bytes_remaining = uncompressed_len; + memset(ctx->tables.maintree_lens, 0, sizeof(ctx->tables.maintree_lens)); + memset(ctx->tables.lentree_lens, 0, sizeof(ctx->tables.lentree_lens)); + lzx_lru_queue_init(&queue); + init_input_bitstream(&istream, compressed_data, compressed_size); - init_input_bitstream(&istream, compressed_data, compressed_len); + e8_preprocessing_done = false; /* Set to true if there may be 0xe8 bytes + in the uncompressed data. */ /* The compressed data will consist of one or more blocks. The * following loop decompresses one block, and it runs until there all * the compressed data has been decompressed, so there are no more * blocks. */ - while (bytes_remaining != 0) { - + for (window_pos = 0; + window_pos < uncompressed_size; + window_pos += block_size) + { LZX_DEBUG("Reading block header."); - ret = lzx_read_block_header(&istream, &block_size, - &block_type, &tables, &queue); - if (ret != 0) + ret = lzx_read_block_header(&istream, ctx->num_main_syms, + ctx->max_window_size, &block_size, + &block_type, &ctx->tables, &queue); + if (ret) return ret; - LZX_DEBUG("block_size = %u, bytes_remaining = %u", - block_size, bytes_remaining); + LZX_DEBUG("block_size = %u, window_pos = %u", + block_size, window_pos); - if (block_size > bytes_remaining) { - ERROR("lzx_decompress(): Expected a block size of at " - "most %u bytes (found %u bytes)", - bytes_remaining, block_size); - return 1; + if (block_size > uncompressed_size - window_pos) { + LZX_DEBUG("Expected a block size of at " + "most %zu bytes (found %u bytes)", + uncompressed_size - window_pos, block_size); + return -1; } switch (block_type) { @@ -807,41 +893,79 @@ int lzx_decompress(const void *compressed_data, unsigned compressed_len, LZX_DEBUG("LZX_BLOCKTYPE_VERBATIM"); else LZX_DEBUG("LZX_BLOCKTYPE_ALIGNED"); - ret = lzx_decompress_block(block_type, block_size, + ctx->num_main_syms, uncompressed_data, - uncompressed_len - - bytes_remaining, - &tables, &queue, &istream); - if (ret != 0) + window_pos, + &ctx->tables, + &queue, + &istream); + if (ret) return ret; + + if (ctx->tables.maintree_lens[0xe8] != 0) + e8_preprocessing_done = true; break; case LZX_BLOCKTYPE_UNCOMPRESSED: LZX_DEBUG("LZX_BLOCKTYPE_UNCOMPRESSED"); - ret = bitstream_read_bytes(&istream, block_size, - uncompressed_data + - uncompressed_len - - bytes_remaining); - if (ret != 0) - return ret; - if (block_size & 1) - align_input_bitstream(&istream, false); - break; - default: - wimlib_assert(0); + if (istream.data_bytes_left < block_size) { + LZX_DEBUG("Unexpected end of input when " + "reading %u bytes from LZX bitstream " + "(only have %u bytes left)", + block_size, istream.data_bytes_left); + return -1; + } + memcpy(&((u8*)uncompressed_data)[window_pos], istream.data, + block_size); + istream.data += block_size; + istream.data_bytes_left -= block_size; + /* Re-align bitstream if an odd number of bytes were + * read. */ + if (istream.data_bytes_left && (block_size & 1)) { + istream.data_bytes_left--; + istream.data++; + } + e8_preprocessing_done = true; break; } + } + if (e8_preprocessing_done) + undo_call_insn_preprocessing(uncompressed_data, uncompressed_size); + return 0; +} - bytes_remaining -= block_size; +static void +lzx_free_decompressor(void *_ctx) +{ + struct lzx_decompressor *ctx = _ctx; - if (bytes_remaining != 0) - LZX_DEBUG("%d bytes remaining.", bytes_remaining); - } + FREE(ctx); +} - if (uncompressed_len >= 10) - undo_call_insn_preprocessing(uncompressed_data, - uncompressed_len); +static int +lzx_create_decompressor(size_t max_window_size, + const struct wimlib_decompressor_params_header *params, + void **ctx_ret) +{ + struct lzx_decompressor *ctx; + + if (!lzx_window_size_valid(max_window_size)) + return WIMLIB_ERR_INVALID_PARAM; + + ctx = MALLOC(sizeof(struct lzx_decompressor)); + if (ctx == NULL) + return WIMLIB_ERR_NOMEM; + ctx->max_window_size = max_window_size; + ctx->num_main_syms = lzx_get_num_main_syms(max_window_size); + + *ctx_ret = ctx; return 0; } + +const struct decompressor_ops lzx_decompressor_ops = { + .create_decompressor = lzx_create_decompressor, + .decompress = lzx_decompress, + .free_decompressor = lzx_free_decompressor, +};