X-Git-Url: https://wimlib.net/git/?a=blobdiff_plain;f=src%2Flzx-decompress.c;h=b7872520890135d8eee0db082746469e715122ba;hb=e176e9731e696562bab8de7b9bd34c019deef3e8;hp=935c6367e9bc769bba03a88fd53e058d2a6de9a2;hpb=9bec19341aac9190cd0bd80a6937b02d923dd566;p=wimlib diff --git a/src/lzx-decompress.c b/src/lzx-decompress.c index 935c6367..b7872520 100644 --- a/src/lzx-decompress.c +++ b/src/lzx-decompress.c @@ -28,31 +28,28 @@ /* * LZX is a LZ77 and Huffman-code based compression format that has many * similarities to the DEFLATE format used in zlib. The compression ratio is as - * good or better than DEFLATE. However, in WIM files only up to 32768 bytes of - * data can ever compressed be in the same LZX block, so a .tar.gz file could - * potentially be smaller than a WIM file that uses LZX compression because it - * can use a larger LZ77 window size. + * good or better than DEFLATE. * * Some notes on the LZX compression format as used in Windows Imaging (WIM) * files: * * A compressed WIM resource consists of a table of chunk offsets followed by * the compressed chunks themselves. All compressed chunks except possibly the - * last decompress to WIM_CHUNK_SIZE (= 32768) bytes. This is quite similar to - * the cabinet (.cab) file format, but they are not the same. According to the - * cabinet format documentation, the LZX block size is independent from the - * CFDATA blocks, and a LZX block may span several CFDATA blocks. However, in - * WIMs, LZX blocks do not appear to ever span multiple WIM chunks. Note that - * this means any WIM chunk may be decompressed or compressed independently from - * any other chunk, which is convenient. + * last decompress to a fixed number of bytes, by default 32768. This is quite + * similar to the cabinet (.cab) file format, but they are not the same. + * According to the cabinet format documentation, the LZX block size is + * independent from the CFDATA blocks, and a LZX block may span several CFDATA + * blocks. However, in WIMs, LZX blocks do not appear to ever span multiple WIM + * chunks. Note that this means any WIM chunk may be decompressed or compressed + * independently from any other chunk, which allows random access. * * A LZX compressed WIM chunk contains one or more LZX blocks of the aligned, * verbatim, or uncompressed block types. For aligned and verbatim blocks, the * size of the block in uncompressed bytes is specified by a bit following the 3 * bits that specify the block type, possibly followed by an additional 16 bits. - * '1' means to use the default block size (equal to 32768, the size of a WIM - * chunk--- and this seems to only be valid for the first LZX block in a WIM - * chunk), while '0' means that the block size is provided by the next 16 bits. + * '1' means to use the default block size (equal to 32768, the default size of + * a WIM chunk), while '0' means that the block size is provided by the next 16 + * bits. * * The cabinet format, as documented, allows for the possibility that a * compressed CFDATA chunk is up to 6144 bytes larger than the data it @@ -85,13 +82,13 @@ * defined in the specification. * * The LZX document states that aligned offset blocks have their aligned offset - * huffman tree AFTER the main and length trees. The implementation suggests + * Huffman tree AFTER the main and length trees. The implementation suggests * that the aligned offset tree is BEFORE the main and length trees. * * The LZX document decoding algorithm states that, in an aligned offset block, * if an extra_bits value is 1, 2 or 3, then that number of bits should be read * and the result added to the match offset. This is correct for 1 and 2, but - * not 3, where just a huffman symbol (using the aligned tree) should be read. + * not 3, where just a Huffman symbol (using the aligned tree) should be read. * * Regarding the E8 preprocessing, the LZX document states 'No translation may * be performed on the last 6 bytes of the input block'. This is correct. @@ -100,7 +97,7 @@ * would cause the next four bytes to be modified, at least one of which would * be in the last 6 bytes, which is not allowed according to the spec. * - * The specification states that the huffman trees must always contain at least + * The specification states that the Huffman trees must always contain at least * one element. However, many CAB files contain blocks where the length tree is * completely empty (because there are no matches), and this is expected to * succeed. @@ -121,9 +118,9 @@ struct lzx_tables { u16 maintree_decode_table[(1 << LZX_MAINCODE_TABLEBITS) + - (LZX_MAINCODE_NUM_SYMBOLS * 2)] + (LZX_MAINCODE_MAX_NUM_SYMBOLS * 2)] _aligned_attribute(DECODE_TABLE_ALIGNMENT); - u8 maintree_lens[LZX_MAINCODE_NUM_SYMBOLS]; + u8 maintree_lens[LZX_MAINCODE_MAX_NUM_SYMBOLS]; u16 lentree_decode_table[(1 << LZX_LENCODE_TABLEBITS) + @@ -156,10 +153,11 @@ read_huffsym_using_pretree(struct input_bitstream *istream, static inline int read_huffsym_using_maintree(struct input_bitstream *istream, const struct lzx_tables *tables, - unsigned *n) + unsigned *n, + unsigned num_main_syms) { return read_huffsym(istream, tables->maintree_decode_table, - tables->maintree_lens, LZX_MAINCODE_NUM_SYMBOLS, + tables->maintree_lens, num_main_syms, LZX_MAINCODE_TABLEBITS, n, LZX_MAX_MAIN_CODEWORD_LEN); } @@ -192,10 +190,10 @@ read_huffsym_using_alignedtree(struct input_bitstream *istream, * code length values from the input. * * @istream: The bit stream for the input. It is positioned on the beginning - * of the pretree for the code length values. + * of the pretree for the code length values. * @lens: An array that contains the length values from the previous time - * the code lengths for this Huffman tree were read, or all - * 0's if this is the first time. + * the code lengths for this Huffman tree were read, or all + * 0's if this is the first time. * @num_lens: Number of length values to decode and return. * */ @@ -209,7 +207,7 @@ lzx_read_code_lens(struct input_bitstream *istream, u8 lens[], _aligned_attribute(DECODE_TABLE_ALIGNMENT); u8 pretree_lens[LZX_PRECODE_NUM_SYMBOLS]; unsigned i; - unsigned len; + u32 len; int ret; /* Read the code lengths of the pretree codes. There are 20 lengths of @@ -243,9 +241,9 @@ lzx_read_code_lens(struct input_bitstream *istream, u8 lens[], * the next lengths are all equal to the next symbol in the * input. */ unsigned tree_code; - unsigned num_zeroes; + u32 num_zeroes; unsigned code; - unsigned num_same; + u32 num_same; signed char value; ret = read_huffsym_using_pretree(istream, pretree_decode_table, @@ -312,18 +310,20 @@ lzx_read_code_lens(struct input_bitstream *istream, u8 lens[], * * @istream: The input bitstream. * @block_size_ret: A pointer to an int into which the size of the block, - * in bytes, will be returned. + * in bytes, will be returned. * @block_type_ret: A pointer to an int into which the type of the block - * (LZX_BLOCKTYPE_*) will be returned. + * (LZX_BLOCKTYPE_*) will be returned. * @tables: A pointer to a lzx_tables structure in which the - * main tree, the length tree, and possibly the - * aligned offset tree will be constructed. + * main tree, the length tree, and possibly the + * aligned offset tree will be constructed. * @queue: A pointer to the least-recently-used queue into which - * R0, R1, and R2 will be written (only for uncompressed - * blocks, which contain this information in the header) + * R0, R1, and R2 will be written (only for uncompressed + * blocks, which contain this information in the header) */ static int lzx_read_block_header(struct input_bitstream *istream, + unsigned num_main_syms, + unsigned max_window_size, unsigned *block_size_ret, unsigned *block_type_ret, struct lzx_tables *tables, @@ -332,32 +332,42 @@ lzx_read_block_header(struct input_bitstream *istream, int ret; unsigned block_type; unsigned block_size; - unsigned s; - unsigned i; - unsigned len; - ret = bitstream_ensure_bits(istream, LZX_BLOCKTYPE_NBITS + 1); - if (ret) { - LZX_DEBUG("LZX input stream overrun"); + ret = bitstream_ensure_bits(istream, 4); + if (ret) return ret; - } /* The first three bits tell us what kind of block it is, and are one * of the LZX_BLOCKTYPE_* values. */ - block_type = bitstream_read_bits_nocheck(istream, LZX_BLOCKTYPE_NBITS); - - /* The next bit indicates whether the block size is the default (32768), - * indicated by a 1 bit, or whether the block size is given by the next - * 16 bits, indicated by a 0 bit. */ - s = bitstream_read_bits_nocheck(istream, 1); + block_type = bitstream_read_bits_nocheck(istream, 3); - if (s) { - block_size = 32768; + /* Read the block size. This mirrors the behavior + * lzx_write_compressed_block() in lzx-compress.c; see that for more + * details. */ + if (bitstream_read_bits_nocheck(istream, 1)) { + block_size = LZX_DEFAULT_BLOCK_SIZE; } else { - ret = bitstream_read_bits(istream, LZX_BLOCKSIZE_NBITS, &block_size); + u32 tmp; + block_size = 0; + + ret = bitstream_read_bits(istream, 8, &tmp); + if (ret) + return ret; + block_size |= tmp; + + ret = bitstream_read_bits(istream, 8, &tmp); if (ret) return ret; - block_size = le16_to_cpu(block_size); + block_size <<= 8; + block_size |= tmp; + + if (max_window_size >= 65536) { + ret = bitstream_read_bits(istream, 8, &tmp); + if (ret) + return ret; + block_size <<= 8; + block_size |= tmp; + } } switch (block_type) { @@ -365,7 +375,9 @@ lzx_read_block_header(struct input_bitstream *istream, /* Read the path lengths for the elements of the aligned tree, * then build it. */ - for (i = 0; i < LZX_ALIGNEDCODE_NUM_SYMBOLS; i++) { + for (unsigned i = 0; i < LZX_ALIGNEDCODE_NUM_SYMBOLS; i++) { + u32 len; + ret = bitstream_read_bits(istream, LZX_ALIGNEDCODE_ELEMENT_SIZE, &len); @@ -408,10 +420,10 @@ lzx_read_block_header(struct input_bitstream *istream, * tree. */ LZX_DEBUG("Reading path lengths for remaining elements of " "main tree (%d elements).", - LZX_MAINCODE_NUM_SYMBOLS - LZX_NUM_CHARS); + num_main_syms - LZX_NUM_CHARS); ret = lzx_read_code_lens(istream, tables->maintree_lens + LZX_NUM_CHARS, - LZX_MAINCODE_NUM_SYMBOLS - LZX_NUM_CHARS); + num_main_syms - LZX_NUM_CHARS); if (ret) { LZX_DEBUG("Failed to read the path lengths for the " "remaining elements of the main tree"); @@ -422,7 +434,7 @@ lzx_read_block_header(struct input_bitstream *istream, "table for the main tree."); ret = make_huffman_decode_table(tables->maintree_decode_table, - LZX_MAINCODE_NUM_SYMBOLS, + num_main_syms, LZX_MAINCODE_TABLEBITS, tables->maintree_lens, LZX_MAX_MAIN_CODEWORD_LEN); @@ -479,9 +491,9 @@ lzx_read_block_header(struct input_bitstream *istream, istream->bitsleft = 0; istream->bitbuf = 0; } - queue->R[0] = le32_to_cpu(*(u32*)(istream->data + 0)); - queue->R[1] = le32_to_cpu(*(u32*)(istream->data + 4)); - queue->R[2] = le32_to_cpu(*(u32*)(istream->data + 8)); + queue->R[0] = le32_to_cpu(*(le32*)(istream->data + 0)); + queue->R[1] = le32_to_cpu(*(le32*)(istream->data + 4)); + queue->R[2] = le32_to_cpu(*(le32*)(istream->data + 8)); istream->data += 12; istream->data_bytes_left -= 12; /* The uncompressed data of this block directly follows and will @@ -506,14 +518,14 @@ lzx_read_block_header(struct input_bitstream *istream, * tree. * * @block_type: The type of the block (LZX_BLOCKTYPE_ALIGNED or - * LZX_BLOCKTYPE_VERBATIM) + * LZX_BLOCKTYPE_VERBATIM) * * @bytes_remaining: The amount of uncompressed data remaining to be - * uncompressed in this block. It is an error if the match - * is longer than this number. + * uncompressed in this block. It is an error if the match + * is longer than this number. * * @window: A pointer to the window into which the uncompressed - * data is being written. + * data is being written. * * @window_pos: The current byte offset in the window. * @@ -527,9 +539,9 @@ lzx_read_block_header(struct input_bitstream *istream, * * Returns the length of the match, or a negative number on error. The possible * error cases are: - * - Match would exceed the amount of data remaining to be uncompressed. - * - Match refers to data before the window. - * - The input bitstream ended unexpectedly. + * - Match would exceed the amount of data remaining to be uncompressed. + * - Match refers to data before the window. + * - The input bitstream ended unexpectedly. */ static int lzx_decode_match(unsigned main_element, int block_type, @@ -545,8 +557,8 @@ lzx_decode_match(unsigned main_element, int block_type, unsigned match_offset; unsigned additional_len; unsigned num_extra_bits; - unsigned verbatim_bits; - unsigned aligned_bits; + u32 verbatim_bits; + u32 aligned_bits; unsigned i; int ret; u8 *match_dest; @@ -752,18 +764,20 @@ undo_call_insn_preprocessing(u8 uncompressed_data[], int uncompressed_data_len) * been read. * * @block_type: The type of the block (LZX_BLOCKTYPE_VERBATIM or - * LZX_BLOCKTYPE_ALIGNED) + * LZX_BLOCKTYPE_ALIGNED) * @block_size: The size of the block, in bytes. + * @num_main_syms: Number of symbols in the main alphabet. * @window: Pointer to the decompression window. * @window_pos: The current position in the window. Will be 0 for the first - * block. + * block. * @tables: The Huffman decoding tables for the block (main, length, and - * aligned offset, the latter only for LZX_BLOCKTYPE_ALIGNED) + * aligned offset, the latter only for LZX_BLOCKTYPE_ALIGNED) * @queue: The least-recently-used queue for match offsets. * @istream: The input bitstream for the compressed literals. */ static int lzx_decompress_block(int block_type, unsigned block_size, + unsigned num_main_syms, u8 *window, unsigned window_pos, const struct lzx_tables *tables, @@ -778,7 +792,8 @@ lzx_decompress_block(int block_type, unsigned block_size, end = window_pos + block_size; while (window_pos < end) { ret = read_huffsym_using_maintree(istream, tables, - &main_element); + &main_element, + num_main_syms); if (ret) return ret; @@ -786,7 +801,7 @@ lzx_decompress_block(int block_type, unsigned block_size, /* literal: 0 to LZX_NUM_CHARS - 1 */ window[window_pos++] = main_element; } else { - /* match: LZX_NUM_CHARS to LZX_MAINCODE_NUM_SYMBOLS - 1 */ + /* match: LZX_NUM_CHARS to num_main_syms - 1 */ match_len = lzx_decode_match(main_element, block_type, end - window_pos, @@ -805,8 +820,9 @@ lzx_decompress_block(int block_type, unsigned block_size, /* API function documented in wimlib.h */ WIMLIBAPI int -wimlib_lzx_decompress(const void *compressed_data, unsigned compressed_len, - void *uncompressed_data, unsigned uncompressed_len) +wimlib_lzx_decompress2(const void *compressed_data, unsigned compressed_len, + void *uncompressed_data, unsigned uncompressed_len, + u32 max_window_size) { struct lzx_tables tables; struct input_bitstream istream; @@ -814,15 +830,30 @@ wimlib_lzx_decompress(const void *compressed_data, unsigned compressed_len, unsigned window_pos; unsigned block_size; unsigned block_type; + unsigned num_main_syms; int ret; bool e8_preprocessing_done; - LZX_DEBUG("lzx_decompress (compressed_data = %p, compressed_len = %d, " - "uncompressed_data = %p, uncompressed_len = %d).", + LZX_DEBUG("compressed_data = %p, compressed_len = %u, " + "uncompressed_data = %p, uncompressed_len = %u, " + "max_window_size=%u).", compressed_data, compressed_len, - uncompressed_data, uncompressed_len); + uncompressed_data, uncompressed_len, max_window_size); - wimlib_assert(uncompressed_len <= 32768); + if (!lzx_window_size_valid(max_window_size)) { + LZX_DEBUG("Window size of %u is invalid!", + max_window_size); + return -1; + } + + num_main_syms = lzx_get_num_main_syms(max_window_size); + + if (uncompressed_len > max_window_size) { + LZX_DEBUG("Uncompressed chunk size of %u exceeds " + "window size of %u!", + uncompressed_len, max_window_size); + return -1; + } memset(tables.maintree_lens, 0, sizeof(tables.maintree_lens)); memset(tables.lentree_lens, 0, sizeof(tables.lentree_lens)); @@ -842,7 +873,8 @@ wimlib_lzx_decompress(const void *compressed_data, unsigned compressed_len, window_pos += block_size) { LZX_DEBUG("Reading block header."); - ret = lzx_read_block_header(&istream, &block_size, + ret = lzx_read_block_header(&istream, num_main_syms, + max_window_size, &block_size, &block_type, &tables, &queue); if (ret) return ret; @@ -866,6 +898,7 @@ wimlib_lzx_decompress(const void *compressed_data, unsigned compressed_len, LZX_DEBUG("LZX_BLOCKTYPE_ALIGNED"); ret = lzx_decompress_block(block_type, block_size, + num_main_syms, uncompressed_data, window_pos, &tables, @@ -873,6 +906,7 @@ wimlib_lzx_decompress(const void *compressed_data, unsigned compressed_len, &istream); if (ret) return ret; + if (tables.maintree_lens[0xe8] != 0) e8_preprocessing_done = true; break; @@ -903,3 +937,13 @@ wimlib_lzx_decompress(const void *compressed_data, unsigned compressed_len, undo_call_insn_preprocessing(uncompressed_data, uncompressed_len); return 0; } + +/* API function documented in wimlib.h */ +WIMLIBAPI int +wimlib_lzx_decompress(const void *compressed_data, unsigned compressed_len, + void *uncompressed_data, unsigned uncompressed_len) +{ + return wimlib_lzx_decompress2(compressed_data, compressed_len, + uncompressed_data, uncompressed_len, + 32768); +}