4 * A very fast decompressor for LZX, as used in WIM files.
8 * Copyright (C) 2012, 2013, 2014 Eric Biggers
10 * This file is part of wimlib, a library for working with WIM files.
12 * wimlib is free software; you can redistribute it and/or modify it under the
13 * terms of the GNU General Public License as published by the Free
14 * Software Foundation; either version 3 of the License, or (at your option)
17 * wimlib is distributed in the hope that it will be useful, but WITHOUT ANY
18 * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
19 * A PARTICULAR PURPOSE. See the GNU General Public License for more
22 * You should have received a copy of the GNU General Public License
23 * along with wimlib; if not, see http://www.gnu.org/licenses/.
27 * LZX is an LZ77 and Huffman-code based compression format that has many
28 * similarities to DEFLATE (the format used by zlib/gzip). The compression
29 * ratio is as good or better than DEFLATE. See lzx-compress.c for a format
30 * overview, and see https://en.wikipedia.org/wiki/LZX_(algorithm) for a
31 * historical overview. Here I make some pragmatic notes.
33 * The old specification for LZX is the document "Microsoft LZX Data Compression
34 * Format" (1997). It defines the LZX format as used in cabinet files. Allowed
35 * window sizes are 2^n where 15 <= n <= 21. However, this document contains
36 * several errors, so don't read too much into it...
38 * The new specification for LZX is the document "[MS-PATCH]: LZX DELTA
39 * Compression and Decompression" (2014). It defines the LZX format as used by
40 * Microsoft's binary patcher. It corrects several errors in the 1997 document
41 * and extends the format in several ways --- namely, optional reference data,
42 * up to 2^25 byte windows, and longer match lengths.
44 * WIM files use a more restricted form of LZX. No LZX DELTA extensions are
45 * present, the window is not "sliding", E8 preprocessing is done
46 * unconditionally with a fixed file size, and the maximum window size is always
47 * 2^15 bytes (equal to the size of each "chunk" in a compressed WIM resource).
48 * This code is primarily intended to implement this form of LZX. But although
49 * not compatible with WIMGAPI, this code also supports maximum window sizes up
52 * TODO: Add support for window sizes up to 2^25 bytes.
59 #include "wimlib/decompressor_ops.h"
60 #include "wimlib/decompress_common.h"
61 #include "wimlib/error.h"
62 #include "wimlib/lzx.h"
63 #include "wimlib/util.h"
67 /* These values are chosen for fast decompression. */
68 #define LZX_MAINCODE_TABLEBITS 11
69 #define LZX_LENCODE_TABLEBITS 10
70 #define LZX_PRECODE_TABLEBITS 6
71 #define LZX_ALIGNEDCODE_TABLEBITS 7
73 /* Huffman decoding tables, and arrays that map symbols to codeword lengths. */
76 u16 maincode_decode_table[(1 << LZX_MAINCODE_TABLEBITS) +
77 (LZX_MAINCODE_MAX_NUM_SYMBOLS * 2)]
78 _aligned_attribute(DECODE_TABLE_ALIGNMENT);
79 u8 maincode_lens[LZX_MAINCODE_MAX_NUM_SYMBOLS];
82 u16 lencode_decode_table[(1 << LZX_LENCODE_TABLEBITS) +
83 (LZX_LENCODE_NUM_SYMBOLS * 2)]
84 _aligned_attribute(DECODE_TABLE_ALIGNMENT);
85 u8 lencode_lens[LZX_LENCODE_NUM_SYMBOLS];
88 u16 alignedcode_decode_table[(1 << LZX_ALIGNEDCODE_TABLEBITS) +
89 (LZX_ALIGNEDCODE_NUM_SYMBOLS * 2)]
90 _aligned_attribute(DECODE_TABLE_ALIGNMENT);
91 u8 alignedcode_lens[LZX_ALIGNEDCODE_NUM_SYMBOLS];
92 } _aligned_attribute(DECODE_TABLE_ALIGNMENT);
94 /* The main LZX decompressor structure.
96 * Note: we keep track of most of the decompression state outside this
97 * structure. This structure only exists so that (1) we can store
98 * @max_window_size and @num_main_syms for multiple calls to lzx_decompress();
99 * and (2) so that we don't have to allocate the large 'struct lzx_tables' on
101 struct lzx_decompressor {
103 unsigned num_main_syms;
104 struct lzx_tables tables;
107 /* Read a Huffman-encoded symbol using the precode. */
109 read_huffsym_using_precode(struct input_bitstream *istream,
110 const u16 precode_decode_table[])
112 return read_huffsym(istream, precode_decode_table,
113 LZX_PRECODE_TABLEBITS, LZX_MAX_PRE_CODEWORD_LEN);
116 /* Read a Huffman-encoded symbol using the main code. */
118 read_huffsym_using_maincode(struct input_bitstream *istream,
119 const struct lzx_tables *tables)
121 return read_huffsym(istream, tables->maincode_decode_table,
122 LZX_MAINCODE_TABLEBITS, LZX_MAX_MAIN_CODEWORD_LEN);
125 /* Read a Huffman-encoded symbol using the length code. */
127 read_huffsym_using_lencode(struct input_bitstream *istream,
128 const struct lzx_tables *tables)
130 return read_huffsym(istream, tables->lencode_decode_table,
131 LZX_LENCODE_TABLEBITS, LZX_MAX_LEN_CODEWORD_LEN);
134 /* Read a Huffman-encoded symbol using the aligned offset code. */
136 read_huffsym_using_alignedcode(struct input_bitstream *istream,
137 const struct lzx_tables *tables)
139 return read_huffsym(istream, tables->alignedcode_decode_table,
140 LZX_ALIGNEDCODE_TABLEBITS, LZX_MAX_ALIGNED_CODEWORD_LEN);
144 * Read the precode from the compressed input bitstream, then use it to decode
145 * @num_lens codeword length values.
148 * The input bitstream.
151 * An array that contains the length values from the previous time the
152 * codeword lengths for this Huffman code were read, or all 0's if this is
156 * Number of length values to decode.
158 * Returns 0 on success, or -1 if the data was invalid.
161 lzx_read_codeword_lens(struct input_bitstream *istream, u8 lens[], unsigned num_lens)
163 /* Declare the decoding table and length table for the precode. */
164 u16 precode_decode_table[(1 << LZX_PRECODE_TABLEBITS) +
165 (LZX_PRECODE_NUM_SYMBOLS * 2)]
166 _aligned_attribute(DECODE_TABLE_ALIGNMENT);
167 u8 precode_lens[LZX_PRECODE_NUM_SYMBOLS];
170 /* Read the lengths of the precode codewords. These are given
172 for (int i = 0; i < LZX_PRECODE_NUM_SYMBOLS; i++) {
173 precode_lens[i] = bitstream_read_bits(istream,
174 LZX_PRECODE_ELEMENT_SIZE);
177 /* Make the decoding table for the precode. */
178 ret = make_huffman_decode_table(precode_decode_table,
179 LZX_PRECODE_NUM_SYMBOLS,
180 LZX_PRECODE_TABLEBITS,
182 LZX_MAX_PRE_CODEWORD_LEN);
186 /* Pointer past the last length value that needs to be filled in. */
187 u8 *lens_end = lens + num_lens;
195 /* Decode a symbol from the input.
197 * If the symbol is between 0 and 16, it is the difference from
198 * the old length, modulo 17.
200 * If the symbol is between 17 and 19, it is a special symbol
201 * that indicates that some number of the next lengths are all
202 * 0, or that some number of the next lengths are all equal to
203 * the next symbol. */
205 presym = read_huffsym_using_precode(istream,
206 precode_decode_table);
209 case 17: /* Run of 0's */
210 run_len = 4 + bitstream_read_bits(istream, 4);
213 if (++lens == lens_end)
218 case 18: /* Longer run of 0's */
219 run_len = 20 + bitstream_read_bits(istream, 5);
222 if (++lens == lens_end)
227 case 19: /* Run of identical lengths */
228 run_len = 4 + bitstream_read_bits(istream, 1);
229 presym = read_huffsym_using_precode(istream,
230 precode_decode_table);
231 value = (signed char)*lens - (signed char)presym;
236 if (++lens == lens_end)
241 default: /* Difference from old length */
242 value = (signed char)*lens - (signed char)presym;
246 if (++lens == lens_end)
254 * Read the header of an LZX block and save the block type and size in
255 * *block_type_ret and *block_size_ret, respectively.
257 * If the block is compressed, also update the Huffman decode @tables with the
260 * If the block is uncompressed, also update the match offset @queue with the
263 * Return 0 on success, or -1 if the data was invalid.
266 lzx_read_block_header(struct input_bitstream *istream,
267 unsigned num_main_syms,
271 struct lzx_tables *tables,
272 struct lzx_lru_queue *queue)
278 bitstream_ensure_bits(istream, 4);
280 /* The first three bits tell us what kind of block it is, and should be
281 * one of the LZX_BLOCKTYPE_* values. */
282 block_type = bitstream_pop_bits(istream, 3);
284 /* Read the block size. This mirrors the behavior of
285 * lzx_write_compressed_block() in lzx-compress.c; see that for more
287 if (bitstream_pop_bits(istream, 1)) {
288 block_size = LZX_DEFAULT_BLOCK_SIZE;
293 tmp = bitstream_read_bits(istream, 8);
295 tmp = bitstream_read_bits(istream, 8);
299 if (max_window_size >= 65536) {
300 tmp = bitstream_read_bits(istream, 8);
306 switch (block_type) {
308 case LZX_BLOCKTYPE_ALIGNED:
310 /* Read the aligned offset code and prepare its decode table.
313 for (int i = 0; i < LZX_ALIGNEDCODE_NUM_SYMBOLS; i++) {
314 tables->alignedcode_lens[i] =
315 bitstream_read_bits(istream,
316 LZX_ALIGNEDCODE_ELEMENT_SIZE);
319 ret = make_huffman_decode_table(tables->alignedcode_decode_table,
320 LZX_ALIGNEDCODE_NUM_SYMBOLS,
321 LZX_ALIGNEDCODE_TABLEBITS,
322 tables->alignedcode_lens,
323 LZX_MAX_ALIGNED_CODEWORD_LEN);
327 /* Fall though, since the rest of the header for aligned offset
328 * blocks is the same as that for verbatim blocks. */
330 case LZX_BLOCKTYPE_VERBATIM:
332 /* Read the main code and prepare its decode table.
334 * Note that the codeword lengths in the main code are encoded
335 * in two parts: one part for literal symbols, and one part for
338 ret = lzx_read_codeword_lens(istream, tables->maincode_lens,
343 ret = lzx_read_codeword_lens(istream,
344 tables->maincode_lens + LZX_NUM_CHARS,
345 num_main_syms - LZX_NUM_CHARS);
349 ret = make_huffman_decode_table(tables->maincode_decode_table,
351 LZX_MAINCODE_TABLEBITS,
352 tables->maincode_lens,
353 LZX_MAX_MAIN_CODEWORD_LEN);
357 /* Read the length code and prepare its decode table. */
359 ret = lzx_read_codeword_lens(istream, tables->lencode_lens,
360 LZX_LENCODE_NUM_SYMBOLS);
364 ret = make_huffman_decode_table(tables->lencode_decode_table,
365 LZX_LENCODE_NUM_SYMBOLS,
366 LZX_LENCODE_TABLEBITS,
367 tables->lencode_lens,
368 LZX_MAX_LEN_CODEWORD_LEN);
374 case LZX_BLOCKTYPE_UNCOMPRESSED:
376 /* Before reading the three LRU match offsets from the
377 * uncompressed block header, the stream must be aligned on a
378 * 16-bit boundary. But, unexpectedly, if the stream is
379 * *already* aligned, the correct thing to do is to throw away
380 * the next 16 bits. */
382 if (istream->bitsleft == 0) {
383 if (istream->data_bytes_left < 14)
386 istream->data_bytes_left -= 2;
388 if (istream->data_bytes_left < 12)
390 istream->bitsleft = 0;
393 queue->R[0] = le32_to_cpu(*(le32*)(istream->data + 0));
394 queue->R[1] = le32_to_cpu(*(le32*)(istream->data + 4));
395 queue->R[2] = le32_to_cpu(*(le32*)(istream->data + 8));
397 istream->data_bytes_left -= 12;
401 /* Unrecognized block type. */
405 *block_type_ret = block_type;
406 *block_size_ret = block_size;
411 * Decode a match and copy its bytes into the decompression window.
413 * Return the length of the match in bytes, or 0 if the match underflowed the
414 * window or overflowed the current block.
417 lzx_decode_match(unsigned main_symbol, int block_type,
418 u32 bytes_remaining, u8 *window, u32 window_pos,
419 const struct lzx_tables *tables,
420 struct lzx_lru_queue *queue,
421 struct input_bitstream *istream)
423 unsigned length_header;
424 unsigned position_slot;
427 unsigned num_extra_bits;
431 /* The main symbol is offset by 256 because values under 256 indicate a
433 main_symbol -= LZX_NUM_CHARS;
435 /* The length header consists of the lower 3 bits of the main element.
436 * The position slot is the rest of it. */
437 length_header = main_symbol & LZX_NUM_PRIMARY_LENS;
438 position_slot = main_symbol >> 3;
440 /* If the length_header is less than LZX_NUM_PRIMARY_LENS (= 7), it
441 * gives the match length as the offset from LZX_MIN_MATCH_LEN.
442 * Otherwise, the length is given by an additional symbol encoded using
443 * the length code, offset by 9 (LZX_MIN_MATCH_LEN +
444 * LZX_NUM_PRIMARY_LENS) */
445 match_len = LZX_MIN_MATCH_LEN + length_header;
446 if (length_header == LZX_NUM_PRIMARY_LENS)
447 match_len += read_huffsym_using_lencode(istream, tables);
449 /* If the position_slot is 0, 1, or 2, the match offset is retrieved
450 * from the LRU queue. Otherwise, the match offset is not in the LRU
452 if (position_slot <= 2) {
453 /* Note: This isn't a real LRU queue, since using the R2 offset
454 * doesn't bump the R1 offset down to R2. This quirk allows all
455 * 3 recent offsets to be handled by the same code. (For R0,
456 * the swap is a no-op.) */
457 match_offset = queue->R[position_slot];
458 queue->R[position_slot] = queue->R[0];
459 queue->R[0] = match_offset;
461 /* Otherwise, the offset was not encoded as one the offsets in
462 * the queue. Depending on the position slot, there is a
463 * certain number of extra bits that need to be read to fully
464 * decode the match offset. */
466 /* Look up the number of extra bits that need to be read. */
467 num_extra_bits = lzx_get_num_extra_bits(position_slot);
469 /* For aligned blocks, if there are at least 3 extra bits, the
470 * actual number of extra bits is 3 less, and they encode a
471 * number of 8-byte words that are added to the offset; there
472 * is then an additional symbol read using the aligned offset
473 * code that specifies the actual byte alignment. */
474 if (block_type == LZX_BLOCKTYPE_ALIGNED && num_extra_bits >= 3) {
476 /* There is an error in the LZX "specification" at this
477 * point; it indicates that a Huffman symbol is to be
478 * read only if num_extra_bits is greater than 3, but
479 * actually it is if num_extra_bits is greater than or
480 * equal to 3. (Note that in the case with
481 * num_extra_bits == 3, the assignment to verbatim_bits
482 * will just set it to 0. ) */
483 verbatim_bits = bitstream_read_bits(istream,
486 aligned_bits = read_huffsym_using_alignedcode(istream,
489 /* For non-aligned blocks, or for aligned blocks with
490 * less than 3 extra bits, the extra bits are added
491 * directly to the match offset, and the correction for
492 * the alignment is taken to be 0. */
493 verbatim_bits = bitstream_read_bits(istream, num_extra_bits);
497 /* Calculate the match offset. */
498 match_offset = lzx_position_base[position_slot] +
499 verbatim_bits + aligned_bits - LZX_OFFSET_OFFSET;
501 /* Update the LRU queue. */
502 queue->R[2] = queue->R[1];
503 queue->R[1] = queue->R[0];
504 queue->R[0] = match_offset;
507 /* Validate the match, then copy it to the current position. */
509 if (unlikely(match_len > bytes_remaining))
512 if (unlikely(match_offset > window_pos))
515 lz_copy(&window[window_pos], match_len, match_offset,
516 &window[window_pos + bytes_remaining]);
522 * Decompress an LZX-compressed block of data.
525 * The type of the block (LZX_BLOCKTYPE_VERBATIM or LZX_BLOCKTYPE_ALIGNED).
528 * The size of the block, in bytes.
531 * Pointer to the beginning of the decompression window.
534 * The position in the window at which the block starts.
537 * The Huffman decoding tables for the block.
540 * The least-recently-used queue for match offsets.
543 * The input bitstream, positioned at the start of the block data.
545 * Returns 0 on success, or -1 if the data was invalid.
548 lzx_decompress_block(int block_type, u32 block_size,
549 u8 *window, u32 window_pos,
550 const struct lzx_tables *tables,
551 struct lzx_lru_queue *queue,
552 struct input_bitstream *istream)
555 unsigned main_symbol;
558 block_end = window_pos + block_size;
559 while (window_pos < block_end) {
560 main_symbol = read_huffsym_using_maincode(istream, tables);
561 if (main_symbol < LZX_NUM_CHARS) {
563 window[window_pos++] = main_symbol;
566 match_len = lzx_decode_match(main_symbol,
568 block_end - window_pos,
574 if (unlikely(match_len == 0))
576 window_pos += match_len;
583 lzx_decompress(const void *compressed_data, size_t compressed_size,
584 void *uncompressed_data, size_t uncompressed_size,
587 struct lzx_decompressor *dec = _dec;
588 struct input_bitstream istream;
589 struct lzx_lru_queue queue;
593 bool may_have_e8_byte;
596 if (uncompressed_size > dec->max_window_size)
599 init_input_bitstream(&istream, compressed_data, compressed_size);
601 /* Initialize the recent offsets queue. */
602 lzx_lru_queue_init(&queue);
604 /* Codeword lengths begin as all 0's for delta encoding purposes. */
605 memset(dec->tables.maincode_lens, 0, sizeof(dec->tables.maincode_lens));
606 memset(dec->tables.lencode_lens, 0, sizeof(dec->tables.lencode_lens));
608 /* Set this to true if there may be 0xe8 bytes in the uncompressed data.
610 may_have_e8_byte = false;
612 /* The compressed data will consist of one or more blocks. The
613 * following loop decompresses one block, and it runs until there all
614 * the compressed data has been decompressed, so there are no more
618 window_pos < uncompressed_size;
619 window_pos += block_size)
621 ret = lzx_read_block_header(&istream, dec->num_main_syms,
622 dec->max_window_size, &block_type,
623 &block_size, &dec->tables, &queue);
627 if (block_size > uncompressed_size - window_pos)
630 if (block_type != LZX_BLOCKTYPE_UNCOMPRESSED) {
632 /* Compressed block. */
634 ret = lzx_decompress_block(block_type,
644 /* If the first 0xe8 byte was in this block, it must
645 * have been encoded as a literal using mainsym 0xe8. */
646 if (dec->tables.maincode_lens[0xe8] != 0)
647 may_have_e8_byte = true;
650 /* Uncompressed block. */
652 if (istream.data_bytes_left < block_size)
655 memcpy(&((u8*)uncompressed_data)[window_pos], istream.data,
657 istream.data += block_size;
658 istream.data_bytes_left -= block_size;
660 /* Re-align the bitstream if an odd number of bytes was
662 if (istream.data_bytes_left && (block_size & 1)) {
663 istream.data_bytes_left--;
666 may_have_e8_byte = true;
670 /* Postprocess the data unless it cannot possibly contain 0xe8 bytes */
671 if (may_have_e8_byte)
672 lzx_undo_e8_preprocessing(uncompressed_data, uncompressed_size);
678 lzx_free_decompressor(void *_dec)
680 struct lzx_decompressor *dec = _dec;
686 lzx_create_decompressor(size_t max_window_size, void **dec_ret)
688 struct lzx_decompressor *dec;
690 if (!lzx_window_size_valid(max_window_size))
691 return WIMLIB_ERR_INVALID_PARAM;
693 /* The aligned allocation is needed to ensure that the lzx_tables are
694 * aligned properly. */
695 dec = ALIGNED_MALLOC(sizeof(struct lzx_decompressor),
696 DECODE_TABLE_ALIGNMENT);
698 return WIMLIB_ERR_NOMEM;
700 dec->max_window_size = max_window_size;
701 dec->num_main_syms = lzx_get_num_main_syms(max_window_size);
707 const struct decompressor_ops lzx_decompressor_ops = {
708 .create_decompressor = lzx_create_decompressor,
709 .decompress = lzx_decompress,
710 .free_decompressor = lzx_free_decompressor,