4 * A decompressor for the LZX compression format, as used in WIM files.
8 * Copyright (C) 2012, 2013, 2014, 2015 Eric Biggers
10 * This file is free software; you can redistribute it and/or modify it under
11 * the terms of the GNU Lesser General Public License as published by the Free
12 * Software Foundation; either version 3 of the License, or (at your option) any
15 * This file is distributed in the hope that it will be useful, but WITHOUT
16 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
17 * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
20 * You should have received a copy of the GNU Lesser General Public License
21 * along with this file; if not, see http://www.gnu.org/licenses/.
25 * LZX is an LZ77 and Huffman-code based compression format that has many
26 * similarities to DEFLATE (the format used by zlib/gzip). The compression
27 * ratio is as good or better than DEFLATE. See lzx_compress.c for a format
28 * overview, and see https://en.wikipedia.org/wiki/LZX_(algorithm) for a
29 * historical overview. Here I make some pragmatic notes.
31 * The old specification for LZX is the document "Microsoft LZX Data Compression
32 * Format" (1997). It defines the LZX format as used in cabinet files. Allowed
33 * window sizes are 2^n where 15 <= n <= 21. However, this document contains
34 * several errors, so don't read too much into it...
36 * The new specification for LZX is the document "[MS-PATCH]: LZX DELTA
37 * Compression and Decompression" (2014). It defines the LZX format as used by
38 * Microsoft's binary patcher. It corrects several errors in the 1997 document
39 * and extends the format in several ways --- namely, optional reference data,
40 * up to 2^25 byte windows, and longer match lengths.
42 * WIM files use a more restricted form of LZX. No LZX DELTA extensions are
43 * present, the window is not "sliding", E8 preprocessing is done
44 * unconditionally with a fixed file size, and the maximum window size is always
45 * 2^15 bytes (equal to the size of each "chunk" in a compressed WIM resource).
46 * This code is primarily intended to implement this form of LZX. But although
47 * not compatible with WIMGAPI, this code also supports maximum window sizes up
50 * TODO: Add support for window sizes up to 2^25 bytes.
59 #include "wimlib/decompressor_ops.h"
60 #include "wimlib/decompress_common.h"
61 #include "wimlib/error.h"
62 #include "wimlib/lzx_common.h"
63 #include "wimlib/util.h"
65 /* These values are chosen for fast decompression. */
66 #define LZX_MAINCODE_TABLEBITS 11
67 #define LZX_LENCODE_TABLEBITS 10
68 #define LZX_PRECODE_TABLEBITS 6
69 #define LZX_ALIGNEDCODE_TABLEBITS 7
71 #define LZX_READ_LENS_MAX_OVERRUN 50
73 /* Huffman decoding tables, and arrays that map symbols to codeword lengths. */
76 u16 maincode_decode_table[(1 << LZX_MAINCODE_TABLEBITS) +
77 (LZX_MAINCODE_MAX_NUM_SYMBOLS * 2)]
78 _aligned_attribute(DECODE_TABLE_ALIGNMENT);
79 u8 maincode_lens[LZX_MAINCODE_MAX_NUM_SYMBOLS + LZX_READ_LENS_MAX_OVERRUN];
82 u16 lencode_decode_table[(1 << LZX_LENCODE_TABLEBITS) +
83 (LZX_LENCODE_NUM_SYMBOLS * 2)]
84 _aligned_attribute(DECODE_TABLE_ALIGNMENT);
85 u8 lencode_lens[LZX_LENCODE_NUM_SYMBOLS + LZX_READ_LENS_MAX_OVERRUN];
88 u16 alignedcode_decode_table[(1 << LZX_ALIGNEDCODE_TABLEBITS) +
89 (LZX_ALIGNEDCODE_NUM_SYMBOLS * 2)]
90 _aligned_attribute(DECODE_TABLE_ALIGNMENT);
91 u8 alignedcode_lens[LZX_ALIGNEDCODE_NUM_SYMBOLS];
92 } _aligned_attribute(DECODE_TABLE_ALIGNMENT);
94 /* Least-recently used queue for match offsets. */
95 struct lzx_lru_queue {
96 u32 R[LZX_NUM_RECENT_OFFSETS];
100 lzx_lru_queue_init(struct lzx_lru_queue *queue)
102 for (unsigned i = 0; i < LZX_NUM_RECENT_OFFSETS; i++)
106 /* The main LZX decompressor structure.
108 * Note: we keep track of most of the decompression state outside this
109 * structure. This structure only exists so that (1) we can store @window_order
110 * and @num_main_syms for multiple calls to lzx_decompress(); and (2) so that we
111 * don't have to allocate the large 'struct lzx_tables' on the stack. */
112 struct lzx_decompressor {
113 unsigned window_order;
114 unsigned num_main_syms;
115 struct lzx_tables tables;
118 /* Read a Huffman-encoded symbol using the precode. */
119 static inline unsigned
120 read_huffsym_using_precode(struct input_bitstream *istream,
121 const u16 precode_decode_table[])
123 return read_huffsym(istream, precode_decode_table,
124 LZX_PRECODE_TABLEBITS, LZX_MAX_PRE_CODEWORD_LEN);
127 /* Read a Huffman-encoded symbol using the main code. */
128 static inline unsigned
129 read_huffsym_using_maincode(struct input_bitstream *istream,
130 const struct lzx_tables *tables)
132 return read_huffsym(istream, tables->maincode_decode_table,
133 LZX_MAINCODE_TABLEBITS, LZX_MAX_MAIN_CODEWORD_LEN);
136 /* Read a Huffman-encoded symbol using the length code. */
137 static inline unsigned
138 read_huffsym_using_lencode(struct input_bitstream *istream,
139 const struct lzx_tables *tables)
141 return read_huffsym(istream, tables->lencode_decode_table,
142 LZX_LENCODE_TABLEBITS, LZX_MAX_LEN_CODEWORD_LEN);
145 /* Read a Huffman-encoded symbol using the aligned offset code. */
146 static inline unsigned
147 read_huffsym_using_alignedcode(struct input_bitstream *istream,
148 const struct lzx_tables *tables)
150 return read_huffsym(istream, tables->alignedcode_decode_table,
151 LZX_ALIGNEDCODE_TABLEBITS, LZX_MAX_ALIGNED_CODEWORD_LEN);
155 * Read the precode from the compressed input bitstream, then use it to decode
156 * @num_lens codeword length values.
159 * The input bitstream.
162 * An array that contains the length values from the previous time the
163 * codeword lengths for this Huffman code were read, or all 0's if this is
164 * the first time. This array must have at least (@num_lens +
165 * LZX_READ_LENS_MAX_OVERRUN) entries.
168 * Number of length values to decode.
170 * Returns 0 on success, or -1 if the data was invalid.
173 lzx_read_codeword_lens(struct input_bitstream *istream, u8 *lens, unsigned num_lens)
175 u16 precode_decode_table[(1 << LZX_PRECODE_TABLEBITS) +
176 (LZX_PRECODE_NUM_SYMBOLS * 2)]
177 _aligned_attribute(DECODE_TABLE_ALIGNMENT);
178 u8 precode_lens[LZX_PRECODE_NUM_SYMBOLS];
180 u8 *lens_end = lens + num_lens;
183 /* Read the lengths of the precode codewords. These are given
185 for (int i = 0; i < LZX_PRECODE_NUM_SYMBOLS; i++) {
186 precode_lens[i] = bitstream_read_bits(istream,
187 LZX_PRECODE_ELEMENT_SIZE);
190 /* Make the decoding table for the precode. */
191 ret = make_huffman_decode_table(precode_decode_table,
192 LZX_PRECODE_NUM_SYMBOLS,
193 LZX_PRECODE_TABLEBITS,
195 LZX_MAX_PRE_CODEWORD_LEN);
199 /* Decode the codeword lengths. */
204 /* Read the next precode symbol. */
205 presym = read_huffsym_using_precode(istream,
206 precode_decode_table);
208 /* Difference from old length */
209 len = *len_ptr - presym;
214 /* Special RLE values */
220 run_len = 4 + bitstream_read_bits(istream, 4);
222 } else if (presym == 18) {
223 /* Longer run of 0's */
224 run_len = 20 + bitstream_read_bits(istream, 5);
227 /* Run of identical lengths */
228 run_len = 4 + bitstream_read_bits(istream, 1);
229 presym = read_huffsym_using_precode(istream,
230 precode_decode_table);
231 if (unlikely(presym > 17))
233 len = *len_ptr - presym;
241 /* Worst case overrun is when presym == 18,
242 * run_len == 20 + 31, and only 1 length was remaining.
243 * So LZX_READ_LENS_MAX_OVERRUN == 50.
245 * Overrun while reading the first half of maincode_lens
246 * can corrupt the previous values in the second half.
247 * This doesn't really matter because the resulting
248 * lengths will still be in range, and data that
249 * generates overruns is invalid anyway. */
251 } while (len_ptr < lens_end);
256 * Read the header of an LZX block and save the block type and size in
257 * *block_type_ret and *block_size_ret, respectively.
259 * If the block is compressed, also update the Huffman decode @tables with the
262 * If the block is uncompressed, also update the match offset @queue with the
265 * Return 0 on success, or -1 if the data was invalid.
268 lzx_read_block_header(struct input_bitstream *istream,
269 unsigned num_main_syms,
270 unsigned window_order,
273 struct lzx_tables *tables,
274 struct lzx_lru_queue *queue)
280 bitstream_ensure_bits(istream, 4);
282 /* The first three bits tell us what kind of block it is, and should be
283 * one of the LZX_BLOCKTYPE_* values. */
284 block_type = bitstream_pop_bits(istream, 3);
286 /* Read the block size. This mirrors the behavior of
287 * lzx_write_compressed_block() in lzx_compress.c; see that for more
289 if (bitstream_pop_bits(istream, 1)) {
290 block_size = LZX_DEFAULT_BLOCK_SIZE;
295 tmp = bitstream_read_bits(istream, 8);
297 tmp = bitstream_read_bits(istream, 8);
301 if (window_order >= 16) {
302 tmp = bitstream_read_bits(istream, 8);
308 switch (block_type) {
310 case LZX_BLOCKTYPE_ALIGNED:
312 /* Read the aligned offset code and prepare its decode table.
315 for (int i = 0; i < LZX_ALIGNEDCODE_NUM_SYMBOLS; i++) {
316 tables->alignedcode_lens[i] =
317 bitstream_read_bits(istream,
318 LZX_ALIGNEDCODE_ELEMENT_SIZE);
321 ret = make_huffman_decode_table(tables->alignedcode_decode_table,
322 LZX_ALIGNEDCODE_NUM_SYMBOLS,
323 LZX_ALIGNEDCODE_TABLEBITS,
324 tables->alignedcode_lens,
325 LZX_MAX_ALIGNED_CODEWORD_LEN);
329 /* Fall though, since the rest of the header for aligned offset
330 * blocks is the same as that for verbatim blocks. */
332 case LZX_BLOCKTYPE_VERBATIM:
334 /* Read the main code and prepare its decode table.
336 * Note that the codeword lengths in the main code are encoded
337 * in two parts: one part for literal symbols, and one part for
340 ret = lzx_read_codeword_lens(istream, tables->maincode_lens,
345 ret = lzx_read_codeword_lens(istream,
346 tables->maincode_lens + LZX_NUM_CHARS,
347 num_main_syms - LZX_NUM_CHARS);
351 ret = make_huffman_decode_table(tables->maincode_decode_table,
353 LZX_MAINCODE_TABLEBITS,
354 tables->maincode_lens,
355 LZX_MAX_MAIN_CODEWORD_LEN);
359 /* Read the length code and prepare its decode table. */
361 ret = lzx_read_codeword_lens(istream, tables->lencode_lens,
362 LZX_LENCODE_NUM_SYMBOLS);
366 ret = make_huffman_decode_table(tables->lencode_decode_table,
367 LZX_LENCODE_NUM_SYMBOLS,
368 LZX_LENCODE_TABLEBITS,
369 tables->lencode_lens,
370 LZX_MAX_LEN_CODEWORD_LEN);
376 case LZX_BLOCKTYPE_UNCOMPRESSED:
378 /* Before reading the three LRU match offsets from the
379 * uncompressed block header, the stream must be aligned on a
380 * 16-bit boundary. But, unexpectedly, if the stream is
381 * *already* aligned, the correct thing to do is to throw away
382 * the next 16 bits. */
384 bitstream_ensure_bits(istream, 1);
385 bitstream_align(istream);
386 queue->R[0] = bitstream_read_u32(istream);
387 queue->R[1] = bitstream_read_u32(istream);
388 queue->R[2] = bitstream_read_u32(istream);
390 /* Offsets of 0 are invalid. */
391 if (queue->R[0] == 0 || queue->R[1] == 0 || queue->R[2] == 0)
396 /* Unrecognized block type. */
400 *block_type_ret = block_type;
401 *block_size_ret = block_size;
406 * Decompress a block of LZX-compressed data.
409 * The type of the block (LZX_BLOCKTYPE_VERBATIM or LZX_BLOCKTYPE_ALIGNED).
411 * The beginning of the (uncompressed) output buffer.
413 * Pointer to the location in the (uncompressed) output buffer at which
414 * this block will start.
416 * Pointer to the location in the (uncompressed) output buffer at which
417 * this block will end.
419 * The Huffman decoding tables for this block.
421 * The least-recently-used queue for match offsets.
423 * The input bitstream, positioned at the start of the block data.
425 * Returns 0 on success, or -1 if the data was invalid.
428 lzx_decompress_block(int block_type, u8 * const out_begin,
429 u8 * out_next, u8 * const out_block_end,
430 const struct lzx_tables *tables,
431 struct lzx_lru_queue *queue,
432 struct input_bitstream *istream)
436 unsigned offset_slot;
438 unsigned num_extra_bits;
439 unsigned ones_if_aligned = 0U - (block_type == LZX_BLOCKTYPE_ALIGNED);
441 while (out_next != out_block_end) {
443 mainsym = read_huffsym_using_maincode(istream, tables);
444 if (mainsym < LZX_NUM_CHARS) {
446 *out_next++ = mainsym;
452 /* Decode the length header and offset slot. */
453 mainsym -= LZX_NUM_CHARS;
454 match_len = mainsym % LZX_NUM_LEN_HEADERS;
455 offset_slot = mainsym / LZX_NUM_LEN_HEADERS;
457 /* If needed, read a length symbol to decode the full length. */
458 if (match_len == LZX_NUM_PRIMARY_LENS)
459 match_len += read_huffsym_using_lencode(istream, tables);
460 match_len += LZX_MIN_MATCH_LEN;
462 if (offset_slot < LZX_NUM_RECENT_OFFSETS) {
465 /* Note: This isn't a real LRU queue, since using the R2
466 * offset doesn't bump the R1 offset down to R2. This
467 * quirk allows all 3 recent offsets to be handled by
468 * the same code. (For R0, the swap is a no-op.) */
469 match_offset = queue->R[offset_slot];
470 queue->R[offset_slot] = queue->R[0];
471 queue->R[0] = match_offset;
473 /* Explicit offset */
475 /* Look up the number of extra bits that need to be read
476 * to decode offsets with this offset slot. */
477 num_extra_bits = lzx_extra_offset_bits[offset_slot];
479 /* Start with the offset slot base value. */
480 match_offset = lzx_offset_slot_base[offset_slot];
482 /* In aligned offset blocks, the low-order 3 bits of
483 * each offset are encoded using the aligned offset
484 * code. Otherwise, all the extra bits are literal. */
486 if ((num_extra_bits & ones_if_aligned) >= LZX_NUM_ALIGNED_OFFSET_BITS) {
488 bitstream_read_bits(istream,
490 LZX_NUM_ALIGNED_OFFSET_BITS)
491 << LZX_NUM_ALIGNED_OFFSET_BITS;
492 match_offset += read_huffsym_using_alignedcode(istream, tables);
494 match_offset += bitstream_read_bits(istream, num_extra_bits);
497 /* Adjust the offset. */
498 match_offset -= LZX_OFFSET_ADJUSTMENT;
500 /* Update the match offset LRU queue. */
501 BUILD_BUG_ON(LZX_NUM_RECENT_OFFSETS != 3);
502 queue->R[2] = queue->R[1];
503 queue->R[1] = queue->R[0];
504 queue->R[0] = match_offset;
507 /* Validate the match, then copy it to the current position. */
509 if (unlikely(match_len > out_block_end - out_next))
512 if (unlikely(match_offset > out_next - out_begin))
515 lz_copy(out_next, match_len, match_offset, out_block_end,
518 out_next += match_len;
524 lzx_decompress(const void *restrict compressed_data, size_t compressed_size,
525 void *restrict uncompressed_data, size_t uncompressed_size,
528 struct lzx_decompressor *dec = _dec;
529 struct input_bitstream istream;
530 struct lzx_lru_queue queue;
531 u8 * const out_begin = uncompressed_data;
532 u8 *out_next = out_begin;
533 u8 * const out_end = out_begin + uncompressed_size;
536 bool may_have_e8_byte;
539 init_input_bitstream(&istream, compressed_data, compressed_size);
541 /* Initialize the recent offsets queue. */
542 lzx_lru_queue_init(&queue);
544 /* Codeword lengths begin as all 0's for delta encoding purposes. */
545 memset(dec->tables.maincode_lens, 0, dec->num_main_syms);
546 memset(dec->tables.lencode_lens, 0, LZX_LENCODE_NUM_SYMBOLS);
548 /* Set this to true if there may be 0xe8 bytes in the uncompressed data.
550 may_have_e8_byte = false;
552 /* The compressed data will consist of one or more blocks. The
553 * following loop decompresses one block, and it runs until there all
554 * the compressed data has been decompressed, so there are no more
557 while (out_next != out_end) {
559 ret = lzx_read_block_header(&istream, dec->num_main_syms,
560 dec->window_order, &block_type,
561 &block_size, &dec->tables, &queue);
565 if (block_size > out_end - out_next)
568 if (block_type != LZX_BLOCKTYPE_UNCOMPRESSED) {
570 /* Compressed block. */
572 ret = lzx_decompress_block(block_type,
575 out_next + block_size,
582 /* If the first 0xe8 byte was in this block, it must
583 * have been encoded as a literal using mainsym 0xe8. */
584 if (dec->tables.maincode_lens[0xe8] != 0)
585 may_have_e8_byte = true;
587 out_next += block_size;
590 /* Uncompressed block. */
591 out_next = bitstream_read_bytes(&istream, out_next, block_size);
595 /* Re-align the bitstream if an odd number of bytes was
598 bitstream_read_byte(&istream);
600 may_have_e8_byte = true;
604 /* Postprocess the data unless it cannot possibly contain 0xe8 bytes */
605 if (may_have_e8_byte)
606 lzx_undo_e8_preprocessing(uncompressed_data, uncompressed_size);
612 lzx_free_decompressor(void *_dec)
614 struct lzx_decompressor *dec = _dec;
620 lzx_create_decompressor(size_t max_block_size, void **dec_ret)
622 struct lzx_decompressor *dec;
623 unsigned window_order;
625 window_order = lzx_get_window_order(max_block_size);
626 if (window_order == 0)
627 return WIMLIB_ERR_INVALID_PARAM;
629 /* The aligned allocation is needed to ensure that the lzx_tables are
630 * aligned properly. */
631 dec = ALIGNED_MALLOC(sizeof(struct lzx_decompressor),
632 DECODE_TABLE_ALIGNMENT);
634 return WIMLIB_ERR_NOMEM;
636 dec->window_order = window_order;
637 dec->num_main_syms = lzx_get_num_main_syms(window_order);
643 const struct decompressor_ops lzx_decompressor_ops = {
644 .create_decompressor = lzx_create_decompressor,
645 .decompress = lzx_decompress,
646 .free_decompressor = lzx_free_decompressor,