4 * A very fast decompressor for LZX, as used in WIM files.
8 * Copyright (C) 2012, 2013, 2014 Eric Biggers
10 * This file is part of wimlib, a library for working with WIM files.
12 * wimlib is free software; you can redistribute it and/or modify it under the
13 * terms of the GNU General Public License as published by the Free
14 * Software Foundation; either version 3 of the License, or (at your option)
17 * wimlib is distributed in the hope that it will be useful, but WITHOUT ANY
18 * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
19 * A PARTICULAR PURPOSE. See the GNU General Public License for more
22 * You should have received a copy of the GNU General Public License
23 * along with wimlib; if not, see http://www.gnu.org/licenses/.
27 * LZX is an LZ77 and Huffman-code based compression format that has many
28 * similarities to DEFLATE (the format used by zlib/gzip). The compression
29 * ratio is as good or better than DEFLATE. See lzx-compress.c for a format
30 * overview, and see https://en.wikipedia.org/wiki/LZX_(algorithm) for a
31 * historical overview. Here I make some pragmatic notes.
33 * The old specification for LZX is the document "Microsoft LZX Data Compression
34 * Format" (1997). It defines the LZX format as used in cabinet files. Allowed
35 * window sizes are 2^n where 15 <= n <= 21. However, this document contains
36 * several errors, so don't read too much into it...
38 * The new specification for LZX is the document "[MS-PATCH]: LZX DELTA
39 * Compression and Decompression" (2014). It defines the LZX format as used by
40 * Microsoft's binary patcher. It corrects several errors in the 1997 document
41 * and extends the format in several ways --- namely, optional reference data,
42 * up to 2^25 byte windows, and longer match lengths.
44 * WIM files use a more restricted form of LZX. No LZX DELTA extensions are
45 * present, the window is not "sliding", E8 preprocessing is done
46 * unconditionally with a fixed file size, and the maximum window size is always
47 * 2^15 bytes (equal to the size of each "chunk" in a compressed WIM resource).
48 * This code is primarily intended to implement this form of LZX. But although
49 * not compatible with WIMGAPI, this code also supports maximum window sizes up
52 * TODO: Add support for window sizes up to 2^25 bytes.
59 #include "wimlib/decompressor_ops.h"
60 #include "wimlib/decompress_common.h"
61 #include "wimlib/error.h"
62 #include "wimlib/lzx.h"
63 #include "wimlib/util.h"
67 /* These values are chosen for fast decompression. */
68 #define LZX_MAINCODE_TABLEBITS 11
69 #define LZX_LENCODE_TABLEBITS 10
70 #define LZX_PRECODE_TABLEBITS 6
71 #define LZX_ALIGNEDCODE_TABLEBITS 7
73 #define LZX_READ_LENS_MAX_OVERRUN 50
75 /* Huffman decoding tables, and arrays that map symbols to codeword lengths. */
78 u16 maincode_decode_table[(1 << LZX_MAINCODE_TABLEBITS) +
79 (LZX_MAINCODE_MAX_NUM_SYMBOLS * 2)]
80 _aligned_attribute(DECODE_TABLE_ALIGNMENT);
81 u8 maincode_lens[LZX_MAINCODE_MAX_NUM_SYMBOLS + LZX_READ_LENS_MAX_OVERRUN];
84 u16 lencode_decode_table[(1 << LZX_LENCODE_TABLEBITS) +
85 (LZX_LENCODE_NUM_SYMBOLS * 2)]
86 _aligned_attribute(DECODE_TABLE_ALIGNMENT);
87 u8 lencode_lens[LZX_LENCODE_NUM_SYMBOLS + LZX_READ_LENS_MAX_OVERRUN];
90 u16 alignedcode_decode_table[(1 << LZX_ALIGNEDCODE_TABLEBITS) +
91 (LZX_ALIGNEDCODE_NUM_SYMBOLS * 2)]
92 _aligned_attribute(DECODE_TABLE_ALIGNMENT);
93 u8 alignedcode_lens[LZX_ALIGNEDCODE_NUM_SYMBOLS];
94 } _aligned_attribute(DECODE_TABLE_ALIGNMENT);
96 /* The main LZX decompressor structure.
98 * Note: we keep track of most of the decompression state outside this
99 * structure. This structure only exists so that (1) we can store @window_order
100 * and @num_main_syms for multiple calls to lzx_decompress(); and (2) so that we
101 * don't have to allocate the large 'struct lzx_tables' on the stack. */
102 struct lzx_decompressor {
103 unsigned window_order;
104 unsigned num_main_syms;
105 struct lzx_tables tables;
108 /* Read a Huffman-encoded symbol using the precode. */
110 read_huffsym_using_precode(struct input_bitstream *istream,
111 const u16 precode_decode_table[])
113 return read_huffsym(istream, precode_decode_table,
114 LZX_PRECODE_TABLEBITS, LZX_MAX_PRE_CODEWORD_LEN);
117 /* Read a Huffman-encoded symbol using the main code. */
119 read_huffsym_using_maincode(struct input_bitstream *istream,
120 const struct lzx_tables *tables)
122 return read_huffsym(istream, tables->maincode_decode_table,
123 LZX_MAINCODE_TABLEBITS, LZX_MAX_MAIN_CODEWORD_LEN);
126 /* Read a Huffman-encoded symbol using the length code. */
128 read_huffsym_using_lencode(struct input_bitstream *istream,
129 const struct lzx_tables *tables)
131 return read_huffsym(istream, tables->lencode_decode_table,
132 LZX_LENCODE_TABLEBITS, LZX_MAX_LEN_CODEWORD_LEN);
135 /* Read a Huffman-encoded symbol using the aligned offset code. */
137 read_huffsym_using_alignedcode(struct input_bitstream *istream,
138 const struct lzx_tables *tables)
140 return read_huffsym(istream, tables->alignedcode_decode_table,
141 LZX_ALIGNEDCODE_TABLEBITS, LZX_MAX_ALIGNED_CODEWORD_LEN);
145 * Read the precode from the compressed input bitstream, then use it to decode
146 * @num_lens codeword length values.
149 * The input bitstream.
152 * An array that contains the length values from the previous time the
153 * codeword lengths for this Huffman code were read, or all 0's if this is
154 * the first time. This array must have at least (@num_lens +
155 * LZX_READ_LENS_MAX_OVERRUN) entries.
158 * Number of length values to decode.
160 * Returns 0 on success, or -1 if the data was invalid.
163 lzx_read_codeword_lens(struct input_bitstream *istream, u8 *lens, unsigned num_lens)
165 u16 precode_decode_table[(1 << LZX_PRECODE_TABLEBITS) +
166 (LZX_PRECODE_NUM_SYMBOLS * 2)]
167 _aligned_attribute(DECODE_TABLE_ALIGNMENT);
168 u8 precode_lens[LZX_PRECODE_NUM_SYMBOLS];
170 u8 *lens_end = lens + num_lens;
173 /* Read the lengths of the precode codewords. These are given
175 for (int i = 0; i < LZX_PRECODE_NUM_SYMBOLS; i++) {
176 precode_lens[i] = bitstream_read_bits(istream,
177 LZX_PRECODE_ELEMENT_SIZE);
180 /* Make the decoding table for the precode. */
181 ret = make_huffman_decode_table(precode_decode_table,
182 LZX_PRECODE_NUM_SYMBOLS,
183 LZX_PRECODE_TABLEBITS,
185 LZX_MAX_PRE_CODEWORD_LEN);
189 /* Decode the codeword lengths. */
194 /* Read the next precode symbol. */
195 presym = read_huffsym_using_precode(istream,
196 precode_decode_table);
198 /* Difference from old length */
199 len = *len_ptr - presym;
204 /* Special RLE values */
210 run_len = 4 + bitstream_read_bits(istream, 4);
212 } else if (presym == 18) {
213 /* Longer run of 0's */
214 run_len = 20 + bitstream_read_bits(istream, 5);
217 /* Run of identical lengths */
218 run_len = 4 + bitstream_read_bits(istream, 1);
219 presym = read_huffsym_using_precode(istream,
220 precode_decode_table);
221 len = *len_ptr - presym;
229 /* Worst case overrun is when presym == 18,
230 * run_len == 20 + 31, and only 1 length was remaining.
231 * So LZX_READ_LENS_MAX_OVERRUN == 50.
233 * Overrun while reading the first half of maincode_lens
234 * can corrupt the previous values in the second half.
235 * This doesn't really matter because the resulting
236 * lengths will still be in range, and data that
237 * generates overruns is invalid anyway. */
239 } while (len_ptr < lens_end);
244 * Read the header of an LZX block and save the block type and size in
245 * *block_type_ret and *block_size_ret, respectively.
247 * If the block is compressed, also update the Huffman decode @tables with the
250 * If the block is uncompressed, also update the match offset @queue with the
253 * Return 0 on success, or -1 if the data was invalid.
256 lzx_read_block_header(struct input_bitstream *istream,
257 unsigned num_main_syms,
258 unsigned window_order,
261 struct lzx_tables *tables,
262 struct lzx_lru_queue *queue)
268 bitstream_ensure_bits(istream, 4);
270 /* The first three bits tell us what kind of block it is, and should be
271 * one of the LZX_BLOCKTYPE_* values. */
272 block_type = bitstream_pop_bits(istream, 3);
274 /* Read the block size. This mirrors the behavior of
275 * lzx_write_compressed_block() in lzx-compress.c; see that for more
277 if (bitstream_pop_bits(istream, 1)) {
278 block_size = LZX_DEFAULT_BLOCK_SIZE;
283 tmp = bitstream_read_bits(istream, 8);
285 tmp = bitstream_read_bits(istream, 8);
289 if (window_order >= 16) {
290 tmp = bitstream_read_bits(istream, 8);
296 switch (block_type) {
298 case LZX_BLOCKTYPE_ALIGNED:
300 /* Read the aligned offset code and prepare its decode table.
303 for (int i = 0; i < LZX_ALIGNEDCODE_NUM_SYMBOLS; i++) {
304 tables->alignedcode_lens[i] =
305 bitstream_read_bits(istream,
306 LZX_ALIGNEDCODE_ELEMENT_SIZE);
309 ret = make_huffman_decode_table(tables->alignedcode_decode_table,
310 LZX_ALIGNEDCODE_NUM_SYMBOLS,
311 LZX_ALIGNEDCODE_TABLEBITS,
312 tables->alignedcode_lens,
313 LZX_MAX_ALIGNED_CODEWORD_LEN);
317 /* Fall though, since the rest of the header for aligned offset
318 * blocks is the same as that for verbatim blocks. */
320 case LZX_BLOCKTYPE_VERBATIM:
322 /* Read the main code and prepare its decode table.
324 * Note that the codeword lengths in the main code are encoded
325 * in two parts: one part for literal symbols, and one part for
328 ret = lzx_read_codeword_lens(istream, tables->maincode_lens,
333 ret = lzx_read_codeword_lens(istream,
334 tables->maincode_lens + LZX_NUM_CHARS,
335 num_main_syms - LZX_NUM_CHARS);
339 ret = make_huffman_decode_table(tables->maincode_decode_table,
341 LZX_MAINCODE_TABLEBITS,
342 tables->maincode_lens,
343 LZX_MAX_MAIN_CODEWORD_LEN);
347 /* Read the length code and prepare its decode table. */
349 ret = lzx_read_codeword_lens(istream, tables->lencode_lens,
350 LZX_LENCODE_NUM_SYMBOLS);
354 ret = make_huffman_decode_table(tables->lencode_decode_table,
355 LZX_LENCODE_NUM_SYMBOLS,
356 LZX_LENCODE_TABLEBITS,
357 tables->lencode_lens,
358 LZX_MAX_LEN_CODEWORD_LEN);
364 case LZX_BLOCKTYPE_UNCOMPRESSED:
366 /* Before reading the three LRU match offsets from the
367 * uncompressed block header, the stream must be aligned on a
368 * 16-bit boundary. But, unexpectedly, if the stream is
369 * *already* aligned, the correct thing to do is to throw away
370 * the next 16 bits. */
372 bitstream_ensure_bits(istream, 1);
373 bitstream_align(istream);
374 queue->R[0] = bitstream_read_u32(istream);
375 queue->R[1] = bitstream_read_u32(istream);
376 queue->R[2] = bitstream_read_u32(istream);
378 /* Offsets of 0 are invalid. */
379 if (queue->R[0] == 0 || queue->R[1] == 0 || queue->R[2] == 0)
384 /* Unrecognized block type. */
388 *block_type_ret = block_type;
389 *block_size_ret = block_size;
394 * Decompress an LZX-compressed block of data.
397 * The type of the block (LZX_BLOCKTYPE_VERBATIM or LZX_BLOCKTYPE_ALIGNED).
400 * The size of the block, in bytes.
403 * Pointer to the beginning of the decompression window.
406 * The position in the window at which the block starts.
409 * The Huffman decoding tables for the block.
412 * The least-recently-used queue for match offsets.
415 * The input bitstream, positioned at the start of the block data.
417 * Returns 0 on success, or -1 if the data was invalid.
420 lzx_decompress_block(int block_type, u32 block_size,
421 u8 *window, u32 window_pos,
422 const struct lzx_tables *tables,
423 struct lzx_lru_queue *queue,
424 struct input_bitstream *istream)
426 u8 *window_ptr = &window[window_pos];
427 u8 *window_end = window_ptr + block_size;
430 unsigned position_slot;
432 unsigned num_extra_bits;
433 unsigned ones_if_aligned = 0U - (block_type == LZX_BLOCKTYPE_ALIGNED);
435 while (window_ptr != window_end) {
437 mainsym = read_huffsym_using_maincode(istream, tables);
438 if (mainsym < LZX_NUM_CHARS) {
440 *window_ptr++ = mainsym;
446 /* Decode the length header and position slot. */
447 mainsym -= LZX_NUM_CHARS;
448 match_len = mainsym & 0x7;
449 position_slot = mainsym >> 3;
451 /* If needed, read a length symbol to decode the full length. */
452 if (match_len == 0x7)
453 match_len += read_huffsym_using_lencode(istream, tables);
454 match_len += LZX_MIN_MATCH_LEN;
456 if (position_slot <= 2) {
459 /* Note: This isn't a real LRU queue, since using the R2
460 * offset doesn't bump the R1 offset down to R2. This
461 * quirk allows all 3 recent offsets to be handled by
462 * the same code. (For R0, the swap is a no-op.) */
463 match_offset = queue->R[position_slot];
464 queue->R[position_slot] = queue->R[0];
465 queue->R[0] = match_offset;
467 /* Explicit offset */
469 /* Look up the number of extra bits that need to be read
470 * to decode offsets with this position slot. */
471 num_extra_bits = lzx_get_num_extra_bits(position_slot);
473 /* Start with the position slot base value. */
474 match_offset = lzx_position_base[position_slot];
476 /* In aligned offset blocks, the low-order 3 bits of
477 * each offset are encoded using the aligned offset
478 * code. Otherwise, all the extra bits are literal. */
480 /*if (block_type == LZX_BLOCKTYPE_ALIGNED && num_extra_bits >= 3) {*/
481 if ((num_extra_bits & ones_if_aligned) >= 3) {
482 match_offset += bitstream_read_bits(istream, num_extra_bits - 3) << 3;
483 match_offset += read_huffsym_using_alignedcode(istream, tables);
485 match_offset += bitstream_read_bits(istream, num_extra_bits);
488 /* Adjust the offset. */
489 match_offset -= LZX_OFFSET_OFFSET;
491 /* Update the match offset LRU queue. */
492 queue->R[2] = queue->R[1];
493 queue->R[1] = queue->R[0];
494 queue->R[0] = match_offset;
497 /* Validate the match, then copy it to the current position. */
499 if (unlikely(match_len > window_end - window_ptr))
502 if (unlikely(match_offset > window_ptr - window))
505 lz_copy(window_ptr, match_len, match_offset, window_end);
507 window_ptr += match_len;
513 lzx_decompress(const void *compressed_data, size_t compressed_size,
514 void *uncompressed_data, size_t uncompressed_size,
517 struct lzx_decompressor *dec = _dec;
518 struct input_bitstream istream;
519 struct lzx_lru_queue queue;
523 bool may_have_e8_byte;
526 init_input_bitstream(&istream, compressed_data, compressed_size);
528 /* Initialize the recent offsets queue. */
529 lzx_lru_queue_init(&queue);
531 /* Codeword lengths begin as all 0's for delta encoding purposes. */
532 memset(dec->tables.maincode_lens, 0, dec->num_main_syms);
533 memset(dec->tables.lencode_lens, 0, LZX_LENCODE_NUM_SYMBOLS);
535 /* Set this to true if there may be 0xe8 bytes in the uncompressed data.
537 may_have_e8_byte = false;
539 /* The compressed data will consist of one or more blocks. The
540 * following loop decompresses one block, and it runs until there all
541 * the compressed data has been decompressed, so there are no more
545 window_pos < uncompressed_size;
546 window_pos += block_size)
548 ret = lzx_read_block_header(&istream, dec->num_main_syms,
549 dec->window_order, &block_type,
550 &block_size, &dec->tables, &queue);
554 if (block_size > uncompressed_size - window_pos)
557 if (block_type != LZX_BLOCKTYPE_UNCOMPRESSED) {
559 /* Compressed block. */
561 ret = lzx_decompress_block(block_type,
571 /* If the first 0xe8 byte was in this block, it must
572 * have been encoded as a literal using mainsym 0xe8. */
573 if (dec->tables.maincode_lens[0xe8] != 0)
574 may_have_e8_byte = true;
577 /* Uncompressed block. */
580 p = bitstream_read_bytes(&istream, block_size);
584 memcpy(&((u8*)uncompressed_data)[window_pos], p, block_size);
586 /* Re-align the bitstream if an odd number of bytes was
589 bitstream_read_byte(&istream);
591 may_have_e8_byte = true;
595 /* Postprocess the data unless it cannot possibly contain 0xe8 bytes */
596 if (may_have_e8_byte)
597 lzx_undo_e8_preprocessing(uncompressed_data, uncompressed_size);
603 lzx_free_decompressor(void *_dec)
605 struct lzx_decompressor *dec = _dec;
611 lzx_create_decompressor(size_t max_block_size, void **dec_ret)
613 struct lzx_decompressor *dec;
614 unsigned window_order;
616 window_order = lzx_get_window_order(max_block_size);
617 if (window_order == 0)
618 return WIMLIB_ERR_INVALID_PARAM;
620 /* The aligned allocation is needed to ensure that the lzx_tables are
621 * aligned properly. */
622 dec = ALIGNED_MALLOC(sizeof(struct lzx_decompressor),
623 DECODE_TABLE_ALIGNMENT);
625 return WIMLIB_ERR_NOMEM;
627 dec->window_order = window_order;
628 dec->num_main_syms = lzx_get_num_main_syms(window_order);
634 const struct decompressor_ops lzx_decompressor_ops = {
635 .create_decompressor = lzx_create_decompressor,
636 .decompress = lzx_decompress,
637 .free_decompressor = lzx_free_decompressor,