4 * A decompressor for the LZX compression format, as used in WIM files.
8 * Copyright (C) 2012-2016 Eric Biggers
10 * This file is free software; you can redistribute it and/or modify it under
11 * the terms of the GNU Lesser General Public License as published by the Free
12 * Software Foundation; either version 3 of the License, or (at your option) any
15 * This file is distributed in the hope that it will be useful, but WITHOUT
16 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
17 * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
20 * You should have received a copy of the GNU Lesser General Public License
21 * along with this file; if not, see http://www.gnu.org/licenses/.
25 * LZX is an LZ77 and Huffman-code based compression format that has many
26 * similarities to DEFLATE (the format used by zlib/gzip). The compression
27 * ratio is as good or better than DEFLATE. See lzx_compress.c for a format
28 * overview, and see https://en.wikipedia.org/wiki/LZX_(algorithm) for a
29 * historical overview. Here I make some pragmatic notes.
31 * The old specification for LZX is the document "Microsoft LZX Data Compression
32 * Format" (1997). It defines the LZX format as used in cabinet files. Allowed
33 * window sizes are 2^n where 15 <= n <= 21. However, this document contains
34 * several errors, so don't read too much into it...
36 * The new specification for LZX is the document "[MS-PATCH]: LZX DELTA
37 * Compression and Decompression" (2014). It defines the LZX format as used by
38 * Microsoft's binary patcher. It corrects several errors in the 1997 document
39 * and extends the format in several ways --- namely, optional reference data,
40 * up to 2^25 byte windows, and longer match lengths.
42 * WIM files use a more restricted form of LZX. No LZX DELTA extensions are
43 * present, the window is not "sliding", E8 preprocessing is done
44 * unconditionally with a fixed file size, and the maximum window size is always
45 * 2^15 bytes (equal to the size of each "chunk" in a compressed WIM resource).
46 * This code is primarily intended to implement this form of LZX. But although
47 * not compatible with WIMGAPI, this code also supports maximum window sizes up
50 * TODO: Add support for window sizes up to 2^25 bytes.
59 #include "wimlib/decompressor_ops.h"
60 #include "wimlib/decompress_common.h"
61 #include "wimlib/error.h"
62 #include "wimlib/lzx_common.h"
63 #include "wimlib/util.h"
65 /* These values are chosen for fast decompression. */
66 #define LZX_MAINCODE_TABLEBITS 11
67 #define LZX_LENCODE_TABLEBITS 9
68 #define LZX_PRECODE_TABLEBITS 6
69 #define LZX_ALIGNEDCODE_TABLEBITS 7
71 #define LZX_READ_LENS_MAX_OVERRUN 50
73 struct lzx_decompressor {
75 DECODE_TABLE(maincode_decode_table, LZX_MAINCODE_MAX_NUM_SYMBOLS,
76 LZX_MAINCODE_TABLEBITS, LZX_MAX_MAIN_CODEWORD_LEN);
77 u8 maincode_lens[LZX_MAINCODE_MAX_NUM_SYMBOLS + LZX_READ_LENS_MAX_OVERRUN];
79 DECODE_TABLE(lencode_decode_table, LZX_LENCODE_NUM_SYMBOLS,
80 LZX_LENCODE_TABLEBITS, LZX_MAX_LEN_CODEWORD_LEN);
81 u8 lencode_lens[LZX_LENCODE_NUM_SYMBOLS + LZX_READ_LENS_MAX_OVERRUN];
84 DECODE_TABLE(alignedcode_decode_table, LZX_ALIGNEDCODE_NUM_SYMBOLS,
85 LZX_ALIGNEDCODE_TABLEBITS, LZX_MAX_ALIGNED_CODEWORD_LEN);
86 u8 alignedcode_lens[LZX_ALIGNEDCODE_NUM_SYMBOLS];
90 DECODE_TABLE(precode_decode_table, LZX_PRECODE_NUM_SYMBOLS,
91 LZX_PRECODE_TABLEBITS, LZX_MAX_PRE_CODEWORD_LEN);
92 u8 precode_lens[LZX_PRECODE_NUM_SYMBOLS];
93 u8 extra_offset_bits[LZX_MAX_OFFSET_SLOTS];
96 unsigned window_order;
97 unsigned num_main_syms;
99 /* Like lzx_extra_offset_bits[], but does not include the entropy-coded
100 * bits of aligned offset blocks */
101 u8 extra_offset_bits_minus_aligned[LZX_MAX_OFFSET_SLOTS];
103 } _aligned_attribute(DECODE_TABLE_ALIGNMENT);
105 /* Read a Huffman-encoded symbol using the precode. */
106 static inline unsigned
107 read_presym(const struct lzx_decompressor *d, struct input_bitstream *is)
109 return read_huffsym(is, d->precode_decode_table,
110 LZX_PRECODE_TABLEBITS, LZX_MAX_PRE_CODEWORD_LEN);
113 /* Read a Huffman-encoded symbol using the main code. */
114 static inline unsigned
115 read_mainsym(const struct lzx_decompressor *d, struct input_bitstream *is)
117 return read_huffsym(is, d->maincode_decode_table,
118 LZX_MAINCODE_TABLEBITS, LZX_MAX_MAIN_CODEWORD_LEN);
121 /* Read a Huffman-encoded symbol using the length code. */
122 static inline unsigned
123 read_lensym(const struct lzx_decompressor *d, struct input_bitstream *is)
125 return read_huffsym(is, d->lencode_decode_table,
126 LZX_LENCODE_TABLEBITS, LZX_MAX_LEN_CODEWORD_LEN);
129 /* Read a Huffman-encoded symbol using the aligned offset code. */
130 static inline unsigned
131 read_alignedsym(const struct lzx_decompressor *d, struct input_bitstream *is)
133 return read_huffsym(is, d->alignedcode_decode_table,
134 LZX_ALIGNEDCODE_TABLEBITS, LZX_MAX_ALIGNED_CODEWORD_LEN);
138 * Read a precode from the compressed input bitstream, then use it to decode
139 * @num_lens codeword length values and write them to @lens.
142 lzx_read_codeword_lens(struct lzx_decompressor *d, struct input_bitstream *is,
143 u8 *lens, unsigned num_lens)
146 u8 *lens_end = lens + num_lens;
148 /* Read the lengths of the precode codewords. These are stored
150 for (int i = 0; i < LZX_PRECODE_NUM_SYMBOLS; i++) {
152 bitstream_read_bits(is, LZX_PRECODE_ELEMENT_SIZE);
155 /* Build the decoding table for the precode. */
156 if (make_huffman_decode_table(d->precode_decode_table,
157 LZX_PRECODE_NUM_SYMBOLS,
158 LZX_PRECODE_TABLEBITS,
160 LZX_MAX_PRE_CODEWORD_LEN))
163 /* Decode the codeword lengths. */
168 /* Read the next precode symbol. */
169 presym = read_presym(d, is);
171 /* Difference from old length */
172 len = *len_ptr - presym;
177 /* Special RLE values */
183 run_len = 4 + bitstream_read_bits(is, 4);
185 } else if (presym == 18) {
186 /* Longer run of 0's */
187 run_len = 20 + bitstream_read_bits(is, 5);
190 /* Run of identical lengths */
191 run_len = 4 + bitstream_read_bits(is, 1);
192 presym = read_presym(d, is);
193 if (unlikely(presym > 17))
195 len = *len_ptr - presym;
204 * The worst case overrun is when presym == 18,
205 * run_len == 20 + 31, and only 1 length was remaining.
206 * So LZX_READ_LENS_MAX_OVERRUN == 50.
208 * Overrun while reading the first half of maincode_lens
209 * can corrupt the previous values in the second half.
210 * This doesn't really matter because the resulting
211 * lengths will still be in range, and data that
212 * generates overruns is invalid anyway.
215 } while (len_ptr < lens_end);
221 * Read the header of an LZX block. For all block types, the block type and
222 * size is saved in *block_type_ret and *block_size_ret, respectively. For
223 * compressed blocks, the codeword lengths are also saved. For uncompressed
224 * blocks, the recent offsets queue is also updated.
227 lzx_read_block_header(struct lzx_decompressor *d, struct input_bitstream *is,
228 u32 recent_offsets[], int *block_type_ret,
234 bitstream_ensure_bits(is, 4);
236 /* Read the block type. */
237 block_type = bitstream_pop_bits(is, 3);
239 /* Read the block size. */
240 if (bitstream_pop_bits(is, 1)) {
241 block_size = LZX_DEFAULT_BLOCK_SIZE;
243 block_size = bitstream_read_bits(is, 16);
244 if (d->window_order >= 16) {
246 block_size |= bitstream_read_bits(is, 8);
250 switch (block_type) {
252 case LZX_BLOCKTYPE_ALIGNED:
254 /* Read the aligned offset codeword lengths. */
256 for (int i = 0; i < LZX_ALIGNEDCODE_NUM_SYMBOLS; i++) {
257 d->alignedcode_lens[i] =
258 bitstream_read_bits(is,
259 LZX_ALIGNEDCODE_ELEMENT_SIZE);
262 /* Fall though, since the rest of the header for aligned offset
263 * blocks is the same as that for verbatim blocks. */
265 case LZX_BLOCKTYPE_VERBATIM:
267 /* Read the main codeword lengths, which are divided into two
268 * parts: literal symbols and match headers. */
270 if (lzx_read_codeword_lens(d, is, d->maincode_lens,
274 if (lzx_read_codeword_lens(d, is, d->maincode_lens + LZX_NUM_CHARS,
275 d->num_main_syms - LZX_NUM_CHARS))
279 /* Read the length codeword lengths. */
281 if (lzx_read_codeword_lens(d, is, d->lencode_lens,
282 LZX_LENCODE_NUM_SYMBOLS))
287 case LZX_BLOCKTYPE_UNCOMPRESSED:
289 * The header of an uncompressed block contains new values for
290 * the recent offsets queue, starting on the next 16-bit
291 * boundary in the bitstream. Careful: if the stream is
292 * *already* aligned, the correct thing to do is to throw away
293 * the next 16 bits (this is probably a mistake in the format).
295 bitstream_ensure_bits(is, 1);
297 recent_offsets[0] = bitstream_read_u32(is);
298 recent_offsets[1] = bitstream_read_u32(is);
299 recent_offsets[2] = bitstream_read_u32(is);
301 /* Offsets of 0 are invalid. */
302 if (recent_offsets[0] == 0 || recent_offsets[1] == 0 ||
303 recent_offsets[2] == 0)
308 /* Unrecognized block type. */
312 *block_type_ret = block_type;
313 *block_size_ret = block_size;
317 /* Decompress a block of LZX-compressed data. */
319 lzx_decompress_block(struct lzx_decompressor *d, struct input_bitstream *is,
320 int block_type, u32 block_size,
321 u8 * const out_begin, u8 *out_next, u32 recent_offsets[])
323 u8 * const block_end = out_next + block_size;
324 unsigned min_aligned_offset_slot;
327 * Build the Huffman decode tables. We always need to build the main
328 * and length decode tables. For aligned blocks we additionally need to
329 * build the aligned offset decode table.
332 if (make_huffman_decode_table(d->maincode_decode_table,
334 LZX_MAINCODE_TABLEBITS,
336 LZX_MAX_MAIN_CODEWORD_LEN))
339 if (make_huffman_decode_table(d->lencode_decode_table,
340 LZX_LENCODE_NUM_SYMBOLS,
341 LZX_LENCODE_TABLEBITS,
343 LZX_MAX_LEN_CODEWORD_LEN))
346 if (block_type == LZX_BLOCKTYPE_ALIGNED) {
347 if (make_huffman_decode_table(d->alignedcode_decode_table,
348 LZX_ALIGNEDCODE_NUM_SYMBOLS,
349 LZX_ALIGNEDCODE_TABLEBITS,
351 LZX_MAX_ALIGNED_CODEWORD_LEN))
353 min_aligned_offset_slot = LZX_MIN_ALIGNED_OFFSET_SLOT;
354 memcpy(d->extra_offset_bits, d->extra_offset_bits_minus_aligned,
355 sizeof(lzx_extra_offset_bits));
357 min_aligned_offset_slot = LZX_MAX_OFFSET_SLOTS;
358 memcpy(d->extra_offset_bits, lzx_extra_offset_bits,
359 sizeof(lzx_extra_offset_bits));
362 /* Decode the literals and matches. */
368 unsigned offset_slot;
370 mainsym = read_mainsym(d, is);
371 if (mainsym < LZX_NUM_CHARS) {
373 *out_next++ = mainsym;
379 /* Decode the length header and offset slot. */
380 STATIC_ASSERT(LZX_NUM_CHARS % LZX_NUM_LEN_HEADERS == 0);
381 length = mainsym % LZX_NUM_LEN_HEADERS;
382 offset_slot = (mainsym - LZX_NUM_CHARS) / LZX_NUM_LEN_HEADERS;
384 /* If needed, read a length symbol to decode the full length. */
385 if (length == LZX_NUM_PRIMARY_LENS)
386 length += read_lensym(d, is);
387 length += LZX_MIN_MATCH_LEN;
389 if (offset_slot < LZX_NUM_RECENT_OFFSETS) {
392 /* Note: This isn't a real LRU queue, since using the R2
393 * offset doesn't bump the R1 offset down to R2. */
394 offset = recent_offsets[offset_slot];
395 recent_offsets[offset_slot] = recent_offsets[0];
397 /* Explicit offset */
398 offset = bitstream_read_bits(is, d->extra_offset_bits[offset_slot]);
399 if (offset_slot >= min_aligned_offset_slot) {
400 offset = (offset << LZX_NUM_ALIGNED_OFFSET_BITS) |
401 read_alignedsym(d, is);
403 offset += lzx_offset_slot_base[offset_slot];
405 /* Update the match offset LRU queue. */
406 STATIC_ASSERT(LZX_NUM_RECENT_OFFSETS == 3);
407 recent_offsets[2] = recent_offsets[1];
408 recent_offsets[1] = recent_offsets[0];
410 recent_offsets[0] = offset;
412 /* Validate the match and copy it to the current position. */
413 if (unlikely(lz_copy(length, offset, out_begin,
414 out_next, block_end, LZX_MIN_MATCH_LEN)))
417 } while (out_next != block_end);
423 lzx_decompress(const void *restrict compressed_data, size_t compressed_size,
424 void *restrict uncompressed_data, size_t uncompressed_size,
427 struct lzx_decompressor *d = _d;
428 u8 * const out_begin = uncompressed_data;
429 u8 *out_next = out_begin;
430 u8 * const out_end = out_begin + uncompressed_size;
431 struct input_bitstream is;
432 STATIC_ASSERT(LZX_NUM_RECENT_OFFSETS == 3);
433 u32 recent_offsets[LZX_NUM_RECENT_OFFSETS] = {1, 1, 1};
434 unsigned may_have_e8_byte = 0;
436 init_input_bitstream(&is, compressed_data, compressed_size);
438 /* Codeword lengths begin as all 0's for delta encoding purposes. */
439 memset(d->maincode_lens, 0, d->num_main_syms);
440 memset(d->lencode_lens, 0, LZX_LENCODE_NUM_SYMBOLS);
442 /* Decompress blocks until we have all the uncompressed data. */
444 while (out_next != out_end) {
448 if (lzx_read_block_header(d, &is, recent_offsets,
449 &block_type, &block_size))
452 if (block_size < 1 || block_size > out_end - out_next)
455 if (likely(block_type != LZX_BLOCKTYPE_UNCOMPRESSED)) {
457 /* Compressed block */
458 if (lzx_decompress_block(d, &is, block_type, block_size,
463 /* If the first E8 byte was in this block, then it must
464 * have been encoded as a literal using mainsym E8. */
465 may_have_e8_byte |= d->maincode_lens[0xE8];
468 /* Uncompressed block */
469 if (bitstream_read_bytes(&is, out_next, block_size))
472 /* Re-align the bitstream if needed. */
474 bitstream_read_byte(&is);
476 /* There may have been an E8 byte in the block. */
477 may_have_e8_byte = 1;
479 out_next += block_size;
482 /* Postprocess the data unless it cannot possibly contain E8 bytes. */
483 if (may_have_e8_byte)
484 lzx_postprocess(uncompressed_data, uncompressed_size);
490 lzx_create_decompressor(size_t max_block_size, void **d_ret)
492 unsigned window_order;
493 struct lzx_decompressor *d;
495 window_order = lzx_get_window_order(max_block_size);
496 if (window_order == 0)
497 return WIMLIB_ERR_INVALID_PARAM;
499 d = ALIGNED_MALLOC(sizeof(*d), DECODE_TABLE_ALIGNMENT);
501 return WIMLIB_ERR_NOMEM;
503 d->window_order = window_order;
504 d->num_main_syms = lzx_get_num_main_syms(window_order);
506 /* Initialize 'd->extra_offset_bits_minus_aligned'. */
507 STATIC_ASSERT(sizeof(d->extra_offset_bits_minus_aligned) ==
508 sizeof(lzx_extra_offset_bits));
509 STATIC_ASSERT(sizeof(d->extra_offset_bits) ==
510 sizeof(lzx_extra_offset_bits));
511 memcpy(d->extra_offset_bits_minus_aligned, lzx_extra_offset_bits,
512 sizeof(lzx_extra_offset_bits));
513 for (unsigned offset_slot = LZX_MIN_ALIGNED_OFFSET_SLOT;
514 offset_slot < LZX_MAX_OFFSET_SLOTS; offset_slot++)
516 d->extra_offset_bits_minus_aligned[offset_slot] -=
517 LZX_NUM_ALIGNED_OFFSET_BITS;
525 lzx_free_decompressor(void *_d)
530 const struct decompressor_ops lzx_decompressor_ops = {
531 .create_decompressor = lzx_create_decompressor,
532 .decompress = lzx_decompress,
533 .free_decompressor = lzx_free_decompressor,