Adjust naming of (de)compression files
[wimlib] / src / lzx_decompress.c
1 /*
2  * lzx_decompress.c
3  *
4  * A decompressor for the LZX compression format, as used in WIM files.
5  */
6
7 /*
8  * Copyright (C) 2012, 2013, 2014 Eric Biggers
9  *
10  * This file is free software; you can redistribute it and/or modify it under
11  * the terms of the GNU Lesser General Public License as published by the Free
12  * Software Foundation; either version 3 of the License, or (at your option) any
13  * later version.
14  *
15  * This file is distributed in the hope that it will be useful, but WITHOUT
16  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
17  * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
18  * details.
19  *
20  * You should have received a copy of the GNU Lesser General Public License
21  * along with this file; if not, see http://www.gnu.org/licenses/.
22  */
23
24 /*
25  * LZX is an LZ77 and Huffman-code based compression format that has many
26  * similarities to DEFLATE (the format used by zlib/gzip).  The compression
27  * ratio is as good or better than DEFLATE.  See lzx_compress.c for a format
28  * overview, and see https://en.wikipedia.org/wiki/LZX_(algorithm) for a
29  * historical overview.  Here I make some pragmatic notes.
30  *
31  * The old specification for LZX is the document "Microsoft LZX Data Compression
32  * Format" (1997).  It defines the LZX format as used in cabinet files.  Allowed
33  * window sizes are 2^n where 15 <= n <= 21.  However, this document contains
34  * several errors, so don't read too much into it...
35  *
36  * The new specification for LZX is the document "[MS-PATCH]: LZX DELTA
37  * Compression and Decompression" (2014).  It defines the LZX format as used by
38  * Microsoft's binary patcher.  It corrects several errors in the 1997 document
39  * and extends the format in several ways --- namely, optional reference data,
40  * up to 2^25 byte windows, and longer match lengths.
41  *
42  * WIM files use a more restricted form of LZX.  No LZX DELTA extensions are
43  * present, the window is not "sliding", E8 preprocessing is done
44  * unconditionally with a fixed file size, and the maximum window size is always
45  * 2^15 bytes (equal to the size of each "chunk" in a compressed WIM resource).
46  * This code is primarily intended to implement this form of LZX.  But although
47  * not compatible with WIMGAPI, this code also supports maximum window sizes up
48  * to 2^21 bytes.
49  *
50  * TODO: Add support for window sizes up to 2^25 bytes.
51  */
52
53 #ifdef HAVE_CONFIG_H
54 #  include "config.h"
55 #endif
56
57 #include "wimlib/decompressor_ops.h"
58 #include "wimlib/decompress_common.h"
59 #include "wimlib/error.h"
60 #include "wimlib/lzx_common.h"
61 #include "wimlib/util.h"
62
63 #include <string.h>
64
65 /* These values are chosen for fast decompression.  */
66 #define LZX_MAINCODE_TABLEBITS          11
67 #define LZX_LENCODE_TABLEBITS           10
68 #define LZX_PRECODE_TABLEBITS           6
69 #define LZX_ALIGNEDCODE_TABLEBITS       7
70
71 #define LZX_READ_LENS_MAX_OVERRUN 50
72
73 /* Huffman decoding tables, and arrays that map symbols to codeword lengths.  */
74 struct lzx_tables {
75
76         u16 maincode_decode_table[(1 << LZX_MAINCODE_TABLEBITS) +
77                                         (LZX_MAINCODE_MAX_NUM_SYMBOLS * 2)]
78                                         _aligned_attribute(DECODE_TABLE_ALIGNMENT);
79         u8 maincode_lens[LZX_MAINCODE_MAX_NUM_SYMBOLS + LZX_READ_LENS_MAX_OVERRUN];
80
81
82         u16 lencode_decode_table[(1 << LZX_LENCODE_TABLEBITS) +
83                                         (LZX_LENCODE_NUM_SYMBOLS * 2)]
84                                         _aligned_attribute(DECODE_TABLE_ALIGNMENT);
85         u8 lencode_lens[LZX_LENCODE_NUM_SYMBOLS + LZX_READ_LENS_MAX_OVERRUN];
86
87
88         u16 alignedcode_decode_table[(1 << LZX_ALIGNEDCODE_TABLEBITS) +
89                                         (LZX_ALIGNEDCODE_NUM_SYMBOLS * 2)]
90                                         _aligned_attribute(DECODE_TABLE_ALIGNMENT);
91         u8 alignedcode_lens[LZX_ALIGNEDCODE_NUM_SYMBOLS];
92 } _aligned_attribute(DECODE_TABLE_ALIGNMENT);
93
94 /* The main LZX decompressor structure.
95  *
96  * Note: we keep track of most of the decompression state outside this
97  * structure.  This structure only exists so that (1) we can store @window_order
98  * and @num_main_syms for multiple calls to lzx_decompress(); and (2) so that we
99  * don't have to allocate the large 'struct lzx_tables' on the stack.  */
100 struct lzx_decompressor {
101         unsigned window_order;
102         unsigned num_main_syms;
103         struct lzx_tables tables;
104 };
105
106 /* Read a Huffman-encoded symbol using the precode.  */
107 static inline u16
108 read_huffsym_using_precode(struct input_bitstream *istream,
109                            const u16 precode_decode_table[])
110 {
111         return read_huffsym(istream, precode_decode_table,
112                             LZX_PRECODE_TABLEBITS, LZX_MAX_PRE_CODEWORD_LEN);
113 }
114
115 /* Read a Huffman-encoded symbol using the main code.  */
116 static inline u16
117 read_huffsym_using_maincode(struct input_bitstream *istream,
118                             const struct lzx_tables *tables)
119 {
120         return read_huffsym(istream, tables->maincode_decode_table,
121                             LZX_MAINCODE_TABLEBITS, LZX_MAX_MAIN_CODEWORD_LEN);
122 }
123
124 /* Read a Huffman-encoded symbol using the length code.  */
125 static inline u16
126 read_huffsym_using_lencode(struct input_bitstream *istream,
127                            const struct lzx_tables *tables)
128 {
129         return read_huffsym(istream, tables->lencode_decode_table,
130                             LZX_LENCODE_TABLEBITS, LZX_MAX_LEN_CODEWORD_LEN);
131 }
132
133 /* Read a Huffman-encoded symbol using the aligned offset code.  */
134 static inline u16
135 read_huffsym_using_alignedcode(struct input_bitstream *istream,
136                                const struct lzx_tables *tables)
137 {
138         return read_huffsym(istream, tables->alignedcode_decode_table,
139                             LZX_ALIGNEDCODE_TABLEBITS, LZX_MAX_ALIGNED_CODEWORD_LEN);
140 }
141
142 /*
143  * Read the precode from the compressed input bitstream, then use it to decode
144  * @num_lens codeword length values.
145  *
146  * @istream:
147  *      The input bitstream.
148  *
149  * @lens:
150  *      An array that contains the length values from the previous time the
151  *      codeword lengths for this Huffman code were read, or all 0's if this is
152  *      the first time.  This array must have at least (@num_lens +
153  *      LZX_READ_LENS_MAX_OVERRUN) entries.
154  *
155  * @num_lens:
156  *      Number of length values to decode.
157  *
158  * Returns 0 on success, or -1 if the data was invalid.
159  */
160 static int
161 lzx_read_codeword_lens(struct input_bitstream *istream, u8 *lens, unsigned num_lens)
162 {
163         u16 precode_decode_table[(1 << LZX_PRECODE_TABLEBITS) +
164                                         (LZX_PRECODE_NUM_SYMBOLS * 2)]
165                                         _aligned_attribute(DECODE_TABLE_ALIGNMENT);
166         u8 precode_lens[LZX_PRECODE_NUM_SYMBOLS];
167         u8 *len_ptr = lens;
168         u8 *lens_end = lens + num_lens;
169         int ret;
170
171         /* Read the lengths of the precode codewords.  These are given
172          * explicitly.  */
173         for (int i = 0; i < LZX_PRECODE_NUM_SYMBOLS; i++) {
174                 precode_lens[i] = bitstream_read_bits(istream,
175                                                       LZX_PRECODE_ELEMENT_SIZE);
176         }
177
178         /* Make the decoding table for the precode.  */
179         ret = make_huffman_decode_table(precode_decode_table,
180                                         LZX_PRECODE_NUM_SYMBOLS,
181                                         LZX_PRECODE_TABLEBITS,
182                                         precode_lens,
183                                         LZX_MAX_PRE_CODEWORD_LEN);
184         if (ret)
185                 return ret;
186
187         /* Decode the codeword lengths.  */
188         do {
189                 unsigned presym;
190                 u8 len;
191
192                 /* Read the next precode symbol.  */
193                 presym = read_huffsym_using_precode(istream,
194                                                     precode_decode_table);
195                 if (presym < 17) {
196                         /* Difference from old length  */
197                         len = *len_ptr - presym;
198                         if ((s8)len < 0)
199                                 len += 17;
200                         *len_ptr++ = len;
201                 } else {
202                         /* Special RLE values  */
203
204                         unsigned run_len;
205
206                         if (presym == 17) {
207                                 /* Run of 0's  */
208                                 run_len = 4 + bitstream_read_bits(istream, 4);
209                                 len = 0;
210                         } else if (presym == 18) {
211                                 /* Longer run of 0's  */
212                                 run_len = 20 + bitstream_read_bits(istream, 5);
213                                 len = 0;
214                         } else {
215                                 /* Run of identical lengths  */
216                                 run_len = 4 + bitstream_read_bits(istream, 1);
217                                 presym = read_huffsym_using_precode(istream,
218                                                                     precode_decode_table);
219                                 len = *len_ptr - presym;
220                                 if ((s8)len < 0)
221                                         len += 17;
222                         }
223
224                         do {
225                                 *len_ptr++ = len;
226                         } while (--run_len);
227                         /* Worst case overrun is when presym == 18,
228                          * run_len == 20 + 31, and only 1 length was remaining.
229                          * So LZX_READ_LENS_MAX_OVERRUN == 50.
230                          *
231                          * Overrun while reading the first half of maincode_lens
232                          * can corrupt the previous values in the second half.
233                          * This doesn't really matter because the resulting
234                          * lengths will still be in range, and data that
235                          * generates overruns is invalid anyway.  */
236                 }
237         } while (len_ptr < lens_end);
238         return 0;
239 }
240
241 /*
242  * Read the header of an LZX block and save the block type and size in
243  * *block_type_ret and *block_size_ret, respectively.
244  *
245  * If the block is compressed, also update the Huffman decode @tables with the
246  * new Huffman codes.
247  *
248  * If the block is uncompressed, also update the match offset @queue with the
249  * new match offsets.
250  *
251  * Return 0 on success, or -1 if the data was invalid.
252  */
253 static int
254 lzx_read_block_header(struct input_bitstream *istream,
255                       unsigned num_main_syms,
256                       unsigned window_order,
257                       int *block_type_ret,
258                       u32 *block_size_ret,
259                       struct lzx_tables *tables,
260                       struct lzx_lru_queue *queue)
261 {
262         int block_type;
263         u32 block_size;
264         int ret;
265
266         bitstream_ensure_bits(istream, 4);
267
268         /* The first three bits tell us what kind of block it is, and should be
269          * one of the LZX_BLOCKTYPE_* values.  */
270         block_type = bitstream_pop_bits(istream, 3);
271
272         /* Read the block size.  This mirrors the behavior of
273          * lzx_write_compressed_block() in lzx_compress.c; see that for more
274          * details.  */
275         if (bitstream_pop_bits(istream, 1)) {
276                 block_size = LZX_DEFAULT_BLOCK_SIZE;
277         } else {
278                 u32 tmp;
279                 block_size = 0;
280
281                 tmp = bitstream_read_bits(istream, 8);
282                 block_size |= tmp;
283                 tmp = bitstream_read_bits(istream, 8);
284                 block_size <<= 8;
285                 block_size |= tmp;
286
287                 if (window_order >= 16) {
288                         tmp = bitstream_read_bits(istream, 8);
289                         block_size <<= 8;
290                         block_size |= tmp;
291                 }
292         }
293
294         switch (block_type) {
295
296         case LZX_BLOCKTYPE_ALIGNED:
297
298                 /* Read the aligned offset code and prepare its decode table.
299                  */
300
301                 for (int i = 0; i < LZX_ALIGNEDCODE_NUM_SYMBOLS; i++) {
302                         tables->alignedcode_lens[i] =
303                                 bitstream_read_bits(istream,
304                                                     LZX_ALIGNEDCODE_ELEMENT_SIZE);
305                 }
306
307                 ret = make_huffman_decode_table(tables->alignedcode_decode_table,
308                                                 LZX_ALIGNEDCODE_NUM_SYMBOLS,
309                                                 LZX_ALIGNEDCODE_TABLEBITS,
310                                                 tables->alignedcode_lens,
311                                                 LZX_MAX_ALIGNED_CODEWORD_LEN);
312                 if (ret)
313                         return ret;
314
315                 /* Fall though, since the rest of the header for aligned offset
316                  * blocks is the same as that for verbatim blocks.  */
317
318         case LZX_BLOCKTYPE_VERBATIM:
319
320                 /* Read the main code and prepare its decode table.
321                  *
322                  * Note that the codeword lengths in the main code are encoded
323                  * in two parts: one part for literal symbols, and one part for
324                  * match symbols.  */
325
326                 ret = lzx_read_codeword_lens(istream, tables->maincode_lens,
327                                              LZX_NUM_CHARS);
328                 if (ret)
329                         return ret;
330
331                 ret = lzx_read_codeword_lens(istream,
332                                              tables->maincode_lens + LZX_NUM_CHARS,
333                                              num_main_syms - LZX_NUM_CHARS);
334                 if (ret)
335                         return ret;
336
337                 ret = make_huffman_decode_table(tables->maincode_decode_table,
338                                                 num_main_syms,
339                                                 LZX_MAINCODE_TABLEBITS,
340                                                 tables->maincode_lens,
341                                                 LZX_MAX_MAIN_CODEWORD_LEN);
342                 if (ret)
343                         return ret;
344
345                 /* Read the length code and prepare its decode table.  */
346
347                 ret = lzx_read_codeword_lens(istream, tables->lencode_lens,
348                                              LZX_LENCODE_NUM_SYMBOLS);
349                 if (ret)
350                         return ret;
351
352                 ret = make_huffman_decode_table(tables->lencode_decode_table,
353                                                 LZX_LENCODE_NUM_SYMBOLS,
354                                                 LZX_LENCODE_TABLEBITS,
355                                                 tables->lencode_lens,
356                                                 LZX_MAX_LEN_CODEWORD_LEN);
357                 if (ret)
358                         return ret;
359
360                 break;
361
362         case LZX_BLOCKTYPE_UNCOMPRESSED:
363
364                 /* Before reading the three LRU match offsets from the
365                  * uncompressed block header, the stream must be aligned on a
366                  * 16-bit boundary.  But, unexpectedly, if the stream is
367                  * *already* aligned, the correct thing to do is to throw away
368                  * the next 16 bits.  */
369
370                 bitstream_ensure_bits(istream, 1);
371                 bitstream_align(istream);
372                 queue->R[0] = bitstream_read_u32(istream);
373                 queue->R[1] = bitstream_read_u32(istream);
374                 queue->R[2] = bitstream_read_u32(istream);
375
376                 /* Offsets of 0 are invalid.  */
377                 if (queue->R[0] == 0 || queue->R[1] == 0 || queue->R[2] == 0)
378                         return -1;
379                 break;
380
381         default:
382                 /* Unrecognized block type.  */
383                 return -1;
384         }
385
386         *block_type_ret = block_type;
387         *block_size_ret = block_size;
388         return 0;
389 }
390
391 /*
392  * Decompress an LZX-compressed block of data.
393  *
394  * @block_type:
395  *      The type of the block (LZX_BLOCKTYPE_VERBATIM or LZX_BLOCKTYPE_ALIGNED).
396  *
397  * @block_size:
398  *      The size of the block, in bytes.
399  *
400  * @window:
401  *      Pointer to the beginning of the decompression window.
402  *
403  * @window_pos:
404  *      The position in the window at which the block starts.
405  *
406  * @tables:
407  *      The Huffman decoding tables for the block.
408  *
409  * @queue:
410  *      The least-recently-used queue for match offsets.
411  *
412  * @istream:
413  *      The input bitstream, positioned at the start of the block data.
414  *
415  * Returns 0 on success, or -1 if the data was invalid.
416  */
417 static int
418 lzx_decompress_block(int block_type, u32 block_size,
419                      u8 *window, u32 window_pos,
420                      const struct lzx_tables *tables,
421                      struct lzx_lru_queue *queue,
422                      struct input_bitstream *istream)
423 {
424         u8 *window_ptr = &window[window_pos];
425         u8 *window_end = window_ptr + block_size;
426         unsigned mainsym;
427         u32 match_len;
428         unsigned offset_slot;
429         u32 match_offset;
430         unsigned num_extra_bits;
431         unsigned ones_if_aligned = 0U - (block_type == LZX_BLOCKTYPE_ALIGNED);
432
433         while (window_ptr != window_end) {
434
435                 mainsym = read_huffsym_using_maincode(istream, tables);
436                 if (mainsym < LZX_NUM_CHARS) {
437                         /* Literal  */
438                         *window_ptr++ = mainsym;
439                         continue;
440                 }
441
442                 /* Match  */
443
444                 /* Decode the length header and offset slot.  */
445                 mainsym -= LZX_NUM_CHARS;
446                 match_len = mainsym & 0x7;
447                 offset_slot = mainsym >> 3;
448
449                 /* If needed, read a length symbol to decode the full length. */
450                 if (match_len == 0x7)
451                         match_len += read_huffsym_using_lencode(istream, tables);
452                 match_len += LZX_MIN_MATCH_LEN;
453
454                 if (offset_slot <= 2) {
455                         /* Repeat offset  */
456
457                         /* Note: This isn't a real LRU queue, since using the R2
458                          * offset doesn't bump the R1 offset down to R2.  This
459                          * quirk allows all 3 recent offsets to be handled by
460                          * the same code.  (For R0, the swap is a no-op.)  */
461                         match_offset = queue->R[offset_slot];
462                         queue->R[offset_slot] = queue->R[0];
463                         queue->R[0] = match_offset;
464                 } else {
465                         /* Explicit offset  */
466
467                         /* Look up the number of extra bits that need to be read
468                          * to decode offsets with this offset slot.  */
469                         num_extra_bits = lzx_extra_offset_bits[offset_slot];
470
471                         /* Start with the offset slot base value.  */
472                         match_offset = lzx_offset_slot_base[offset_slot];
473
474                         /* In aligned offset blocks, the low-order 3 bits of
475                          * each offset are encoded using the aligned offset
476                          * code.  Otherwise, all the extra bits are literal.  */
477
478                         /*if (block_type == LZX_BLOCKTYPE_ALIGNED && num_extra_bits >= 3) {*/
479                         if ((num_extra_bits & ones_if_aligned) >= 3) {
480                                 match_offset += bitstream_read_bits(istream, num_extra_bits - 3) << 3;
481                                 match_offset += read_huffsym_using_alignedcode(istream, tables);
482                         } else {
483                                 match_offset += bitstream_read_bits(istream, num_extra_bits);
484                         }
485
486                         /* Adjust the offset.  */
487                         match_offset -= LZX_OFFSET_OFFSET;
488
489                         /* Update the match offset LRU queue.  */
490                         queue->R[2] = queue->R[1];
491                         queue->R[1] = queue->R[0];
492                         queue->R[0] = match_offset;
493                 }
494
495                 /* Validate the match, then copy it to the current position.  */
496
497                 if (unlikely(match_len > window_end - window_ptr))
498                         return -1;
499
500                 if (unlikely(match_offset > window_ptr - window))
501                         return -1;
502
503                 lz_copy(window_ptr, match_len, match_offset, window_end,
504                         LZX_MIN_MATCH_LEN);
505
506                 window_ptr += match_len;
507         }
508         return 0;
509 }
510
511 static int
512 lzx_decompress(const void *compressed_data, size_t compressed_size,
513                void *uncompressed_data, size_t uncompressed_size,
514                void *_dec)
515 {
516         struct lzx_decompressor *dec = _dec;
517         struct input_bitstream istream;
518         struct lzx_lru_queue queue;
519         u32 window_pos;
520         int block_type;
521         u32 block_size;
522         bool may_have_e8_byte;
523         int ret;
524
525         init_input_bitstream(&istream, compressed_data, compressed_size);
526
527         /* Initialize the recent offsets queue.  */
528         lzx_lru_queue_init(&queue);
529
530         /* Codeword lengths begin as all 0's for delta encoding purposes.  */
531         memset(dec->tables.maincode_lens, 0, dec->num_main_syms);
532         memset(dec->tables.lencode_lens, 0, LZX_LENCODE_NUM_SYMBOLS);
533
534         /* Set this to true if there may be 0xe8 bytes in the uncompressed data.
535          */
536         may_have_e8_byte = false;
537
538         /* The compressed data will consist of one or more blocks.  The
539          * following loop decompresses one block, and it runs until there all
540          * the compressed data has been decompressed, so there are no more
541          * blocks.  */
542
543         for (window_pos = 0;
544              window_pos < uncompressed_size;
545              window_pos += block_size)
546         {
547                 ret = lzx_read_block_header(&istream, dec->num_main_syms,
548                                             dec->window_order, &block_type,
549                                             &block_size, &dec->tables, &queue);
550                 if (ret)
551                         return ret;
552
553                 if (block_size > uncompressed_size - window_pos)
554                         return -1;
555
556                 if (block_type != LZX_BLOCKTYPE_UNCOMPRESSED) {
557
558                         /* Compressed block.  */
559
560                         ret = lzx_decompress_block(block_type,
561                                                    block_size,
562                                                    uncompressed_data,
563                                                    window_pos,
564                                                    &dec->tables,
565                                                    &queue,
566                                                    &istream);
567                         if (ret)
568                                 return ret;
569
570                         /* If the first 0xe8 byte was in this block, it must
571                          * have been encoded as a literal using mainsym 0xe8. */
572                         if (dec->tables.maincode_lens[0xe8] != 0)
573                                 may_have_e8_byte = true;
574                 } else {
575
576                         /* Uncompressed block.  */
577                         const u8 *p;
578
579                         p = bitstream_read_bytes(&istream, block_size);
580                         if (!p)
581                                 return -1;
582
583                         memcpy(&((u8*)uncompressed_data)[window_pos], p, block_size);
584
585                         /* Re-align the bitstream if an odd number of bytes was
586                          * read.  */
587                         if (block_size & 1)
588                                 bitstream_read_byte(&istream);
589
590                         may_have_e8_byte = true;
591                 }
592         }
593
594         /* Postprocess the data unless it cannot possibly contain 0xe8 bytes  */
595         if (may_have_e8_byte)
596                 lzx_undo_e8_preprocessing(uncompressed_data, uncompressed_size);
597
598         return 0;
599 }
600
601 static void
602 lzx_free_decompressor(void *_dec)
603 {
604         struct lzx_decompressor *dec = _dec;
605
606         ALIGNED_FREE(dec);
607 }
608
609 static int
610 lzx_create_decompressor(size_t max_block_size, void **dec_ret)
611 {
612         struct lzx_decompressor *dec;
613         unsigned window_order;
614
615         window_order = lzx_get_window_order(max_block_size);
616         if (window_order == 0)
617                 return WIMLIB_ERR_INVALID_PARAM;
618
619         /* The aligned allocation is needed to ensure that the lzx_tables are
620          * aligned properly.  */
621         dec = ALIGNED_MALLOC(sizeof(struct lzx_decompressor),
622                              DECODE_TABLE_ALIGNMENT);
623         if (!dec)
624                 return WIMLIB_ERR_NOMEM;
625
626         dec->window_order = window_order;
627         dec->num_main_syms = lzx_get_num_main_syms(window_order);
628
629         *dec_ret = dec;
630         return 0;
631 }
632
633 const struct decompressor_ops lzx_decompressor_ops = {
634         .create_decompressor = lzx_create_decompressor,
635         .decompress          = lzx_decompress,
636         .free_decompressor   = lzx_free_decompressor,
637 };