wimlib.net Git - wimlib/blob - src/lzx_decompress.c

   1 /*
   2  * lzx_decompress.c
   3  *
   4  * A decompressor for the LZX compression format, as used in WIM files.
   5  */
   6
   7 /*
   8  * Copyright (C) 2012-2016 Eric Biggers
   9  *
  10  * This file is free software; you can redistribute it and/or modify it under
  11  * the terms of the GNU Lesser General Public License as published by the Free
  12  * Software Foundation; either version 3 of the License, or (at your option) any
  13  * later version.
  14  *
  15  * This file is distributed in the hope that it will be useful, but WITHOUT
  16  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
  17  * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
  18  * details.
  19  *
  20  * You should have received a copy of the GNU Lesser General Public License
  21  * along with this file; if not, see http://www.gnu.org/licenses/.
  22  */
  23
  24 /*
  25  * LZX is an LZ77 and Huffman-code based compression format that has many
  26  * similarities to DEFLATE (the format used by zlib/gzip).  The compression
  27  * ratio is as good or better than DEFLATE.  See lzx_compress.c for a format
  28  * overview, and see https://en.wikipedia.org/wiki/LZX_(algorithm) for a
  29  * historical overview.  Here I make some pragmatic notes.
  30  *
  31  * The old specification for LZX is the document "Microsoft LZX Data Compression
  32  * Format" (1997).  It defines the LZX format as used in cabinet files.  Allowed
  33  * window sizes are 2^n where 15 <= n <= 21.  However, this document contains
  34  * several errors, so don't read too much into it...
  35  *
  36  * The new specification for LZX is the document "[MS-PATCH]: LZX DELTA
  37  * Compression and Decompression" (2014).  It defines the LZX format as used by
  38  * Microsoft's binary patcher.  It corrects several errors in the 1997 document
  39  * and extends the format in several ways --- namely, optional reference data,
  40  * up to 2^25 byte windows, and longer match lengths.
  41  *
  42  * WIM files use a more restricted form of LZX.  No LZX DELTA extensions are
  43  * present, the window is not "sliding", E8 preprocessing is done
  44  * unconditionally with a fixed file size, and the maximum window size is always
  45  * 2^15 bytes (equal to the size of each "chunk" in a compressed WIM resource).
  46  * This code is primarily intended to implement this form of LZX.  But although
  47  * not compatible with WIMGAPI, this code also supports maximum window sizes up
  48  * to 2^21 bytes.
  49  *
  50  * TODO: Add support for window sizes up to 2^25 bytes.
  51  */
  52
  53 #ifdef HAVE_CONFIG_H
  54 #  include "config.h"
  55 #endif
  56
  57 #include <string.h>
  58
  59 #include "wimlib/decompressor_ops.h"
  60 #include "wimlib/decompress_common.h"
  61 #include "wimlib/error.h"
  62 #include "wimlib/lzx_common.h"
  63 #include "wimlib/util.h"
  64
  65 /* These values are chosen for fast decompression.  */
  66 #define LZX_MAINCODE_TABLEBITS          11
  67 #define LZX_LENCODE_TABLEBITS           10
  68 #define LZX_PRECODE_TABLEBITS           6
  69 #define LZX_ALIGNEDCODE_TABLEBITS       7
  70
  71 static void _unused_attribute
  72 check_enough_values(void)
  73 {
  74 /* ./enough 656 11 16 */
  75 #define LZX_MAINCODE_ENOUGH             2726
  76         STATIC_ASSERT(LZX_MAINCODE_MAX_NUM_SYMBOLS == 656);
  77         STATIC_ASSERT(LZX_MAINCODE_TABLEBITS == 11);
  78         STATIC_ASSERT(LZX_MAX_MAIN_CODEWORD_LEN == 16);
  79
  80 /* ./enough 249 10 16 */
  81 #define LZX_LENCODE_ENOUGH              1326
  82         STATIC_ASSERT(LZX_LENCODE_NUM_SYMBOLS == 249);
  83         STATIC_ASSERT(LZX_LENCODE_TABLEBITS == 10);
  84         STATIC_ASSERT(LZX_MAX_LEN_CODEWORD_LEN == 16);
  85
  86 /* ./enough 20 6 15 */
  87 #define LZX_PRECODE_ENOUGH              582
  88         STATIC_ASSERT(LZX_PRECODE_NUM_SYMBOLS == 20);
  89         STATIC_ASSERT(LZX_PRECODE_TABLEBITS == 6);
  90         STATIC_ASSERT(LZX_MAX_PRE_CODEWORD_LEN == 15);
  91
  92 /* ./enough 8 7 7 */
  93 #define LZX_ALIGNEDCODE_ENOUGH          128
  94         STATIC_ASSERT(LZX_ALIGNEDCODE_NUM_SYMBOLS == 8);
  95         STATIC_ASSERT(LZX_ALIGNEDCODE_TABLEBITS == 7);
  96         STATIC_ASSERT(LZX_MAX_ALIGNED_CODEWORD_LEN == 7);
  97 }
  98
  99
 100 #define LZX_READ_LENS_MAX_OVERRUN 50
 101
 102 /* Reusable heap-allocated memory for LZX decompression */
 103 struct lzx_decompressor {
 104
 105         /* Huffman decoding tables and codeword lengths */
 106
 107         u16 maincode_decode_table[LZX_MAINCODE_ENOUGH]
 108                         _aligned_attribute(DECODE_TABLE_ALIGNMENT);
 109         u8 maincode_lens[LZX_MAINCODE_MAX_NUM_SYMBOLS + LZX_READ_LENS_MAX_OVERRUN];
 110
 111         u16 lencode_decode_table[LZX_LENCODE_ENOUGH]
 112                         _aligned_attribute(DECODE_TABLE_ALIGNMENT);
 113         u8 lencode_lens[LZX_LENCODE_NUM_SYMBOLS + LZX_READ_LENS_MAX_OVERRUN];
 114
 115         union {
 116                 u16 alignedcode_decode_table[LZX_ALIGNEDCODE_ENOUGH]
 117                         _aligned_attribute(DECODE_TABLE_ALIGNMENT);
 118                 u8 alignedcode_lens[LZX_ALIGNEDCODE_NUM_SYMBOLS];
 119         };
 120
 121         union {
 122                 u16 precode_decode_table[LZX_PRECODE_ENOUGH]
 123                         _aligned_attribute(DECODE_TABLE_ALIGNMENT);
 124                 u8 precode_lens[LZX_PRECODE_NUM_SYMBOLS];
 125         };
 126
 127         unsigned window_order;
 128         unsigned num_main_syms;
 129 } _aligned_attribute(DECODE_TABLE_ALIGNMENT);
 130
 131 /* Read a Huffman-encoded symbol using the precode. */
 132 static inline unsigned
 133 read_presym(const struct lzx_decompressor *d, struct input_bitstream *is)
 134 {
 135         return read_huffsym(is, d->precode_decode_table,
 136                             LZX_PRECODE_TABLEBITS, LZX_MAX_PRE_CODEWORD_LEN);
 137 }
 138
 139 /* Read a Huffman-encoded symbol using the main code. */
 140 static inline unsigned
 141 read_mainsym(const struct lzx_decompressor *d, struct input_bitstream *is)
 142 {
 143         return read_huffsym(is, d->maincode_decode_table,
 144                             LZX_MAINCODE_TABLEBITS, LZX_MAX_MAIN_CODEWORD_LEN);
 145 }
 146
 147 /* Read a Huffman-encoded symbol using the length code. */
 148 static inline unsigned
 149 read_lensym(const struct lzx_decompressor *d, struct input_bitstream *is)
 150 {
 151         return read_huffsym(is, d->lencode_decode_table,
 152                             LZX_LENCODE_TABLEBITS, LZX_MAX_LEN_CODEWORD_LEN);
 153 }
 154
 155 /* Read a Huffman-encoded symbol using the aligned offset code. */
 156 static inline unsigned
 157 read_alignedsym(const struct lzx_decompressor *d, struct input_bitstream *is)
 158 {
 159         return read_huffsym(is, d->alignedcode_decode_table,
 160                             LZX_ALIGNEDCODE_TABLEBITS, LZX_MAX_ALIGNED_CODEWORD_LEN);
 161 }
 162
 163 /*
 164  * Read a precode from the compressed input bitstream, then use it to decode
 165  * @num_lens codeword length values and write them to @lens.
 166  */
 167 static int
 168 lzx_read_codeword_lens(struct lzx_decompressor *d, struct input_bitstream *is,
 169                        u8 *lens, unsigned num_lens)
 170 {
 171         u8 *len_ptr = lens;
 172         u8 *lens_end = lens + num_lens;
 173
 174         /* Read the lengths of the precode codewords, which are stored
 175          * explicitly. */
 176         for (int i = 0; i < LZX_PRECODE_NUM_SYMBOLS; i++) {
 177                 d->precode_lens[i] =
 178                         bitstream_read_bits(is, LZX_PRECODE_ELEMENT_SIZE);
 179         }
 180
 181         /* Build the decoding table for the precode. */
 182         if (make_huffman_decode_table(d->precode_decode_table,
 183                                       LZX_PRECODE_NUM_SYMBOLS,
 184                                       LZX_PRECODE_TABLEBITS,
 185                                       d->precode_lens,
 186                                       LZX_MAX_PRE_CODEWORD_LEN))
 187                 return -1;
 188
 189         /* Decode the codeword lengths. */
 190         do {
 191                 unsigned presym;
 192                 u8 len;
 193
 194                 /* Read the next precode symbol. */
 195                 presym = read_presym(d, is);
 196                 if (presym < 17) {
 197                         /* Difference from old length */
 198                         len = *len_ptr - presym;
 199                         if ((s8)len < 0)
 200                                 len += 17;
 201                         *len_ptr++ = len;
 202                 } else {
 203                         /* Special RLE values */
 204
 205                         unsigned run_len;
 206
 207                         if (presym == 17) {
 208                                 /* Run of 0's */
 209                                 run_len = 4 + bitstream_read_bits(is, 4);
 210                                 len = 0;
 211                         } else if (presym == 18) {
 212                                 /* Longer run of 0's */
 213                                 run_len = 20 + bitstream_read_bits(is, 5);
 214                                 len = 0;
 215                         } else {
 216                                 /* Run of identical lengths */
 217                                 run_len = 4 + bitstream_read_bits(is, 1);
 218                                 presym = read_presym(d, is);
 219                                 if (unlikely(presym > 17))
 220                                         return -1;
 221                                 len = *len_ptr - presym;
 222                                 if ((s8)len < 0)
 223                                         len += 17;
 224                         }
 225
 226                         do {
 227                                 *len_ptr++ = len;
 228                         } while (--run_len);
 229                         /*
 230                          * The worst case overrun is when presym == 18,
 231                          * run_len == 20 + 31, and only 1 length was remaining.
 232                          * So LZX_READ_LENS_MAX_OVERRUN == 50.
 233                          *
 234                          * Overrun while reading the first half of maincode_lens
 235                          * can corrupt the previous values in the second half.
 236                          * This doesn't really matter because the resulting
 237                          * lengths will still be in range, and data that
 238                          * generates overruns is invalid anyway.
 239                          */
 240                 }
 241         } while (len_ptr < lens_end);
 242
 243         return 0;
 244 }
 245
 246 /*
 247  * Read the header of an LZX block and save the block type and size in
 248  * *block_type_ret and *block_size_ret, respectively.
 249  *
 250  * If the block is compressed, the nalso initialize the decode tables for the
 251  * new Huffman codes.
 252  *
 253  * If the block is uncompressed, then also update the recent offsets queue.
 254  *
 255  * Return 0 on success, or -1 if the data was invalid.
 256  */
 257 static int
 258 lzx_read_block_header(struct lzx_decompressor *d, struct input_bitstream *is,
 259                       u32 recent_offsets[], int *block_type_ret,
 260                       u32 *block_size_ret)
 261 {
 262         int block_type;
 263         u32 block_size;
 264
 265         bitstream_ensure_bits(is, 4);
 266
 267         /* Read the block type. */
 268         block_type = bitstream_pop_bits(is, 3);
 269
 270         /* Read the block size. */
 271         if (bitstream_pop_bits(is, 1)) {
 272                 block_size = LZX_DEFAULT_BLOCK_SIZE;
 273         } else {
 274                 u32 tmp;
 275                 block_size = 0;
 276
 277                 tmp = bitstream_read_bits(is, 8);
 278                 block_size |= tmp;
 279                 tmp = bitstream_read_bits(is, 8);
 280                 block_size <<= 8;
 281                 block_size |= tmp;
 282
 283                 if (d->window_order >= 16) {
 284                         tmp = bitstream_read_bits(is, 8);
 285                         block_size <<= 8;
 286                         block_size |= tmp;
 287                 }
 288         }
 289
 290         switch (block_type) {
 291
 292         case LZX_BLOCKTYPE_ALIGNED:
 293
 294                 /* Read the aligned offset code and build its decode table. */
 295
 296                 for (int i = 0; i < LZX_ALIGNEDCODE_NUM_SYMBOLS; i++) {
 297                         d->alignedcode_lens[i] =
 298                                 bitstream_read_bits(is,
 299                                                     LZX_ALIGNEDCODE_ELEMENT_SIZE);
 300                 }
 301
 302                 if (make_huffman_decode_table(d->alignedcode_decode_table,
 303                                               LZX_ALIGNEDCODE_NUM_SYMBOLS,
 304                                               LZX_ALIGNEDCODE_TABLEBITS,
 305                                               d->alignedcode_lens,
 306                                               LZX_MAX_ALIGNED_CODEWORD_LEN))
 307                         return -1;
 308
 309                 /* Fall though, since the rest of the header for aligned offset
 310                  * blocks is the same as that for verbatim blocks.  */
 311
 312         case LZX_BLOCKTYPE_VERBATIM:
 313
 314                 /* Read the main code and build its decode table.  The codeword
 315                  * lengths in the main code are encoded in two parts: one part
 316                  * for literal symbols, and one part for match symbols. */
 317
 318                 if (lzx_read_codeword_lens(d, is, d->maincode_lens,
 319                                            LZX_NUM_CHARS))
 320                         return -1;
 321
 322                 if (lzx_read_codeword_lens(d, is, d->maincode_lens + LZX_NUM_CHARS,
 323                                            d->num_main_syms - LZX_NUM_CHARS))
 324                         return -1;
 325
 326                 if (make_huffman_decode_table(d->maincode_decode_table,
 327                                               d->num_main_syms,
 328                                               LZX_MAINCODE_TABLEBITS,
 329                                               d->maincode_lens,
 330                                               LZX_MAX_MAIN_CODEWORD_LEN))
 331                         return -1;
 332
 333                 /* Read the length code and build its decode table. */
 334
 335                 if (lzx_read_codeword_lens(d, is, d->lencode_lens,
 336                                            LZX_LENCODE_NUM_SYMBOLS))
 337                         return -1;
 338
 339                 if (make_huffman_decode_table(d->lencode_decode_table,
 340                                               LZX_LENCODE_NUM_SYMBOLS,
 341                                               LZX_LENCODE_TABLEBITS,
 342                                               d->lencode_lens,
 343                                               LZX_MAX_LEN_CODEWORD_LEN))
 344                         return -1;
 345
 346                 break;
 347
 348         case LZX_BLOCKTYPE_UNCOMPRESSED:
 349                 /*
 350                  * The header of an uncompressed block contains new values for
 351                  * the recent offsets queue, starting on the next 16-bit
 352                  * boundary in the bitstream.  Careful: if the stream is
 353                  * *already* aligned, the correct thing to do is to throw away
 354                  * the next 16 bits (this is probably a mistake in the format).
 355                  */
 356                 bitstream_ensure_bits(is, 1);
 357                 bitstream_align(is);
 358                 recent_offsets[0] = bitstream_read_u32(is);
 359                 recent_offsets[1] = bitstream_read_u32(is);
 360                 recent_offsets[2] = bitstream_read_u32(is);
 361
 362                 /* Offsets of 0 are invalid. */
 363                 if (recent_offsets[0] == 0 || recent_offsets[1] == 0 ||
 364                     recent_offsets[2] == 0)
 365                         return -1;
 366                 break;
 367
 368         default:
 369                 /* Unrecognized block type */
 370                 return -1;
 371         }
 372
 373         *block_type_ret = block_type;
 374         *block_size_ret = block_size;
 375         return 0;
 376 }
 377
 378 /* Decompress a block of LZX-compressed data. */
 379 static int
 380 lzx_decompress_block(const struct lzx_decompressor *d,
 381                      struct input_bitstream *is, int block_type, u32 block_size,
 382                      u8 * const out_begin, u8 *out_next, u32 recent_offsets[])
 383 {
 384         bool is_aligned_block = (block_type == LZX_BLOCKTYPE_ALIGNED);
 385         u8 * const block_end = out_next + block_size;
 386
 387         do {
 388                 unsigned mainsym;
 389                 unsigned length;
 390                 u32 offset;
 391                 unsigned offset_slot;
 392                 unsigned num_extra_bits;
 393
 394                 mainsym = read_mainsym(d, is);
 395                 if (mainsym < LZX_NUM_CHARS) {
 396                         /* Literal */
 397                         *out_next++ = mainsym;
 398                         continue;
 399                 }
 400
 401                 /* Match */
 402
 403                 /* Decode the length header and offset slot.  */
 404                 mainsym -= LZX_NUM_CHARS;
 405                 length = mainsym % LZX_NUM_LEN_HEADERS;
 406                 offset_slot = mainsym / LZX_NUM_LEN_HEADERS;
 407
 408                 /* If needed, read a length symbol to decode the full length. */
 409                 if (length == LZX_NUM_PRIMARY_LENS)
 410                         length += read_lensym(d, is);
 411                 length += LZX_MIN_MATCH_LEN;
 412
 413                 if (offset_slot < LZX_NUM_RECENT_OFFSETS) {
 414                         /* Repeat offset  */
 415
 416                         /* Note: This isn't a real LRU queue, since using the R2
 417                          * offset doesn't bump the R1 offset down to R2.  This
 418                          * quirk allows all 3 recent offsets to be handled by
 419                          * the same code.  (For R0, the swap is a no-op.)  */
 420                         offset = recent_offsets[offset_slot];
 421                         recent_offsets[offset_slot] = recent_offsets[0];
 422                 } else {
 423                         /* Explicit offset  */
 424
 425                         /* Look up the number of extra bits that need to be read
 426                          * to decode offsets with this offset slot.  */
 427                         num_extra_bits = lzx_extra_offset_bits[offset_slot];
 428
 429                         /* Start with the offset slot base value.  */
 430                         offset = lzx_offset_slot_base[offset_slot];
 431
 432                         /* In aligned offset blocks, the low-order 3 bits of
 433                          * each offset are encoded using the aligned offset
 434                          * code.  Otherwise, all the extra bits are literal.  */
 435
 436                         if (is_aligned_block && offset_slot >= 8) {
 437                                 offset +=
 438                                         bitstream_read_bits(is,
 439                                                             num_extra_bits -
 440                                                                 LZX_NUM_ALIGNED_OFFSET_BITS)
 441                                                         << LZX_NUM_ALIGNED_OFFSET_BITS;
 442                                 offset += read_alignedsym(d, is);
 443                         } else {
 444                                 offset += bitstream_read_bits(is, num_extra_bits);
 445                         }
 446
 447                         /* Adjust the offset.  */
 448                         offset -= LZX_OFFSET_ADJUSTMENT;
 449
 450                         /* Update the match offset LRU queue.  */
 451                         STATIC_ASSERT(LZX_NUM_RECENT_OFFSETS == 3);
 452                         recent_offsets[2] = recent_offsets[1];
 453                         recent_offsets[1] = recent_offsets[0];
 454                 }
 455                 recent_offsets[0] = offset;
 456
 457                 /* Validate the match, then copy it to the current position.  */
 458
 459                 if (unlikely(length > block_end - out_next))
 460                         return -1;
 461
 462                 if (unlikely(offset > out_next - out_begin))
 463                         return -1;
 464
 465                 lz_copy(out_next, length, offset, block_end, LZX_MIN_MATCH_LEN);
 466
 467                 out_next += length;
 468
 469         } while (out_next != block_end);
 470
 471         return 0;
 472 }
 473
 474 static int
 475 lzx_decompress(const void *restrict compressed_data, size_t compressed_size,
 476                void *restrict uncompressed_data, size_t uncompressed_size,
 477                void *restrict _d)
 478 {
 479         struct lzx_decompressor *d = _d;
 480         u8 * const out_begin = uncompressed_data;
 481         u8 *out_next = out_begin;
 482         u8 * const out_end = out_begin + uncompressed_size;
 483         struct input_bitstream is;
 484         STATIC_ASSERT(LZX_NUM_RECENT_OFFSETS == 3);
 485         u32 recent_offsets[] = {1, 1, 1};
 486         unsigned e8_status = 0;
 487
 488         init_input_bitstream(&is, compressed_data, compressed_size);
 489
 490         /* Codeword lengths begin as all 0's for delta encoding purposes. */
 491         memset(d->maincode_lens, 0, d->num_main_syms);
 492         memset(d->lencode_lens, 0, LZX_LENCODE_NUM_SYMBOLS);
 493
 494         /* Decompress blocks until we have all the uncompressed data. */
 495
 496         while (out_next != out_end) {
 497                 int block_type;
 498                 u32 block_size;
 499
 500                 if (lzx_read_block_header(d, &is, recent_offsets,
 501                                           &block_type, &block_size))
 502                         return -1;
 503
 504                 if (block_size < 1 || block_size > out_end - out_next)
 505                         return -1;
 506
 507                 if (block_type != LZX_BLOCKTYPE_UNCOMPRESSED) {
 508
 509                         /* Compressed block */
 510                         if (lzx_decompress_block(d, &is, block_type, block_size,
 511                                                  out_begin, out_next,
 512                                                  recent_offsets))
 513                                 return -1;
 514
 515                         /* If the first E8 byte was in this block, then it must
 516                          * have been encoded as a literal using mainsym E8. */
 517                         e8_status |= d->maincode_lens[0xE8];
 518                 } else {
 519
 520                         /* Uncompressed block */
 521                         if (bitstream_read_bytes(&is, out_next, block_size))
 522                                 return -1;
 523
 524                         /* Re-align the bitstream if needed. */
 525                         if (block_size & 1)
 526                                 bitstream_read_byte(&is);
 527
 528                         /* There may have been an E8 byte in the block. */
 529                         e8_status = 1;
 530                 }
 531                 out_next += block_size;
 532         }
 533
 534         /* Postprocess the data unless it cannot possibly contain E8 bytes. */
 535         if (e8_status)
 536                 lzx_postprocess(uncompressed_data, uncompressed_size);
 537
 538         return 0;
 539 }
 540
 541 static void
 542 lzx_free_decompressor(void *_d)
 543 {
 544         ALIGNED_FREE(_d);
 545 }
 546
 547 static int
 548 lzx_create_decompressor(size_t max_block_size, void **d_ret)
 549 {
 550         unsigned window_order;
 551         struct lzx_decompressor *d;
 552
 553         window_order = lzx_get_window_order(max_block_size);
 554         if (window_order == 0)
 555                 return WIMLIB_ERR_INVALID_PARAM;
 556
 557         d = ALIGNED_MALLOC(sizeof(*d), DECODE_TABLE_ALIGNMENT);
 558         if (!d)
 559                 return WIMLIB_ERR_NOMEM;
 560
 561         d->window_order = window_order;
 562         d->num_main_syms = lzx_get_num_main_syms(window_order);
 563
 564         *d_ret = d;
 565         return 0;
 566 }
 567
 568 const struct decompressor_ops lzx_decompressor_ops = {
 569         .create_decompressor = lzx_create_decompressor,
 570         .decompress          = lzx_decompress,
 571         .free_decompressor   = lzx_free_decompressor,
 572 };