[wimlib] / src / lzx_decompress.c

/*
 * lzx_decompress.c
 *
 * A decompressor for the LZX compression format, as used in WIM files.
 */

/*
 * Copyright (C) 2012, 2013, 2014, 2015 Eric Biggers
 *
 * This file is free software; you can redistribute it and/or modify it under
 * the terms of the GNU Lesser General Public License as published by the Free
 * Software Foundation; either version 3 of the License, or (at your option) any
 * later version.
 *
 * This file is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
 * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
 * details.
 *
 * You should have received a copy of the GNU Lesser General Public License
 * along with this file; if not, see http://www.gnu.org/licenses/.
 */

/*
 * LZX is an LZ77 and Huffman-code based compression format that has many
 * similarities to DEFLATE (the format used by zlib/gzip).  The compression
 * ratio is as good or better than DEFLATE.  See lzx_compress.c for a format
 * overview, and see https://en.wikipedia.org/wiki/LZX_(algorithm) for a
 * historical overview.  Here I make some pragmatic notes.
 *
 * The old specification for LZX is the document "Microsoft LZX Data Compression
 * Format" (1997).  It defines the LZX format as used in cabinet files.  Allowed
 * window sizes are 2^n where 15 <= n <= 21.  However, this document contains
 * several errors, so don't read too much into it...
 *
 * The new specification for LZX is the document "[MS-PATCH]: LZX DELTA
 * Compression and Decompression" (2014).  It defines the LZX format as used by
 * Microsoft's binary patcher.  It corrects several errors in the 1997 document
 * and extends the format in several ways --- namely, optional reference data,
 * up to 2^25 byte windows, and longer match lengths.
 *
 * WIM files use a more restricted form of LZX.  No LZX DELTA extensions are
 * present, the window is not "sliding", E8 preprocessing is done
 * unconditionally with a fixed file size, and the maximum window size is always
 * 2^15 bytes (equal to the size of each "chunk" in a compressed WIM resource).
 * This code is primarily intended to implement this form of LZX.  But although
 * not compatible with WIMGAPI, this code also supports maximum window sizes up
 * to 2^21 bytes.
 *
 * TODO: Add support for window sizes up to 2^25 bytes.
 */

#ifdef HAVE_CONFIG_H
#  include "config.h"
#endif

#include "wimlib/decompressor_ops.h"
#include "wimlib/decompress_common.h"
#include "wimlib/error.h"
#include "wimlib/lzx_common.h"
#include "wimlib/util.h"

#include <string.h>

/* These values are chosen for fast decompression.  */
#define LZX_MAINCODE_TABLEBITS          11
#define LZX_LENCODE_TABLEBITS           10
#define LZX_PRECODE_TABLEBITS           6
#define LZX_ALIGNEDCODE_TABLEBITS       7

#define LZX_READ_LENS_MAX_OVERRUN 50

/* Huffman decoding tables, and arrays that map symbols to codeword lengths.  */
struct lzx_tables {

        u16 maincode_decode_table[(1 << LZX_MAINCODE_TABLEBITS) +
                                        (LZX_MAINCODE_MAX_NUM_SYMBOLS * 2)]
                                        _aligned_attribute(DECODE_TABLE_ALIGNMENT);
        u8 maincode_lens[LZX_MAINCODE_MAX_NUM_SYMBOLS + LZX_READ_LENS_MAX_OVERRUN];


        u16 lencode_decode_table[(1 << LZX_LENCODE_TABLEBITS) +
                                        (LZX_LENCODE_NUM_SYMBOLS * 2)]
                                        _aligned_attribute(DECODE_TABLE_ALIGNMENT);
        u8 lencode_lens[LZX_LENCODE_NUM_SYMBOLS + LZX_READ_LENS_MAX_OVERRUN];


        u16 alignedcode_decode_table[(1 << LZX_ALIGNEDCODE_TABLEBITS) +
                                        (LZX_ALIGNEDCODE_NUM_SYMBOLS * 2)]
                                        _aligned_attribute(DECODE_TABLE_ALIGNMENT);
        u8 alignedcode_lens[LZX_ALIGNEDCODE_NUM_SYMBOLS];
} _aligned_attribute(DECODE_TABLE_ALIGNMENT);

/* Least-recently used queue for match offsets.  */
struct lzx_lru_queue {
        u32 R[LZX_NUM_RECENT_OFFSETS];
};

static inline void
lzx_lru_queue_init(struct lzx_lru_queue *queue)
{
        for (unsigned i = 0; i < LZX_NUM_RECENT_OFFSETS; i++)
                queue->R[i] = 1;
}

/* The main LZX decompressor structure.
 *
 * Note: we keep track of most of the decompression state outside this
 * structure.  This structure only exists so that (1) we can store @window_order
 * and @num_main_syms for multiple calls to lzx_decompress(); and (2) so that we
 * don't have to allocate the large 'struct lzx_tables' on the stack.  */
struct lzx_decompressor {
        unsigned window_order;
        unsigned num_main_syms;
        struct lzx_tables tables;
};

/* Read a Huffman-encoded symbol using the precode.  */
static inline u16
read_huffsym_using_precode(struct input_bitstream *istream,
                           const u16 precode_decode_table[])
{
        return read_huffsym(istream, precode_decode_table,
                            LZX_PRECODE_TABLEBITS, LZX_MAX_PRE_CODEWORD_LEN);
}

/* Read a Huffman-encoded symbol using the main code.  */
static inline u16
read_huffsym_using_maincode(struct input_bitstream *istream,
                            const struct lzx_tables *tables)
{
        return read_huffsym(istream, tables->maincode_decode_table,
                            LZX_MAINCODE_TABLEBITS, LZX_MAX_MAIN_CODEWORD_LEN);
}

/* Read a Huffman-encoded symbol using the length code.  */
static inline u16
read_huffsym_using_lencode(struct input_bitstream *istream,
                           const struct lzx_tables *tables)
{
        return read_huffsym(istream, tables->lencode_decode_table,
                            LZX_LENCODE_TABLEBITS, LZX_MAX_LEN_CODEWORD_LEN);
}

/* Read a Huffman-encoded symbol using the aligned offset code.  */
static inline u16
read_huffsym_using_alignedcode(struct input_bitstream *istream,
                               const struct lzx_tables *tables)
{
        return read_huffsym(istream, tables->alignedcode_decode_table,
                            LZX_ALIGNEDCODE_TABLEBITS, LZX_MAX_ALIGNED_CODEWORD_LEN);
}

/*
 * Read the precode from the compressed input bitstream, then use it to decode
 * @num_lens codeword length values.
 *
 * @istream:
 *      The input bitstream.
 *
 * @lens:
 *      An array that contains the length values from the previous time the
 *      codeword lengths for this Huffman code were read, or all 0's if this is
 *      the first time.  This array must have at least (@num_lens +
 *      LZX_READ_LENS_MAX_OVERRUN) entries.
 *
 * @num_lens:
 *      Number of length values to decode.
 *
 * Returns 0 on success, or -1 if the data was invalid.
 */
static int
lzx_read_codeword_lens(struct input_bitstream *istream, u8 *lens, unsigned num_lens)
{
        u16 precode_decode_table[(1 << LZX_PRECODE_TABLEBITS) +
                                        (LZX_PRECODE_NUM_SYMBOLS * 2)]
                                        _aligned_attribute(DECODE_TABLE_ALIGNMENT);
        u8 precode_lens[LZX_PRECODE_NUM_SYMBOLS];
        u8 *len_ptr = lens;
        u8 *lens_end = lens + num_lens;
        int ret;

        /* Read the lengths of the precode codewords.  These are given
         * explicitly.  */
        for (int i = 0; i < LZX_PRECODE_NUM_SYMBOLS; i++) {
                precode_lens[i] = bitstream_read_bits(istream,
                                                      LZX_PRECODE_ELEMENT_SIZE);
        }

        /* Make the decoding table for the precode.  */
        ret = make_huffman_decode_table(precode_decode_table,
                                        LZX_PRECODE_NUM_SYMBOLS,
                                        LZX_PRECODE_TABLEBITS,
                                        precode_lens,
                                        LZX_MAX_PRE_CODEWORD_LEN);
        if (ret)
                return ret;

        /* Decode the codeword lengths.  */
        do {
                unsigned presym;
                u8 len;

                /* Read the next precode symbol.  */
                presym = read_huffsym_using_precode(istream,
                                                    precode_decode_table);
                if (presym < 17) {
                        /* Difference from old length  */
                        len = *len_ptr - presym;
                        if ((s8)len < 0)
                                len += 17;
                        *len_ptr++ = len;
                } else {
                        /* Special RLE values  */

                        unsigned run_len;

                        if (presym == 17) {
                                /* Run of 0's  */
                                run_len = 4 + bitstream_read_bits(istream, 4);
                                len = 0;
                        } else if (presym == 18) {
                                /* Longer run of 0's  */
                                run_len = 20 + bitstream_read_bits(istream, 5);
                                len = 0;
                        } else {
                                /* Run of identical lengths  */
                                run_len = 4 + bitstream_read_bits(istream, 1);
                                presym = read_huffsym_using_precode(istream,
                                                                    precode_decode_table);
                                len = *len_ptr - presym;
                                if ((s8)len < 0)
                                        len += 17;
                        }

                        do {
                                *len_ptr++ = len;
                        } while (--run_len);
                        /* Worst case overrun is when presym == 18,
                         * run_len == 20 + 31, and only 1 length was remaining.
                         * So LZX_READ_LENS_MAX_OVERRUN == 50.
                         *
                         * Overrun while reading the first half of maincode_lens
                         * can corrupt the previous values in the second half.
                         * This doesn't really matter because the resulting
                         * lengths will still be in range, and data that
                         * generates overruns is invalid anyway.  */
                }
        } while (len_ptr < lens_end);
        return 0;
}

/*
 * Read the header of an LZX block and save the block type and size in
 * *block_type_ret and *block_size_ret, respectively.
 *
 * If the block is compressed, also update the Huffman decode @tables with the
 * new Huffman codes.
 *
 * If the block is uncompressed, also update the match offset @queue with the
 * new match offsets.
 *
 * Return 0 on success, or -1 if the data was invalid.
 */
static int
lzx_read_block_header(struct input_bitstream *istream,
                      unsigned num_main_syms,
                      unsigned window_order,
                      int *block_type_ret,
                      u32 *block_size_ret,
                      struct lzx_tables *tables,
                      struct lzx_lru_queue *queue)
{
        int block_type;
        u32 block_size;
        int ret;

        bitstream_ensure_bits(istream, 4);

        /* The first three bits tell us what kind of block it is, and should be
         * one of the LZX_BLOCKTYPE_* values.  */
        block_type = bitstream_pop_bits(istream, 3);

        /* Read the block size.  This mirrors the behavior of
         * lzx_write_compressed_block() in lzx_compress.c; see that for more
         * details.  */
        if (bitstream_pop_bits(istream, 1)) {
                block_size = LZX_DEFAULT_BLOCK_SIZE;
        } else {
                u32 tmp;
                block_size = 0;

                tmp = bitstream_read_bits(istream, 8);
                block_size |= tmp;
                tmp = bitstream_read_bits(istream, 8);
                block_size <<= 8;
                block_size |= tmp;

                if (window_order >= 16) {
                        tmp = bitstream_read_bits(istream, 8);
                        block_size <<= 8;
                        block_size |= tmp;
                }
        }

        switch (block_type) {

        case LZX_BLOCKTYPE_ALIGNED:

                /* Read the aligned offset code and prepare its decode table.
                 */

                for (int i = 0; i < LZX_ALIGNEDCODE_NUM_SYMBOLS; i++) {
                        tables->alignedcode_lens[i] =
                                bitstream_read_bits(istream,
                                                    LZX_ALIGNEDCODE_ELEMENT_SIZE);
                }

                ret = make_huffman_decode_table(tables->alignedcode_decode_table,
                                                LZX_ALIGNEDCODE_NUM_SYMBOLS,
                                                LZX_ALIGNEDCODE_TABLEBITS,
                                                tables->alignedcode_lens,
                                                LZX_MAX_ALIGNED_CODEWORD_LEN);
                if (ret)
                        return ret;

                /* Fall though, since the rest of the header for aligned offset
                 * blocks is the same as that for verbatim blocks.  */

        case LZX_BLOCKTYPE_VERBATIM:

                /* Read the main code and prepare its decode table.
                 *
                 * Note that the codeword lengths in the main code are encoded
                 * in two parts: one part for literal symbols, and one part for
                 * match symbols.  */

                ret = lzx_read_codeword_lens(istream, tables->maincode_lens,
                                             LZX_NUM_CHARS);
                if (ret)
                        return ret;

                ret = lzx_read_codeword_lens(istream,
                                             tables->maincode_lens + LZX_NUM_CHARS,
                                             num_main_syms - LZX_NUM_CHARS);
                if (ret)
                        return ret;

                ret = make_huffman_decode_table(tables->maincode_decode_table,
                                                num_main_syms,
                                                LZX_MAINCODE_TABLEBITS,
                                                tables->maincode_lens,
                                                LZX_MAX_MAIN_CODEWORD_LEN);
                if (ret)
                        return ret;

                /* Read the length code and prepare its decode table.  */

                ret = lzx_read_codeword_lens(istream, tables->lencode_lens,
                                             LZX_LENCODE_NUM_SYMBOLS);
                if (ret)
                        return ret;

                ret = make_huffman_decode_table(tables->lencode_decode_table,
                                                LZX_LENCODE_NUM_SYMBOLS,
                                                LZX_LENCODE_TABLEBITS,
                                                tables->lencode_lens,
                                                LZX_MAX_LEN_CODEWORD_LEN);
                if (ret)
                        return ret;

                break;

        case LZX_BLOCKTYPE_UNCOMPRESSED:

                /* Before reading the three LRU match offsets from the
                 * uncompressed block header, the stream must be aligned on a
                 * 16-bit boundary.  But, unexpectedly, if the stream is
                 * *already* aligned, the correct thing to do is to throw away
                 * the next 16 bits.  */

                bitstream_ensure_bits(istream, 1);
                bitstream_align(istream);
                queue->R[0] = bitstream_read_u32(istream);
                queue->R[1] = bitstream_read_u32(istream);
                queue->R[2] = bitstream_read_u32(istream);

                /* Offsets of 0 are invalid.  */
                if (queue->R[0] == 0 || queue->R[1] == 0 || queue->R[2] == 0)
                        return -1;
                break;

        default:
                /* Unrecognized block type.  */
                return -1;
        }

        *block_type_ret = block_type;
        *block_size_ret = block_size;
        return 0;
}

/*
 * Decompress an LZX-compressed block of data.
 *
 * @block_type:
 *      The type of the block (LZX_BLOCKTYPE_VERBATIM or LZX_BLOCKTYPE_ALIGNED).
 *
 * @block_size:
 *      The size of the block, in bytes.
 *
 * @window:
 *      Pointer to the beginning of the decompression window.
 *
 * @window_pos:
 *      The position in the window at which the block starts.
 *
 * @tables:
 *      The Huffman decoding tables for the block.
 *
 * @queue:
 *      The least-recently-used queue for match offsets.
 *
 * @istream:
 *      The input bitstream, positioned at the start of the block data.
 *
 * Returns 0 on success, or -1 if the data was invalid.
 */
static int
lzx_decompress_block(int block_type, u32 block_size,
                     u8 *window, u32 window_pos,
                     const struct lzx_tables *tables,
                     struct lzx_lru_queue *queue,
                     struct input_bitstream *istream)
{
        u8 *window_ptr = &window[window_pos];
        u8 *window_end = window_ptr + block_size;
        unsigned mainsym;
        u32 match_len;
        unsigned offset_slot;
        u32 match_offset;
        unsigned num_extra_bits;
        unsigned ones_if_aligned = 0U - (block_type == LZX_BLOCKTYPE_ALIGNED);

        while (window_ptr != window_end) {

                mainsym = read_huffsym_using_maincode(istream, tables);
                if (mainsym < LZX_NUM_CHARS) {
                        /* Literal  */
                        *window_ptr++ = mainsym;
                        continue;
                }

                /* Match  */

                /* Decode the length header and offset slot.  */
                mainsym -= LZX_NUM_CHARS;
                match_len = mainsym % LZX_NUM_LEN_HEADERS;
                offset_slot = mainsym / LZX_NUM_LEN_HEADERS;

                /* If needed, read a length symbol to decode the full length. */
                if (match_len == LZX_NUM_PRIMARY_LENS)
                        match_len += read_huffsym_using_lencode(istream, tables);
                match_len += LZX_MIN_MATCH_LEN;

                if (offset_slot < LZX_NUM_RECENT_OFFSETS) {
                        /* Repeat offset  */

                        /* Note: This isn't a real LRU queue, since using the R2
                         * offset doesn't bump the R1 offset down to R2.  This
                         * quirk allows all 3 recent offsets to be handled by
                         * the same code.  (For R0, the swap is a no-op.)  */
                        match_offset = queue->R[offset_slot];
                        queue->R[offset_slot] = queue->R[0];
                        queue->R[0] = match_offset;
                } else {
                        /* Explicit offset  */

                        /* Look up the number of extra bits that need to be read
                         * to decode offsets with this offset slot.  */
                        num_extra_bits = lzx_extra_offset_bits[offset_slot];

                        /* Start with the offset slot base value.  */
                        match_offset = lzx_offset_slot_base[offset_slot];

                        /* In aligned offset blocks, the low-order 3 bits of
                         * each offset are encoded using the aligned offset
                         * code.  Otherwise, all the extra bits are literal.  */

                        if ((num_extra_bits & ones_if_aligned) >= LZX_NUM_ALIGNED_OFFSET_BITS) {
                                match_offset +=
                                        bitstream_read_bits(istream,
                                                            num_extra_bits -
                                                                LZX_NUM_ALIGNED_OFFSET_BITS)
                                                        << LZX_NUM_ALIGNED_OFFSET_BITS;
                                match_offset += read_huffsym_using_alignedcode(istream, tables);
                        } else {
                                match_offset += bitstream_read_bits(istream, num_extra_bits);
                        }

                        /* Adjust the offset.  */
                        match_offset -= LZX_OFFSET_ADJUSTMENT;

                        /* Update the match offset LRU queue.  */
                        BUILD_BUG_ON(LZX_NUM_RECENT_OFFSETS != 3);
                        queue->R[2] = queue->R[1];
                        queue->R[1] = queue->R[0];
                        queue->R[0] = match_offset;
                }

                /* Validate the match, then copy it to the current position.  */

                if (unlikely(match_len > window_end - window_ptr))
                        return -1;

                if (unlikely(match_offset > window_ptr - window))
                        return -1;

                lz_copy(window_ptr, match_len, match_offset, window_end,
                        LZX_MIN_MATCH_LEN);

                window_ptr += match_len;
        }
        return 0;
}

static int
lzx_decompress(const void *compressed_data, size_t compressed_size,
               void *uncompressed_data, size_t uncompressed_size,
               void *_dec)
{
        struct lzx_decompressor *dec = _dec;
        struct input_bitstream istream;
        struct lzx_lru_queue queue;
        u32 window_pos;
        int block_type;
        u32 block_size;
        bool may_have_e8_byte;
        int ret;

        init_input_bitstream(&istream, compressed_data, compressed_size);

        /* Initialize the recent offsets queue.  */
        lzx_lru_queue_init(&queue);

        /* Codeword lengths begin as all 0's for delta encoding purposes.  */
        memset(dec->tables.maincode_lens, 0, dec->num_main_syms);
        memset(dec->tables.lencode_lens, 0, LZX_LENCODE_NUM_SYMBOLS);

        /* Set this to true if there may be 0xe8 bytes in the uncompressed data.
         */
        may_have_e8_byte = false;

        /* The compressed data will consist of one or more blocks.  The
         * following loop decompresses one block, and it runs until there all
         * the compressed data has been decompressed, so there are no more
         * blocks.  */

        for (window_pos = 0;
             window_pos < uncompressed_size;
             window_pos += block_size)
        {
                ret = lzx_read_block_header(&istream, dec->num_main_syms,
                                            dec->window_order, &block_type,
                                            &block_size, &dec->tables, &queue);
                if (ret)
                        return ret;

                if (block_size > uncompressed_size - window_pos)
                        return -1;

                if (block_type != LZX_BLOCKTYPE_UNCOMPRESSED) {

                        /* Compressed block.  */

                        ret = lzx_decompress_block(block_type,
                                                   block_size,
                                                   uncompressed_data,
                                                   window_pos,
                                                   &dec->tables,
                                                   &queue,
                                                   &istream);
                        if (ret)
                                return ret;

                        /* If the first 0xe8 byte was in this block, it must
                         * have been encoded as a literal using mainsym 0xe8. */
                        if (dec->tables.maincode_lens[0xe8] != 0)
                                may_have_e8_byte = true;
                } else {

                        /* Uncompressed block.  */
                        const u8 *p;

                        p = bitstream_read_bytes(&istream, block_size);
                        if (!p)
                                return -1;

                        memcpy(&((u8*)uncompressed_data)[window_pos], p, block_size);

                        /* Re-align the bitstream if an odd number of bytes was
                         * read.  */
                        if (block_size & 1)
                                bitstream_read_byte(&istream);

                        may_have_e8_byte = true;
                }
        }

        /* Postprocess the data unless it cannot possibly contain 0xe8 bytes  */
        if (may_have_e8_byte)
                lzx_undo_e8_preprocessing(uncompressed_data, uncompressed_size);

        return 0;
}

static void
lzx_free_decompressor(void *_dec)
{
        struct lzx_decompressor *dec = _dec;

        ALIGNED_FREE(dec);
}

static int
lzx_create_decompressor(size_t max_block_size, void **dec_ret)
{
        struct lzx_decompressor *dec;
        unsigned window_order;

        window_order = lzx_get_window_order(max_block_size);
        if (window_order == 0)
                return WIMLIB_ERR_INVALID_PARAM;

        /* The aligned allocation is needed to ensure that the lzx_tables are
         * aligned properly.  */
        dec = ALIGNED_MALLOC(sizeof(struct lzx_decompressor),
                             DECODE_TABLE_ALIGNMENT);
        if (!dec)
                return WIMLIB_ERR_NOMEM;

        dec->window_order = window_order;
        dec->num_main_syms = lzx_get_num_main_syms(window_order);

        *dec_ret = dec;
        return 0;
}

const struct decompressor_ops lzx_decompressor_ops = {
        .create_decompressor = lzx_create_decompressor,
        .decompress          = lzx_decompress,
        .free_decompressor   = lzx_free_decompressor,
};