* about dealing with the unaligned case. */
#define DECODE_TABLE_ALIGNMENT 16
-/* Maximum supported symbol count for make_huffman_decode_table().
- *
- * Reason: In direct mapping entries, we store the symbol in 11 bits. */
-#define DECODE_TABLE_MAX_SYMBOLS 2048
+#define DECODE_TABLE_SYMBOL_SHIFT 4
+#define DECODE_TABLE_LENGTH_MASK DECODE_TABLE_MAX_LENGTH
-/* Maximum supported table bits for make_huffman_decode_table().
- *
- * Reason: In internal binary tree nodes, offsets are encoded in 14 bits.
- * But the real limit is 13, because we allocate entries past the end of
- * the direct lookup part of the table for binary tree nodes. (Note: if
- * needed this limit could be removed by encoding the offsets relative to
- * &decode_table[1 << table_bits].) */
-#define DECODE_TABLE_MAX_TABLE_BITS 13
-
-/* Maximum supported codeword length for make_huffman_decode_table().
+#define DECODE_TABLE_MAX_NUM_SYMS ((1 << (16 - DECODE_TABLE_SYMBOL_SHIFT)) - 1)
+#define DECODE_TABLE_MAX_LENGTH ((1 << DECODE_TABLE_SYMBOL_SHIFT) - 1)
+
+/*
+ * Reads and returns the next Huffman-encoded symbol from a bitstream.
*
- * Reason: In direct mapping entries, we encode the codeword length in 5
- * bits, and the top 2 bits can't both be set because that has special
- * meaning. */
-#define DECODE_TABLE_MAX_CODEWORD_LEN 23
-
-/* Reads and returns the next Huffman-encoded symbol from a bitstream. If the
- * input data is exhausted, the Huffman symbol is decoded as if the missing bits
- * are all zeroes.
+ * If the input data is exhausted, the Huffman symbol is decoded as if the
+ * missing bits are all zeroes.
*
* XXX: This is mostly duplicated in lzms_decode_huffman_symbol() in
- * lzms_decompress.c. */
+ * lzms_decompress.c.
+ */
static inline unsigned
-read_huffsym(struct input_bitstream *istream, const u16 decode_table[],
+read_huffsym(struct input_bitstream *is, const u16 decode_table[],
unsigned table_bits, unsigned max_codeword_len)
{
unsigned entry;
- unsigned key_bits;
-
- bitstream_ensure_bits(istream, max_codeword_len);
-
- /* Index the decode table by the next table_bits bits of the input. */
- key_bits = bitstream_peek_bits(istream, table_bits);
- entry = decode_table[key_bits];
- if (likely(entry < 0xC000)) {
- /* Fast case: The decode table directly provided the
- * symbol and codeword length. The low 11 bits are the
- * symbol, and the high 5 bits are the codeword length. */
- bitstream_remove_bits(istream, entry >> 11);
- return entry & 0x7FF;
- } else {
- /* Slow case: The codeword for the symbol is longer than
- * table_bits, so the symbol does not have an entry
- * directly in the first (1 << table_bits) entries of the
- * decode table. Traverse the appropriate binary tree
- * bit-by-bit to decode the symbol. */
- bitstream_remove_bits(istream, table_bits);
- do {
- key_bits = (entry & 0x3FFF) + bitstream_pop_bits(istream, 1);
- } while ((entry = decode_table[key_bits]) >= 0xC000);
- return entry;
+ unsigned sym;
+ unsigned len;
+
+ /* If the bitbuffer contains fewer bits than might be required by a
+ * single codeword, then refill it. */
+ bitstream_ensure_bits(is, max_codeword_len);
+
+ /* Index the root table by the next 'table_bits' bits of input. */
+ entry = decode_table[bitstream_peek_bits(is, table_bits)];
+
+ /* Extract the symbol and length from the entry. */
+ sym = entry >> DECODE_TABLE_SYMBOL_SHIFT;
+ len = entry & DECODE_TABLE_LENGTH_MASK;
+
+ /* If the root table is indexed by the full 'max_codeword_len' bits,
+ * then there cannot be any subtables. This will be known at compile
+ * time. Otherwise, we must check whether the decoded symbol is really
+ * a subtable pointer. If so, we must discard the bits with which the
+ * root table was indexed, then index the subtable by the next 'len'
+ * bits of input to get the real entry. */
+ if (max_codeword_len > table_bits &&
+ entry >= (1U << (table_bits + DECODE_TABLE_SYMBOL_SHIFT)))
+ {
+ /* Subtable required */
+ bitstream_remove_bits(is, table_bits);
+ entry = decode_table[sym + bitstream_peek_bits(is, len)];
+ sym = entry >> DECODE_TABLE_SYMBOL_SHIFT;;
+ len = entry & DECODE_TABLE_LENGTH_MASK;
}
+
+ /* Discard the bits (or the remaining bits, if a subtable was required)
+ * of the codeword. */
+ bitstream_remove_bits(is, len);
+
+ /* Return the decoded symbol. */
+ return sym;
}
+/*
+ * The ENOUGH() macro returns the maximum number of decode table entries,
+ * including all subtable entries, that may be required for decoding a given
+ * Huffman code. This depends on three parameters:
+ *
+ * num_syms: the maximum number of symbols in the code
+ * table_bits: the number of bits with which the root table will be indexed
+ * max_codeword_len: the maximum allowed codeword length
+ *
+ * Given these parameters, the utility program 'enough' from zlib, when run as
+ * './enough num_syms table_bits max_codeword_len', will compute the maximum
+ * number of entries required. This has already been done for the combinations
+ * we need (or may need) and incorporated into the macro below so that the
+ * mapping can be done at compilation time. If an unknown combination is used,
+ * then a compilation error will result. To fix this, use 'enough' to find the
+ * missing value and add it below.
+ */
+#define ENOUGH(num_syms, table_bits, max_codeword_len) ( \
+ ((num_syms) == 8 && (table_bits) == 7 && (max_codeword_len) == 15) ? 128 : \
+ ((num_syms) == 8 && (table_bits) == 5 && (max_codeword_len) == 7) ? 36 : \
+ ((num_syms) == 8 && (table_bits) == 6 && (max_codeword_len) == 7) ? 66 : \
+ ((num_syms) == 8 && (table_bits) == 7 && (max_codeword_len) == 7) ? 128 : \
+ ((num_syms) == 20 && (table_bits) == 5 && (max_codeword_len) == 15) ? 1062 : \
+ ((num_syms) == 20 && (table_bits) == 6 && (max_codeword_len) == 15) ? 582 : \
+ ((num_syms) == 20 && (table_bits) == 7 && (max_codeword_len) == 15) ? 390 : \
+ ((num_syms) == 54 && (table_bits) == 9 && (max_codeword_len) == 15) ? 618 : \
+ ((num_syms) == 54 && (table_bits) == 10 && (max_codeword_len) == 15) ? 1098 : \
+ ((num_syms) == 249 && (table_bits) == 9 && (max_codeword_len) == 16) ? 878 : \
+ ((num_syms) == 249 && (table_bits) == 10 && (max_codeword_len) == 16) ? 1326 : \
+ ((num_syms) == 249 && (table_bits) == 11 && (max_codeword_len) == 16) ? 2318 : \
+ ((num_syms) == 256 && (table_bits) == 9 && (max_codeword_len) == 15) ? 822 : \
+ ((num_syms) == 256 && (table_bits) == 10 && (max_codeword_len) == 15) ? 1302 : \
+ ((num_syms) == 256 && (table_bits) == 11 && (max_codeword_len) == 15) ? 2310 : \
+ ((num_syms) == 512 && (table_bits) == 10 && (max_codeword_len) == 15) ? 1558 : \
+ ((num_syms) == 512 && (table_bits) == 11 && (max_codeword_len) == 15) ? 2566 : \
+ ((num_syms) == 512 && (table_bits) == 12 && (max_codeword_len) == 15) ? 4606 : \
+ ((num_syms) == 656 && (table_bits) == 10 && (max_codeword_len) == 16) ? 1734 : \
+ ((num_syms) == 656 && (table_bits) == 11 && (max_codeword_len) == 16) ? 2726 : \
+ ((num_syms) == 656 && (table_bits) == 12 && (max_codeword_len) == 16) ? 4758 : \
+ ((num_syms) == 799 && (table_bits) == 9 && (max_codeword_len) == 15) ? 1366 : \
+ ((num_syms) == 799 && (table_bits) == 10 && (max_codeword_len) == 15) ? 1846 : \
+ ((num_syms) == 799 && (table_bits) == 11 && (max_codeword_len) == 15) ? 2854 : \
+ -1)
+
+/* Wrapper around ENOUGH() that does additional compile-time validation. */
+#define DECODE_TABLE_LENGTH(num_syms, table_bits, max_codeword_len) ( \
+ \
+ /* Every possible symbol value must fit into the symbol portion \
+ * of a decode table entry. */ \
+ STATIC_ASSERT_ZERO((num_syms) <= DECODE_TABLE_MAX_NUM_SYMS) + \
+ \
+ /* There cannot be more symbols than possible codewords. */ \
+ STATIC_ASSERT_ZERO((num_syms) <= 1U << (max_codeword_len)) + \
+ \
+ /* It doesn't make sense to use a table_bits more than the \
+ * maximum codeword length. */ \
+ STATIC_ASSERT_ZERO((max_codeword_len) >= (table_bits)) + \
+ \
+ /* The maximum length in the root table must fit into the \
+ * length portion of a decode table entry. */ \
+ STATIC_ASSERT_ZERO((table_bits) <= DECODE_TABLE_MAX_LENGTH) + \
+ \
+ /* The maximum length in a subtable must fit into the length
+ * portion of a decode table entry. */ \
+ STATIC_ASSERT_ZERO((max_codeword_len) - (table_bits) <= \
+ DECODE_TABLE_MAX_LENGTH) + \
+ \
+ /* The needed 'enough' value must have been defined. */ \
+ STATIC_ASSERT_ZERO(ENOUGH((num_syms), (table_bits), \
+ (max_codeword_len)) >= 0) + \
+ \
+ /* The maximum subtable index must fit in the field which would \
+ * normally hold a symbol value. */ \
+ STATIC_ASSERT_ZERO(ENOUGH((num_syms), (table_bits), \
+ (max_codeword_len)) <= \
+ DECODE_TABLE_MAX_NUM_SYMS) + \
+ \
+ /* The minimum subtable index must be greater than the greatest \
+ * possible symbol value. */ \
+ STATIC_ASSERT_ZERO((1U << table_bits) >= num_syms) + \
+ \
+ ENOUGH(num_syms, table_bits, max_codeword_len) \
+)
+
+/*
+ * Declare the decode table for a Huffman code, given several compile-time
+ * constants that describe that code (see ENOUGH() for details).
+ *
+ * Decode tables must be aligned to a DECODE_TABLE_ALIGNMENT-boundary. This
+ * implies that if a decode table is nested a dynamically allocated structure,
+ * then the outer structure must be allocated on a DECODE_TABLE_ALIGNMENT-byte
+ * boundary as well.
+ */
+#define DECODE_TABLE(name, num_syms, table_bits, max_codeword_len) \
+ u16 name[DECODE_TABLE_LENGTH((num_syms), (table_bits), \
+ (max_codeword_len))] \
+ _aligned_attribute(DECODE_TABLE_ALIGNMENT)
+
extern int
make_huffman_decode_table(u16 decode_table[], unsigned num_syms,
- unsigned num_bits, const u8 lens[],
+ unsigned table_bits, const u8 lens[],
unsigned max_codeword_len);
static inline void
}
static inline machine_word_t
-repeat_byte(u8 b)
+repeat_u16(u16 b)
{
- machine_word_t v;
+ machine_word_t v = b;
STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
-
- v = b;
- v |= v << 8;
v |= v << 16;
v |= v << ((WORDBITS == 64) ? 32 : 0);
return v;
}
+static inline machine_word_t
+repeat_u8(u8 b)
+{
+ return repeat_u16(((u16)b << 8) | b);
+}
+
/*
* Copy an LZ77 match at (dst - offset) to dst.
*
* encoding of the previous byte. This case is common
* if the data contains many repeated bytes. */
- machine_word_t v = repeat_byte(*(dst - 1));
+ machine_word_t v = repeat_u8(*(dst - 1));
do {
store_word_unaligned(v, dst);
src += WORDBYTES;
*
* The following copying information applies to this specific source code file:
*
- * Written in 2012-2015 by Eric Biggers <ebiggers3@gmail.com>
+ * Written in 2012-2016 by Eric Biggers <ebiggers3@gmail.com>
*
* To the extent possible under law, the author(s) have dedicated all copyright
* and related and neighboring rights to this software to the public domain
# include "config.h"
#endif
-#include "wimlib/decompress_common.h"
-
#include <string.h>
-#define USE_WORD_FILL
-
-#ifdef __GNUC__
-# ifdef __SSE2__
-# undef USE_WORD_FILL
-# define USE_SSE2_FILL
-# include <emmintrin.h>
-# endif
+#ifdef __SSE2__
+# include <emmintrin.h>
#endif
-/* Construct a direct mapping entry in the lookup table. */
-#define MAKE_DIRECT_ENTRY(symbol, length) ((symbol) | ((length) << 11))
+#include "wimlib/decompress_common.h"
+
+#define MAKE_ENTRY(sym, len) (((sym) << DECODE_TABLE_SYMBOL_SHIFT) | (len))
/*
* make_huffman_decode_table() -
*
- * Build a decoding table for a canonical prefix code, or "Huffman code".
- *
- * This takes as input the length of the codeword for each symbol in the
- * alphabet and produces as output a table that can be used for fast
- * decoding of prefix-encoded symbols using read_huffsym().
+ * Build a decoding table for a canonical prefix code, or "Huffman code". This
+ * takes as input the length of the codeword for each symbol in the code and
+ * produces as output a table for fast symbol decoding with read_huffsym().
*
- * Strictly speaking, a canonical prefix code might not be a Huffman
- * code. But this algorithm will work either way; and in fact, since
- * Huffman codes are defined in terms of symbol frequencies, there is no
- * way for the decompressor to know whether the code is a true Huffman
- * code or not until all symbols have been decoded.
+ * Because the code is assumed to be "canonical", it can be reconstructed
+ * directly from the codeword lengths. A prefix code is canonical if and only
+ * if a longer codeword never lexicographically precedes a shorter codeword, and
+ * the lexicographic ordering of codewords of the same length is the same as the
+ * lexicographic ordering of the corresponding symbols. Consequently, we can
+ * sort the symbols primarily by codeword length and secondarily by symbol
+ * value, then reconstruct the code by generating codewords lexicographically in
+ * that order.
*
- * Because the prefix code is assumed to be "canonical", it can be
- * reconstructed directly from the codeword lengths. A prefix code is
- * canonical if and only if a longer codeword never lexicographically
- * precedes a shorter codeword, and the lexicographic ordering of
- * codewords of the same length is the same as the lexicographic ordering
- * of the corresponding symbols. Consequently, we can sort the symbols
- * primarily by codeword length and secondarily by symbol value, then
- * reconstruct the prefix code by generating codewords lexicographically
- * in that order.
+ * This function does not, however, generate the code explicitly. Instead, it
+ * directly builds a table for decoding symbols using the code. The basic idea
+ * is this: given the next 'max_codeword_len' bits in the input, we can look up
+ * the decoded symbol by indexing a table containing 2**max_codeword_len
+ * entries. A codeword with length 'max_codeword_len' will have exactly one
+ * entry in this table, whereas a codeword shorter than 'max_codeword_len' will
+ * have multiple entries in this table. Precisely, a codeword of length n will
+ * be represented by 2**(max_codeword_len - n) entries in this table. The
+ * 0-based index of each such entry will contain the corresponding codeword as a
+ * prefix when zero-padded on the left to 'max_codeword_len' binary digits.
*
- * This function does not, however, generate the prefix code explicitly.
- * Instead, it directly builds a table for decoding symbols using the
- * code. The basic idea is this: given the next 'max_codeword_len' bits
- * in the input, we can look up the decoded symbol by indexing a table
- * containing 2**max_codeword_len entries. A codeword with length
- * 'max_codeword_len' will have exactly one entry in this table, whereas
- * a codeword shorter than 'max_codeword_len' will have multiple entries
- * in this table. Precisely, a codeword of length n will be represented
- * by 2**(max_codeword_len - n) entries in this table. The 0-based index
- * of each such entry will contain the corresponding codeword as a prefix
- * when zero-padded on the left to 'max_codeword_len' binary digits.
+ * That's the basic idea, but we implement two optimizations:
*
- * That's the basic idea, but we implement two optimizations regarding
- * the format of the decode table itself:
+ * - Often the maximum codeword length is too long for it to be efficient to
+ * build the full decoding table whenever a new code is used. Instead, we can
+ * build the table using only 2**table_bits entries, where 'table_bits <=
+ * max_codeword_len'. Then, a lookup of 'table_bits' bits will produce either
+ * a codeword directly (for codewords not longer than 'table_bits') or the
+ * index of a subtable which must be indexed with additional bits of input to
+ * decode the full codeword (for codewords longer than 'table_bits').
*
- * - For many compression formats, the maximum codeword length is too
- * long for it to be efficient to build the full decoding table
- * whenever a new prefix code is used. Instead, we can build the table
- * using only 2**table_bits entries, where 'table_bits' is some number
- * less than or equal to 'max_codeword_len'. Then, only codewords of
- * length 'table_bits' and shorter can be directly looked up. For
- * longer codewords, the direct lookup instead produces the root of a
- * binary tree. Using this tree, the decoder can do traditional
- * bit-by-bit decoding of the remainder of the codeword. Child nodes
- * are allocated in extra entries at the end of the table; leaf nodes
- * contain symbols. Note that the long-codeword case is, in general,
- * not performance critical, since in Huffman codes the most frequently
- * used symbols are assigned the shortest codeword lengths.
- *
- * - When we decode a symbol using a direct lookup of the table, we still
- * need to know its length so that the bitstream can be advanced by the
- * appropriate number of bits. The simple solution is to simply retain
- * the 'lens' array and use the decoded symbol as an index into it.
- * However, this requires two separate array accesses in the fast path.
- * The optimization is to store the length directly in the decode
- * table. We use the bottom 11 bits for the symbol and the top 5 bits
- * for the length. In addition, to combine this optimization with the
- * previous one, we introduce a special case where the top 2 bits of
- * the length are both set if the entry is actually the root of a
- * binary tree.
+ * - When we decode a symbol, we still need to know its codeword length so that
+ * the bitstream can be advanced by the appropriate number of bits. The
+ * obvious solution is to simply retain the 'lens' array and use the decoded
+ * symbol as an index into it. However, this requires two separate array
+ * accesses in the fast path. The optimization is to store the length
+ * directly in the decode table, along with the symbol.
*
* @decode_table:
- * The array in which to create the decoding table. This must be
- * 16-byte aligned and must have a length of at least
- * ((2**table_bits) + 2 * num_syms) entries. This is permitted to
- * alias @lens, since all information from @lens is consumed before
-* anything is written to @decode_table.
+ * The array in which to build the decode table. This must have been
+ * declared by the DECODE_TABLE() macro. This may alias @lens, since all
+ * @lens are consumed before the decode table is written to.
*
* @num_syms:
- * The number of symbols in the alphabet; also, the length of the
- * 'lens' array. Must be less than or equal to
- * DECODE_TABLE_MAX_SYMBOLS.
+ * The number of symbols in the alphabet.
*
* @table_bits:
- * The order of the decode table size, as explained above. Must be
- * less than or equal to DECODE_TABLE_MAX_TABLE_BITS.
+ * The log base 2 of the number of entries in the root table.
*
* @lens:
- * An array of length @num_syms, indexable by symbol, that gives the
- * length of the codeword, in bits, for that symbol. The length can
- * be 0, which means that the symbol does not have a codeword
- * assigned. This is permitted to alias @decode_table, since all
- * information from @lens is consumed before anything is written to
- * @decode_table.
+ * An array of length @num_syms, indexable by symbol, that gives the length
+ * of the codeword, in bits, for that symbol. The length can be 0, which
+ * means that the symbol does not have a codeword assigned. In addition,
+ * @lens may alias @decode_table, as noted above.
*
* @max_codeword_len:
- * The longest codeword length allowed in the compression format.
- * All entries in 'lens' must be less than or equal to this value.
- * This must be less than or equal to DECODE_TABLE_MAX_CODEWORD_LEN.
+ * The maximum codeword length permitted for this code. All entries in
+ * 'lens' must be less than or equal to this value.
*
- * Returns 0 on success, or -1 if the lengths do not form a valid prefix
- * code.
+ * Returns 0 on success, or -1 if the lengths do not form a valid prefix code.
*/
int
-make_huffman_decode_table(u16 decode_table[const],
- const unsigned num_syms,
- const unsigned table_bits,
- const u8 lens[const],
- const unsigned max_codeword_len)
+make_huffman_decode_table(u16 decode_table[], unsigned num_syms,
+ unsigned table_bits, const u8 lens[],
+ unsigned max_codeword_len)
{
- const unsigned table_num_entries = 1 << table_bits;
- unsigned len_counts[max_codeword_len + 1];
+ u16 offsets[max_codeword_len + 1];
+ u16 len_counts[max_codeword_len + 1];
u16 sorted_syms[num_syms];
- int left;
- void *decode_table_ptr;
+ s32 remainder = 1;
+ void *entry_ptr = decode_table;
+ unsigned codeword_len = 1;
unsigned sym_idx;
- unsigned codeword_len;
- unsigned stores_per_loop;
- unsigned decode_table_pos;
-
-#ifdef USE_WORD_FILL
- const unsigned entries_per_word = WORDBYTES / sizeof(decode_table[0]);
-#endif
+ unsigned codeword;
+ unsigned subtable_pos;
+ unsigned subtable_bits;
+ unsigned subtable_prefix;
-#ifdef USE_SSE2_FILL
- const unsigned entries_per_xmm = sizeof(__m128i) / sizeof(decode_table[0]);
-#endif
-
- /* Count how many symbols have each possible codeword length.
- * Note that a length of 0 indicates the corresponding symbol is not
- * used in the code and therefore does not have a codeword. */
+ /* Count how many codewords have each length, including 0. */
for (unsigned len = 0; len <= max_codeword_len; len++)
len_counts[len] = 0;
for (unsigned sym = 0; sym < num_syms; sym++)
len_counts[lens[sym]]++;
- /* We can assume all lengths are <= max_codeword_len, but we
- * cannot assume they form a valid prefix code. A codeword of
- * length n should require a proportion of the codespace equaling
- * (1/2)^n. The code is valid if and only if the codespace is
- * exactly filled by the lengths, by this measure. */
- left = 1;
+ /* It is already guaranteed that all lengths are <= max_codeword_len,
+ * but it cannot be assumed they form a complete prefix code. A
+ * codeword of length n should require a proportion of the codespace
+ * equaling (1/2)^n. The code is complete if and only if, by this
+ * measure, the codespace is exactly filled by the lengths. */
for (unsigned len = 1; len <= max_codeword_len; len++) {
- left <<= 1;
- left -= len_counts[len];
- if (unlikely(left < 0)) {
- /* The lengths overflow the codespace; that is, the code
- * is over-subscribed. */
+ remainder = (remainder << 1) - len_counts[len];
+ /* Do the lengths overflow the codespace? */
+ if (unlikely(remainder < 0))
return -1;
- }
}
- if (unlikely(left != 0)) {
+ if (remainder != 0) {
/* The lengths do not fill the codespace; that is, they form an
- * incomplete set. */
- if (left == (1 << max_codeword_len)) {
- /* The code is completely empty. This is arguably
- * invalid, but in fact it is valid in LZX and XPRESS,
- * so we must allow it. By definition, no symbols can
- * be decoded with an empty code. Consequently, we
- * technically don't even need to fill in the decode
- * table. However, to avoid accessing uninitialized
- * memory if the algorithm nevertheless attempts to
- * decode symbols using such a code, we zero out the
- * decode table. */
- memset(decode_table, 0,
- table_num_entries * sizeof(decode_table[0]));
- return 0;
- }
- return -1;
+ * incomplete code. This is permitted only if the code is empty
+ * (contains no symbols). */
+
+ if (unlikely(remainder != (s32)1 << max_codeword_len))
+ return -1;
+
+ /* The code is empty. When processing a well-formed stream, the
+ * decode table need not be initialized in this case. However,
+ * we cannot assume the stream is well-formed, so we must
+ * initialize the decode table anyway. Setting all entries to 0
+ * makes this table always produce symbol '0' without consuming
+ * any bits, which is good enough. */
+ memset(decode_table, 0,
+ (1U << table_bits) * sizeof(decode_table[0]));
+ return 0;
}
- /* Sort the symbols primarily by length and secondarily by symbol order.
- */
- {
- unsigned offsets[max_codeword_len + 1];
+ /* Sort the symbols primarily by increasing codeword length and
+ * secondarily by increasing symbol value. */
- /* Initialize 'offsets' so that offsets[len] for 1 <= len <=
- * max_codeword_len is the number of codewords shorter than
- * 'len' bits. */
- offsets[1] = 0;
- for (unsigned len = 1; len < max_codeword_len; len++)
- offsets[len + 1] = offsets[len] + len_counts[len];
+ /* Initialize 'offsets' so that 'offsets[len]' is the number of
+ * codewords shorter than 'len' bits, including length 0. */
+ offsets[0] = 0;
+ for (unsigned len = 0; len < max_codeword_len; len++)
+ offsets[len + 1] = offsets[len] + len_counts[len];
- /* Use the 'offsets' array to sort the symbols.
- * Note that we do not include symbols that are not used in the
- * code. Consequently, fewer than 'num_syms' entries in
- * 'sorted_syms' may be filled. */
- for (unsigned sym = 0; sym < num_syms; sym++)
- if (lens[sym] != 0)
- sorted_syms[offsets[lens[sym]]++] = sym;
- }
+ /* Use the 'offsets' array to sort the symbols. */
+ for (unsigned sym = 0; sym < num_syms; sym++)
+ sorted_syms[offsets[lens[sym]]++] = sym;
- /* Fill entries for codewords with length <= table_bits
+ /*
+ * Fill entries for codewords with length <= table_bits
* --- that is, those short enough for a direct mapping.
*
* The table will start with entries for the shortest codeword(s), which
* codeword will decrease. As an optimization, we may begin filling
* entries with SSE2 vector accesses (8 entries/store), then change to
* 'machine_word_t' accesses (2 or 4 entries/store), then change to
- * 16-bit accesses (1 entry/store). */
- decode_table_ptr = decode_table;
- sym_idx = 0;
- codeword_len = 1;
-#ifdef USE_SSE2_FILL
- /* Fill the entries one 128-bit vector at a time.
- * This is 8 entries per store. */
- stores_per_loop = (1 << (table_bits - codeword_len)) / entries_per_xmm;
- for (; stores_per_loop != 0; codeword_len++, stores_per_loop >>= 1) {
+ * 16-bit accesses (1 entry/store).
+ */
+ sym_idx = offsets[0];
+#ifdef __SSE2__
+ /* Fill entries one 128-bit vector (8 entries) at a time. */
+ for (unsigned stores_per_loop = (1U << (table_bits - codeword_len)) /
+ (sizeof(__m128i) / sizeof(decode_table[0]));
+ stores_per_loop != 0; codeword_len++, stores_per_loop >>= 1)
+ {
unsigned end_sym_idx = sym_idx + len_counts[codeword_len];
for (; sym_idx < end_sym_idx; sym_idx++) {
- /* Note: unlike in the machine_word_t version below, the
- * __m128i type already has __attribute__((may_alias)),
- * so using it to access the decode table, which is an
- * array of unsigned shorts, will not violate strict
+ /* Note: unlike in the "word" version below, the __m128i
+ * type already has __attribute__((may_alias)), so using
+ * it to access an array of u16 will not violate strict
* aliasing. */
- u16 entry;
- __m128i v;
- __m128i *p;
- unsigned n;
-
- entry = MAKE_DIRECT_ENTRY(sorted_syms[sym_idx], codeword_len);
-
- v = _mm_set1_epi16(entry);
- p = (__m128i*)decode_table_ptr;
- n = stores_per_loop;
+ __m128i v = _mm_set1_epi16(
+ MAKE_ENTRY(sorted_syms[sym_idx], codeword_len));
+ unsigned n = stores_per_loop;
do {
- *p++ = v;
+ *(__m128i *)entry_ptr = v;
+ entry_ptr += sizeof(v);
} while (--n);
- decode_table_ptr = p;
}
}
-#endif /* USE_SSE2_FILL */
+#endif /* __SSE2__ */
-#ifdef USE_WORD_FILL
- /* Fill the entries one machine word at a time.
- * On 32-bit systems this is 2 entries per store, while on 64-bit
- * systems this is 4 entries per store. */
- stores_per_loop = (1 << (table_bits - codeword_len)) / entries_per_word;
- for (; stores_per_loop != 0; codeword_len++, stores_per_loop >>= 1) {
+#ifdef __GNUC__
+ /* Fill entries one word (2 or 4 entries) at a time. */
+ for (unsigned stores_per_loop = (1U << (table_bits - codeword_len)) /
+ (WORDBYTES / sizeof(decode_table[0]));
+ stores_per_loop != 0; codeword_len++, stores_per_loop >>= 1)
+ {
unsigned end_sym_idx = sym_idx + len_counts[codeword_len];
for (; sym_idx < end_sym_idx; sym_idx++) {
* the code with -fno-strict-aliasing to guarantee
* correctness. To work around this problem, use the
* gcc 'may_alias' extension. */
- typedef machine_word_t _may_alias_attribute aliased_word_t;
-
- machine_word_t v;
- aliased_word_t *p;
- unsigned n;
-
- STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
-
- v = MAKE_DIRECT_ENTRY(sorted_syms[sym_idx], codeword_len);
- v |= v << 16;
- v |= v << (WORDBITS == 64 ? 32 : 0);
-
- p = (aliased_word_t *)decode_table_ptr;
- n = stores_per_loop;
+ typedef machine_word_t
+ __attribute__((may_alias)) aliased_word_t;
+ aliased_word_t v = repeat_u16(
+ MAKE_ENTRY(sorted_syms[sym_idx], codeword_len));
+ unsigned n = stores_per_loop;
do {
- *p++ = v;
+ *(aliased_word_t *)entry_ptr = v;
+ entry_ptr += sizeof(v);
} while (--n);
- decode_table_ptr = p;
}
}
-#endif /* USE_WORD_FILL */
+#endif /* __GNUC__ */
- /* Fill the entries one 16-bit integer at a time. */
- stores_per_loop = (1 << (table_bits - codeword_len));
- for (; stores_per_loop != 0; codeword_len++, stores_per_loop >>= 1) {
+ /* Fill entries one at a time. */
+ for (unsigned stores_per_loop = (1U << (table_bits - codeword_len));
+ stores_per_loop != 0; codeword_len++, stores_per_loop >>= 1)
+ {
unsigned end_sym_idx = sym_idx + len_counts[codeword_len];
for (; sym_idx < end_sym_idx; sym_idx++) {
- u16 entry;
- u16 *p;
- unsigned n;
-
- entry = MAKE_DIRECT_ENTRY(sorted_syms[sym_idx], codeword_len);
-
- p = (u16*)decode_table_ptr;
- n = stores_per_loop;
-
+ u16 v = MAKE_ENTRY(sorted_syms[sym_idx], codeword_len);
+ unsigned n = stores_per_loop;
do {
- *p++ = entry;
+ *(u16 *)entry_ptr = v;
+ entry_ptr += sizeof(v);
} while (--n);
-
- decode_table_ptr = p;
}
}
- /* If we've filled in the entire table, we are done. Otherwise,
- * there are codewords longer than table_bits for which we must
- * generate binary trees. */
-
- decode_table_pos = (u16*)decode_table_ptr - decode_table;
- if (decode_table_pos != table_num_entries) {
- unsigned j;
- unsigned next_free_tree_slot;
- unsigned cur_codeword;
-
- /* First, zero out the remaining entries. This is
- * necessary so that these entries appear as
- * "unallocated" in the next part. Each of these entries
- * will eventually be filled with the representation of
- * the root node of a binary tree. */
- j = decode_table_pos;
- do {
- decode_table[j] = 0;
- } while (++j != table_num_entries);
-
- /* We allocate child nodes starting at the end of the
- * direct lookup table. Note that there should be
- * 2*num_syms extra entries for this purpose, although
- * fewer than this may actually be needed. */
- next_free_tree_slot = table_num_entries;
-
- /* Iterate through each codeword with length greater than
- * 'table_bits', primarily in order of codeword length
- * and secondarily in order of symbol. */
- for (cur_codeword = decode_table_pos << 1;
- codeword_len <= max_codeword_len;
- codeword_len++, cur_codeword <<= 1)
- {
- unsigned end_sym_idx = sym_idx + len_counts[codeword_len];
- for (; sym_idx < end_sym_idx; sym_idx++, cur_codeword++)
- {
- /* 'sym' is the symbol represented by the
- * codeword. */
- unsigned sym = sorted_syms[sym_idx];
-
- unsigned extra_bits = codeword_len - table_bits;
+ /* If all symbols were processed, then no subtables are required. */
+ if (sym_idx == num_syms)
+ return 0;
+
+ /* At least one subtable is required. Process the remaining symbols. */
+ codeword = ((u16 *)entry_ptr - decode_table) << 1;
+ subtable_pos = 1U << table_bits;
+ subtable_bits = table_bits;
+ subtable_prefix = -1;
+ do {
+ while (len_counts[codeword_len] == 0) {
+ codeword_len++;
+ codeword <<= 1;
+ }
- unsigned node_idx = cur_codeword >> extra_bits;
+ unsigned prefix = codeword >> (codeword_len - table_bits);
+
+ /* Start a new subtable if the first 'table_bits' bits of the
+ * codeword don't match the prefix for the previous subtable, or
+ * if this will be the first subtable. */
+ if (prefix != subtable_prefix) {
+
+ subtable_prefix = prefix;
+
+ /* Calculate the subtable length. If the codeword
+ * length exceeds 'table_bits' by n, the subtable needs
+ * at least 2**n entries. But it may need more; if
+ * there are fewer than 2**n codewords of length
+ * 'table_bits + n' remaining, then n will need to be
+ * incremented to bring in longer codewords until the
+ * subtable can be filled completely. Note that it
+ * always will, eventually, be possible to fill the
+ * subtable, since it was previously verified that the
+ * code is complete. */
+ subtable_bits = codeword_len - table_bits;
+ remainder = (s32)1 << subtable_bits;
+ for (;;) {
+ remainder -= len_counts[table_bits +
+ subtable_bits];
+ if (remainder <= 0)
+ break;
+ subtable_bits++;
+ remainder <<= 1;
+ }
- /* Go through each bit of the current codeword
- * beyond the prefix of length @table_bits and
- * walk the appropriate binary tree, allocating
- * any slots that have not yet been allocated.
- *
- * Note that the 'pointer' entry to the binary
- * tree, which is stored in the direct lookup
- * portion of the table, is represented
- * identically to other internal (non-leaf)
- * nodes of the binary tree; it can be thought
- * of as simply the root of the tree. The
- * representation of these internal nodes is
- * simply the index of the left child combined
- * with the special bits 0xC000 to distinguish
- * the entry from direct mapping and leaf node
- * entries. */
- do {
+ /* Create the entry that points from the root table to
+ * the subtable. This entry contains the index of the
+ * start of the subtable and the number of bits with
+ * which the subtable is indexed (the log base 2 of the
+ * number of entries it contains). */
+ decode_table[subtable_prefix] =
+ MAKE_ENTRY(subtable_pos, subtable_bits);
+ }
- /* At least one bit remains in the
- * codeword, but the current node is an
- * unallocated leaf. Change it to an
- * internal node. */
- if (decode_table[node_idx] == 0) {
- decode_table[node_idx] =
- next_free_tree_slot | 0xC000;
- decode_table[next_free_tree_slot++] = 0;
- decode_table[next_free_tree_slot++] = 0;
- }
+ u16 entry = MAKE_ENTRY(sorted_syms[sym_idx],
+ codeword_len - table_bits);
+ unsigned n = 1U << (subtable_bits - (codeword_len -
+ table_bits));
+ do {
+ decode_table[subtable_pos++] = entry;
+ } while (--n);
- /* Go to the left child if the next bit
- * in the codeword is 0; otherwise go to
- * the right child. */
- node_idx = decode_table[node_idx] & 0x3FFF;
- --extra_bits;
- node_idx += (cur_codeword >> extra_bits) & 1;
- } while (extra_bits != 0);
+ len_counts[codeword_len]--;
+ codeword++;
+ } while (++sym_idx < num_syms);
- /* We've traversed the tree using the entire
- * codeword, and we're now at the entry where
- * the actual symbol will be stored. This is
- * distinguished from internal nodes by not
- * having its high two bits set. */
- decode_table[node_idx] = sym;
- }
- }
- }
return 0;
}
*/
/*
- * Copyright (C) 2013, 2014, 2015 Eric Biggers
+ * Copyright (C) 2013-2016 Eric Biggers
*
* This file is free software; you can redistribute it and/or modify it under
* the terms of the GNU Lesser General Public License as published by the Free
/* The TABLEBITS values can be changed; they only affect decoding speed. */
#define LZMS_LITERAL_TABLEBITS 10
-#define LZMS_LENGTH_TABLEBITS 10
-#define LZMS_LZ_OFFSET_TABLEBITS 10
-#define LZMS_DELTA_OFFSET_TABLEBITS 10
-#define LZMS_DELTA_POWER_TABLEBITS 8
+#define LZMS_LENGTH_TABLEBITS 9
+#define LZMS_LZ_OFFSET_TABLEBITS 11
+#define LZMS_DELTA_OFFSET_TABLEBITS 11
+#define LZMS_DELTA_POWER_TABLEBITS 7
struct lzms_range_decoder {
struct lzms_probabilites probs;
- u16 literal_decode_table[(1 << LZMS_LITERAL_TABLEBITS) +
- (2 * LZMS_NUM_LITERAL_SYMS)]
- _aligned_attribute(DECODE_TABLE_ALIGNMENT);
+ DECODE_TABLE(literal_decode_table, LZMS_NUM_LITERAL_SYMS,
+ LZMS_LITERAL_TABLEBITS, LZMS_MAX_CODEWORD_LENGTH);
u32 literal_freqs[LZMS_NUM_LITERAL_SYMS];
struct lzms_huffman_rebuild_info literal_rebuild_info;
- u16 lz_offset_decode_table[(1 << LZMS_LZ_OFFSET_TABLEBITS) +
- ( 2 * LZMS_MAX_NUM_OFFSET_SYMS)]
- _aligned_attribute(DECODE_TABLE_ALIGNMENT);
+ DECODE_TABLE(lz_offset_decode_table, LZMS_MAX_NUM_OFFSET_SYMS,
+ LZMS_LZ_OFFSET_TABLEBITS, LZMS_MAX_CODEWORD_LENGTH);
u32 lz_offset_freqs[LZMS_MAX_NUM_OFFSET_SYMS];
struct lzms_huffman_rebuild_info lz_offset_rebuild_info;
- u16 length_decode_table[(1 << LZMS_LENGTH_TABLEBITS) +
- (2 * LZMS_NUM_LENGTH_SYMS)]
- _aligned_attribute(DECODE_TABLE_ALIGNMENT);
+ DECODE_TABLE(length_decode_table, LZMS_NUM_LENGTH_SYMS,
+ LZMS_LENGTH_TABLEBITS, LZMS_MAX_CODEWORD_LENGTH);
u32 length_freqs[LZMS_NUM_LENGTH_SYMS];
struct lzms_huffman_rebuild_info length_rebuild_info;
- u16 delta_offset_decode_table[(1 << LZMS_DELTA_OFFSET_TABLEBITS) +
- (2 * LZMS_MAX_NUM_OFFSET_SYMS)]
- _aligned_attribute(DECODE_TABLE_ALIGNMENT);
+ DECODE_TABLE(delta_offset_decode_table, LZMS_MAX_NUM_OFFSET_SYMS,
+ LZMS_DELTA_OFFSET_TABLEBITS, LZMS_MAX_CODEWORD_LENGTH);
u32 delta_offset_freqs[LZMS_MAX_NUM_OFFSET_SYMS];
struct lzms_huffman_rebuild_info delta_offset_rebuild_info;
- u16 delta_power_decode_table[(1 << LZMS_DELTA_POWER_TABLEBITS) +
- (2 * LZMS_NUM_DELTA_POWER_SYMS)]
- _aligned_attribute(DECODE_TABLE_ALIGNMENT);
+ DECODE_TABLE(delta_power_decode_table, LZMS_NUM_DELTA_POWER_SYMS,
+ LZMS_DELTA_POWER_TABLEBITS, LZMS_MAX_CODEWORD_LENGTH);
u32 delta_power_freqs[LZMS_NUM_DELTA_POWER_SYMS];
struct lzms_huffman_rebuild_info delta_power_rebuild_info;
unsigned table_bits, u32 freqs[],
struct lzms_huffman_rebuild_info *rebuild_info)
{
- unsigned key_bits;
unsigned entry;
unsigned sym;
+ unsigned len;
lzms_ensure_bits(is, LZMS_MAX_CODEWORD_LENGTH);
- /* Index the decode table by the next table_bits bits of the input. */
- key_bits = lzms_peek_bits(is, table_bits);
- entry = decode_table[key_bits];
- if (likely(entry < 0xC000)) {
- /* Fast case: The decode table directly provided the symbol and
- * codeword length. The low 11 bits are the symbol, and the
- * high 5 bits are the codeword length. */
- lzms_remove_bits(is, entry >> 11);
- sym = entry & 0x7FF;
- } else {
- /* Slow case: The codeword for the symbol is longer than
- * table_bits, so the symbol does not have an entry directly in
- * the first (1 << table_bits) entries of the decode table.
- * Traverse the appropriate binary tree bit-by-bit in order to
- * decode the symbol. */
+ entry = decode_table[lzms_peek_bits(is, table_bits)];
+ sym = entry >> DECODE_TABLE_SYMBOL_SHIFT;
+ len = entry & DECODE_TABLE_LENGTH_MASK;
+
+ if (entry >= (1U << (table_bits + DECODE_TABLE_SYMBOL_SHIFT))) {
+ /* Subtable required */
lzms_remove_bits(is, table_bits);
- do {
- key_bits = (entry & 0x3FFF) + lzms_pop_bits(is, 1);
- } while ((entry = decode_table[key_bits]) >= 0xC000);
- sym = entry;
+ entry = decode_table[sym + lzms_peek_bits(is, len)];
+ sym = entry >> DECODE_TABLE_SYMBOL_SHIFT;;
+ len = entry & DECODE_TABLE_LENGTH_MASK;
}
+ lzms_remove_bits(is, len);
+
freqs[sym]++;
if (--rebuild_info->num_syms_until_rebuild == 0)
lzms_rebuild_huffman_code(rebuild_info);