X-Git-Url: https://wimlib.net/git/?p=wimlib;a=blobdiff_plain;f=src%2Fdecompress_common.c;h=fa605f0cc08b21b301fc641a3e43088fd805b6a2;hp=973f467cc6e0c7ffe3a8bb557988f29e2f8bf8f7;hb=908381d2809a48acd9490ec080e51087ae1529fd;hpb=ed92ad52377e0ee686faec69ec5cbca291ab83c1

diff --git a/src/decompress_common.c b/src/decompress_common.c
index 973f467c..fa605f0c 100644
--- a/src/decompress_common.c
+++ b/src/decompress_common.c
@@ -5,7 +5,7 @@
  *
  * The following copying information applies to this specific source code file:
  *
- * Written in 2012-2015 by Eric Biggers <ebiggers3@gmail.com>
+ * Written in 2012-2016 by Eric Biggers <ebiggers3@gmail.com>
  *
  * To the extent possible under law, the author(s) have dedicated all copyright
  * and related and neighboring rights to this software to the public domain
@@ -24,259 +24,202 @@
 #  include "config.h"
 #endif
 
-#include "wimlib/decompress_common.h"
-
 #include <string.h>
 
-#define USE_WORD_FILL
-
-#ifdef __GNUC__
-#  ifdef __SSE2__
-#    undef USE_WORD_FILL
-#    define USE_SSE2_FILL
-#    include <emmintrin.h>
-#  endif
+#ifdef __SSE2__
+#  include <emmintrin.h>
 #endif
 
-/* Construct a direct mapping entry in the lookup table.  */
-#define MAKE_DIRECT_ENTRY(symbol, length) ((symbol) | ((length) << 11))
+#include "wimlib/decompress_common.h"
 
 /*
  * make_huffman_decode_table() -
  *
- * Build a decoding table for a canonical prefix code, or "Huffman code".
+ * Given an alphabet of symbols and the length of each symbol's codeword in a
+ * canonical prefix code, build a table for quickly decoding symbols that were
+ * encoded with that code.
  *
- * This takes as input the length of the codeword for each symbol in the
- * alphabet and produces as output a table that can be used for fast
- * decoding of prefix-encoded symbols using read_huffsym().
+ * A _prefix code_ is an assignment of bitstrings called _codewords_ to symbols
+ * such that no whole codeword is a prefix of any other.  A prefix code might be
+ * a _Huffman code_, which means that it is an optimum prefix code for a given
+ * list of symbol frequencies and was generated by the Huffman algorithm.
+ * Although the prefix codes processed here will ordinarily be "Huffman codes",
+ * strictly speaking the decoder cannot know whether a given code was actually
+ * generated by the Huffman algorithm or not.
  *
- * Strictly speaking, a canonical prefix code might not be a Huffman
- * code.  But this algorithm will work either way; and in fact, since
- * Huffman codes are defined in terms of symbol frequencies, there is no
- * way for the decompressor to know whether the code is a true Huffman
- * code or not until all symbols have been decoded.
+ * A prefix code is _canonical_ if and only if a longer codeword never
+ * lexicographically precedes a shorter codeword, and the lexicographic ordering
+ * of codewords of equal length is the same as the lexicographic ordering of the
+ * corresponding symbols.  The advantage of using a canonical prefix code is
+ * that the codewords can be reconstructed from only the symbol => codeword
+ * length mapping.  This eliminates the need to transmit the codewords
+ * explicitly.  Instead, they can be enumerated in lexicographic order after
+ * sorting the symbols primarily by increasing codeword length and secondarily
+ * by increasing symbol value.
  *
- * Because the prefix code is assumed to be "canonical", it can be
- * reconstructed directly from the codeword lengths.  A prefix code is
- * canonical if and only if a longer codeword never lexicographically
- * precedes a shorter codeword, and the lexicographic ordering of
- * codewords of the same length is the same as the lexicographic ordering
- * of the corresponding symbols.  Consequently, we can sort the symbols
- * primarily by codeword length and secondarily by symbol value, then
- * reconstruct the prefix code by generating codewords lexicographically
- * in that order.
+ * However, the decoder's real goal is to decode symbols with the code, not just
+ * generate the list of codewords.  Consequently, this function directly builds
+ * a table for efficiently decoding symbols using the code.  The basic idea is
+ * that given the next 'max_codeword_len' bits of input, the decoder can look up
+ * the next decoded symbol by indexing a table containing '2^max_codeword_len'
+ * entries.  A codeword with length 'max_codeword_len' will have exactly one
+ * entry in this table, whereas a codeword shorter than 'max_codeword_len' will
+ * have multiple entries in this table.  Precisely, a codeword of length 'n'
+ * will have '2^(max_codeword_len - n)' entries.  The index of each such entry,
+ * considered as a bitstring of length 'max_codeword_len', will contain the
+ * corresponding codeword as a prefix.
  *
- * This function does not, however, generate the prefix code explicitly.
- * Instead, it directly builds a table for decoding symbols using the
- * code.  The basic idea is this: given the next 'max_codeword_len' bits
- * in the input, we can look up the decoded symbol by indexing a table
- * containing 2**max_codeword_len entries.  A codeword with length
- * 'max_codeword_len' will have exactly one entry in this table, whereas
- * a codeword shorter than 'max_codeword_len' will have multiple entries
- * in this table.  Precisely, a codeword of length n will be represented
- * by 2**(max_codeword_len - n) entries in this table.  The 0-based index
- * of each such entry will contain the corresponding codeword as a prefix
- * when zero-padded on the left to 'max_codeword_len' binary digits.
+ * That's the basic idea, but we extend it in two ways:
  *
- * That's the basic idea, but we implement two optimizations regarding
- * the format of the decode table itself:
+ * - Often the maximum codeword length is too long for it to be efficient to
+ *   build the full decode table whenever a new code is used.  Instead, we build
+ *   a "root" table using only '2^table_bits' entries, where 'table_bits <=
+ *   max_codeword_len'.  Then, a lookup of 'table_bits' bits produces either a
+ *   symbol directly (for codewords not longer than 'table_bits'), or the index
+ *   of a subtable which must be indexed with additional bits of input to fully
+ *   decode the symbol (for codewords longer than 'table_bits').
  *
- * - For many compression formats, the maximum codeword length is too
- *   long for it to be efficient to build the full decoding table
- *   whenever a new prefix code is used.  Instead, we can build the table
- *   using only 2**table_bits entries, where 'table_bits' is some number
- *   less than or equal to 'max_codeword_len'.  Then, only codewords of
- *   length 'table_bits' and shorter can be directly looked up.  For
- *   longer codewords, the direct lookup instead produces the root of a
- *   binary tree.  Using this tree, the decoder can do traditional
- *   bit-by-bit decoding of the remainder of the codeword.  Child nodes
- *   are allocated in extra entries at the end of the table; leaf nodes
- *   contain symbols.  Note that the long-codeword case is, in general,
- *   not performance critical, since in Huffman codes the most frequently
- *   used symbols are assigned the shortest codeword lengths.
+ * - Whenever the decoder decodes a symbol, it needs to know the codeword length
+ *   so that it can remove the appropriate number of input bits.  The obvious
+ *   solution would be to simply retain the codeword lengths array and use the
+ *   decoded symbol as an index into it.  However, that would require two array
+ *   accesses when decoding each symbol.  Our strategy is to instead store the
+ *   codeword length directly in the decode table entry along with the symbol.
  *
- * - When we decode a symbol using a direct lookup of the table, we still
- *   need to know its length so that the bitstream can be advanced by the
- *   appropriate number of bits.  The simple solution is to simply retain
- *   the 'lens' array and use the decoded symbol as an index into it.
- *   However, this requires two separate array accesses in the fast path.
- *   The optimization is to store the length directly in the decode
- *   table.  We use the bottom 11 bits for the symbol and the top 5 bits
- *   for the length.  In addition, to combine this optimization with the
- *   previous one, we introduce a special case where the top 2 bits of
- *   the length are both set if the entry is actually the root of a
- *   binary tree.
+ * See MAKE_DECODE_TABLE_ENTRY() for full details on the format of decode table
+ * entries, and see read_huffsym() for full details on how symbols are decoded.
  *
  * @decode_table:
- *	The array in which to create the decoding table.  This must be
- *	16-byte aligned and must have a length of at least
- *	((2**table_bits) + 2 * num_syms) entries.  This is permitted to
- *	alias @lens, since all information from @lens is consumed before
-*	anything is written to @decode_table.
+ *	The array in which to build the decode table.  This must have been
+ *	declared by the DECODE_TABLE() macro.  This may alias @lens, since all
+ *	@lens are consumed before the decode table is written to.
  *
  * @num_syms:
- *	The number of symbols in the alphabet; also, the length of the
- *	'lens' array.  Must be less than or equal to
- *	DECODE_TABLE_MAX_SYMBOLS.
+ *	The number of symbols in the alphabet.
  *
  * @table_bits:
- *	The order of the decode table size, as explained above.  Must be
- *	less than or equal to DECODE_TABLE_MAX_TABLE_BITS.
+ *	The log base 2 of the number of entries in the root table.
  *
  * @lens:
- *	An array of length @num_syms, indexable by symbol, that gives the
- *	length of the codeword, in bits, for that symbol.  The length can
- *	be 0, which means that the symbol does not have a codeword
- *	assigned.  This is permitted to alias @decode_table, since all
- *	information from @lens is consumed before anything is written to
- *	@decode_table.
+ *	An array of length @num_syms, indexed by symbol, that gives the length
+ *	of the codeword, in bits, for each symbol.  The length can be 0, which
+ *	means that the symbol does not have a codeword assigned.  In addition,
+ *	@lens may alias @decode_table, as noted above.
  *
  * @max_codeword_len:
- *	The longest codeword length allowed in the compression format.
- *	All entries in 'lens' must be less than or equal to this value.
- *	This must be less than or equal to DECODE_TABLE_MAX_CODEWORD_LEN.
+ *	The maximum codeword length permitted for this code.  All entries in
+ *	'lens' must be less than or equal to this value.
  *
- * Returns 0 on success, or -1 if the lengths do not form a valid prefix
- * code.
+ * Returns 0 on success, or -1 if the lengths do not form a valid prefix code.
  */
 int
-make_huffman_decode_table(u16 decode_table[const],
-			  const unsigned num_syms,
-			  const unsigned table_bits,
-			  const u8 lens[const],
-			  const unsigned max_codeword_len)
+make_huffman_decode_table(u16 decode_table[], unsigned num_syms,
+			  unsigned table_bits, const u8 lens[],
+			  unsigned max_codeword_len)
 {
-	const unsigned table_num_entries = 1 << table_bits;
-	unsigned len_counts[max_codeword_len + 1];
+	u16 len_counts[max_codeword_len + 1];
+	u16 offsets[max_codeword_len + 1];
 	u16 sorted_syms[num_syms];
-	int left;
-	void *decode_table_ptr;
+	s32 remainder = 1;
+	void *entry_ptr = decode_table;
+	unsigned codeword_len = 1;
 	unsigned sym_idx;
-	unsigned codeword_len;
-	unsigned stores_per_loop;
-	unsigned decode_table_pos;
-
-#ifdef USE_WORD_FILL
-	const unsigned entries_per_word = WORDBYTES / sizeof(decode_table[0]);
-#endif
-
-#ifdef USE_SSE2_FILL
-	const unsigned entries_per_xmm = sizeof(__m128i) / sizeof(decode_table[0]);
-#endif
+	unsigned codeword;
+	unsigned subtable_pos;
+	unsigned subtable_bits;
+	unsigned subtable_prefix;
 
-	/* Count how many symbols have each possible codeword length.
-	 * Note that a length of 0 indicates the corresponding symbol is not
-	 * used in the code and therefore does not have a codeword.  */
+	/* Count how many codewords have each length, including 0.  */
 	for (unsigned len = 0; len <= max_codeword_len; len++)
 		len_counts[len] = 0;
 	for (unsigned sym = 0; sym < num_syms; sym++)
 		len_counts[lens[sym]]++;
 
-	/* We can assume all lengths are <= max_codeword_len, but we
-	 * cannot assume they form a valid prefix code.  A codeword of
-	 * length n should require a proportion of the codespace equaling
-	 * (1/2)^n.  The code is valid if and only if the codespace is
-	 * exactly filled by the lengths, by this measure.  */
-	left = 1;
+	/* It is already guaranteed that all lengths are <= max_codeword_len,
+	 * but it cannot be assumed they form a complete prefix code.  A
+	 * codeword of length n should require a proportion of the codespace
+	 * equaling (1/2)^n.  The code is complete if and only if, by this
+	 * measure, the codespace is exactly filled by the lengths.  */
 	for (unsigned len = 1; len <= max_codeword_len; len++) {
-		left <<= 1;
-		left -= len_counts[len];
-		if (unlikely(left < 0)) {
-			/* The lengths overflow the codespace; that is, the code
-			 * is over-subscribed.  */
+		remainder = (remainder << 1) - len_counts[len];
+		/* Do the lengths overflow the codespace? */
+		if (unlikely(remainder < 0))
 			return -1;
-		}
 	}
 
-	if (unlikely(left != 0)) {
+	if (remainder != 0) {
 		/* The lengths do not fill the codespace; that is, they form an
-		 * incomplete set.  */
-		if (left == (1 << max_codeword_len)) {
-			/* The code is completely empty.  This is arguably
-			 * invalid, but in fact it is valid in LZX and XPRESS,
-			 * so we must allow it.  By definition, no symbols can
-			 * be decoded with an empty code.  Consequently, we
-			 * technically don't even need to fill in the decode
-			 * table.  However, to avoid accessing uninitialized
-			 * memory if the algorithm nevertheless attempts to
-			 * decode symbols using such a code, we zero out the
-			 * decode table.  */
-			memset(decode_table, 0,
-			       table_num_entries * sizeof(decode_table[0]));
-			return 0;
-		}
-		return -1;
+		 * incomplete code.  This is permitted only if the code is empty
+		 * (contains no symbols). */
+
+		if (unlikely(remainder != 1U << max_codeword_len))
+			return -1;
+
+		/* The code is empty.  When processing a well-formed stream, the
+		 * decode table need not be initialized in this case.  However,
+		 * we cannot assume the stream is well-formed, so we must
+		 * initialize the decode table anyway.  Setting all entries to 0
+		 * makes the decode table always produce symbol '0' without
+		 * consuming any bits, which is good enough. */
+		memset(decode_table, 0, sizeof(decode_table[0]) << table_bits);
+		return 0;
 	}
 
-	/* Sort the symbols primarily by length and secondarily by symbol order.
-	 */
-	{
-		unsigned offsets[max_codeword_len + 1];
+	/* Sort the symbols primarily by increasing codeword length and
+	 * secondarily by increasing symbol value. */
 
-		/* Initialize 'offsets' so that offsets[len] for 1 <= len <=
-		 * max_codeword_len is the number of codewords shorter than
-		 * 'len' bits.  */
-		offsets[1] = 0;
-		for (unsigned len = 1; len < max_codeword_len; len++)
-			offsets[len + 1] = offsets[len] + len_counts[len];
+	/* Initialize 'offsets' so that 'offsets[len]' is the number of
+	 * codewords shorter than 'len' bits, including length 0. */
+	offsets[0] = 0;
+	for (unsigned len = 0; len < max_codeword_len; len++)
+		offsets[len + 1] = offsets[len] + len_counts[len];
 
-		/* Use the 'offsets' array to sort the symbols.
-		 * Note that we do not include symbols that are not used in the
-		 * code.  Consequently, fewer than 'num_syms' entries in
-		 * 'sorted_syms' may be filled.  */
-		for (unsigned sym = 0; sym < num_syms; sym++)
-			if (lens[sym] != 0)
-				sorted_syms[offsets[lens[sym]]++] = sym;
-	}
+	/* Use the 'offsets' array to sort the symbols. */
+	for (unsigned sym = 0; sym < num_syms; sym++)
+		sorted_syms[offsets[lens[sym]]++] = sym;
 
-	/* Fill entries for codewords with length <= table_bits
-	 * --- that is, those short enough for a direct mapping.
+	/*
+	 * Fill the root table entries for codewords no longer than table_bits.
 	 *
 	 * The table will start with entries for the shortest codeword(s), which
-	 * have the most entries.  From there, the number of entries per
+	 * will have the most entries.  From there, the number of entries per
 	 * codeword will decrease.  As an optimization, we may begin filling
 	 * entries with SSE2 vector accesses (8 entries/store), then change to
-	 * 'machine_word_t' accesses (2 or 4 entries/store), then change to
-	 * 16-bit accesses (1 entry/store).  */
-	decode_table_ptr = decode_table;
-	sym_idx = 0;
-	codeword_len = 1;
-#ifdef USE_SSE2_FILL
-	/* Fill the entries one 128-bit vector at a time.
-	 * This is 8 entries per store.  */
-	stores_per_loop = (1 << (table_bits - codeword_len)) / entries_per_xmm;
-	for (; stores_per_loop != 0; codeword_len++, stores_per_loop >>= 1) {
+	 * word accesses (2 or 4 entries/store), then change to 16-bit accesses
+	 * (1 entry/store).
+	 */
+	sym_idx = offsets[0];
+
+#ifdef __SSE2__
+	/* Fill entries one 128-bit vector (8 entries) at a time. */
+	for (unsigned stores_per_loop = (1U << (table_bits - codeword_len)) /
+				    (sizeof(__m128i) / sizeof(decode_table[0]));
+	     stores_per_loop != 0; codeword_len++, stores_per_loop >>= 1)
+	{
 		unsigned end_sym_idx = sym_idx + len_counts[codeword_len];
 		for (; sym_idx < end_sym_idx; sym_idx++) {
-			/* Note: unlike in the machine_word_t version below, the
-			 * __m128i type already has __attribute__((may_alias)),
-			 * so using it to access the decode table, which is an
-			 * array of unsigned shorts, will not violate strict
+			/* Note: unlike in the "word" version below, the __m128i
+			 * type already has __attribute__((may_alias)), so using
+			 * it to access an array of u16 will not violate strict
 			 * aliasing.  */
-			u16 entry;
-			__m128i v;
-			__m128i *p;
-			unsigned n;
-
-			entry = MAKE_DIRECT_ENTRY(sorted_syms[sym_idx], codeword_len);
-
-			v = _mm_set1_epi16(entry);
-			p = (__m128i*)decode_table_ptr;
-			n = stores_per_loop;
+			__m128i v = _mm_set1_epi16(
+				MAKE_DECODE_TABLE_ENTRY(sorted_syms[sym_idx],
+							codeword_len));
+			unsigned n = stores_per_loop;
 			do {
-				*p++ = v;
+				*(__m128i *)entry_ptr = v;
+				entry_ptr += sizeof(v);
 			} while (--n);
-			decode_table_ptr = p;
 		}
 	}
-#endif /* USE_SSE2_FILL */
+#endif /* __SSE2__ */
 
-#ifdef USE_WORD_FILL
-	/* Fill the entries one machine word at a time.
-	 * On 32-bit systems this is 2 entries per store, while on 64-bit
-	 * systems this is 4 entries per store.  */
-	stores_per_loop = (1 << (table_bits - codeword_len)) / entries_per_word;
-	for (; stores_per_loop != 0; codeword_len++, stores_per_loop >>= 1) {
+#ifdef __GNUC__
+	/* Fill entries one word (2 or 4 entries) at a time. */
+	for (unsigned stores_per_loop = (1U << (table_bits - codeword_len)) /
+					(WORDBYTES / sizeof(decode_table[0]));
+	     stores_per_loop != 0; codeword_len++, stores_per_loop >>= 1)
+	{
 		unsigned end_sym_idx = sym_idx + len_counts[codeword_len];
 		for (; sym_idx < end_sym_idx; sym_idx++) {
 
@@ -285,140 +228,105 @@ make_huffman_decode_table(u16 decode_table[const],
 			 * the code with -fno-strict-aliasing to guarantee
 			 * correctness.  To work around this problem, use the
 			 * gcc 'may_alias' extension.  */
-			typedef machine_word_t _may_alias_attribute aliased_word_t;
-
-			machine_word_t v;
-			aliased_word_t *p;
-			unsigned n;
-
-			STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
-
-			v = MAKE_DIRECT_ENTRY(sorted_syms[sym_idx], codeword_len);
-			v |= v << 16;
-			v |= v << (WORDBITS == 64 ? 32 : 0);
-
-			p = (aliased_word_t *)decode_table_ptr;
-			n = stores_per_loop;
-
+			typedef machine_word_t
+				__attribute__((may_alias)) aliased_word_t;
+			aliased_word_t v = repeat_u16(
+				MAKE_DECODE_TABLE_ENTRY(sorted_syms[sym_idx],
+							codeword_len));
+			unsigned n = stores_per_loop;
 			do {
-				*p++ = v;
+				*(aliased_word_t *)entry_ptr = v;
+				entry_ptr += sizeof(v);
 			} while (--n);
-			decode_table_ptr = p;
 		}
 	}
-#endif /* USE_WORD_FILL */
+#endif /* __GNUC__ */
 
-	/* Fill the entries one 16-bit integer at a time.  */
-	stores_per_loop = (1 << (table_bits - codeword_len));
-	for (; stores_per_loop != 0; codeword_len++, stores_per_loop >>= 1) {
+	/* Fill entries one at a time. */
+	for (unsigned stores_per_loop = (1U << (table_bits - codeword_len));
+	     stores_per_loop != 0; codeword_len++, stores_per_loop >>= 1)
+	{
 		unsigned end_sym_idx = sym_idx + len_counts[codeword_len];
 		for (; sym_idx < end_sym_idx; sym_idx++) {
-			u16 entry;
-			u16 *p;
-			unsigned n;
-
-			entry = MAKE_DIRECT_ENTRY(sorted_syms[sym_idx], codeword_len);
-
-			p = (u16*)decode_table_ptr;
-			n = stores_per_loop;
-
+			u16 v = MAKE_DECODE_TABLE_ENTRY(sorted_syms[sym_idx],
+							codeword_len);
+			unsigned n = stores_per_loop;
 			do {
-				*p++ = entry;
+				*(u16 *)entry_ptr = v;
+				entry_ptr += sizeof(v);
 			} while (--n);
-
-			decode_table_ptr = p;
 		}
 	}
 
-	/* If we've filled in the entire table, we are done.  Otherwise,
-	 * there are codewords longer than table_bits for which we must
-	 * generate binary trees.  */
-
-	decode_table_pos = (u16*)decode_table_ptr - decode_table;
-	if (decode_table_pos != table_num_entries) {
-		unsigned j;
-		unsigned next_free_tree_slot;
-		unsigned cur_codeword;
-
-		/* First, zero out the remaining entries.  This is
-		 * necessary so that these entries appear as
-		 * "unallocated" in the next part.  Each of these entries
-		 * will eventually be filled with the representation of
-		 * the root node of a binary tree.  */
-		j = decode_table_pos;
-		do {
-			decode_table[j] = 0;
-		} while (++j != table_num_entries);
-
-		/* We allocate child nodes starting at the end of the
-		 * direct lookup table.  Note that there should be
-		 * 2*num_syms extra entries for this purpose, although
-		 * fewer than this may actually be needed.  */
-		next_free_tree_slot = table_num_entries;
-
-		/* Iterate through each codeword with length greater than
-		 * 'table_bits', primarily in order of codeword length
-		 * and secondarily in order of symbol.  */
-		for (cur_codeword = decode_table_pos << 1;
-		     codeword_len <= max_codeword_len;
-		     codeword_len++, cur_codeword <<= 1)
-		{
-			unsigned end_sym_idx = sym_idx + len_counts[codeword_len];
-			for (; sym_idx < end_sym_idx; sym_idx++, cur_codeword++)
-			{
-				/* 'sym' is the symbol represented by the
-				 * codeword.  */
-				unsigned sym = sorted_syms[sym_idx];
-
-				unsigned extra_bits = codeword_len - table_bits;
+	/* If all symbols were processed, then no subtables are required. */
+	if (sym_idx == num_syms)
+		return 0;
+
+	/* At least one subtable is required.  Process the remaining symbols. */
+	codeword = ((u16 *)entry_ptr - decode_table) << 1;
+	subtable_pos = 1U << table_bits;
+	subtable_bits = table_bits;
+	subtable_prefix = -1;
+	do {
+		while (len_counts[codeword_len] == 0) {
+			codeword_len++;
+			codeword <<= 1;
+		}
 
-				unsigned node_idx = cur_codeword >> extra_bits;
+		unsigned prefix = codeword >> (codeword_len - table_bits);
+
+		/* Start a new subtable if the first 'table_bits' bits of the
+		 * codeword don't match the prefix for the previous subtable, or
+		 * if this will be the first subtable. */
+		if (prefix != subtable_prefix) {
+
+			subtable_prefix = prefix;
+
+			/*
+			 * Calculate the subtable length.  If the codeword
+			 * length exceeds 'table_bits' by n, then the subtable
+			 * needs at least 2^n entries.  But it may need more; if
+			 * there are fewer than 2^n codewords of length
+			 * 'table_bits + n' remaining, then n will need to be
+			 * incremented to bring in longer codewords until the
+			 * subtable can be filled completely.  Note that it
+			 * always will, eventually, be possible to fill the
+			 * subtable, since it was previously verified that the
+			 * code is complete.
+			 */
+			subtable_bits = codeword_len - table_bits;
+			remainder = (s32)1 << subtable_bits;
+			for (;;) {
+				remainder -= len_counts[table_bits +
+							subtable_bits];
+				if (remainder <= 0)
+					break;
+				subtable_bits++;
+				remainder <<= 1;
+			}
 
-				/* Go through each bit of the current codeword
-				 * beyond the prefix of length @table_bits and
-				 * walk the appropriate binary tree, allocating
-				 * any slots that have not yet been allocated.
-				 *
-				 * Note that the 'pointer' entry to the binary
-				 * tree, which is stored in the direct lookup
-				 * portion of the table, is represented
-				 * identically to other internal (non-leaf)
-				 * nodes of the binary tree; it can be thought
-				 * of as simply the root of the tree.  The
-				 * representation of these internal nodes is
-				 * simply the index of the left child combined
-				 * with the special bits 0xC000 to distinguish
-				 * the entry from direct mapping and leaf node
-				 * entries.  */
-				do {
+			/* Create the entry that points from the root table to
+			 * the subtable.  This entry contains the index of the
+			 * start of the subtable and the number of bits with
+			 * which the subtable is indexed (the log base 2 of the
+			 * number of entries it contains).  */
+			decode_table[subtable_prefix] =
+				MAKE_DECODE_TABLE_ENTRY(subtable_pos,
+							subtable_bits);
+		}
 
-					/* At least one bit remains in the
-					 * codeword, but the current node is an
-					 * unallocated leaf.  Change it to an
-					 * internal node.  */
-					if (decode_table[node_idx] == 0) {
-						decode_table[node_idx] =
-							next_free_tree_slot | 0xC000;
-						decode_table[next_free_tree_slot++] = 0;
-						decode_table[next_free_tree_slot++] = 0;
-					}
+		/* Fill the subtable entries for this symbol. */
+		u16 entry = MAKE_DECODE_TABLE_ENTRY(sorted_syms[sym_idx],
+						    codeword_len - table_bits);
+		unsigned n = 1U << (subtable_bits - (codeword_len -
+						     table_bits));
+		do {
+			decode_table[subtable_pos++] = entry;
+		} while (--n);
 
-					/* Go to the left child if the next bit
-					 * in the codeword is 0; otherwise go to
-					 * the right child.  */
-					node_idx = decode_table[node_idx] & 0x3FFF;
-					--extra_bits;
-					node_idx += (cur_codeword >> extra_bits) & 1;
-				} while (extra_bits != 0);
+		len_counts[codeword_len]--;
+		codeword++;
+	} while (++sym_idx < num_syms);
 
-				/* We've traversed the tree using the entire
-				 * codeword, and we're now at the entry where
-				 * the actual symbol will be stored.  This is
-				 * distinguished from internal nodes by not
-				 * having its high two bits set.  */
-				decode_table[node_idx] = sym;
-			}
-		}
-	}
 	return 0;
 }