4 * Functions too long to declare as inline in comp.h.
6 * Copyright (C) 2012 Eric Biggers
8 * wimlib - Library for working with WIM files
10 * This library is free software; you can redistribute it and/or modify it under
11 * the terms of the GNU Lesser General Public License as published by the Free
12 * Software Foundation; either version 2.1 of the License, or (at your option) any
15 * This library is distributed in the hope that it will be useful, but WITHOUT ANY
16 * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
17 * PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details.
19 * You should have received a copy of the GNU Lesser General Public License along
20 * with this library; if not, write to the Free Software Foundation, Inc., 59
21 * Temple Place, Suite 330, Boston, MA 02111-1307 USA
28 static inline void flush_bits(struct output_bitstream *ostream)
30 *(u16*)ostream->bit_output = to_le16(ostream->bitbuf);
31 ostream->bit_output = ostream->next_bit_output;
32 ostream->next_bit_output = ostream->output;
34 ostream->num_bytes_remaining -= 2;
37 /* Writes @num_bits bits, given by the @num_bits least significant bits of
38 * @bits, to the output @ostream. */
39 int bitstream_put_bits(struct output_bitstream *ostream, output_bitbuf_t bits,
44 wimlib_assert(num_bits <= 16);
45 if (num_bits <= ostream->free_bits) {
46 ostream->bitbuf = (ostream->bitbuf << num_bits) | bits;
47 ostream->free_bits -= num_bits;
50 if (ostream->num_bytes_remaining + (ostream->output -
51 ostream->bit_output) < 2)
54 /* It is tricky to output the bits correctly. The correct way
55 * is to output little-endian 2-byte words, such that the bits
56 * in the SECOND byte logically precede those in the FIRST byte.
57 * While the byte order is little-endian, the bit order is
58 * big-endian; the first bit in a byte is the high-order one.
59 * Any multi-bit numbers are in bit-big-endian form, so the
60 * low-order bit of a multi-bit number is the LAST bit to be
62 rem_bits = num_bits - ostream->free_bits;
63 ostream->bitbuf <<= ostream->free_bits;
64 ostream->bitbuf |= bits >> rem_bits;
66 ostream->free_bits = 16 - rem_bits;
67 ostream->bitbuf = bits;
73 /* Flushes any remaining bits in the output buffer to the output byte stream. */
74 int flush_output_bitstream(struct output_bitstream *ostream)
76 if (ostream->num_bytes_remaining + (ostream->output -
77 ostream->bit_output) < 2)
79 if (ostream->free_bits != 16) {
80 ostream->bitbuf <<= ostream->free_bits;
86 /* Initializes an output bit buffer to write its output to the memory location
87 * pointer to by @data. */
88 void init_output_bitstream(struct output_bitstream *ostream, void *data,
92 ostream->free_bits = 16;
93 ostream->bit_output = (u8*)data;
94 ostream->next_bit_output = (u8*)data + 2;
95 ostream->output = (u8*)data + 4;
96 ostream->num_bytes_remaining = num_bytes - 4;
99 /* Intermediate (non-leaf) node in a Huffman tree. */
100 typedef struct HuffmanNode {
107 struct HuffmanNode *left_child;
108 struct HuffmanNode *right_child;
111 /* Leaf node in a Huffman tree. The fields are in the same order as the
112 * HuffmanNode, so it can be cast to a HuffmanNode. There are no pointers to
113 * the children in the leaf node. */
123 /* Comparator function for HuffmanLeafNodes. Sorts primarily by symbol
124 * frequency and secondarily by symbol value. */
125 static int cmp_leaves_by_freq(const void *__leaf1, const void *__leaf2)
127 const HuffmanLeafNode *leaf1 = __leaf1;
128 const HuffmanLeafNode *leaf2 = __leaf2;
130 int freq_diff = (int)leaf1->freq - (int)leaf2->freq;
133 return (int)leaf1->sym - (int)leaf2->sym;
138 /* Comparator function for HuffmanLeafNodes. Sorts primarily by code length and
139 * secondarily by symbol value. */
140 static int cmp_leaves_by_code_len(const void *__leaf1, const void *__leaf2)
142 const HuffmanLeafNode *leaf1 = __leaf1;
143 const HuffmanLeafNode *leaf2 = __leaf2;
145 int code_len_diff = (int)leaf1->path_len - (int)leaf2->path_len;
147 if (code_len_diff == 0)
148 return (int)leaf1->sym - (int)leaf2->sym;
150 return code_len_diff;
153 /* Recursive function to calculate the depth of the leaves in a Huffman tree.
155 static void huffman_tree_compute_path_lengths(HuffmanNode *node, u16 cur_len)
157 if (node->sym == (u16)(-1)) {
158 /* Intermediate node. */
159 huffman_tree_compute_path_lengths(node->left_child, cur_len + 1);
160 huffman_tree_compute_path_lengths(node->right_child, cur_len + 1);
163 node->path_len = cur_len;
167 /* Creates a canonical Huffman code from an array of symbol frequencies.
169 * The algorithm used is similar to the well-known algorithm that builds a
170 * Huffman tree using a minheap. In that algorithm, the leaf nodes are
171 * initialized and inserted into the minheap with the frequency as the key.
172 * Repeatedly, the top two nodes (nodes with the lowest frequency) are taken out
173 * of the heap and made the children of a new node that has a frequency equal to
174 * the sum of the two frequencies of its children. This new node is inserted
175 * into the heap. When all the nodes have been removed from the heap, what
176 * remains is the Huffman tree. The Huffman code for a symbol is given by the
177 * path to it in the tree, where each left pointer is mapped to a 0 bit and each
178 * right pointer is mapped to a 1 bit.
180 * The algorithm used here uses an optimization that removes the need to
181 * actually use a heap. The leaf nodes are first sorted by frequency, as
182 * opposed to being made into a heap. Note that this sorting step takes O(n log
183 * n) time vs. O(n) time for heapifying the array, where n is the number of
184 * symbols. However, the heapless method is probably faster overall, due to the
185 * time saved later. In the heapless method, whenever an intermediate node is
186 * created, it is not inserted into the sorted array. Instead, the intermediate
187 * nodes are kept in a separate array, which is easily kept sorted because every
188 * time an intermediate node is initialized, it will have a frequency at least
189 * as high as that of the previous intermediate node that was initialized. So
190 * whenever we want the 2 nodes, leaf or intermediate, that have the lowest
191 * frequency, we check the low-frequency ends of both arrays, which is an O(1)
194 * The function builds a canonical Huffman code, not just any Huffman code. A
195 * Huffman code is canonical if the codeword for each symbol numerically
196 * precedes the codeword for all other symbols of the same length that are
197 * numbered higher than the symbol, and additionally, all shorter codewords,
198 * 0-extended, numerically precede longer codewords. A canonical Huffman code
199 * is useful because it can be reconstructed by only knowing the path lengths in
200 * the tree. See the make_huffman_decode_table() function to see how to
201 * reconstruct a canonical Huffman code from only the lengths of the codes.
203 * @num_syms: The number of symbols in the alphabet.
205 * @max_codeword_len: The maximum allowed length of a codeword in the code.
206 * Note that if the code being created runs up against
207 * this restriction, the code ultimately created will be
208 * suboptimal, although there are some advantages for
209 * limiting the length of the codewords.
211 * @freq_tab: An array of length @num_syms that contains the frequencies
212 * of each symbol in the uncompressed data.
214 * @lens: An array of length @num_syms into which the lengths of the
215 * codewords for each symbol will be written.
217 * @codewords: An array of @num_syms short integers into which the
218 * codewords for each symbol will be written. The first
219 * lens[i] bits of codewords[i] will contain the codeword
222 void make_canonical_huffman_code(uint num_syms, uint max_codeword_len,
223 const u32 freq_tab[], u8 lens[],
226 /* We require at least 2 possible symbols in the alphabet to produce a
227 * valid Huffman decoding table. It is allowed that fewer than 2 symbols
228 * are actually used, though. */
229 wimlib_assert(num_syms >= 2);
231 /* Initialize the lengths and codewords to 0 */
232 memset(lens, 0, num_syms * sizeof(lens[0]));
233 memset(codewords, 0, num_syms * sizeof(codewords[0]));
235 /* Calculate how many symbols have non-zero frequency. These are the
236 * symbols that actually appeared in the input. */
237 uint num_used_symbols = 0;
238 for (uint i = 0; i < num_syms; i++)
239 if (freq_tab[i] != 0)
243 /* It is impossible to make a code for num_used_symbols symbols if there
244 * aren't enough code bits to uniquely represent all of them. */
245 wimlib_assert((1 << max_codeword_len) > num_used_symbols);
247 /* Initialize the array of leaf nodes with the symbols and their
249 HuffmanLeafNode leaves[num_used_symbols];
251 for (uint i = 0; i < num_syms; i++) {
252 if (freq_tab[i] != 0) {
253 leaves[leaf_idx].freq = freq_tab[i];
254 leaves[leaf_idx].sym = i;
255 leaves[leaf_idx].height = 0;
260 /* Deal with the special cases where num_used_symbols < 2. */
261 if (num_used_symbols < 2) {
262 if (num_used_symbols == 0) {
263 /* If num_used_symbols is 0, there are no symbols in the
264 * input, so it must be empty. This should be an error,
265 * but the LZX format expects this case to succeed. All
266 * the codeword lengths are simply marked as 0 (which
267 * was already done.) */
269 /* If only one symbol is present, the LZX format
270 * requires that the Huffman code include two codewords.
271 * One is not used. Note that this doesn't make the
272 * encoded data take up more room anyway, since binary
273 * data itself has 2 symbols. */
275 uint sym = leaves[0].sym;
280 /* dummy symbol is 1, real symbol is 0 */
284 /* dummy symbol is 0, real symbol is sym */
292 /* Otherwise, there are at least 2 symbols in the input, so we need to
293 * find a real Huffman code. */
296 /* Declare the array of intermediate nodes. An intermediate node is not
297 * associated with a symbol. Instead, it represents some binary code
298 * prefix that is shared between at least 2 codewords. There can be at
299 * most num_used_symbols - 1 intermediate nodes when creating a Huffman
300 * code. This is because if there were at least num_used_symbols nodes,
301 * the code would be suboptimal because there would be at least one
302 * unnecessary intermediate node.
304 * The worst case (greatest number of intermediate nodes) would be if
305 * all the intermediate nodes were chained together. This results in
306 * num_used_symbols - 1 intermediate nodes. If num_used_symbols is at
307 * least 17, this configuration would not be allowed because the LZX
308 * format constrains codes to 16 bits or less each. However, it is
309 * still possible for there to be more than 16 intermediate nodes, as
310 * long as no leaf has a depth of more than 16. */
311 HuffmanNode inodes[num_used_symbols - 1];
314 /* Pointer to the leaf node of lowest frequency that hasn't already been
315 * added as the child of some intermediate note. */
316 HuffmanLeafNode *cur_leaf = &leaves[0];
318 /* Pointer past the end of the array of leaves. */
319 HuffmanLeafNode *end_leaf = &leaves[num_used_symbols];
321 /* Pointer to the intermediate node of lowest frequency. */
322 HuffmanNode *cur_inode = &inodes[0];
324 /* Pointer to the next unallocated intermediate node. */
325 HuffmanNode *next_inode = &inodes[0];
327 /* Only jump back to here if the maximum length of the codewords allowed
328 * by the LZX format (16 bits) is exceeded. */
329 try_building_tree_again:
331 /* Sort the leaves from those that correspond to the least frequent
332 * symbol, to those that correspond to the most frequent symbol. If two
333 * leaves have the same frequency, they are sorted by symbol. */
334 qsort(leaves, num_used_symbols, sizeof(leaves[0]), cmp_leaves_by_freq);
336 cur_leaf = &leaves[0];
337 cur_inode = &inodes[0];
338 next_inode = &inodes[0];
340 /* The following loop takes the two lowest frequency nodes of those
341 * remaining and makes them the children of the next available
342 * intermediate node. It continues until all the leaf nodes and
343 * intermediate nodes have been used up, or the maximum allowed length
344 * for the codewords is exceeded. For the latter case, we must adjust
345 * the frequencies to be more equal and then execute this loop again. */
348 /* Lowest frequency node. */
349 HuffmanNode *f1 = NULL;
351 /* Second lowest frequency node. */
352 HuffmanNode *f2 = NULL;
354 /* Get the lowest and second lowest frequency nodes from
355 * the remaining leaves or from the intermediate nodes.
358 if (cur_leaf != end_leaf && (cur_inode == next_inode ||
359 cur_leaf->freq <= cur_inode->freq)) {
360 f1 = (HuffmanNode*)cur_leaf++;
361 } else if (cur_inode != next_inode) {
365 if (cur_leaf != end_leaf && (cur_inode == next_inode ||
366 cur_leaf->freq <= cur_inode->freq)) {
367 f2 = (HuffmanNode*)cur_leaf++;
368 } else if (cur_inode != next_inode) {
372 /* All nodes used up! */
373 if (f1 == NULL || f2 == NULL)
376 /* next_inode becomes the parent of f1 and f2. */
378 next_inode->freq = f1->freq + f2->freq;
379 next_inode->sym = (u16)(-1); /* Invalid symbol. */
380 next_inode->left_child = f1;
381 next_inode->right_child = f2;
383 /* We need to keep track of the height so that we can detect if
384 * the length of a codeword has execeed max_codeword_len. The
385 * parent node has a height one higher than the maximum height
386 * of its children. */
387 next_inode->height = max(f1->height, f2->height) + 1;
389 /* Check to see if the code length of the leaf farthest away
390 * from next_inode has exceeded the maximum code length. */
391 if (next_inode->height > max_codeword_len) {
392 /* The code lengths can be made more uniform by making
393 * the frequencies more uniform. Divide all the
394 * frequencies by 2, leaving 1 as the minimum frequency.
395 * If this keeps happening, the symbol frequencies will
396 * approach equality, which makes their Huffman
397 * codewords approach the length
398 * log_2(num_used_symbols).
400 for (uint i = 0; i < num_used_symbols; i++)
401 if (leaves[i].freq > 1)
402 leaves[i].freq >>= 1;
403 goto try_building_tree_again;
408 /* The Huffman tree is now complete, and its height is no more than
409 * max_codeword_len. */
411 HuffmanNode *root = next_inode - 1;
412 wimlib_assert(root->height <= max_codeword_len);
414 /* Compute the path lengths for the leaf nodes. */
415 huffman_tree_compute_path_lengths(root, 0);
417 /* Sort the leaf nodes primarily by code length and secondarily by
419 qsort(leaves, num_used_symbols, sizeof(leaves[0]), cmp_leaves_by_code_len);
421 u16 cur_codeword = 0;
422 uint cur_codeword_len = 0;
423 for (uint i = 0; i < num_used_symbols; i++) {
425 /* Each time a codeword becomes one longer, the current codeword
426 * is left shifted by one place. This is part of the procedure
427 * for enumerating the canonical Huffman code. Additionally,
428 * whenever a codeword is used, 1 is added to the current
431 uint len_diff = leaves[i].path_len - cur_codeword_len;
432 cur_codeword <<= len_diff;
433 cur_codeword_len += len_diff;
435 u16 sym = leaves[i].sym;
436 codewords[sym] = cur_codeword;
437 lens[sym] = cur_codeword_len;