4 * Functions used for compression.
8 * Copyright (C) 2012, 2013 Eric Biggers
10 * This file is part of wimlib, a library for working with WIM files.
12 * wimlib is free software; you can redistribute it and/or modify it under the
13 * terms of the GNU General Public License as published by the Free
14 * Software Foundation; either version 3 of the License, or (at your option)
17 * wimlib is distributed in the hope that it will be useful, but WITHOUT ANY
18 * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
19 * A PARTICULAR PURPOSE. See the GNU General Public License for more
22 * You should have received a copy of the GNU General Public License
23 * along with wimlib; if not, see http://www.gnu.org/licenses/.
30 #include "wimlib/assert.h"
31 #include "wimlib/compiler.h"
32 #include "wimlib/compress.h"
33 #include "wimlib/util.h"
38 /* Writes @num_bits bits, given by the @num_bits least significant bits of
39 * @bits, to the output @ostream. */
41 bitstream_put_bits(struct output_bitstream *ostream, u32 bits,
44 bits &= (1U << num_bits) - 1;
45 while (num_bits > ostream->free_bits) {
46 /* Buffer variable does not have space for the new bits. It
47 * needs to be flushed as a 16-bit integer. Bits in the second
48 * byte logically precede those in the first byte
49 * (little-endian), but within each byte the bits are ordered
50 * from high to low. This is true for both XPRESS and LZX
53 /* There must be at least 2 bytes of space remaining. */
54 if (unlikely(ostream->bytes_remaining < 2)) {
55 ostream->overrun = true;
59 /* Fill the buffer with as many bits that fit. */
60 unsigned fill_bits = ostream->free_bits;
62 ostream->bitbuf <<= fill_bits;
63 ostream->bitbuf |= bits >> (num_bits - fill_bits);
65 *(le16*)ostream->bit_output = cpu_to_le16(ostream->bitbuf);
66 ostream->bit_output = ostream->next_bit_output;
67 ostream->next_bit_output = ostream->output;
69 ostream->bytes_remaining -= 2;
71 ostream->free_bits = 16;
72 num_bits -= fill_bits;
73 bits &= (1U << num_bits) - 1;
76 /* Buffer variable has space for the new bits. */
77 ostream->bitbuf = (ostream->bitbuf << num_bits) | bits;
78 ostream->free_bits -= num_bits;
82 bitstream_put_byte(struct output_bitstream *ostream, u8 n)
84 if (unlikely(ostream->bytes_remaining < 1)) {
85 ostream->overrun = true;
88 *ostream->output++ = n;
89 ostream->bytes_remaining--;
92 /* Flushes any remaining bits to the output bitstream.
94 * Returns -1 if the stream has overrun; otherwise returns the total number of
95 * bytes in the output. */
97 flush_output_bitstream(struct output_bitstream *ostream)
99 if (unlikely(ostream->overrun))
100 return ~(input_idx_t)0;
102 *(le16*)ostream->bit_output =
103 cpu_to_le16((u16)((u32)ostream->bitbuf << ostream->free_bits));
104 *(le16*)ostream->next_bit_output =
107 return ostream->output - ostream->output_start;
110 /* Initializes an output bit buffer to write its output to the memory location
111 * pointer to by @data. */
113 init_output_bitstream(struct output_bitstream *ostream,
114 void *data, unsigned num_bytes)
116 wimlib_assert(num_bytes >= 4);
119 ostream->free_bits = 16;
120 ostream->output_start = data;
121 ostream->bit_output = data;
122 ostream->next_bit_output = data + 2;
123 ostream->output = data + 4;
124 ostream->bytes_remaining = num_bytes;
125 ostream->overrun = false;
137 typedef struct HuffmanIntermediateNode {
138 HuffmanNode node_base;
139 HuffmanNode *left_child;
140 HuffmanNode *right_child;
141 } HuffmanIntermediateNode;
144 /* Comparator function for HuffmanNodes. Sorts primarily by symbol
145 * frequency and secondarily by symbol value. */
147 cmp_nodes_by_freq(const void *_leaf1, const void *_leaf2)
149 const HuffmanNode *leaf1 = _leaf1;
150 const HuffmanNode *leaf2 = _leaf2;
152 if (leaf1->freq > leaf2->freq)
154 else if (leaf1->freq < leaf2->freq)
157 return (int)leaf1->sym - (int)leaf2->sym;
160 /* Comparator function for HuffmanNodes. Sorts primarily by code length and
161 * secondarily by symbol value. */
163 cmp_nodes_by_code_len(const void *_leaf1, const void *_leaf2)
165 const HuffmanNode *leaf1 = _leaf1;
166 const HuffmanNode *leaf2 = _leaf2;
168 int code_len_diff = (int)leaf1->path_len - (int)leaf2->path_len;
170 if (code_len_diff == 0)
171 return (int)leaf1->sym - (int)leaf2->sym;
173 return code_len_diff;
176 #define INVALID_SYMBOL 0xffff
178 /* Recursive function to calculate the depth of the leaves in a Huffman tree.
181 huffman_tree_compute_path_lengths(HuffmanNode *base_node, u16 cur_len)
183 if (base_node->sym == INVALID_SYMBOL) {
184 /* Intermediate node. */
185 HuffmanIntermediateNode *node = (HuffmanIntermediateNode*)base_node;
186 huffman_tree_compute_path_lengths(node->left_child, cur_len + 1);
187 huffman_tree_compute_path_lengths(node->right_child, cur_len + 1);
190 base_node->path_len = cur_len;
194 /* make_canonical_huffman_code: - Creates a canonical Huffman code from an array
195 * of symbol frequencies.
197 * The algorithm used is similar to the well-known algorithm that builds a
198 * Huffman tree using a minheap. In that algorithm, the leaf nodes are
199 * initialized and inserted into the minheap with the frequency as the key.
200 * Repeatedly, the top two nodes (nodes with the lowest frequency) are taken out
201 * of the heap and made the children of a new node that has a frequency equal to
202 * the sum of the two frequencies of its children. This new node is inserted
203 * into the heap. When all the nodes have been removed from the heap, what
204 * remains is the Huffman tree. The Huffman code for a symbol is given by the
205 * path to it in the tree, where each left pointer is mapped to a 0 bit and each
206 * right pointer is mapped to a 1 bit.
208 * The algorithm used here uses an optimization that removes the need to
209 * actually use a heap. The leaf nodes are first sorted by frequency, as
210 * opposed to being made into a heap. Note that this sorting step takes O(n log
211 * n) time vs. O(n) time for heapifying the array, where n is the number of
212 * symbols. However, the heapless method is probably faster overall, due to the
213 * time saved later. In the heapless method, whenever an intermediate node is
214 * created, it is not inserted into the sorted array. Instead, the intermediate
215 * nodes are kept in a separate array, which is easily kept sorted because every
216 * time an intermediate node is initialized, it will have a frequency at least
217 * as high as that of the previous intermediate node that was initialized. So
218 * whenever we want the 2 nodes, leaf or intermediate, that have the lowest
219 * frequency, we check the low-frequency ends of both arrays, which is an O(1)
222 * The function builds a canonical Huffman code, not just any Huffman code. A
223 * Huffman code is canonical if the codeword for each symbol numerically
224 * precedes the codeword for all other symbols of the same length that are
225 * numbered higher than the symbol, and additionally, all shorter codewords,
226 * 0-extended, numerically precede longer codewords. A canonical Huffman code
227 * is useful because it can be reconstructed by only knowing the path lengths in
228 * the tree. See the make_huffman_decode_table() function to see how to
229 * reconstruct a canonical Huffman code from only the lengths of the codes.
231 * @num_syms: The number of symbols in the alphabet.
233 * @max_codeword_len: The maximum allowed length of a codeword in the code.
234 * Note that if the code being created runs up against
235 * this restriction, the code ultimately created will be
236 * suboptimal, although there are some advantages for
237 * limiting the length of the codewords.
239 * @freq_tab: An array of length @num_syms that contains the frequencies
240 * of each symbol in the uncompressed data.
242 * @lens: An array of length @num_syms into which the lengths of the
243 * codewords for each symbol will be written.
245 * @codewords: An array of @num_syms short integers into which the
246 * codewords for each symbol will be written. The first
247 * lens[i] bits of codewords[i] will contain the codeword
251 make_canonical_huffman_code(unsigned num_syms,
252 unsigned max_codeword_len,
253 const freq_t freq_tab[restrict],
255 u16 codewords[restrict])
257 /* We require at least 2 possible symbols in the alphabet to produce a
258 * valid Huffman decoding table. It is allowed that fewer than 2 symbols
259 * are actually used, though. */
260 wimlib_assert(num_syms >= 2 && num_syms < INVALID_SYMBOL);
262 /* Initialize the lengths and codewords to 0 */
263 memset(lens, 0, num_syms * sizeof(lens[0]));
264 memset(codewords, 0, num_syms * sizeof(codewords[0]));
266 /* Calculate how many symbols have non-zero frequency. These are the
267 * symbols that actually appeared in the input. */
268 unsigned num_used_symbols = 0;
269 for (unsigned i = 0; i < num_syms; i++)
270 if (freq_tab[i] != 0)
274 /* It is impossible to make a code for num_used_symbols symbols if there
275 * aren't enough code bits to uniquely represent all of them. */
276 wimlib_assert((1 << max_codeword_len) > num_used_symbols);
278 /* Initialize the array of leaf nodes with the symbols and their
280 HuffmanNode leaves[num_used_symbols];
281 unsigned leaf_idx = 0;
282 for (unsigned i = 0; i < num_syms; i++) {
283 if (freq_tab[i] != 0) {
284 leaves[leaf_idx].freq = freq_tab[i];
285 leaves[leaf_idx].sym = i;
286 leaves[leaf_idx].height = 0;
291 /* Deal with the special cases where num_used_symbols < 2. */
292 if (num_used_symbols < 2) {
293 if (num_used_symbols == 0) {
294 /* If num_used_symbols is 0, there are no symbols in the
295 * input, so it must be empty. This should be an error,
296 * but the LZX format expects this case to succeed. All
297 * the codeword lengths are simply marked as 0 (which
298 * was already done.) */
300 /* If only one symbol is present, the LZX format
301 * requires that the Huffman code include two codewords.
302 * One is not used. Note that this doesn't make the
303 * encoded data take up more room anyway, since binary
304 * data itself has 2 symbols. */
306 unsigned sym = leaves[0].sym;
311 /* dummy symbol is 1, real symbol is 0 */
315 /* dummy symbol is 0, real symbol is sym */
323 /* Otherwise, there are at least 2 symbols in the input, so we need to
324 * find a real Huffman code. */
327 /* Declare the array of intermediate nodes. An intermediate node is not
328 * associated with a symbol. Instead, it represents some binary code
329 * prefix that is shared between at least 2 codewords. There can be at
330 * most num_used_symbols - 1 intermediate nodes when creating a Huffman
331 * code. This is because if there were at least num_used_symbols nodes,
332 * the code would be suboptimal because there would be at least one
333 * unnecessary intermediate node.
335 * The worst case (greatest number of intermediate nodes) would be if
336 * all the intermediate nodes were chained together. This results in
337 * num_used_symbols - 1 intermediate nodes. If num_used_symbols is at
338 * least 17, this configuration would not be allowed because the LZX
339 * format constrains codes to 16 bits or less each. However, it is
340 * still possible for there to be more than 16 intermediate nodes, as
341 * long as no leaf has a depth of more than 16. */
342 HuffmanIntermediateNode inodes[num_used_symbols - 1];
345 /* Pointer to the leaf node of lowest frequency that hasn't already been
346 * added as the child of some intermediate note. */
347 HuffmanNode *cur_leaf;
349 /* Pointer past the end of the array of leaves. */
350 HuffmanNode *end_leaf = &leaves[num_used_symbols];
352 /* Pointer to the intermediate node of lowest frequency. */
353 HuffmanIntermediateNode *cur_inode;
355 /* Pointer to the next unallocated intermediate node. */
356 HuffmanIntermediateNode *next_inode;
358 /* Only jump back to here if the maximum length of the codewords allowed
359 * by the LZX format (16 bits) is exceeded. */
360 try_building_tree_again:
362 /* Sort the leaves from those that correspond to the least frequent
363 * symbol, to those that correspond to the most frequent symbol. If two
364 * leaves have the same frequency, they are sorted by symbol. */
365 qsort(leaves, num_used_symbols, sizeof(leaves[0]), cmp_nodes_by_freq);
367 cur_leaf = &leaves[0];
368 cur_inode = &inodes[0];
369 next_inode = &inodes[0];
371 /* The following loop takes the two lowest frequency nodes of those
372 * remaining and makes them the children of the next available
373 * intermediate node. It continues until all the leaf nodes and
374 * intermediate nodes have been used up, or the maximum allowed length
375 * for the codewords is exceeded. For the latter case, we must adjust
376 * the frequencies to be more equal and then execute this loop again. */
379 /* Lowest frequency node. */
382 /* Second lowest frequency node. */
385 /* Get the lowest and second lowest frequency nodes from the
386 * remaining leaves or from the intermediate nodes. */
388 if (cur_leaf != end_leaf && (cur_inode == next_inode ||
389 cur_leaf->freq <= cur_inode->node_base.freq)) {
391 } else if (cur_inode != next_inode) {
392 f1 = (HuffmanNode*)cur_inode++;
395 if (cur_leaf != end_leaf && (cur_inode == next_inode ||
396 cur_leaf->freq <= cur_inode->node_base.freq)) {
398 } else if (cur_inode != next_inode) {
399 f2 = (HuffmanNode*)cur_inode++;
401 /* All nodes used up! */
405 /* next_inode becomes the parent of f1 and f2. */
407 next_inode->node_base.freq = f1->freq + f2->freq;
408 next_inode->node_base.sym = INVALID_SYMBOL;
409 next_inode->left_child = f1;
410 next_inode->right_child = f2;
412 /* We need to keep track of the height so that we can detect if
413 * the length of a codeword has execeed max_codeword_len. The
414 * parent node has a height one higher than the maximum height
415 * of its children. */
416 next_inode->node_base.height = max(f1->height, f2->height) + 1;
418 /* Check to see if the code length of the leaf farthest away
419 * from next_inode has exceeded the maximum code length. */
420 if (next_inode->node_base.height > max_codeword_len) {
421 /* The code lengths can be made more uniform by making
422 * the frequencies more uniform. Divide all the
423 * frequencies by 2, leaving 1 as the minimum frequency.
424 * If this keeps happening, the symbol frequencies will
425 * approach equality, which makes their Huffman
426 * codewords approach the length
427 * log_2(num_used_symbols).
429 for (unsigned i = 0; i < num_used_symbols; i++)
430 if (leaves[i].freq > 1)
431 leaves[i].freq >>= 1;
432 goto try_building_tree_again;
437 /* The Huffman tree is now complete, and its height is no more than
438 * max_codeword_len. */
440 HuffmanIntermediateNode *root = next_inode - 1;
441 wimlib_assert(root->node_base.height <= max_codeword_len);
443 /* Compute the path lengths for the leaf nodes. */
444 huffman_tree_compute_path_lengths(&root->node_base, 0);
446 /* Sort the leaf nodes primarily by code length and secondarily by
448 qsort(leaves, num_used_symbols, sizeof(leaves[0]), cmp_nodes_by_code_len);
450 u16 cur_codeword = 0;
451 unsigned cur_codeword_len = 0;
452 for (unsigned i = 0; i < num_used_symbols; i++) {
454 /* Each time a codeword becomes one longer, the current codeword
455 * is left shifted by one place. This is part of the procedure
456 * for enumerating the canonical Huffman code. Additionally,
457 * whenever a codeword is used, 1 is added to the current
460 unsigned len_diff = leaves[i].path_len - cur_codeword_len;
461 cur_codeword <<= len_diff;
462 cur_codeword_len += len_diff;
464 u16 sym = leaves[i].sym;
465 codewords[sym] = cur_codeword;
466 lens[sym] = cur_codeword_len;