From ec0ddfdcfb88119f6918a3901ca865a72b89e9fd Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Sun, 24 Jul 2016 14:13:17 -0700 Subject: [PATCH] lzx_compress: make lzx_cost_for_probability() more robust --- src/lzx_compress.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/lzx_compress.c b/src/lzx_compress.c index 21636d49..ca25d92f 100644 --- a/src/lzx_compress.c +++ b/src/lzx_compress.c @@ -1920,12 +1920,17 @@ lzx_cost_for_probability(float prob) * entropy = -log2(probability) * * Use this to get the cost in fractional bits. Then multiply by our - * scaling factor of BIT_COST and truncate to a u32. + * scaling factor of BIT_COST and convert to an integer. * * In addition, the minimum cost is BIT_COST (one bit) because the * entropy coding method will be Huffman codes. + * + * Careful: even though 'prob' should be <= 1.0, 'log2f_fast(prob)' may + * be positive due to inaccuracy in our log2 approximation. Therefore, + * we cannot, in general, assume the computed cost is non-negative, and + * we should make sure negative costs get rounded up correctly. */ - u32 cost = -log2f_fast(prob) * BIT_COST; + s32 cost = -log2f_fast(prob) * BIT_COST; return max(cost, BIT_COST); } -- 2.43.0