From ec0ddfdcfb88119f6918a3901ca865a72b89e9fd Mon Sep 17 00:00:00 2001
From: Eric Biggers
Date: Sun, 24 Jul 2016 14:13:17 -0700
Subject: [PATCH] lzx_compress: make lzx_cost_for_probability() more robust
---
src/lzx_compress.c | 9 +++++++--
1 file changed, 7 insertions(+), 2 deletions(-)
diff --git a/src/lzx_compress.c b/src/lzx_compress.c
index 21636d49..ca25d92f 100644
--- a/src/lzx_compress.c
+++ b/src/lzx_compress.c
@@ -1920,12 +1920,17 @@ lzx_cost_for_probability(float prob)
* entropy = -log2(probability)
*
* Use this to get the cost in fractional bits. Then multiply by our
- * scaling factor of BIT_COST and truncate to a u32.
+ * scaling factor of BIT_COST and convert to an integer.
*
* In addition, the minimum cost is BIT_COST (one bit) because the
* entropy coding method will be Huffman codes.
+ *
+ * Careful: even though 'prob' should be <= 1.0, 'log2f_fast(prob)' may
+ * be positive due to inaccuracy in our log2 approximation. Therefore,
+ * we cannot, in general, assume the computed cost is non-negative, and
+ * we should make sure negative costs get rounded up correctly.
*/
- u32 cost = -log2f_fast(prob) * BIT_COST;
+ s32 cost = -log2f_fast(prob) * BIT_COST;
return max(cost, BIT_COST);
}
--
2.18.2