X-Git-Url: https://wimlib.net/git/?p=wimlib;a=blobdiff_plain;f=include%2Fwimlib%2Flzms_common.h;h=b5071469f443bf3227657ac7ba0c72023fe30b27;hp=69b6e67b4dbbeebd76463be5a39e4322a838ca53;hb=4a20aae0dd8469a352517a0b107416ffa99ccc55;hpb=723d5dbc1705200082f640453f19233a386bc655

diff --git a/include/wimlib/lzms_common.h b/include/wimlib/lzms_common.h
index 69b6e67b..b5071469 100644
--- a/include/wimlib/lzms_common.h
+++ b/include/wimlib/lzms_common.h
@@ -23,14 +23,14 @@ extern unsigned
 lzms_get_slot(u32 value, const u32 slot_base_tab[], unsigned num_slots);
 
 /* Return the offset slot for the specified offset  */
-static inline unsigned
+static forceinline unsigned
 lzms_get_offset_slot(u32 offset)
 {
 	return lzms_get_slot(offset, lzms_offset_slot_base, LZMS_MAX_NUM_OFFSET_SYMS);
 }
 
 /* Return the length slot for the specified length  */
-static inline unsigned
+static forceinline unsigned
 lzms_get_length_slot(u32 length)
 {
 	return lzms_get_slot(length, lzms_length_slot_base, LZMS_NUM_LENGTH_SYMS);
@@ -71,30 +71,62 @@ extern void
 lzms_init_probabilities(struct lzms_probabilites *probs);
 
 /* Given a decoded or encoded bit, update the probability entry.  */
-static inline void
+static forceinline void
 lzms_update_probability_entry(struct lzms_probability_entry *entry, int bit)
 {
-	BUILD_BUG_ON(LZMS_PROBABILITY_DENOMINATOR != sizeof(entry->recent_bits) * 8);
-
-	s32 delta_zero_bits = (s32)(entry->recent_bits >>
-				    (LZMS_PROBABILITY_DENOMINATOR - 1)) - bit;
-
-	entry->num_recent_zero_bits += delta_zero_bits;
-	entry->recent_bits = (entry->recent_bits << 1) | bit;
+	STATIC_ASSERT(LZMS_PROBABILITY_DENOMINATOR == sizeof(entry->recent_bits) * 8);
+
+#ifdef __x86_64__
+	if (__builtin_constant_p(bit)) {
+		/* Optimized implementation for x86_64 using carry flag  */
+		if (bit) {
+		       __asm__("shlq %[recent_bits]                          \n"
+			       "adcl $0xffffffff, %[num_recent_zero_bits]    \n"
+			       "orq $0x1, %[recent_bits]                     \n"
+			       : [recent_bits] "+r" (entry->recent_bits),
+				 [num_recent_zero_bits] "+mr" (entry->num_recent_zero_bits)
+			       :
+			       : "cc");
+		} else {
+		       __asm__("shlq %[recent_bits]                          \n"
+			       "adcl $0x0, %[num_recent_zero_bits]           \n"
+			       : [recent_bits] "+m" (entry->recent_bits),
+				 [num_recent_zero_bits] "+mr" (entry->num_recent_zero_bits)
+			       :
+			       : "cc");
+		}
+	} else
+#endif
+	{
+		s32 delta_zero_bits = (s32)(entry->recent_bits >>
+					    (LZMS_PROBABILITY_DENOMINATOR - 1)) - bit;
+
+		entry->num_recent_zero_bits += delta_zero_bits;
+		entry->recent_bits = (entry->recent_bits << 1) | bit;
+	}
 }
 
 /* Given a probability entry, return the chance out of
  * LZMS_PROBABILITY_DENOMINATOR that the next decoded bit will be a 0.  */
-static inline u32
+static forceinline u32
 lzms_get_probability(const struct lzms_probability_entry *prob_entry)
 {
 	u32 prob = prob_entry->num_recent_zero_bits;
 
 	/* 0% and 100% probabilities aren't allowed.  */
-	if (prob == 0)
-		prob++;
-	else if (prob == LZMS_PROBABILITY_DENOMINATOR)
-		prob--;
+
+	/*
+	 *	if (prob == 0)
+	 *		prob++;
+	 */
+	prob += (u32)(prob - 1) >> 31;
+
+	/*
+	 *	if (prob == LZMS_PROBABILITY_DENOMINATOR)
+	 *		prob--;
+	 */
+	prob -= (prob >> LZMS_PROBABILITY_BITS);
+
 	return prob;
 }