Stop force-inlining everything marked 'inline'

[wimlib] / include / wimlib / lzms_common.h

diff --git a/include/wimlib/lzms_common.h b/include/wimlib/lzms_common.h
index 1cb8c5660f7eff73a99a433e578105e62a9a1234..b5071469f443bf3227657ac7ba0c72023fe30b27 100644 (file)
--- a/include/wimlib/lzms_common.h
+++ b/include/wimlib/lzms_common.h
@@ -23,14 +23,14 @@ extern unsigned
 lzms_get_slot(u32 value, const u32 slot_base_tab[], unsigned num_slots);
 
 /* Return the offset slot for the specified offset  */
-static inline unsigned
+static forceinline unsigned
 lzms_get_offset_slot(u32 offset)
 {
        return lzms_get_slot(offset, lzms_offset_slot_base, LZMS_MAX_NUM_OFFSET_SYMS);
 }
 
 /* Return the length slot for the specified length  */
-static inline unsigned
+static forceinline unsigned
 lzms_get_length_slot(u32 length)
 {
        return lzms_get_slot(length, lzms_length_slot_base, LZMS_NUM_LENGTH_SYMS);
@@ -56,34 +56,77 @@ struct lzms_probability_entry {
        u64 recent_bits;
 };
 
+struct lzms_probabilites {
+       struct lzms_probability_entry main[LZMS_NUM_MAIN_PROBS];
+       struct lzms_probability_entry match[LZMS_NUM_MATCH_PROBS];
+       struct lzms_probability_entry lz[LZMS_NUM_LZ_PROBS];
+       struct lzms_probability_entry delta[LZMS_NUM_DELTA_PROBS];
+       struct lzms_probability_entry lz_rep[LZMS_NUM_LZ_REP_DECISIONS]
+                                           [LZMS_NUM_LZ_REP_PROBS];
+       struct lzms_probability_entry delta_rep[LZMS_NUM_DELTA_REP_DECISIONS]
+                                              [LZMS_NUM_DELTA_REP_PROBS];
+};
+
 extern void
-lzms_init_probability_entries(struct lzms_probability_entry *entries, size_t count);
+lzms_init_probabilities(struct lzms_probabilites *probs);
 
 /* Given a decoded or encoded bit, update the probability entry.  */
-static inline void
+static forceinline void
 lzms_update_probability_entry(struct lzms_probability_entry *entry, int bit)
 {
-       BUILD_BUG_ON(LZMS_PROBABILITY_DENOMINATOR != sizeof(entry->recent_bits) * 8);
-
-       s32 delta_zero_bits = (s32)(entry->recent_bits >>
-                                   (LZMS_PROBABILITY_DENOMINATOR - 1)) - bit;
-
-       entry->num_recent_zero_bits += delta_zero_bits;
-       entry->recent_bits = (entry->recent_bits << 1) | bit;
+       STATIC_ASSERT(LZMS_PROBABILITY_DENOMINATOR == sizeof(entry->recent_bits) * 8);
+
+#ifdef __x86_64__
+       if (__builtin_constant_p(bit)) {
+               /* Optimized implementation for x86_64 using carry flag  */
+               if (bit) {
+                      __asm__("shlq %[recent_bits]                          \n"
+                              "adcl $0xffffffff, %[num_recent_zero_bits]    \n"
+                              "orq $0x1, %[recent_bits]                     \n"
+                              : [recent_bits] "+r" (entry->recent_bits),
+                                [num_recent_zero_bits] "+mr" (entry->num_recent_zero_bits)
+                              :
+                              : "cc");
+               } else {
+                      __asm__("shlq %[recent_bits]                          \n"
+                              "adcl $0x0, %[num_recent_zero_bits]           \n"
+                              : [recent_bits] "+m" (entry->recent_bits),
+                                [num_recent_zero_bits] "+mr" (entry->num_recent_zero_bits)
+                              :
+                              : "cc");
+               }
+       } else
+#endif
+       {
+               s32 delta_zero_bits = (s32)(entry->recent_bits >>
+                                           (LZMS_PROBABILITY_DENOMINATOR - 1)) - bit;
+
+               entry->num_recent_zero_bits += delta_zero_bits;
+               entry->recent_bits = (entry->recent_bits << 1) | bit;
+       }
 }
 
 /* Given a probability entry, return the chance out of
  * LZMS_PROBABILITY_DENOMINATOR that the next decoded bit will be a 0.  */
-static inline u32
+static forceinline u32
 lzms_get_probability(const struct lzms_probability_entry *prob_entry)
 {
        u32 prob = prob_entry->num_recent_zero_bits;
 
        /* 0% and 100% probabilities aren't allowed.  */
-       if (prob == 0)
-               prob++;
-       else if (prob == LZMS_PROBABILITY_DENOMINATOR)
-               prob--;
+
+       /*
+        *      if (prob == 0)
+        *              prob++;
+        */
+       prob += (u32)(prob - 1) >> 31;
+
+       /*
+        *      if (prob == LZMS_PROBABILITY_DENOMINATOR)
+        *              prob--;
+        */
+       prob -= (prob >> LZMS_PROBABILITY_BITS);
+
        return prob;
 }