lzms_get_slot(u32 value, const u32 slot_base_tab[], unsigned num_slots);
/* Return the offset slot for the specified offset */
-static inline unsigned
+static forceinline unsigned
lzms_get_offset_slot(u32 offset)
{
return lzms_get_slot(offset, lzms_offset_slot_base, LZMS_MAX_NUM_OFFSET_SYMS);
}
/* Return the length slot for the specified length */
-static inline unsigned
+static forceinline unsigned
lzms_get_length_slot(u32 length)
{
return lzms_get_slot(length, lzms_length_slot_base, LZMS_NUM_LENGTH_SYMS);
u64 recent_bits;
};
+struct lzms_probabilites {
+ struct lzms_probability_entry main[LZMS_NUM_MAIN_PROBS];
+ struct lzms_probability_entry match[LZMS_NUM_MATCH_PROBS];
+ struct lzms_probability_entry lz[LZMS_NUM_LZ_PROBS];
+ struct lzms_probability_entry delta[LZMS_NUM_DELTA_PROBS];
+ struct lzms_probability_entry lz_rep[LZMS_NUM_LZ_REP_DECISIONS]
+ [LZMS_NUM_LZ_REP_PROBS];
+ struct lzms_probability_entry delta_rep[LZMS_NUM_DELTA_REP_DECISIONS]
+ [LZMS_NUM_DELTA_REP_PROBS];
+};
+
extern void
-lzms_init_probability_entries(struct lzms_probability_entry *entries, size_t count);
+lzms_init_probabilities(struct lzms_probabilites *probs);
/* Given a decoded or encoded bit, update the probability entry. */
-static inline void
+static forceinline void
lzms_update_probability_entry(struct lzms_probability_entry *entry, int bit)
{
- BUILD_BUG_ON(LZMS_PROBABILITY_DENOMINATOR != sizeof(entry->recent_bits) * 8);
-
- s32 delta_zero_bits = (s32)(entry->recent_bits >>
- (LZMS_PROBABILITY_DENOMINATOR - 1)) - bit;
-
- entry->num_recent_zero_bits += delta_zero_bits;
- entry->recent_bits = (entry->recent_bits << 1) | bit;
+ STATIC_ASSERT(LZMS_PROBABILITY_DENOMINATOR == sizeof(entry->recent_bits) * 8);
+
+#ifdef __x86_64__
+ if (__builtin_constant_p(bit)) {
+ /* Optimized implementation for x86_64 using carry flag */
+ if (bit) {
+ __asm__("shlq %[recent_bits] \n"
+ "adcl $0xffffffff, %[num_recent_zero_bits] \n"
+ "orq $0x1, %[recent_bits] \n"
+ : [recent_bits] "+r" (entry->recent_bits),
+ [num_recent_zero_bits] "+mr" (entry->num_recent_zero_bits)
+ :
+ : "cc");
+ } else {
+ __asm__("shlq %[recent_bits] \n"
+ "adcl $0x0, %[num_recent_zero_bits] \n"
+ : [recent_bits] "+m" (entry->recent_bits),
+ [num_recent_zero_bits] "+mr" (entry->num_recent_zero_bits)
+ :
+ : "cc");
+ }
+ } else
+#endif
+ {
+ s32 delta_zero_bits = (s32)(entry->recent_bits >>
+ (LZMS_PROBABILITY_DENOMINATOR - 1)) - bit;
+
+ entry->num_recent_zero_bits += delta_zero_bits;
+ entry->recent_bits = (entry->recent_bits << 1) | bit;
+ }
}
/* Given a probability entry, return the chance out of
* LZMS_PROBABILITY_DENOMINATOR that the next decoded bit will be a 0. */
-static inline u32
+static forceinline u32
lzms_get_probability(const struct lzms_probability_entry *prob_entry)
{
u32 prob = prob_entry->num_recent_zero_bits;
/* 0% and 100% probabilities aren't allowed. */
- if (prob == 0)
- prob++;
- else if (prob == LZMS_PROBABILITY_DENOMINATOR)
- prob--;
+
+ /*
+ * if (prob == 0)
+ * prob++;
+ */
+ prob += (u32)(prob - 1) >> 31;
+
+ /*
+ * if (prob == LZMS_PROBABILITY_DENOMINATOR)
+ * prob--;
+ */
+ prob -= (prob >> LZMS_PROBABILITY_BITS);
+
return prob;
}