Instead, replace 'inline' with 'forceinline' in selected places.
19 files changed:
* input value must be nonzero!
*/
* input value must be nonzero!
*/
+static forceinline unsigned
bsr32(u32 v)
{
#ifdef compiler_bsr32
bsr32(u32 v)
{
#ifdef compiler_bsr32
+static forceinline unsigned
bsr64(u64 v)
{
#ifdef compiler_bsr64
bsr64(u64 v)
{
#ifdef compiler_bsr64
+static forceinline unsigned
bsrw(machine_word_t v)
{
STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
bsrw(machine_word_t v)
{
STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
* input value must be nonzero!
*/
* input value must be nonzero!
*/
+static forceinline unsigned
bsf32(u32 v)
{
#ifdef compiler_bsf32
bsf32(u32 v)
{
#ifdef compiler_bsf32
+static forceinline unsigned
bsf64(u64 v)
{
#ifdef compiler_bsf64
bsf64(u64 v)
{
#ifdef compiler_bsf64
+static forceinline unsigned
bsfw(machine_word_t v)
{
STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
bsfw(machine_word_t v)
{
STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
}
/* Return the log base 2 of 'n', rounded up to the nearest integer. */
}
/* Return the log base 2 of 'n', rounded up to the nearest integer. */
+static forceinline unsigned
ilog2_ceil(size_t n)
{
if (n <= 1)
ilog2_ceil(size_t n)
{
if (n <= 1)
}
/* Round 'n' up to the nearest power of 2 */
}
/* Round 'n' up to the nearest power of 2 */
+static forceinline size_t
roundup_pow_of_2(size_t n)
{
return (size_t)1 << ilog2_ceil(n);
roundup_pow_of_2(size_t n)
{
return (size_t)1 << ilog2_ceil(n);
/* Return the number of bytes that must be allocated for a 'bt_matchfinder' that
* can work with buffers up to the specified size. */
/* Return the number of bytes that must be allocated for a 'bt_matchfinder' that
* can work with buffers up to the specified size. */
+static forceinline size_t
TEMPLATED(bt_matchfinder_size)(size_t max_bufsize)
{
return sizeof(struct TEMPLATED(bt_matchfinder)) +
TEMPLATED(bt_matchfinder_size)(size_t max_bufsize)
{
return sizeof(struct TEMPLATED(bt_matchfinder)) +
}
/* Prepare the matchfinder for a new input buffer. */
}
/* Prepare the matchfinder for a new input buffer. */
TEMPLATED(bt_matchfinder_init)(struct TEMPLATED(bt_matchfinder) *mf)
{
memset(mf, 0, sizeof(*mf));
}
TEMPLATED(bt_matchfinder_init)(struct TEMPLATED(bt_matchfinder) *mf)
{
memset(mf, 0, sizeof(*mf));
}
-static inline mf_pos_t *
+static forceinline mf_pos_t *
TEMPLATED(bt_left_child)(struct TEMPLATED(bt_matchfinder) *mf, u32 node)
{
return &mf->child_tab[(node << 1) + 0];
}
TEMPLATED(bt_left_child)(struct TEMPLATED(bt_matchfinder) *mf, u32 node)
{
return &mf->child_tab[(node << 1) + 0];
}
-static inline mf_pos_t *
+static forceinline mf_pos_t *
TEMPLATED(bt_right_child)(struct TEMPLATED(bt_matchfinder) *mf, u32 node)
{
return &mf->child_tab[(node << 1) + 1];
TEMPLATED(bt_right_child)(struct TEMPLATED(bt_matchfinder) *mf, u32 node)
{
return &mf->child_tab[(node << 1) + 1];
/* Advance the binary tree matchfinder by one byte, optionally recording
* matches. @record_matches should be a compile-time constant. */
/* Advance the binary tree matchfinder by one byte, optionally recording
* matches. @record_matches should be a compile-time constant. */
-static inline struct lz_match *
+static forceinline struct lz_match *
TEMPLATED(bt_matchfinder_advance_one_byte)(struct TEMPLATED(bt_matchfinder) * const restrict mf,
const u8 * const restrict in_begin,
const ptrdiff_t cur_pos,
TEMPLATED(bt_matchfinder_advance_one_byte)(struct TEMPLATED(bt_matchfinder) * const restrict mf,
const u8 * const restrict in_begin,
const ptrdiff_t cur_pos,
* The return value is a pointer to the next available slot in the @lz_matchptr
* array. (If no matches were found, this will be the same as @lz_matchptr.)
*/
* The return value is a pointer to the next available slot in the @lz_matchptr
* array. (If no matches were found, this will be the same as @lz_matchptr.)
*/
-static inline struct lz_match *
+static forceinline struct lz_match *
TEMPLATED(bt_matchfinder_get_matches)(struct TEMPLATED(bt_matchfinder) *mf,
const u8 *in_begin,
ptrdiff_t cur_pos,
TEMPLATED(bt_matchfinder_get_matches)(struct TEMPLATED(bt_matchfinder) *mf,
const u8 *in_begin,
ptrdiff_t cur_pos,
* This is very similar to bt_matchfinder_get_matches() because both functions
* must do hashing and tree re-rooting.
*/
* This is very similar to bt_matchfinder_get_matches() because both functions
* must do hashing and tree re-rooting.
*/
TEMPLATED(bt_matchfinder_skip_position)(struct TEMPLATED(bt_matchfinder) *mf,
const u8 *in_begin,
ptrdiff_t cur_pos,
TEMPLATED(bt_matchfinder_skip_position)(struct TEMPLATED(bt_matchfinder) *mf,
const u8 *in_begin,
ptrdiff_t cur_pos,
# define WIMLIBAPI __attribute__((visibility("default")))
#endif
# define WIMLIBAPI __attribute__((visibility("default")))
#endif
-/* Declare that the annotated function should be inlined. Currently, we force
- * the compiler to honor this because we use 'inline' in highly tuned code, e.g.
- * compression codecs. */
-#define inline inline __attribute__((always_inline))
+/* Declare that the annotated function should always be inlined. This might be
+ * desirable in highly tuned code, e.g. compression codecs. */
+#define forceinline inline __attribute__((always_inline))
/* Declare that the annotated function should *not* be inlined. */
#define noinline __attribute__((noinline))
/* Declare that the annotated function should *not* be inlined. */
#define noinline __attribute__((noinline))
};
/* Initialize a bitstream to read from the specified input buffer. */
};
/* Initialize a bitstream to read from the specified input buffer. */
init_input_bitstream(struct input_bitstream *is, const void *buffer, u32 size)
{
is->bitbuf = 0;
init_input_bitstream(struct input_bitstream *is, const void *buffer, u32 size)
{
is->bitbuf = 0;
/* Ensure the bit buffer variable for the bitstream contains at least @num_bits
* bits. Following this, bitstream_peek_bits() and/or bitstream_remove_bits()
* may be called on the bitstream to peek or remove up to @num_bits bits. */
/* Ensure the bit buffer variable for the bitstream contains at least @num_bits
* bits. Following this, bitstream_peek_bits() and/or bitstream_remove_bits()
* may be called on the bitstream to peek or remove up to @num_bits bits. */
bitstream_ensure_bits(struct input_bitstream *is, const unsigned num_bits)
{
/* This currently works for at most 17 bits. */
bitstream_ensure_bits(struct input_bitstream *is, const unsigned num_bits)
{
/* This currently works for at most 17 bits. */
/* Return the next @num_bits bits from the bitstream, without removing them.
* There must be at least @num_bits remaining in the buffer variable, from a
* previous call to bitstream_ensure_bits(). */
/* Return the next @num_bits bits from the bitstream, without removing them.
* There must be at least @num_bits remaining in the buffer variable, from a
* previous call to bitstream_ensure_bits(). */
bitstream_peek_bits(const struct input_bitstream *is, const unsigned num_bits)
{
return (is->bitbuf >> 1) >> (sizeof(is->bitbuf) * 8 - num_bits - 1);
bitstream_peek_bits(const struct input_bitstream *is, const unsigned num_bits)
{
return (is->bitbuf >> 1) >> (sizeof(is->bitbuf) * 8 - num_bits - 1);
/* Remove @num_bits from the bitstream. There must be at least @num_bits
* remaining in the buffer variable, from a previous call to
* bitstream_ensure_bits(). */
/* Remove @num_bits from the bitstream. There must be at least @num_bits
* remaining in the buffer variable, from a previous call to
* bitstream_ensure_bits(). */
bitstream_remove_bits(struct input_bitstream *is, unsigned num_bits)
{
is->bitbuf <<= num_bits;
bitstream_remove_bits(struct input_bitstream *is, unsigned num_bits)
{
is->bitbuf <<= num_bits;
/* Remove and return @num_bits bits from the bitstream. There must be at least
* @num_bits remaining in the buffer variable, from a previous call to
* bitstream_ensure_bits(). */
/* Remove and return @num_bits bits from the bitstream. There must be at least
* @num_bits remaining in the buffer variable, from a previous call to
* bitstream_ensure_bits(). */
bitstream_pop_bits(struct input_bitstream *is, unsigned num_bits)
{
u32 bits = bitstream_peek_bits(is, num_bits);
bitstream_pop_bits(struct input_bitstream *is, unsigned num_bits)
{
u32 bits = bitstream_peek_bits(is, num_bits);
}
/* Read and return the next @num_bits bits from the bitstream. */
}
/* Read and return the next @num_bits bits from the bitstream. */
bitstream_read_bits(struct input_bitstream *is, unsigned num_bits)
{
bitstream_ensure_bits(is, num_bits);
bitstream_read_bits(struct input_bitstream *is, unsigned num_bits)
{
bitstream_ensure_bits(is, num_bits);
}
/* Read and return the next literal byte embedded in the bitstream. */
}
/* Read and return the next literal byte embedded in the bitstream. */
bitstream_read_byte(struct input_bitstream *is)
{
if (unlikely(is->end == is->next))
bitstream_read_byte(struct input_bitstream *is)
{
if (unlikely(is->end == is->next))
}
/* Read and return the next 16-bit integer embedded in the bitstream. */
}
/* Read and return the next 16-bit integer embedded in the bitstream. */
bitstream_read_u16(struct input_bitstream *is)
{
u16 v;
bitstream_read_u16(struct input_bitstream *is)
{
u16 v;
}
/* Read and return the next 32-bit integer embedded in the bitstream. */
}
/* Read and return the next 32-bit integer embedded in the bitstream. */
bitstream_read_u32(struct input_bitstream *is)
{
u32 v;
bitstream_read_u32(struct input_bitstream *is)
{
u32 v;
/* Read into @dst_buffer an array of literal bytes embedded in the bitstream.
* Return 0 if there were enough bytes remaining in the input, otherwise -1. */
/* Read into @dst_buffer an array of literal bytes embedded in the bitstream.
* Return 0 if there were enough bytes remaining in the input, otherwise -1. */
bitstream_read_bytes(struct input_bitstream *is, void *dst_buffer, size_t count)
{
if (unlikely(is->end - is->next < count))
bitstream_read_bytes(struct input_bitstream *is, void *dst_buffer, size_t count)
{
if (unlikely(is->end - is->next < count))
}
/* Align the input bitstream on a coding-unit boundary. */
}
/* Align the input bitstream on a coding-unit boundary. */
bitstream_align(struct input_bitstream *is)
{
is->bitsleft = 0;
bitstream_align(struct input_bitstream *is)
{
is->bitsleft = 0;
* XXX: This is mostly duplicated in lzms_decode_huffman_symbol() in
* lzms_decompress.c; keep them in sync!
*/
* XXX: This is mostly duplicated in lzms_decode_huffman_symbol() in
* lzms_decompress.c; keep them in sync!
*/
+static forceinline unsigned
read_huffsym(struct input_bitstream *is, const u16 decode_table[],
unsigned table_bits, unsigned max_codeword_len)
{
read_huffsym(struct input_bitstream *is, const u16 decode_table[],
unsigned table_bits, unsigned max_codeword_len)
{
/* LZ match copying */
/*----------------------------------------------------------------------------*/
/* LZ match copying */
/*----------------------------------------------------------------------------*/
copy_word_unaligned(const void *src, void *dst)
{
store_word_unaligned(load_word_unaligned(src), dst);
}
copy_word_unaligned(const void *src, void *dst)
{
store_word_unaligned(load_word_unaligned(src), dst);
}
-static inline machine_word_t
+static forceinline machine_word_t
repeat_u16(u16 b)
{
machine_word_t v = b;
repeat_u16(u16 b)
{
machine_word_t v = b;
-static inline machine_word_t
+static forceinline machine_word_t
repeat_byte(u8 b)
{
return repeat_u16(((u16)b << 8) | b);
repeat_byte(u8 b)
{
return repeat_u16(((u16)b << 8) | b);
* 'min_length' is a hint which specifies the minimum possible match length.
* This should be a compile-time constant.
*/
* 'min_length' is a hint which specifies the minimum possible match length.
* This should be a compile-time constant.
*/
lz_copy(u32 length, u32 offset, u8 *out_begin, u8 *out_next, u8 *out_end,
u32 min_length)
{
lz_copy(u32 length, u32 offset, u8 *out_begin, u8 *out_next, u8 *out_end,
u32 min_length)
{
(((u64)(n) & 0x00FF000000000000) >> 40) | \
(((u64)(n) & 0xFF00000000000000) >> 56))
(((u64)(n) & 0x00FF000000000000) >> 40) | \
(((u64)(n) & 0xFF00000000000000) >> 56))
-static inline u16 do_bswap16(u16 n)
+static forceinline u16 do_bswap16(u16 n)
{
#ifdef compiler_bswap16
return compiler_bswap16(n);
{
#ifdef compiler_bswap16
return compiler_bswap16(n);
-static inline u32 do_bswap32(u32 n)
+static forceinline u32 do_bswap32(u32 n)
{
#ifdef compiler_bswap32
return compiler_bswap32(n);
{
#ifdef compiler_bswap32
return compiler_bswap32(n);
-static inline u64 do_bswap64(u64 n)
+static forceinline u64 do_bswap64(u64 n)
{
#ifdef compiler_bswap64
return compiler_bswap64(n);
{
#ifdef compiler_bswap64
return compiler_bswap64(n);
/* Return the number of bytes that must be allocated for a 'hc_matchfinder' that
* can work with buffers up to the specified size. */
/* Return the number of bytes that must be allocated for a 'hc_matchfinder' that
* can work with buffers up to the specified size. */
+static forceinline size_t
TEMPLATED(hc_matchfinder_size)(size_t max_bufsize)
{
return sizeof(struct TEMPLATED(hc_matchfinder)) +
TEMPLATED(hc_matchfinder_size)(size_t max_bufsize)
{
return sizeof(struct TEMPLATED(hc_matchfinder)) +
}
/* Prepare the matchfinder for a new input buffer. */
}
/* Prepare the matchfinder for a new input buffer. */
TEMPLATED(hc_matchfinder_init)(struct TEMPLATED(hc_matchfinder) *mf)
{
memset(mf, 0, sizeof(*mf));
TEMPLATED(hc_matchfinder_init)(struct TEMPLATED(hc_matchfinder) *mf)
{
memset(mf, 0, sizeof(*mf));
* Return the length of the match found, or 'best_len' if no match longer than
* 'best_len' was found.
*/
* Return the length of the match found, or 'best_len' if no match longer than
* 'best_len' was found.
*/
TEMPLATED(hc_matchfinder_longest_match)(struct TEMPLATED(hc_matchfinder) * const restrict mf,
const u8 * const restrict in_begin,
const ptrdiff_t cur_pos,
TEMPLATED(hc_matchfinder_longest_match)(struct TEMPLATED(hc_matchfinder) * const restrict mf,
const u8 * const restrict in_begin,
const ptrdiff_t cur_pos,
*
* Returns @in_next + @count.
*/
*
* Returns @in_next + @count.
*/
-static inline const u8 *
+static forceinline const u8 *
TEMPLATED(hc_matchfinder_skip_positions)(struct TEMPLATED(hc_matchfinder) * const restrict mf,
const u8 * const restrict in_begin,
const ptrdiff_t cur_pos,
TEMPLATED(hc_matchfinder_skip_positions)(struct TEMPLATED(hc_matchfinder) * const restrict mf,
const u8 * const restrict in_begin,
const ptrdiff_t cur_pos,
* Return the number of bytes at @matchptr that match the bytes at @strptr, up
* to a maximum of @max_len. Initially, @len bytes are matched.
*/
* Return the number of bytes at @matchptr that match the bytes at @strptr, up
* to a maximum of @max_len. Initially, @len bytes are matched.
*/
lz_extend(const u8 * const strptr, const u8 * const matchptr,
u32 len, const u32 max_len)
{
lz_extend(const u8 * const strptr, const u8 * const matchptr,
u32 len, const u32 max_len)
{
* next-highest @num_bits bits of the product as the hash value, as those have
* the most randomness.
*/
* next-highest @num_bits bits of the product as the hash value, as those have
* the most randomness.
*/
lz_hash(u32 seq, unsigned num_bits)
{
return (u32)(seq * 0x1E35A7BD) >> (32 - num_bits);
lz_hash(u32 seq, unsigned num_bits)
{
return (u32)(seq * 0x1E35A7BD) >> (32 - num_bits);
lzms_get_slot(u32 value, const u32 slot_base_tab[], unsigned num_slots);
/* Return the offset slot for the specified offset */
lzms_get_slot(u32 value, const u32 slot_base_tab[], unsigned num_slots);
/* Return the offset slot for the specified offset */
+static forceinline unsigned
lzms_get_offset_slot(u32 offset)
{
return lzms_get_slot(offset, lzms_offset_slot_base, LZMS_MAX_NUM_OFFSET_SYMS);
}
/* Return the length slot for the specified length */
lzms_get_offset_slot(u32 offset)
{
return lzms_get_slot(offset, lzms_offset_slot_base, LZMS_MAX_NUM_OFFSET_SYMS);
}
/* Return the length slot for the specified length */
+static forceinline unsigned
lzms_get_length_slot(u32 length)
{
return lzms_get_slot(length, lzms_length_slot_base, LZMS_NUM_LENGTH_SYMS);
lzms_get_length_slot(u32 length)
{
return lzms_get_slot(length, lzms_length_slot_base, LZMS_NUM_LENGTH_SYMS);
lzms_init_probabilities(struct lzms_probabilites *probs);
/* Given a decoded or encoded bit, update the probability entry. */
lzms_init_probabilities(struct lzms_probabilites *probs);
/* Given a decoded or encoded bit, update the probability entry. */
lzms_update_probability_entry(struct lzms_probability_entry *entry, int bit)
{
STATIC_ASSERT(LZMS_PROBABILITY_DENOMINATOR == sizeof(entry->recent_bits) * 8);
lzms_update_probability_entry(struct lzms_probability_entry *entry, int bit)
{
STATIC_ASSERT(LZMS_PROBABILITY_DENOMINATOR == sizeof(entry->recent_bits) * 8);
/* Given a probability entry, return the chance out of
* LZMS_PROBABILITY_DENOMINATOR that the next decoded bit will be a 0. */
/* Given a probability entry, return the chance out of
* LZMS_PROBABILITY_DENOMINATOR that the next decoded bit will be a 0. */
lzms_get_probability(const struct lzms_probability_entry *prob_entry)
{
u32 prob = prob_entry->num_recent_zero_bits;
lzms_get_probability(const struct lzms_probability_entry *prob_entry)
{
u32 prob = prob_entry->num_recent_zero_bits;
type v; \
} _packed_attribute; \
\
type v; \
} _packed_attribute; \
\
+static forceinline type \
load_##type##_unaligned(const void *p) \
{ \
return ((const struct type##_unaligned *)p)->v; \
} \
\
load_##type##_unaligned(const void *p) \
{ \
return ((const struct type##_unaligned *)p)->v; \
} \
\
+static forceinline void \
store_##type##_unaligned(type val, void *p) \
{ \
((struct type##_unaligned *)p)->v = val; \
store_##type##_unaligned(type val, void *p) \
{ \
((struct type##_unaligned *)p)->v = val; \
#define load_word_unaligned load_machine_word_t_unaligned
#define store_word_unaligned store_machine_word_t_unaligned
#define load_word_unaligned load_machine_word_t_unaligned
#define store_word_unaligned store_machine_word_t_unaligned
get_unaligned_le16(const u8 *p)
{
if (UNALIGNED_ACCESS_IS_FAST)
get_unaligned_le16(const u8 *p)
{
if (UNALIGNED_ACCESS_IS_FAST)
return ((u16)p[1] << 8) | p[0];
}
return ((u16)p[1] << 8) | p[0];
}
get_unaligned_le32(const u8 *p)
{
if (UNALIGNED_ACCESS_IS_FAST)
get_unaligned_le32(const u8 *p)
{
if (UNALIGNED_ACCESS_IS_FAST)
((u32)p[1] << 8) | p[0];
}
((u32)p[1] << 8) | p[0];
}
put_unaligned_le16(u16 v, u8 *p)
{
if (UNALIGNED_ACCESS_IS_FAST) {
put_unaligned_le16(u16 v, u8 *p)
{
if (UNALIGNED_ACCESS_IS_FAST) {
put_unaligned_le32(u32 v, u8 *p)
{
if (UNALIGNED_ACCESS_IS_FAST) {
put_unaligned_le32(u32 v, u8 *p)
{
if (UNALIGNED_ACCESS_IS_FAST) {
* bits contain the first 3 bytes, arranged in octets in a platform-dependent
* order, at the memory location from which the input 32-bit value was loaded.
*/
* bits contain the first 3 bytes, arranged in octets in a platform-dependent
* order, at the memory location from which the input 32-bit value was loaded.
*/
loaded_u32_to_u24(u32 v)
{
if (CPU_IS_LITTLE_ENDIAN)
loaded_u32_to_u24(u32 v)
{
if (CPU_IS_LITTLE_ENDIAN)
* in the 24 bits is platform-dependent. At least LOAD_U24_REQUIRED_NBYTES
* bytes must be available at @p; note that this may be more than 3.
*/
* in the 24 bits is platform-dependent. At least LOAD_U24_REQUIRED_NBYTES
* bytes must be available at @p; note that this may be more than 3.
*/
load_u24_unaligned(const u8 *p)
{
#if UNALIGNED_ACCESS_IS_FAST
load_u24_unaligned(const u8 *p)
{
#if UNALIGNED_ACCESS_IS_FAST
#if (SS_BLOCKSIZE == 0) || (SS_INSERTIONSORT_THRESHOLD < SS_BLOCKSIZE)
#if (SS_BLOCKSIZE == 0) || (SS_INSERTIONSORT_THRESHOLD < SS_BLOCKSIZE)
int
ss_ilg(int n) {
#if SS_BLOCKSIZE == 0
int
ss_ilg(int n) {
#if SS_BLOCKSIZE == 0
247, 248, 248, 249, 249, 250, 250, 251, 251, 252, 252, 253, 253, 254, 254, 255
};
247, 248, 248, 249, 249, 250, 250, 251, 251, 252, 252, 253, 253, 254, 254, 255
};
int
ss_isqrt(int x) {
int y, e;
int
ss_isqrt(int x) {
int y, e;
/*---------------------------------------------------------------------------*/
/* Compares two suffixes. */
/*---------------------------------------------------------------------------*/
/* Compares two suffixes. */
int
ss_compare(const unsigned char *T,
const int *p1, const int *p2,
int
ss_compare(const unsigned char *T,
const int *p1, const int *p2,
#if (SS_BLOCKSIZE == 0) || (SS_INSERTIONSORT_THRESHOLD < SS_BLOCKSIZE)
#if (SS_BLOCKSIZE == 0) || (SS_INSERTIONSORT_THRESHOLD < SS_BLOCKSIZE)
void
ss_fixdown(const unsigned char *Td, const int *PA,
int *SA, int i, int size) {
void
ss_fixdown(const unsigned char *Td, const int *PA,
int *SA, int i, int size) {
/*---------------------------------------------------------------------------*/
/* Returns the median of three elements. */
/*---------------------------------------------------------------------------*/
/* Returns the median of three elements. */
int *
ss_median3(const unsigned char *Td, const int *PA,
int *v1, int *v2, int *v3) {
int *
ss_median3(const unsigned char *Td, const int *PA,
int *v1, int *v2, int *v3) {
}
/* Returns the median of five elements. */
}
/* Returns the median of five elements. */
int *
ss_median5(const unsigned char *Td, const int *PA,
int *v1, int *v2, int *v3, int *v4, int *v5) {
int *
ss_median5(const unsigned char *Td, const int *PA,
int *v1, int *v2, int *v3, int *v4, int *v5) {
}
/* Returns the pivot element. */
}
/* Returns the pivot element. */
int *
ss_pivot(const unsigned char *Td, const int *PA, int *first, int *last) {
int *middle;
int *
ss_pivot(const unsigned char *Td, const int *PA, int *first, int *last) {
int *middle;
/*---------------------------------------------------------------------------*/
/* Binary partition for substrings. */
/*---------------------------------------------------------------------------*/
/* Binary partition for substrings. */
int *
ss_partition(const int *PA,
int *first, int *last, int depth) {
int *
ss_partition(const int *PA,
int *first, int *last, int depth) {
void
ss_blockswap(int *a, int *b, int n) {
int t;
void
ss_blockswap(int *a, int *b, int n) {
int t;
void
ss_rotate(int *first, int *middle, int *last) {
int *a, *b, t;
void
ss_rotate(int *first, int *middle, int *last) {
int *a, *b, t;
/*---------------------------------------------------------------------------*/
/*---------------------------------------------------------------------------*/
int
tr_ilg(int n) {
return (n & 0xffff0000) ?
int
tr_ilg(int n) {
return (n & 0xffff0000) ?
/*---------------------------------------------------------------------------*/
/*---------------------------------------------------------------------------*/
void
tr_fixdown(const int *ISAd, int *SA, int i, int size) {
int j, k;
void
tr_fixdown(const int *ISAd, int *SA, int i, int size) {
int j, k;
/*---------------------------------------------------------------------------*/
/* Returns the median of three elements. */
/*---------------------------------------------------------------------------*/
/* Returns the median of three elements. */
int *
tr_median3(const int *ISAd, int *v1, int *v2, int *v3) {
if(ISAd[*v1] > ISAd[*v2]) { SWAP(v1, v2); }
int *
tr_median3(const int *ISAd, int *v1, int *v2, int *v3) {
if(ISAd[*v1] > ISAd[*v2]) { SWAP(v1, v2); }
}
/* Returns the median of five elements. */
}
/* Returns the median of five elements. */
int *
tr_median5(const int *ISAd,
int *v1, int *v2, int *v3, int *v4, int *v5) {
int *
tr_median5(const int *ISAd,
int *v1, int *v2, int *v3, int *v4, int *v5) {
}
/* Returns the pivot element. */
}
/* Returns the pivot element. */
int *
tr_pivot(const int *ISAd, int *first, int *last) {
int *middle;
int *
tr_pivot(const int *ISAd, int *first, int *last) {
int *middle;
void
trbudget_init(trbudget_t *budget, int chance, int incval) {
budget->chance = chance;
budget->remain = budget->incval = incval;
}
void
trbudget_init(trbudget_t *budget, int chance, int incval) {
budget->chance = chance;
budget->remain = budget->incval = incval;
}
int
trbudget_check(trbudget_t *budget, int size) {
if(size <= budget->remain) { budget->remain -= size; return 1; }
int
trbudget_check(trbudget_t *budget, int size) {
if(size <= budget->remain) { budget->remain -= size; return 1; }
/*---------------------------------------------------------------------------*/
/*---------------------------------------------------------------------------*/
void
tr_partition(const int *ISAd,
int *first, int *middle, int *last,
void
tr_partition(const int *ISAd,
int *first, int *middle, int *last,
/* Encode the Unicode codepoint @c and return the number of bytes used. */
typedef unsigned (*encode_codepoint_fn)(u32 c, u8 *out);
/* Encode the Unicode codepoint @c and return the number of bytes used. */
typedef unsigned (*encode_codepoint_fn)(u32 c, u8 *out);
+static forceinline unsigned
utf8_decode_codepoint(const u8 *in, size_t remaining, bool validate, u32 *c_ret)
{
if (likely(in[0] < 0x80)) { /* U+0...U+7F */
utf8_decode_codepoint(const u8 *in, size_t remaining, bool validate, u32 *c_ret)
{
if (likely(in[0] < 0x80)) { /* U+0...U+7F */
+static forceinline unsigned
utf8_encode_codepoint(u32 c, u8 *out)
{
if (likely(c < 0x80)) {
utf8_encode_codepoint(u32 c, u8 *out)
{
if (likely(c < 0x80)) {
+static forceinline unsigned
utf16le_decode_codepoint(const u8 *in, size_t remaining, bool validate,
u32 *c_ret)
{
utf16le_decode_codepoint(const u8 *in, size_t remaining, bool validate,
u32 *c_ret)
{
return min(remaining, 2);
}
return min(remaining, 2);
}
+static forceinline unsigned
utf16le_encode_codepoint(u32 c, u8 *out)
{
if (likely(c < 0x10000)) {
utf16le_encode_codepoint(u32 c, u8 *out)
{
if (likely(c < 0x10000)) {
* If the input string is malformed, return @ilseq_err with errno set to EILSEQ.
* If out of memory, return WIMLIB_ERR_NOMEM with errno set to ENOMEM.
*/
* If the input string is malformed, return @ilseq_err with errno set to EILSEQ.
* If out of memory, return WIMLIB_ERR_NOMEM with errno set to ENOMEM.
*/
convert_string(const u8 * const in, const size_t in_nbytes,
u8 **out_ret, size_t *out_nbytes_ret,
int ilseq_err,
convert_string(const u8 * const in, const size_t in_nbytes,
u8 **out_ret, size_t *out_nbytes_ret,
int ilseq_err,
* around by just continuing until we get to a link that actually takes us
* higher in the tree. This can be described as a lazy-update scheme.
*/
* around by just continuing until we get to a link that actually takes us
* higher in the tree. This can be described as a lazy-update scheme.
*/
lcpit_advance_one_byte(const u32 cur_pos,
u32 pos_data[restrict],
u32 intervals[restrict],
lcpit_advance_one_byte(const u32 cur_pos,
u32 pos_data[restrict],
u32 intervals[restrict],
/* Like lcpit_advance_one_byte(), but for buffers larger than
* MAX_NORMAL_BUFSIZE. */
/* Like lcpit_advance_one_byte(), but for buffers larger than
* MAX_NORMAL_BUFSIZE. */
lcpit_advance_one_byte_huge(const u32 cur_pos,
u32 pos_data[restrict],
u64 intervals64[restrict],
lcpit_advance_one_byte_huge(const u32 cur_pos,
u32 pos_data[restrict],
u64 intervals64[restrict],
return matchptr - matches;
}
return matchptr - matches;
}
get_pos_data_size(size_t max_bufsize)
{
return (u64)max((u64)max_bufsize + PREFETCH_SAFETY,
DIVSUFSORT_TMP_LEN) * sizeof(u32);
}
get_pos_data_size(size_t max_bufsize)
{
return (u64)max((u64)max_bufsize + PREFETCH_SAFETY,
DIVSUFSORT_TMP_LEN) * sizeof(u32);
}
get_intervals_size(size_t max_bufsize)
{
return ((u64)max_bufsize + PREFETCH_SAFETY) *
get_intervals_size(size_t max_bufsize)
{
return ((u64)max_bufsize + PREFETCH_SAFETY) *
find_next_opcode_sse4_2(u8 *p)
{
const __v16qi potential_opcodes = (__v16qi) {0x48, 0x4C, 0xE8, 0xE9, 0xF0, 0xFF};
find_next_opcode_sse4_2(u8 *p)
{
const __v16qi potential_opcodes = (__v16qi) {0x48, 0x4C, 0xE8, 0xE9, 0xF0, 0xFF};
}
#endif /* __x86_64__ */
}
#endif /* __x86_64__ */
find_next_opcode_default(u8 *p)
{
/*
find_next_opcode_default(u8 *p)
{
/*
translate_if_needed(u8 *data, u8 *p, s32 *last_x86_pos,
s32 last_target_usages[], bool undo)
{
translate_if_needed(u8 *data, u8 *p, s32 *last_x86_pos,
s32 last_target_usages[], bool undo)
{
* Return the length slot for the specified match length, using the compressor's
* acceleration table if the length is small enough.
*/
* Return the length slot for the specified match length, using the compressor's
* acceleration table if the length is small enough.
*/
+static forceinline unsigned
lzms_comp_get_length_slot(const struct lzms_compressor *c, u32 length)
{
if (likely(length <= MAX_FAST_LENGTH))
lzms_comp_get_length_slot(const struct lzms_compressor *c, u32 length)
{
if (likely(length <= MAX_FAST_LENGTH))
* Return the offset slot for the specified match offset, using the compressor's
* acceleration tables to speed up the mapping.
*/
* Return the offset slot for the specified match offset, using the compressor's
* acceleration tables to speed up the mapping.
*/
+static forceinline unsigned
lzms_comp_get_offset_slot(const struct lzms_compressor *c, u32 offset)
{
if (offset < 0xe4a5)
lzms_comp_get_offset_slot(const struct lzms_compressor *c, u32 offset)
{
if (offset < 0xe4a5)
* @prob is the probability out of LZMS_PROBABILITY_DENOMINATOR that the next
* bit is 0 rather than 1.
*/
* @prob is the probability out of LZMS_PROBABILITY_DENOMINATOR that the next
* bit is 0 rather than 1.
*/
lzms_range_encode_bit(struct lzms_range_encoder *rc, int bit, u32 prob)
{
/* Normalize if needed. */
lzms_range_encode_bit(struct lzms_range_encoder *rc, int bit, u32 prob)
{
/* Normalize if needed. */
* Encode a bit. This wraps around lzms_range_encode_bit() to handle using and
* updating the state and its corresponding probability entry.
*/
* Encode a bit. This wraps around lzms_range_encode_bit() to handle using and
* updating the state and its corresponding probability entry.
*/
lzms_encode_bit(int bit, unsigned *state_p, unsigned num_states,
struct lzms_probability_entry *probs,
struct lzms_range_encoder *rc)
lzms_encode_bit(int bit, unsigned *state_p, unsigned num_states,
struct lzms_probability_entry *probs,
struct lzms_range_encoder *rc)
* @max_num_bits is a compile-time constant that specifies the maximum number of
* bits that can ever be written at this call site.
*/
* @max_num_bits is a compile-time constant that specifies the maximum number of
* bits that can ever be written at this call site.
*/
lzms_write_bits(struct lzms_output_bitstream *os, const u32 bits,
const unsigned num_bits, const unsigned max_num_bits)
{
lzms_write_bits(struct lzms_output_bitstream *os, const u32 bits,
const unsigned num_bits, const unsigned max_num_bits)
{
* Encode a symbol using the specified Huffman code. Then, if the Huffman code
* needs to be rebuilt, rebuild it and return true; otherwise return false.
*/
* Encode a symbol using the specified Huffman code. Then, if the Huffman code
* needs to be rebuilt, rebuild it and return true; otherwise return false.
*/
lzms_huffman_encode_symbol(unsigned sym,
const u32 *codewords, const u8 *lens, u32 *freqs,
struct lzms_output_bitstream *os,
lzms_huffman_encode_symbol(unsigned sym,
const u32 *codewords, const u8 *lens, u32 *freqs,
struct lzms_output_bitstream *os,
} while (cur_node != end_node);
}
} while (cur_node != end_node);
}
lzms_encode_item_list(struct lzms_compressor *c,
struct lzms_optimum_node *end_node)
{
lzms_encode_item_list(struct lzms_compressor *c,
struct lzms_optimum_node *end_node)
{
#endif
/* Return the cost to encode a 0 bit in the specified context. */
#endif
/* Return the cost to encode a 0 bit in the specified context. */
lzms_bit_0_cost(unsigned state, const struct lzms_probability_entry *probs)
{
return lzms_bit_costs[probs[state].num_recent_zero_bits];
}
/* Return the cost to encode a 1 bit in the specified context. */
lzms_bit_0_cost(unsigned state, const struct lzms_probability_entry *probs)
{
return lzms_bit_costs[probs[state].num_recent_zero_bits];
}
/* Return the cost to encode a 1 bit in the specified context. */
lzms_bit_1_cost(unsigned state, const struct lzms_probability_entry *probs)
{
return lzms_bit_costs[LZMS_PROBABILITY_DENOMINATOR -
lzms_bit_1_cost(unsigned state, const struct lzms_probability_entry *probs)
{
return lzms_bit_costs[LZMS_PROBABILITY_DENOMINATOR -
}
/* Return the cost to encode a literal, including the main bit. */
}
/* Return the cost to encode a literal, including the main bit. */
lzms_literal_cost(struct lzms_compressor *c, unsigned main_state, unsigned literal)
{
return lzms_bit_0_cost(main_state, c->probs.main) +
lzms_literal_cost(struct lzms_compressor *c, unsigned main_state, unsigned literal)
{
return lzms_bit_0_cost(main_state, c->probs.main) +
/* Return the cost to encode the specified match length, which must not exceed
* MAX_FAST_LENGTH. */
/* Return the cost to encode the specified match length, which must not exceed
* MAX_FAST_LENGTH. */
lzms_fast_length_cost(const struct lzms_compressor *c, u32 length)
{
return c->fast_length_cost_tab[length];
}
/* Return the cost to encode the specified LZ match offset. */
lzms_fast_length_cost(const struct lzms_compressor *c, u32 length)
{
return c->fast_length_cost_tab[length];
}
/* Return the cost to encode the specified LZ match offset. */
lzms_lz_offset_cost(const struct lzms_compressor *c, u32 offset)
{
unsigned slot = lzms_comp_get_offset_slot(c, offset);
lzms_lz_offset_cost(const struct lzms_compressor *c, u32 offset)
{
unsigned slot = lzms_comp_get_offset_slot(c, offset);
}
/* Return the cost to encode the specified delta power and raw offset. */
}
/* Return the cost to encode the specified delta power and raw offset. */
lzms_delta_source_cost(const struct lzms_compressor *c, u32 power, u32 raw_offset)
{
unsigned slot = lzms_comp_get_offset_slot(c, raw_offset);
lzms_delta_source_cost(const struct lzms_compressor *c, u32 power, u32 raw_offset)
{
unsigned slot = lzms_comp_get_offset_slot(c, raw_offset);
state->prev_delta_pair = state->upcoming_delta_pair;
}
state->prev_delta_pair = state->upcoming_delta_pair;
}
lzms_update_state(u8 *state_p, int bit, unsigned num_states)
{
*state_p = ((*state_p << 1) | bit) & (num_states - 1);
}
lzms_update_state(u8 *state_p, int bit, unsigned num_states)
{
*state_p = ((*state_p << 1) | bit) & (num_states - 1);
}
lzms_update_main_state(struct lzms_adaptive_state *state, int is_match)
{
lzms_update_state(&state->main_state, is_match, LZMS_NUM_MAIN_PROBS);
}
lzms_update_main_state(struct lzms_adaptive_state *state, int is_match)
{
lzms_update_state(&state->main_state, is_match, LZMS_NUM_MAIN_PROBS);
}
lzms_update_match_state(struct lzms_adaptive_state *state, int is_delta)
{
lzms_update_state(&state->match_state, is_delta, LZMS_NUM_MATCH_PROBS);
}
lzms_update_match_state(struct lzms_adaptive_state *state, int is_delta)
{
lzms_update_state(&state->match_state, is_delta, LZMS_NUM_MATCH_PROBS);
}
lzms_update_lz_state(struct lzms_adaptive_state *state, int is_rep)
{
lzms_update_state(&state->lz_state, is_rep, LZMS_NUM_LZ_PROBS);
}
lzms_update_lz_state(struct lzms_adaptive_state *state, int is_rep)
{
lzms_update_state(&state->lz_state, is_rep, LZMS_NUM_LZ_PROBS);
}
lzms_update_lz_rep_states(struct lzms_adaptive_state *state, int rep_idx)
{
for (int i = 0; i < rep_idx; i++)
lzms_update_lz_rep_states(struct lzms_adaptive_state *state, int rep_idx)
{
for (int i = 0; i < rep_idx; i++)
lzms_update_state(&state->lz_rep_states[rep_idx], 0, LZMS_NUM_LZ_REP_PROBS);
}
lzms_update_state(&state->lz_rep_states[rep_idx], 0, LZMS_NUM_LZ_REP_PROBS);
}
lzms_update_delta_state(struct lzms_adaptive_state *state, int is_rep)
{
lzms_update_state(&state->delta_state, is_rep, LZMS_NUM_DELTA_PROBS);
}
lzms_update_delta_state(struct lzms_adaptive_state *state, int is_rep)
{
lzms_update_state(&state->delta_state, is_rep, LZMS_NUM_DELTA_PROBS);
}
lzms_update_delta_rep_states(struct lzms_adaptive_state *state, int rep_idx)
{
for (int i = 0; i < rep_idx; i++)
lzms_update_delta_rep_states(struct lzms_adaptive_state *state, int rep_idx)
{
for (int i = 0; i < rep_idx; i++)
* NBYTES_HASHED_FOR_DELTA bytes of the sequence beginning at @p when taken in a
* delta context with the specified @span.
*/
* NBYTES_HASHED_FOR_DELTA bytes of the sequence beginning at @p when taken in a
* delta context with the specified @span.
*/
lzms_delta_hash(const u8 *p, const u32 pos, u32 span)
{
/* A delta match has a certain span and an offset that is a multiple of
lzms_delta_hash(const u8 *p, const u32 pos, u32 span)
{
/* A delta match has a certain span and an offset that is a multiple of
* specified @span and having the initial @len, extend the match as far as
* possible, up to a limit of @max_len.
*/
* specified @span and having the initial @len, extend the match as far as
* possible, up to a limit of @max_len.
*/
lzms_extend_delta_match(const u8 *in_next, const u8 *matchptr,
u32 len, u32 max_len, u32 span)
{
lzms_extend_delta_match(const u8 *in_next, const u8 *matchptr,
u32 len, u32 max_len, u32 span)
{
/* Ensure that at least @num_bits bits are in the bitbuffer variable.
* @num_bits cannot be more than 32. */
/* Ensure that at least @num_bits bits are in the bitbuffer variable.
* @num_bits cannot be more than 32. */
lzms_ensure_bits(struct lzms_input_bitstream *is, unsigned num_bits)
{
unsigned avail;
lzms_ensure_bits(struct lzms_input_bitstream *is, unsigned num_bits)
{
unsigned avail;
}
/* Get @num_bits bits from the bitbuffer variable. */
}
/* Get @num_bits bits from the bitbuffer variable. */
+static forceinline bitbuf_t
lzms_peek_bits(struct lzms_input_bitstream *is, unsigned num_bits)
{
return (is->bitbuf >> 1) >> (BITBUF_NBITS - num_bits - 1);
}
/* Remove @num_bits bits from the bitbuffer variable. */
lzms_peek_bits(struct lzms_input_bitstream *is, unsigned num_bits)
{
return (is->bitbuf >> 1) >> (BITBUF_NBITS - num_bits - 1);
}
/* Remove @num_bits bits from the bitbuffer variable. */
lzms_remove_bits(struct lzms_input_bitstream *is, unsigned num_bits)
{
is->bitbuf <<= num_bits;
lzms_remove_bits(struct lzms_input_bitstream *is, unsigned num_bits)
{
is->bitbuf <<= num_bits;
}
/* Remove and return @num_bits bits from the bitbuffer variable. */
}
/* Remove and return @num_bits bits from the bitbuffer variable. */
+static forceinline bitbuf_t
lzms_pop_bits(struct lzms_input_bitstream *is, unsigned num_bits)
{
bitbuf_t bits = lzms_peek_bits(is, num_bits);
lzms_pop_bits(struct lzms_input_bitstream *is, unsigned num_bits)
{
bitbuf_t bits = lzms_peek_bits(is, num_bits);
}
/* Read @num_bits bits from the input bitstream. */
}
/* Read @num_bits bits from the input bitstream. */
+static forceinline bitbuf_t
lzms_read_bits(struct lzms_input_bitstream *is, unsigned num_bits)
{
lzms_ensure_bits(is, num_bits);
lzms_read_bits(struct lzms_input_bitstream *is, unsigned num_bits)
{
lzms_ensure_bits(is, num_bits);
* probability entry to use. The state and probability entry will be updated
* based on the decoded bit.
*/
* probability entry to use. The state and probability entry will be updated
* based on the decoded bit.
*/
lzms_decode_bit(struct lzms_range_decoder *rd, u32 *state_p, u32 num_states,
struct lzms_probability_entry *probs)
{
lzms_decode_bit(struct lzms_range_decoder *rd, u32 *state_p, u32 num_states,
struct lzms_probability_entry *probs)
{
/* XXX: mostly copied from read_huffsym() in decompress_common.h because LZMS
* needs its own bitstream */
/* XXX: mostly copied from read_huffsym() in decompress_common.h because LZMS
* needs its own bitstream */
+static forceinline unsigned
lzms_decode_huffman_symbol(struct lzms_input_bitstream *is, u16 decode_table[],
unsigned table_bits, u32 freqs[],
struct lzms_huffman_rebuild_info *rebuild_info)
lzms_decode_huffman_symbol(struct lzms_input_bitstream *is, u16 decode_table[],
unsigned table_bits, u32 freqs[],
struct lzms_huffman_rebuild_info *rebuild_info)
+static forceinline unsigned
lzms_decode_literal(struct lzms_decompressor *d,
struct lzms_input_bitstream *is)
{
lzms_decode_literal(struct lzms_decompressor *d,
struct lzms_input_bitstream *is)
{
&d->literal_rebuild_info);
}
&d->literal_rebuild_info);
}
lzms_decode_lz_offset(struct lzms_decompressor *d,
struct lzms_input_bitstream *is)
{
lzms_decode_lz_offset(struct lzms_decompressor *d,
struct lzms_input_bitstream *is)
{
lzms_read_bits(is, lzms_extra_offset_bits[slot]);
}
lzms_read_bits(is, lzms_extra_offset_bits[slot]);
}
lzms_decode_length(struct lzms_decompressor *d,
struct lzms_input_bitstream *is)
{
lzms_decode_length(struct lzms_decompressor *d,
struct lzms_input_bitstream *is)
{
lzms_decode_delta_offset(struct lzms_decompressor *d,
struct lzms_input_bitstream *is)
{
lzms_decode_delta_offset(struct lzms_decompressor *d,
struct lzms_input_bitstream *is)
{
lzms_read_bits(is, lzms_extra_offset_bits[slot]);
}
lzms_read_bits(is, lzms_extra_offset_bits[slot]);
}
+static forceinline unsigned
lzms_decode_delta_power(struct lzms_decompressor *d,
struct lzms_input_bitstream *is)
{
lzms_decode_delta_power(struct lzms_decompressor *d,
struct lzms_input_bitstream *is)
{
* This requires that the limit be no more than the length of offset_slot_tab_1
* (currently 32768).
*/
* This requires that the limit be no more than the length of offset_slot_tab_1
* (currently 32768).
*/
lzx_is_16_bit(size_t max_bufsize)
{
STATIC_ASSERT(ARRAY_LEN(((struct lzx_compressor *)0)->offset_slot_tab_1) == 32768);
lzx_is_16_bit(size_t max_bufsize)
{
STATIC_ASSERT(ARRAY_LEN(((struct lzx_compressor *)0)->offset_slot_tab_1) == 32768);
/*
* Return the offset slot for the specified adjusted match offset.
*/
/*
* Return the offset slot for the specified adjusted match offset.
*/
+static forceinline unsigned
lzx_get_offset_slot(struct lzx_compressor *c, u32 adjusted_offset,
bool is_16_bit)
{
lzx_get_offset_slot(struct lzx_compressor *c, u32 adjusted_offset,
bool is_16_bit)
{
* Add some bits to the bitbuffer variable of the output bitstream. The caller
* must make sure there is enough room.
*/
* Add some bits to the bitbuffer variable of the output bitstream. The caller
* must make sure there is enough room.
*/
lzx_add_bits(struct lzx_output_bitstream *os, u32 bits, unsigned num_bits)
{
os->bitbuf = (os->bitbuf << num_bits) | bits;
lzx_add_bits(struct lzx_output_bitstream *os, u32 bits, unsigned num_bits)
{
os->bitbuf = (os->bitbuf << num_bits) | bits;
* specifies the maximum number of bits that may have been added since the last
* flush.
*/
* specifies the maximum number of bits that may have been added since the last
* flush.
*/
lzx_flush_bits(struct lzx_output_bitstream *os, unsigned max_num_bits)
{
/* Masking the number of bits to shift is only needed to avoid undefined
lzx_flush_bits(struct lzx_output_bitstream *os, unsigned max_num_bits)
{
/* Masking the number of bits to shift is only needed to avoid undefined
}
/* Add at most 16 bits to the bitbuffer and flush it. */
}
/* Add at most 16 bits to the bitbuffer and flush it. */
lzx_write_bits(struct lzx_output_bitstream *os, u32 bits, unsigned num_bits)
{
lzx_add_bits(os, bits, num_bits);
lzx_write_bits(struct lzx_output_bitstream *os, u32 bits, unsigned num_bits)
{
lzx_add_bits(os, bits, num_bits);
/* Literal observation. Heuristic: use the top 2 bits and low 1 bits of the
* literal, for 8 possible literal observation types. */
/* Literal observation. Heuristic: use the top 2 bits and low 1 bits of the
* literal, for 8 possible literal observation types. */
lzx_observe_literal(struct lzx_block_split_stats *stats, u8 lit)
{
stats->new_observations[((lit >> 5) & 0x6) | (lit & 1)]++;
lzx_observe_literal(struct lzx_block_split_stats *stats, u8 lit)
{
stats->new_observations[((lit >> 5) & 0x6) | (lit & 1)]++;
/* Match observation. Heuristic: use one observation type for "short match" and
* one observation type for "long match". */
/* Match observation. Heuristic: use one observation type for "short match" and
* one observation type for "long match". */
lzx_observe_match(struct lzx_block_split_stats *stats, unsigned length)
{
stats->new_observations[NUM_LITERAL_OBSERVATION_TYPES + (length >= 5)]++;
lzx_observe_match(struct lzx_block_split_stats *stats, unsigned length)
{
stats->new_observations[NUM_LITERAL_OBSERVATION_TYPES + (length >= 5)]++;
((u64)1 << LZX_QUEUE_R1_SHIFT) | \
((u64)1 << LZX_QUEUE_R2_SHIFT) }
((u64)1 << LZX_QUEUE_R1_SHIFT) | \
((u64)1 << LZX_QUEUE_R2_SHIFT) }
lzx_lru_queue_R0(struct lzx_lru_queue queue)
{
return (queue.R >> LZX_QUEUE_R0_SHIFT) & LZX_QUEUE_OFFSET_MASK;
}
lzx_lru_queue_R0(struct lzx_lru_queue queue)
{
return (queue.R >> LZX_QUEUE_R0_SHIFT) & LZX_QUEUE_OFFSET_MASK;
}
lzx_lru_queue_R1(struct lzx_lru_queue queue)
{
return (queue.R >> LZX_QUEUE_R1_SHIFT) & LZX_QUEUE_OFFSET_MASK;
}
lzx_lru_queue_R1(struct lzx_lru_queue queue)
{
return (queue.R >> LZX_QUEUE_R1_SHIFT) & LZX_QUEUE_OFFSET_MASK;
}
lzx_lru_queue_R2(struct lzx_lru_queue queue)
{
return (queue.R >> LZX_QUEUE_R2_SHIFT) & LZX_QUEUE_OFFSET_MASK;
}
/* Push a match offset onto the front (most recently used) end of the queue. */
lzx_lru_queue_R2(struct lzx_lru_queue queue)
{
return (queue.R >> LZX_QUEUE_R2_SHIFT) & LZX_QUEUE_OFFSET_MASK;
}
/* Push a match offset onto the front (most recently used) end of the queue. */
-static inline struct lzx_lru_queue
+static forceinline struct lzx_lru_queue
lzx_lru_queue_push(struct lzx_lru_queue queue, u32 offset)
{
return (struct lzx_lru_queue) {
lzx_lru_queue_push(struct lzx_lru_queue queue, u32 offset)
{
return (struct lzx_lru_queue) {
}
/* Swap a match offset to the front of the queue. */
}
/* Swap a match offset to the front of the queue. */
-static inline struct lzx_lru_queue
+static forceinline struct lzx_lru_queue
lzx_lru_queue_swap(struct lzx_lru_queue queue, unsigned idx)
{
unsigned shift = idx * 21;
lzx_lru_queue_swap(struct lzx_lru_queue queue, unsigned idx)
{
unsigned shift = idx * 21;
lzx_walk_item_list(struct lzx_compressor *c, u32 block_size, bool is_16_bit,
bool record)
{
lzx_walk_item_list(struct lzx_compressor *c, u32 block_size, bool is_16_bit,
bool record)
{
* beginning of the block), but this doesn't matter because this function only
* computes frequencies.
*/
* beginning of the block), but this doesn't matter because this function only
* computes frequencies.
*/
lzx_tally_item_list(struct lzx_compressor *c, u32 block_size, bool is_16_bit)
{
lzx_walk_item_list(c, block_size, is_16_bit, false);
lzx_tally_item_list(struct lzx_compressor *c, u32 block_size, bool is_16_bit)
{
lzx_walk_item_list(c, block_size, is_16_bit, false);
* first-to-last order. The return value is the index in c->chosen_sequences at
* which the lzx_sequences begin.
*/
* first-to-last order. The return value is the index in c->chosen_sequences at
* which the lzx_sequences begin.
*/
lzx_record_item_list(struct lzx_compressor *c, u32 block_size, bool is_16_bit)
{
return lzx_walk_item_list(c, block_size, is_16_bit, true);
lzx_record_item_list(struct lzx_compressor *c, u32 block_size, bool is_16_bit)
{
return lzx_walk_item_list(c, block_size, is_16_bit, true);
* one step ahead, with the exception of special consideration for "gap
* matches".
*/
* one step ahead, with the exception of special consideration for "gap
* matches".
*/
-static inline struct lzx_lru_queue
+static forceinline struct lzx_lru_queue
lzx_find_min_cost_path(struct lzx_compressor * const restrict c,
const u8 * const restrict block_begin,
const u32 block_size,
lzx_find_min_cost_path(struct lzx_compressor * const restrict c,
const u8 * const restrict block_begin,
const u32 block_size,
* for the block uses default costs; additional passes use costs derived from
* the Huffman codes computed in the previous pass.
*/
* for the block uses default costs; additional passes use costs derived from
* the Huffman codes computed in the previous pass.
*/
-static inline struct lzx_lru_queue
+static forceinline struct lzx_lru_queue
lzx_optimize_and_flush_block(struct lzx_compressor * const restrict c,
struct lzx_output_bitstream * const restrict os,
const u8 * const restrict block_begin,
lzx_optimize_and_flush_block(struct lzx_compressor * const restrict c,
struct lzx_output_bitstream * const restrict os,
const u8 * const restrict block_begin,
* time, but rather to produce a compression ratio significantly better than a
* simpler "greedy" or "lazy" parse while still being relatively fast.
*/
* time, but rather to produce a compression ratio significantly better than a
* simpler "greedy" or "lazy" parse while still being relatively fast.
*/
lzx_compress_near_optimal(struct lzx_compressor * restrict c,
const u8 * const restrict in_begin, size_t in_nbytes,
struct lzx_output_bitstream * restrict os,
lzx_compress_near_optimal(struct lzx_compressor * restrict c,
const u8 * const restrict in_begin, size_t in_nbytes,
struct lzx_output_bitstream * restrict os,
* Huffman symbol for the literal, increments the current literal run length,
* and "observes" the literal for the block split statistics.
*/
* Huffman symbol for the literal, increments the current literal run length,
* and "observes" the literal for the block split statistics.
*/
lzx_choose_literal(struct lzx_compressor *c, unsigned literal, u32 *litrunlen_p)
{
lzx_observe_literal(&c->split_stats, literal);
lzx_choose_literal(struct lzx_compressor *c, unsigned literal, u32 *litrunlen_p)
{
lzx_observe_literal(&c->split_stats, literal);
* literal run, updates the recent offsets queue, and "observes" the match for
* the block split statistics.
*/
* literal run, updates the recent offsets queue, and "observes" the match for
* the block split statistics.
*/
lzx_choose_match(struct lzx_compressor *c, unsigned length, u32 adjusted_offset,
u32 recent_offsets[LZX_NUM_RECENT_OFFSETS], bool is_16_bit,
u32 *litrunlen_p, struct lzx_sequence **next_seq_p)
lzx_choose_match(struct lzx_compressor *c, unsigned length, u32 adjusted_offset,
u32 recent_offsets[LZX_NUM_RECENT_OFFSETS], bool is_16_bit,
u32 *litrunlen_p, struct lzx_sequence **next_seq_p)
* which is just a literal run with no following match. This literal run might
* be empty.
*/
* which is just a literal run with no following match. This literal run might
* be empty.
*/
lzx_finish_sequence(struct lzx_sequence *last_seq, u32 litrunlen)
{
last_seq->litrunlen = litrunlen;
lzx_finish_sequence(struct lzx_sequence *last_seq, u32 litrunlen)
{
last_seq->litrunlen = litrunlen;
* offset matches, since those require fewer bits to encode.
*/
* offset matches, since those require fewer bits to encode.
*/
+static forceinline unsigned
lzx_explicit_offset_match_score(unsigned len, u32 adjusted_offset)
{
unsigned score = len;
lzx_explicit_offset_match_score(unsigned len, u32 adjusted_offset)
{
unsigned score = len;
+static forceinline unsigned
lzx_repeat_offset_match_score(unsigned rep_len, unsigned rep_idx)
{
return rep_len + 3;
lzx_repeat_offset_match_score(unsigned rep_len, unsigned rep_idx)
{
return rep_len + 3;
* when we decide whether a match is "better" than another, we take the offset
* into consideration as well as the length.
*/
* when we decide whether a match is "better" than another, we take the offset
* into consideration as well as the length.
*/
lzx_compress_lazy(struct lzx_compressor * restrict c,
const u8 * const restrict in_begin, size_t in_nbytes,
struct lzx_output_bitstream * restrict os, bool is_16_bit)
lzx_compress_lazy(struct lzx_compressor * restrict c,
const u8 * const restrict in_begin, size_t in_nbytes,
struct lzx_output_bitstream * restrict os, bool is_16_bit)
} _aligned_attribute(DECODE_TABLE_ALIGNMENT);
/* Read a Huffman-encoded symbol using the precode. */
} _aligned_attribute(DECODE_TABLE_ALIGNMENT);
/* Read a Huffman-encoded symbol using the precode. */
+static forceinline unsigned
read_presym(const struct lzx_decompressor *d, struct input_bitstream *is)
{
return read_huffsym(is, d->precode_decode_table,
read_presym(const struct lzx_decompressor *d, struct input_bitstream *is)
{
return read_huffsym(is, d->precode_decode_table,
}
/* Read a Huffman-encoded symbol using the main code. */
}
/* Read a Huffman-encoded symbol using the main code. */
+static forceinline unsigned
read_mainsym(const struct lzx_decompressor *d, struct input_bitstream *is)
{
return read_huffsym(is, d->maincode_decode_table,
read_mainsym(const struct lzx_decompressor *d, struct input_bitstream *is)
{
return read_huffsym(is, d->maincode_decode_table,
}
/* Read a Huffman-encoded symbol using the length code. */
}
/* Read a Huffman-encoded symbol using the length code. */
+static forceinline unsigned
read_lensym(const struct lzx_decompressor *d, struct input_bitstream *is)
{
return read_huffsym(is, d->lencode_decode_table,
read_lensym(const struct lzx_decompressor *d, struct input_bitstream *is)
{
return read_huffsym(is, d->lencode_decode_table,
}
/* Read a Huffman-encoded symbol using the aligned offset code. */
}
/* Read a Huffman-encoded symbol using the aligned offset code. */
+static forceinline unsigned
read_alignedsym(const struct lzx_decompressor *d, struct input_bitstream *is)
{
return read_huffsym(is, d->alignedcode_decode_table,
read_alignedsym(const struct lzx_decompressor *d, struct input_bitstream *is)
{
return read_huffsym(is, d->alignedcode_decode_table,
* If the output buffer space is exhausted, then the bits will be ignored, and
* xpress_flush_output() will return 0 when it gets called.
*/
* If the output buffer space is exhausted, then the bits will be ignored, and
* xpress_flush_output() will return 0 when it gets called.
*/
xpress_write_bits(struct xpress_output_bitstream *os,
const u32 bits, const unsigned num_bits)
{
xpress_write_bits(struct xpress_output_bitstream *os,
const u32 bits, const unsigned num_bits)
{
/*
* Interweave a literal byte into the output bitstream.
*/
/*
* Interweave a literal byte into the output bitstream.
*/
xpress_write_byte(struct xpress_output_bitstream *os, u8 byte)
{
if (os->next_byte < os->end)
xpress_write_byte(struct xpress_output_bitstream *os, u8 byte)
{
if (os->next_byte < os->end)
/*
* Interweave two literal bytes into the output bitstream.
*/
/*
* Interweave two literal bytes into the output bitstream.
*/
xpress_write_u16(struct xpress_output_bitstream *os, u16 v)
{
if (os->end - os->next_byte >= 2) {
xpress_write_u16(struct xpress_output_bitstream *os, u16 v)
{
if (os->end - os->next_byte >= 2) {
return os->next_byte - os->start;
}
return os->next_byte - os->start;
}
xpress_write_extra_length_bytes(struct xpress_output_bitstream *os,
unsigned adjusted_len)
{
xpress_write_extra_length_bytes(struct xpress_output_bitstream *os,
unsigned adjusted_len)
{
}
/* Output a match or literal. */
}
/* Output a match or literal. */
xpress_write_item(struct xpress_item item, struct xpress_output_bitstream *os,
const u32 codewords[], const u8 lens[])
{
xpress_write_item(struct xpress_item item, struct xpress_output_bitstream *os,
const u32 codewords[], const u8 lens[])
{
/* Tally the Huffman symbol for a literal and return the intermediate
* representation of that literal. */
/* Tally the Huffman symbol for a literal and return the intermediate
* representation of that literal. */
-static inline struct xpress_item
+static forceinline struct xpress_item
xpress_record_literal(struct xpress_compressor *c, unsigned literal)
{
c->freqs[literal]++;
xpress_record_literal(struct xpress_compressor *c, unsigned literal)
{
c->freqs[literal]++;
/* Tally the Huffman symbol for a match and return the intermediate
* representation of that match. */
/* Tally the Huffman symbol for a match and return the intermediate
* representation of that match. */
-static inline struct xpress_item
+static forceinline struct xpress_item
xpress_record_match(struct xpress_compressor *c, unsigned length, unsigned offset)
{
unsigned adjusted_len = length - XPRESS_MIN_MATCH_LEN;
xpress_record_match(struct xpress_compressor *c, unsigned length, unsigned offset)
{
unsigned adjusted_len = length - XPRESS_MIN_MATCH_LEN;
git://wimlib.net / wimlib / commitdiff