X-Git-Url: https://wimlib.net/git/?p=wimlib;a=blobdiff_plain;f=src%2Flzms_common.c;h=fb13a4358c5469a79d0c4c5e101c03ee2f9e59db;hp=1564dfd6bb9a577447349472fa30b57065c4fc1f;hb=HEAD;hpb=a0d3f5bf878ec6b240f5e7f5ea1a816b205a5f11 diff --git a/src/lzms_common.c b/src/lzms_common.c index 1564dfd6..50176847 100644 --- a/src/lzms_common.c +++ b/src/lzms_common.c @@ -3,7 +3,7 @@ */ /* - * Copyright (C) 2013, 2014 Eric Biggers + * Copyright (C) 2013-2016 Eric Biggers * * This file is free software; you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -16,16 +16,21 @@ * details. * * You should have received a copy of the GNU Lesser General Public License - * along with this file; if not, see http://www.gnu.org/licenses/. + * along with this file; if not, see https://www.gnu.org/licenses/. */ #ifdef HAVE_CONFIG_H # include "config.h" #endif +#include "wimlib/cpu_features.h" #include "wimlib/lzms_common.h" #include "wimlib/unaligned.h" +#ifdef __x86_64__ +# include +#endif + /* Table: offset slot => offset slot base value */ const u32 lzms_offset_slot_base[LZMS_MAX_NUM_OFFSET_SYMS + 1] = { 0x00000001, 0x00000002, 0x00000003, 0x00000004, @@ -346,9 +351,13 @@ lzms_get_num_offset_slots(size_t uncompressed_size) } void -lzms_init_probability_entries(struct lzms_probability_entry *entries, size_t count) +lzms_init_probabilities(struct lzms_probabilites *probs) { - for (size_t i = 0; i < count; i++) { + struct lzms_probability_entry *entries = + (struct lzms_probability_entry *)probs; + size_t num_entries = sizeof(struct lzms_probabilites) / + sizeof(struct lzms_probability_entry); + for (size_t i = 0; i < num_entries; i++) { entries[i].num_recent_zero_bits = LZMS_INITIAL_PROBABILITY; entries[i].recent_bits = LZMS_INITIAL_RECENT_BITS; } @@ -368,8 +377,36 @@ lzms_dilute_symbol_frequencies(u32 freqs[], unsigned num_syms) freqs[sym] = (freqs[sym] >> 1) + 1; } -static inline u8 * -find_next_opcode(u8 *p) + +#ifdef __x86_64__ +static forceinline u8 * +find_next_opcode_sse4_2(u8 *p) +{ + const __v16qi potential_opcodes = (__v16qi) {0x48, 0x4C, 0xE8, 0xE9, 0xF0, 0xFF}; + __asm__( + " pcmpestri $0x0, (%[p]), %[potential_opcodes] \n" + " jc 2f \n" + "1: \n" + " add $0x10, %[p] \n" + " pcmpestri $0x0, (%[p]), %[potential_opcodes] \n" + " jnc 1b \n" + "2: \n" + #ifdef __ILP32__ /* x32 ABI (x86_64 with 32-bit pointers) */ + " add %%ecx, %[p] \n" + #else + " add %%rcx, %[p] \n" + #endif + : [p] "+r" (p) + : [potential_opcodes] "x" (potential_opcodes), "a" (6), "d" (16) + : "rcx", "cc" + ); + + return p; +} +#endif /* __x86_64__ */ + +static forceinline u8 * +find_next_opcode_default(u8 *p) { /* * The following table is used to accelerate the common case where the @@ -400,7 +437,7 @@ find_next_opcode(u8 *p) return p; } -static inline u8 * +static forceinline u8 * translate_if_needed(u8 *data, u8 *p, s32 *last_x86_pos, s32 last_target_usages[], bool undo) { @@ -411,62 +448,69 @@ translate_if_needed(u8 *data, u8 *p, s32 *last_x86_pos, max_trans_offset = LZMS_X86_MAX_TRANSLATION_OFFSET; - if ((*p & 0xFE) == 0xE8) { - if (*p & 0x01) { - /* 0xE9: Jump relative */ - p += 4; - } else { - /* 0xE8: Call relative. Note: 'max_trans_offset' must - * be halved for this instruction. This means that we - * must be more confident that we are in a region of x86 - * machine code before we will do a translation for this - * particular instruction. */ - opcode_nbytes = 1; - max_trans_offset /= 2; - goto have_opcode; - } - } else if ((*p & 0xFB) == 0x48) { - if (*p & 0x04) { - /* 0x4C */ - if (*(p + 1) == 0x8D) { - if ((*(p + 2) & 0x7) == 0x5) { - /* Load effective address relative (x86_64) */ - opcode_nbytes = 3; - goto have_opcode; - } - } - } else { - /* 0x48 */ - if (*(p + 1) == 0x8B) { - if (*(p + 2) == 0x5 || *(p + 2) == 0xD) { - /* Load relative (x86_64) */ - opcode_nbytes = 3; - goto have_opcode; - } - } else if (*(p + 1) == 0x8D) { - if ((*(p + 2) & 0x7) == 0x5) { - /* Load effective address relative (x86_64) */ - opcode_nbytes = 3; - goto have_opcode; - } - } - } - } else { - if (*p & 0x0F) { - /* 0xFF */ - if (*(p + 1) == 0x15) { - /* Call indirect */ + /* + * p[0] has one of the following values: + * 0x48 0x4C 0xE8 0xE9 0xF0 0xFF + */ + + if (p[0] >= 0xF0) { + if (p[0] & 0x0F) { + /* 0xFF (instruction group) */ + if (p[1] == 0x15) { + /* Call indirect relative */ opcode_nbytes = 2; goto have_opcode; } } else { - /* 0xF0 */ - if (*(p + 1) == 0x83 && *(p + 2) == 0x05) { + /* 0xF0 (lock prefix) */ + if (p[1] == 0x83 && p[2] == 0x05) { /* Lock add relative */ opcode_nbytes = 3; goto have_opcode; } } + } else if (p[0] <= 0x4C) { + + /* 0x48 or 0x4C. In 64-bit code this is a REX prefix byte with + * W=1, R=[01], X=0, and B=0, and it will be followed by the + * actual opcode, then additional bytes depending on the opcode. + * We are most interested in several common instructions that + * access data relative to the instruction pointer. These use a + * 1-byte opcode, followed by a ModR/M byte, followed by a + * 4-byte displacement. */ + + /* Test: does the ModR/M byte indicate RIP-relative addressing? + * Note: there seems to be a mistake in the format here; the + * mask really should be 0xC7 instead of 0x07 so that both the + * MOD and R/M fields of ModR/M are tested, not just R/M. */ + if ((p[2] & 0x07) == 0x05) { + /* Check for the LEA (load effective address) or MOV + * (move) opcodes. For MOV there are additional + * restrictions, although it seems they are only helpful + * due to the overly lax ModR/M test. */ + if (p[1] == 0x8D || + (p[1] == 0x8B && !(p[0] & 0x04) && !(p[2] & 0xF0))) + { + opcode_nbytes = 3; + goto have_opcode; + } + } + } else { + if (p[0] & 0x01) { + /* 0xE9: Jump relative. Theoretically this would be + * useful to translate, but in fact it's explicitly + * excluded. Most likely it creates too many false + * positives for the detection algorithm. */ + p += 4; + } else { + /* 0xE8: Call relative. This is a common case, so it + * uses a reduced max_trans_offset. In other words, we + * have to be more confident that the data actually is + * x86 machine code before we'll do the translation. */ + opcode_nbytes = 1; + max_trans_offset >>= 1; + goto have_opcode; + } } return p + 1; @@ -476,15 +520,15 @@ have_opcode: p += opcode_nbytes; if (undo) { if (i - *last_x86_pos <= max_trans_offset) { - u32 n = get_unaligned_u32_le(p); - put_unaligned_u32_le(n - i, p); + u32 n = get_unaligned_le32(p); + put_unaligned_le32(n - i, p); } - target16 = i + get_unaligned_u16_le(p); + target16 = i + get_unaligned_le16(p); } else { - target16 = i + get_unaligned_u16_le(p); + target16 = i + get_unaligned_le16(p); if (i - *last_x86_pos <= max_trans_offset) { - u32 n = get_unaligned_u32_le(p); - put_unaligned_u32_le(n + i, p); + u32 n = get_unaligned_le32(p); + put_unaligned_le32(n + i, p); } } @@ -541,8 +585,7 @@ lzms_x86_filter(u8 data[restrict], s32 size, u8 *p; u8 *tail_ptr; - u8 saved_byte; - s32 last_x86_pos; + s32 last_x86_pos = -LZMS_X86_MAX_TRANSLATION_OFFSET - 1; if (size <= 17) return; @@ -565,21 +608,36 @@ lzms_x86_filter(u8 data[restrict], s32 size, * data[(size - 16) + 7] and have no effect on the result, as long * as we restore those bytes later. */ - tail_ptr = &data[size - 16]; - saved_byte = *(tail_ptr + 8); - *(tail_ptr + 8) = 0xE8; - last_x86_pos = -LZMS_X86_MAX_TRANSLATION_OFFSET - 1; /* Note: the very first byte must be ignored completely! */ p = data + 1; - for (;;) { - p = find_next_opcode(p); - - if (p >= tail_ptr) - break; + tail_ptr = &data[size - 16]; - p = translate_if_needed(data, p, &last_x86_pos, last_target_usages, undo); +#ifdef __x86_64__ + if (cpu_features & X86_CPU_FEATURE_SSE4_2) { + u8 saved_byte = *tail_ptr; + *tail_ptr = 0xE8; + for (;;) { + u8 *new_p = find_next_opcode_sse4_2(p); + if (new_p >= tail_ptr - 8) + break; + p = new_p; + p = translate_if_needed(data, p, &last_x86_pos, + last_target_usages, undo); + } + *tail_ptr = saved_byte; + } +#endif + { + u8 saved_byte = *(tail_ptr + 8); + *(tail_ptr + 8) = 0xE8; + for (;;) { + p = find_next_opcode_default(p); + if (p >= tail_ptr) + break; + p = translate_if_needed(data, p, &last_x86_pos, + last_target_usages, undo); + } + *(tail_ptr + 8) = saved_byte; } - - *(tail_ptr + 8) = saved_byte; }