#include "wimlib/lzms_common.h"
#include "wimlib/unaligned.h"
+#include "wimlib/x86_cpu_features.h"
+
+#ifdef __x86_64__
+# include <emmintrin.h>
+#endif
/* Table: offset slot => offset slot base value */
const u32 lzms_offset_slot_base[LZMS_MAX_NUM_OFFSET_SYMS + 1] = {
freqs[sym] = (freqs[sym] >> 1) + 1;
}
+
+#ifdef __x86_64__
+static inline u8 *
+find_next_opcode_sse4_2(u8 *p)
+{
+ const __v16qi potential_opcodes = (__v16qi) {0x48, 0x4C, 0xE8, 0xE9, 0xF0, 0xFF};
+ __asm__(
+ " pcmpestri $0x0, (%[p]), %[potential_opcodes] \n"
+ " jc 2f \n"
+ "1: \n"
+ " add $0x10, %[p] \n"
+ " pcmpestri $0x0, (%[p]), %[potential_opcodes] \n"
+ " jnc 1b \n"
+ "2: \n"
+ " add %%rcx, %[p] \n"
+ : [p] "+r" (p)
+ : [potential_opcodes] "x" (potential_opcodes), "a" (6), "d" (16)
+ : "rcx", "cc"
+ );
+
+ return p;
+}
+#endif /* __x86_64__ */
+
static inline u8 *
-find_next_opcode(u8 *p)
+find_next_opcode_default(u8 *p)
{
/*
* The following table is used to accelerate the common case where the
max_trans_offset = LZMS_X86_MAX_TRANSLATION_OFFSET;
- switch (*p) {
- case 0x48:
- if (*(p + 1) == 0x8B) {
- if (*(p + 2) == 0x5 || *(p + 2) == 0xD) {
- /* Load relative (x86_64) */
- opcode_nbytes = 3;
- goto have_opcode;
+ if ((*p & 0xFE) == 0xE8) {
+ if (*p & 0x01) {
+ /* 0xE9: Jump relative */
+ p += 4;
+ } else {
+ /* 0xE8: Call relative. Note: 'max_trans_offset' must
+ * be halved for this instruction. This means that we
+ * must be more confident that we are in a region of x86
+ * machine code before we will do a translation for this
+ * particular instruction. */
+ opcode_nbytes = 1;
+ max_trans_offset /= 2;
+ goto have_opcode;
+ }
+ } else if ((*p & 0xFB) == 0x48) {
+ if (*p & 0x04) {
+ /* 0x4C */
+ if (*(p + 1) == 0x8D) {
+ if ((*(p + 2) & 0x7) == 0x5) {
+ /* Load effective address relative (x86_64) */
+ opcode_nbytes = 3;
+ goto have_opcode;
+ }
}
- } else if (*(p + 1) == 0x8D) {
- if ((*(p + 2) & 0x7) == 0x5) {
- /* Load effective address relative (x86_64) */
- opcode_nbytes = 3;
- goto have_opcode;
+ } else {
+ /* 0x48 */
+ if (*(p + 1) == 0x8B) {
+ if (*(p + 2) == 0x5 || *(p + 2) == 0xD) {
+ /* Load relative (x86_64) */
+ opcode_nbytes = 3;
+ goto have_opcode;
+ }
+ } else if (*(p + 1) == 0x8D) {
+ if ((*(p + 2) & 0x7) == 0x5) {
+ /* Load effective address relative (x86_64) */
+ opcode_nbytes = 3;
+ goto have_opcode;
+ }
}
}
- break;
- case 0x4C:
- if (*(p + 1) == 0x8D) {
- if ((*(p + 2) & 0x7) == 0x5) {
- /* Load effective address relative (x86_64) */
+ } else {
+ if (*p & 0x0F) {
+ /* 0xFF */
+ if (*(p + 1) == 0x15) {
+ /* Call indirect */
+ opcode_nbytes = 2;
+ goto have_opcode;
+ }
+ } else {
+ /* 0xF0 */
+ if (*(p + 1) == 0x83 && *(p + 2) == 0x05) {
+ /* Lock add relative */
opcode_nbytes = 3;
goto have_opcode;
}
}
- break;
- case 0xE8:
- /* Call relative. Note: 'max_trans_offset' must be
- * halved for this instruction. This means that we must
- * be more confident that we are in a region of x86
- * machine code before we will do a translation for this
- * particular instruction. */
- opcode_nbytes = 1;
- max_trans_offset /= 2;
- goto have_opcode;
- case 0xE9:
- /* Jump relative */
- p += 4;
- break;
- case 0xF0:
- if (*(p + 1) == 0x83 && *(p + 2) == 0x05) {
- /* Lock add relative */
- opcode_nbytes = 3;
- goto have_opcode;
- }
- break;
- case 0xFF:
- if (*(p + 1) == 0x15) {
- /* Call indirect */
- opcode_nbytes = 2;
- goto have_opcode;
- }
- break;
}
return p + 1;
u8 *p;
u8 *tail_ptr;
- u8 saved_byte;
- s32 last_x86_pos;
+ s32 last_x86_pos = -LZMS_X86_MAX_TRANSLATION_OFFSET - 1;
if (size <= 17)
return;
* data[(size - 16) + 7] and have no effect on the result, as long
* as we restore those bytes later.
*/
- tail_ptr = &data[size - 16];
- saved_byte = *(tail_ptr + 8);
- *(tail_ptr + 8) = 0xE8;
- last_x86_pos = -LZMS_X86_MAX_TRANSLATION_OFFSET - 1;
/* Note: the very first byte must be ignored completely! */
p = data + 1;
- for (;;) {
- p = find_next_opcode(p);
-
- if (p >= tail_ptr)
- break;
+ tail_ptr = &data[size - 16];
- p = translate_if_needed(data, p, &last_x86_pos, last_target_usages, undo);
+#ifdef __x86_64__
+ if (x86_have_cpu_feature(X86_CPU_FEATURE_SSE4_2)) {
+ u8 saved_byte = *tail_ptr;
+ *tail_ptr = 0xE8;
+ for (;;) {
+ u8 *new_p = find_next_opcode_sse4_2(p);
+ if (new_p >= tail_ptr - 8)
+ break;
+ p = new_p;
+ p = translate_if_needed(data, p, &last_x86_pos,
+ last_target_usages, undo);
+ }
+ *tail_ptr = saved_byte;
+ }
+#endif
+ {
+ u8 saved_byte = *(tail_ptr + 8);
+ *(tail_ptr + 8) = 0xE8;
+ for (;;) {
+ p = find_next_opcode_default(p);
+ if (p >= tail_ptr)
+ break;
+ p = translate_if_needed(data, p, &last_x86_pos,
+ last_target_usages, undo);
+ }
+ *(tail_ptr + 8) = saved_byte;
}
-
- *(tail_ptr + 8) = saved_byte;
}
git://wimlib.net / wimlib / blobdiff