Emit ModRM Correctly When Register R12 Is Used (vbpf#534)

When register R12 is used in an instruction requiring a ModRM byte, the
semantics and encoding of the instruction are unique. The
emit_modrm_and_displacement function did not handle this uniqueness
properly. This patch modifies the function to handle the use of R12
correctly.

Fixes vbpf#529

Signed-off-by: Will Hawkins <[email protected]>

tests/ldxb-0offset.data

@@ -0,0 +1,19 @@
+# Copyright (c) Big Switch Networks, Inc
+# SPDX-License-Identifier: Apache-2.0
+-- asm
+mov %r0, 0
+mov %r2, %r1
+mov %r7, %r1
+mov %r8, %r1
+add %r7, 1
+add %r8, 2
+ldxb %r0, [%r2+0]
+ldxb %r7, [%r7+0]
+ldxb %r8, [%r8+0]
+add %r0, %r7
+add %r0, %r8
+exit
+-- mem
+01 02 03
+-- result
+0x06

tests/ldxb-large-offset.data

@@ -0,0 +1,33 @@
+# Copyright (c) Big Switch Networks, Inc
+# SPDX-License-Identifier: Apache-2.0
+-- asm
+mov %r2, %r1
+mov %r8, %r1
+ldxb %r2, [%r2+0x80]
+ldxb %r8, [%r8+0x88]
+mov %r0, %r2
+add %r0, %r8
+exit
+-- mem
+aa bb 11 cc dd ee ff 11
+aa bb 11 cc dd ee ff 11
+aa bb 11 cc dd ee ff 11
+aa bb 11 cc dd ee ff 11
+aa bb 11 cc dd ee ff 11
+aa bb 11 cc dd ee ff 11
+aa bb 11 cc dd ee ff 11
+aa bb 11 cc dd ee ff 11
+aa bb 11 cc dd ee ff 11
+aa bb 11 cc dd ee ff 11
+aa bb 11 cc dd ee ff 11
+aa bb 11 cc dd ee ff 11
+aa bb 11 cc dd ee ff 11
+aa bb 11 cc dd ee ff 11
+aa bb 11 cc dd ee ff 11
+aa bb 11 cc dd ee ff 11
+02 bb 11 cc dd ee ff 11
+03 bb 11 cc dd ee ff 11
+-- result
+0x05
+
+

tests/ldxb-small-offset.data

@@ -0,0 +1,31 @@
+# Copyright (c) Big Switch Networks, Inc
+# SPDX-License-Identifier: Apache-2.0
+-- asm
+mov %r2, %r1
+mov %r8, %r1
+ldxb %r2, [%r2+0x08]
+ldxb %r8, [%r8+0x7f]
+mov %r0, %r2
+add %r0, %r8
+exit
+-- mem
+aa bb 11 cc dd ee ff 11
+02 bb 11 cc dd ee ff 11
+aa bb 11 cc dd ee ff 11
+aa bb 11 cc dd ee ff 11
+aa bb 11 cc dd ee ff 11
+aa bb 11 cc dd ee ff 11
+aa bb 11 cc dd ee ff 11
+aa bb 11 cc dd ee ff 11
+aa bb 11 cc dd ee ff 11
+aa bb 11 cc dd ee ff 11
+aa bb 11 cc dd ee ff 11
+aa bb 11 cc dd ee ff 11
+aa bb 11 cc dd ee ff 11
+aa bb 11 cc dd ee ff 11
+aa bb 11 cc dd ee ff 11
+aa bb 11 cc dd ee ff 03
+aa bb 11 cc dd ee ff 11
+aa bb 11 cc dd ee ff 11
+-- result
+0x05

vm/ubpf_jit_x86_64.h

@@ -161,18 +161,47 @@ emit_modrm_reg2reg(struct jit_state* state, int r, int m)
     emit_modrm(state, 0xc0, r, m);
 }
 
+/**
+ * @brief Emit a ModRM byte and accompanying displacement.
+ * Special case for the situation where the displacement is 0.
+ *
+ * @param[in] state The JIT state in which to emit this instruction.
+ * @param[in] reg The value for the reg of the ModRM byte.
+ * @param[in] rm The value for the rm of the ModRM byte.
+ * @param[in] d The displacement value
+ */
 static inline void
-emit_modrm_and_displacement(struct jit_state* state, int r, int m, int32_t d)
+emit_modrm_and_displacement(struct jit_state* state, int reg, int rm, int32_t d)
 {
-    if (d == 0 && (m & 7) != RBP) {
-        emit_modrm(state, 0x00, r, m);
-    } else if (d >= -128 && d <= 127) {
-        emit_modrm(state, 0x40, r, m);
+    rm &= 0xf;
+    reg &= 0xf;
+
+    // Handle 0 displacement special (where we can!).
+    if (d == 0 && rm != RSP && rm != RBP && rm != R12 && rm != R13) {
+        emit_modrm(state, 0x00, reg, rm);
+        return;
+    }
+
+    uint32_t near_disp = (d >= -128 && d <= 127);
+    uint8_t mod = near_disp ? 0x40 : 0x80;
+
+    emit_modrm(state, mod, reg, rm);
+    if (rm == R12) {
+        // When using R12 as the rm in (rm + disp), the actual
+        // rm has to be put in an SIB. SIB value of 0x24 means:
+        // scale (of index): N/A (see below)
+        // index: no index
+        // base: R12
+        // A SIB byte with this value means that the resulting
+        // encoded instruction will mimic the semantics when
+        // using any other register.
+        emit1(state, 0x24);
+    }
+
+    if (near_disp)
         emit1(state, d);
-    } else {
-        emit_modrm(state, 0x80, r, m);
+    else
         emit4(state, d);
-    }
 }
 
 static inline void