interpreter specialization for whamm probe (whamm trampoline)

src/engine/BytecodeIterator.v3

@@ -50,7 +50,7 @@ class BytecodeIterator {
 
 		// Read the first byte of the code
 		var b = codeptr.read1();
-		if (b == InternalOpcode.PROBE.code) { // probe is inserted here
+		if (b == InternalOpcode.PROBE.code || b == InternalOpcode.WHAMM_PROBE.code) { // probe is inserted here
 			b = origptr.reset(func.orig_bytecode, pc, func.orig_bytecode.length).read1();
 		}
 		// Query opcode attributes array
@@ -101,9 +101,10 @@ class BytecodeIterator {
 
 		// Read the first byte of the code
 		var b = codeptr.read1();
-		if (b == InternalOpcode.PROBE.code) { // probe is inserted here
+		if (b == InternalOpcode.PROBE.code || b == InternalOpcode.WHAMM_PROBE.code) { // probe is inserted here
 			v.visitProbe();
 			b = origptr.reset(func.orig_bytecode, pc, func.orig_bytecode.length).read1();
+			codeptr.reset(func.cur_bytecode, pc + 1, func.cur_bytecode.length);
 		}
 		// Query opcode attributes array
 		var opcode: Opcode;

src/engine/CodePtr.v3

@@ -14,7 +14,7 @@ class CodePtr extends DataReader {
 	}
 	def read_opcode_but_skip_probe(func: FuncDecl) -> Opcode {
 		var pc = pos, b = read1();
-		if (b == InternalOpcode.PROBE.code) b = func.orig_bytecode[pc];
+		if (b == InternalOpcode.PROBE.code || b == InternalOpcode.WHAMM_PROBE.code) b = func.orig_bytecode[pc];
 		var op = Opcodes.opcode_by_prefix[b];
 		if (op != Opcode.INVALID) return op;
 		return if(Opcodes.isPrefix(b), Opcodes.find(b, read_uleb32()));

src/engine/Engine.v3

@@ -25,7 +25,7 @@ class Engine {
 		if (data == null) return FileLoadResult.FileNotFound(path);
 		var limits = Limits.new().set(extensions);
 		var bp = BinParser.new(extensions, limits, path);
-		bp.tiering = tiering_override;
+		bp.tiering = if(tiering_override != null, tiering_override, Execute.tiering);
 		var r = bp.push(data, 0, data.length).finish();
 		match (r) {
 			Ok(module) =>

src/engine/Instrumentation.v3

@@ -38,18 +38,23 @@ component Instrumentation {
 		match (probe) {
 			l: ProbeList => {
 				l.add(p);
+				func.activateProbingAt(offset, InternalOpcode.PROBE.code);
 				Execute.tiering.onFuncProbeInsertN(module, func, offset, p);
 			}
 			null => {
 				map[offset] = p;
-				func.activateProbingAt(offset);
+				func.activateProbingAt(offset,
+					if (WhammProbe.?(p) && FastIntTuning.enableWhammProbeTrampoline,
+						InternalOpcode.WHAMM_PROBE.code,
+						InternalOpcode.PROBE.code));
 				Execute.tiering.onFuncProbeInsert1(module, func, offset, p);
 			}
 			_ => {
 				var list = ProbeList.new();
 				list.add(probe);
 				list.add(p);
 				map[offset] = list;
+				func.activateProbingAt(offset, InternalOpcode.PROBE.code);
 				Execute.tiering.onFuncProbeInsert1(module, func, offset, p);
 			}
 		}

src/engine/Module.v3

@@ -131,14 +131,15 @@ class FuncDecl(sig_index: int) extends Decl {
 	def setOrigCode(code: Array<byte>) -> this {
 		cur_bytecode = orig_bytecode = code;
 		var tc: TargetCode;
+		var tr: TargetCode;
 		target_code = tc; // reset target code as well
 		sidetable = Sidetables.NO_SIDETABLE;
 	}
-	def activateProbingAt(pc: int) {
+	def activateProbingAt(pc: int, probe_byte: byte) {
 		if (pc == 0) return void(entry_probed = true); // special case for function entry
 		// "orig" will become a copy of the original code, to allow in-place modification of old code
 		if (cur_bytecode == orig_bytecode) orig_bytecode = Arrays.dup(orig_bytecode);
-		cur_bytecode[pc] = InternalOpcode.PROBE.code;
+		cur_bytecode[pc] = probe_byte;
 	}
 	def deactiveProbingAt(pc: int) {
 		if (pc == 0) return;

src/engine/Opcodes.v3

@@ -677,6 +677,7 @@ component ImmSigs {
 // Internal opcodes used by the interpreter.
 enum InternalOpcode(code: u8, mnemonic: string) {
 	PROBE(0x1E, "<probe>"),	// Used to overwrite a bytecode where a probe has been inserted
+	WHAMM_PROBE(0x1D, "<whamm-probe>"),
 	//PROBE_COUNTER
 	//PROBE_COUNTER_n
 	//PROBE_TOS_i
@@ -1031,7 +1032,7 @@ class InstrTracer {
 			op = Opcodes.find(b, b2);
 			if (op == Opcode.INVALID) out.put2("%x %x <invalid>", b, b2);
 			else out.puts(op.mnemonic);
-		} else if (b == InternalOpcode.PROBE.code) {
+		} else if (b == InternalOpcode.PROBE.code || b == InternalOpcode.WHAMM_PROBE.code) {
 			out.put1("<probe>", b);
 			return;
 		} else {

src/engine/Tuning.v3

@@ -34,6 +34,7 @@ component FastIntTuning {
 	def fourByteSidetable = true;	// sidetable entries are 4-bytes
 	def entryTierUpDecrement = 1;	// "cost" of entering a function in the interpreter
 	def loopTierUpDecrement = 1;	// "cost" of looping in the interpreter
+	def enableWhammProbeTrampoline = true;
 }
 
 // Tuning settings for the single-pass compiler that have no effect on correctness.
@@ -53,4 +54,5 @@ component SpcTuning {
 	var inlineSmallFunc = true;         // inline small functions, currently only applicable for whamm probes
 	def probeCallFreesRegs = true;		// probe calls frees registers in abstract state
 	def runtimeCallFreesRegs = true;	// runtime calls frees registers in abstract state
+	def intrinsifyMemoryProbes = true;
 }

src/engine/Value.v3

@@ -10,6 +10,34 @@ type Value {
 	case F32(bits: u32);
 	case F64(bits: u64);
 	case V128(low: u64, high: u64);
+
+	def equal(that: Value) -> bool {
+		if (this == that) return true;
+		if (Value.Ref.?(this) == Value.Ref.?(that)) {
+			return Value.Ref.!(this).val == Value.Ref.!(that).val;
+		}
+		if (Value.I31.?(this) == Value.I31.?(that)) {
+			return Value.I31.!(this).val == Value.I31.!(that).val;
+		}
+		if (Value.I32.?(this) == Value.I32.?(that)) {
+			return Value.I32.!(this).val == Value.I32.!(that).val;
+		}
+		if (Value.I64.?(this) == Value.I64.?(that)) {
+			return Value.I64.!(this).val == Value.I64.!(that).val;
+		}
+		if (Value.F32.?(this) == Value.F32.?(that)) {
+			return Value.F32.!(this).bits == Value.F32.!(that).bits;
+		}
+		if (Value.F64.?(this) == Value.F64.?(that)) {
+			return Value.F64.!(this).bits == Value.F64.!(that).bits;
+		}
+		if (Value.V128.?(this) == Value.V128.?(that)) {
+			var a = Value.V128.!(this);
+			var b = Value.V128.!(that);
+			return a.low == b.low && a.high == b.high;
+		}
+		return false;
+	}
 }
 
 // Categorization of values into storage kinds.

src/engine/compiler/MacroAssembler.v3

@@ -165,6 +165,7 @@ class MacroAssembler(valuerep: Tagging, regConfig: RegConfig) {
 	}
 
 	// Architecture-specific load and store routines for Wasm load/store.
+	// if `intrinsify_probe` is true, writes the effective address to scratch register.
 	def emit_loadbsx_r_r_r_i(kind: ValueKind, dst: Reg, base: Reg, index: Reg, offset: u32);
 	def emit_loadbzx_r_r_r_i(kind: ValueKind, dst: Reg, base: Reg, index: Reg, offset: u32);
 	def emit_loadwsx_r_r_r_i(kind: ValueKind, dst: Reg, base: Reg, index: Reg, offset: u32);
@@ -173,9 +174,9 @@ class MacroAssembler(valuerep: Tagging, regConfig: RegConfig) {
 	def emit_loaddzx_r_r_r_i(kind: ValueKind, dst: Reg, base: Reg, index: Reg, offset: u32);
 	def emit_load_r_r_r_i(kind: ValueKind, dst: Reg, base: Reg, index: Reg, offset: u32);
 
-	def emit_storeb_r_r_r_i(kind: ValueKind, val: Reg, base: Reg, index: Reg, offset: u32);
-	def emit_storew_r_r_r_i(kind: ValueKind, val: Reg, base: Reg, index: Reg, offset: u32);
-	def emit_store_r_r_r_i(kind: ValueKind, val: Reg, base: Reg, index: Reg, offset: u32);
+	def emit_storeb_r_r_r_i(kind: ValueKind, val: Reg, base: Reg, mirror_base: Reg, index: Reg, offset: u32);
+	def emit_storew_r_r_r_i(kind: ValueKind, val: Reg, base: Reg, mirror_base: Reg, index: Reg, offset: u32);
+	def emit_store_r_r_r_i(kind: ValueKind, val: Reg, base: Reg, mirror_base: Reg, index: Reg, offset: u32);
 
 	def emit_mov_r_r(kind: ValueKind, reg: Reg, reg2: Reg);
 	def emit_mov_r_m(kind: ValueKind, reg: Reg, addr: MasmAddr);
@@ -246,6 +247,8 @@ class MacroAssembler(valuerep: Tagging, regConfig: RegConfig) {
 	def emit_call_runtime_getFrameAccessorMetaRef();
 	def emit_increment_CountProbe(tmp: Reg, probe: CountProbe, increment: u64);
 	def emit_call_OperandProbe_i_v_fire(probe: OperandProbe_i_v, value_reg: Reg);
+	def emit_call_MemoryReadProbe_fire(probe: MemoryReadProbe);
+	def emit_call_MemoryWriteProbe_fire(probe: MemoryWriteProbe);
 
 	def emit_debugger_breakpoint();