Optimize for common drop_keep cases in wasmi executor

crates/wasmi/src/engine/executor.rs

@@ -86,9 +86,9 @@ impl<'ctx, 'engine, 'func, HostData> Executor<'ctx, 'engine, 'func, HostData> {
                 Instr::LocalGet { local_depth } => self.visit_local_get(local_depth),
                 Instr::LocalSet { local_depth } => self.visit_local_set(local_depth),
                 Instr::LocalTee { local_depth } => self.visit_local_tee(local_depth),
-                Instr::Br(target) => self.visit_br(target),
-                Instr::BrIfEqz(target) => self.visit_br_if_eqz(target),
-                Instr::BrIfNez(target) => self.visit_br_if_nez(target),
+                Instr::Br(params) => self.visit_br(params),
+                Instr::BrIfEqz(params) => self.visit_br_if_eqz(params),
+                Instr::BrIfNez(params) => self.visit_br_if_nez(params),
                 Instr::ReturnIfNez(drop_keep) => {
                     if let MaybeReturn::Return = self.visit_return_if_nez(drop_keep) {
                         return Ok(CallOutcome::Return);
@@ -447,7 +447,8 @@ impl<'ctx, 'engine, 'func, HostData> Executor<'ctx, 'engine, 'func, HostData> {
         f: fn(UntypedValue) -> Result<UntypedValue, TrapCode>,
     ) -> Result<(), TrapCode> {
         self.value_stack.try_eval_top(f)?;
-        self.try_next_instr()
+        self.next_instr();
+        Ok(())
     }
 
     fn execute_binary(&mut self, f: fn(UntypedValue, UntypedValue) -> UntypedValue) {
@@ -460,7 +461,8 @@ impl<'ctx, 'engine, 'func, HostData> Executor<'ctx, 'engine, 'func, HostData> {
         f: fn(UntypedValue, UntypedValue) -> Result<UntypedValue, TrapCode>,
     ) -> Result<(), TrapCode> {
         self.value_stack.try_eval_top2(f)?;
-        self.try_next_instr()
+        self.next_instr();
+        Ok(())
     }
 
     fn execute_reinterpret<T, U>(&mut self)
@@ -472,33 +474,24 @@ impl<'ctx, 'engine, 'func, HostData> Executor<'ctx, 'engine, 'func, HostData> {
         self.next_instr()
     }
 
-    fn try_next_instr(&mut self) -> Result<(), TrapCode> {
-        self.next_instr();
-        Ok(())
+    fn next_instr(&mut self) {
+        self.ip_add(1)
     }
 
-    fn next_instr(&mut self) {
-        // Safety: This is safe since we carefully constructed the `wasmi`
-        //         bytecode in conjunction with Wasm validation so that the
-        //         offsets of the instruction pointer within the sequence of
-        //         instructions never make the instruction pointer point out
-        //         of bounds of the instructions that belong to the function
-        //         that is currently executed.
-        unsafe {
-            self.ip.offset(1);
-        };
+    fn branch_to(&mut self, params: BranchParams) {
+        self.value_stack.drop_keep(params.drop_keep());
+        self.ip_add(params.offset().into_i32() as isize)
     }
 
-    fn branch_to(&mut self, target: BranchParams) {
-        self.value_stack.drop_keep(target.drop_keep());
+    fn ip_add(&mut self, delta: isize) {
         // Safety: This is safe since we carefully constructed the `wasmi`
         //         bytecode in conjunction with Wasm validation so that the
         //         offsets of the instruction pointer within the sequence of
         //         instructions never make the instruction pointer point out
         //         of bounds of the instructions that belong to the function
         //         that is currently executed.
         unsafe {
-            self.ip.offset(target.offset().into_i32() as isize);
+            self.ip.offset(delta);
         }
     }
 
@@ -530,23 +523,23 @@ impl<'ctx, 'engine, 'func, HostData> Executor<'ctx, 'engine, 'func, HostData> {
         Err(TrapCode::Unreachable).map_err(Into::into)
     }
 
-    fn visit_br(&mut self, target: BranchParams) {
-        self.branch_to(target)
+    fn visit_br(&mut self, params: BranchParams) {
+        self.branch_to(params)
     }
 
-    fn visit_br_if_eqz(&mut self, target: BranchParams) {
+    fn visit_br_if_eqz(&mut self, params: BranchParams) {
         let condition = self.value_stack.pop_as();
         if condition {
             self.next_instr()
         } else {
-            self.branch_to(target)
+            self.branch_to(params)
         }
     }
 
-    fn visit_br_if_nez(&mut self, target: BranchParams) {
+    fn visit_br_if_nez(&mut self, params: BranchParams) {
         let condition = self.value_stack.pop_as();
         if condition {
-            self.branch_to(target)
+            self.branch_to(params)
         } else {
             self.next_instr()
         }

crates/wasmi/src/engine/stack/values/vref.rs

@@ -111,12 +111,20 @@ impl<'a> ValueStackRef<'a> {
             return;
         }
         let keep = drop_keep.keep();
-        // Copy kept values over to their new place on the stack.
-        // Note: We cannot use `memcpy` since the slices may overlap.
-        let src = self.stack_ptr - keep;
-        let dst = self.stack_ptr - keep - drop;
-        for i in 0..keep {
-            *self.get_release_unchecked_mut(dst + i) = self.get_release_unchecked(src + i);
+        if keep == 0 {
+            // Bail out early when there are no values to keep.
+        } else if keep == 1 {
+            // Bail out early when there is only one value to copy.
+            *self.get_release_unchecked_mut(self.stack_ptr - 1 - drop) =
+                self.get_release_unchecked(self.stack_ptr - 1);
+        } else {
+            // Copy kept values over to their new place on the stack.
+            // Note: We cannot use `memcpy` since the slices may overlap.
+            let src = self.stack_ptr - keep;
+            let dst = self.stack_ptr - keep - drop;
+            for i in 0..keep {
+                *self.get_release_unchecked_mut(dst + i) = self.get_release_unchecked(src + i);
+            }
         }
         self.stack_ptr -= drop;
     }