From 5a2db7764927430f99a8f0c688adc84c7355a64e Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 28 Apr 2025 11:16:38 +0200 Subject: [PATCH 001/343] create stack2 module for stack based translation --- .../src/engine/translator/stack2/consts.rs | 2 + .../src/engine/translator/stack2/locals.rs | 15 +++ .../wasmi/src/engine/translator/stack2/mod.rs | 124 ++++++++++++++++++ 3 files changed, 141 insertions(+) create mode 100644 crates/wasmi/src/engine/translator/stack2/consts.rs create mode 100644 crates/wasmi/src/engine/translator/stack2/locals.rs create mode 100644 crates/wasmi/src/engine/translator/stack2/mod.rs diff --git a/crates/wasmi/src/engine/translator/stack2/consts.rs b/crates/wasmi/src/engine/translator/stack2/consts.rs new file mode 100644 index 0000000000..eafac51f16 --- /dev/null +++ b/crates/wasmi/src/engine/translator/stack2/consts.rs @@ -0,0 +1,2 @@ +#[derive(Debug, Clone)] +pub struct ConstRegistry {} diff --git a/crates/wasmi/src/engine/translator/stack2/locals.rs b/crates/wasmi/src/engine/translator/stack2/locals.rs new file mode 100644 index 0000000000..13866cd70a --- /dev/null +++ b/crates/wasmi/src/engine/translator/stack2/locals.rs @@ -0,0 +1,15 @@ +use crate::core::ValType; +use alloc::vec::Vec; + +#[derive(Debug, Clone)] +pub struct LocalsRegistry { + groups: Vec, + len_locals: usize, +} + +#[derive(Debug, Copy, Clone)] +struct LocalGroup { + start_idx: usize, + len: usize, + ty: ValType, +} diff --git a/crates/wasmi/src/engine/translator/stack2/mod.rs b/crates/wasmi/src/engine/translator/stack2/mod.rs new file mode 100644 index 0000000000..7e42220657 --- /dev/null +++ b/crates/wasmi/src/engine/translator/stack2/mod.rs @@ -0,0 +1,124 @@ +mod consts; +mod locals; + +use self::{consts::ConstRegistry, locals::LocalsRegistry}; +use crate::core::{TypedVal, ValType}; +use alloc::vec::Vec; +use core::num::NonZeroUsize; + +#[derive(Debug, Clone)] +pub struct ValueStack { + operands: Vec, + consts: ConstRegistry, + locals: LocalsRegistry, + first_local: Option, + last_local: Option, + max_stack_height: usize, +} + +#[derive(Debug, Copy, Clone)] +pub struct OperandIdx(NonZeroUsize); + +impl From for usize { + fn from(value: OperandIdx) -> Self { + value.0.get() + } +} + +impl From for OperandIdx { + fn from(value: usize) -> Self { + let Some(operand_idx) = NonZeroUsize::new(value.wrapping_add(1)) else { + panic!("out of bounds `OperandIdx`: {value}") + }; + Self(operand_idx) + } +} + +#[derive(Debug, Copy, Clone)] +enum StackOperand { + Local { + index: LocalIdx, + prev_local: Option, + next_local: Option, + }, + Temp { + ty: ValType, + }, + Immediate { + val: TypedVal, + }, +} + +#[derive(Debug, Copy, Clone)] +pub enum Operand { + Local(LocalOperand), + Temp(TempOperand), + Immediate(ImmediateOperand), +} + +impl Operand { + pub fn is_local(self) -> bool { + matches!(self, Self::Local(_)) + } + + pub fn is_temp(self) -> bool { + matches!(self, Self::Temp(_)) + } + + pub fn is_immediate(self) -> bool { + matches!(self, Self::Immediate(_)) + } + + pub fn ty(self) -> ValType { + match self { + Self::Local(local_operand) => local_operand.ty(), + Self::Temp(temp_operand) => temp_operand.ty(), + Self::Immediate(immediate_operand) => immediate_operand.ty(), + } + } +} + +#[derive(Debug, Copy, Clone)] +pub struct LocalOperand { + index: LocalIdx, + ty: ValType, +} + +impl LocalOperand { + pub fn index(self) -> LocalIdx { + self.index + } + + pub fn ty(self) -> ValType { + self.ty + } +} + +#[derive(Debug, Copy, Clone)] +pub struct TempOperand { + ty: ValType, +} + +impl TempOperand { + pub fn ty(self) -> ValType { + self.ty + } +} + +#[derive(Debug, Copy, Clone)] +pub struct ImmediateOperand { + val: TypedVal, +} + +impl ImmediateOperand { + pub fn val(self) -> TypedVal { + self.val + } + + pub fn ty(self) -> ValType { + self.val.ty() + } +} + +#[derive(Debug, Copy, Clone)] +pub struct LocalIdx(usize); From 5175bceb434b5d7e9911472878b83af410c10e9c Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Sat, 10 May 2025 12:23:16 +0200 Subject: [PATCH 002/343] add docs --- .../wasmi/src/engine/translator/stack2/mod.rs | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/crates/wasmi/src/engine/translator/stack2/mod.rs b/crates/wasmi/src/engine/translator/stack2/mod.rs index 7e42220657..539884e6fe 100644 --- a/crates/wasmi/src/engine/translator/stack2/mod.rs +++ b/crates/wasmi/src/engine/translator/stack2/mod.rs @@ -8,11 +8,18 @@ use core::num::NonZeroUsize; #[derive(Debug, Clone)] pub struct ValueStack { +/// The Wasm value stack during translation from Wasm to Wasmi bytecode. + /// The stack of operands. operands: Vec, + /// All function local constants. consts: ConstRegistry, + /// All function parameters and locals and their types. locals: LocalsRegistry, + /// The index of the first [`StackOperand::Local`] on the [`Stack`]. first_local: Option, + /// The index of the last [`StackOperand::Local`] on the [`Stack`]. last_local: Option, + /// The maximum size of the [`Stack`]. max_stack_height: usize, } @@ -34,41 +41,61 @@ impl From for OperandIdx { } } +/// An [`Operand`] on the [`Stack`]. +/// +/// This is the internal version of [`Operand`] with information that shall remain +/// hidden to the outside. #[derive(Debug, Copy, Clone)] enum StackOperand { + /// A local variable. Local { + /// The index of the local variable. index: LocalIdx, + /// The previous [`StackOperand::Local`] on the [`Stack`]. prev_local: Option, + /// The next [`StackOperand::Local`] on the [`Stack`]. next_local: Option, }, + /// A temporary value on the [`Stack`]. Temp { + /// The type of the temporary value. ty: ValType, }, + /// An immediate value on the [`Stack`]. Immediate { + /// The value (and type) of the immediate value. val: TypedVal, }, } +/// An operand on the [`Stack`]. #[derive(Debug, Copy, Clone)] pub enum Operand { + /// A local variable operand. Local(LocalOperand), + /// A temporary operand. Temp(TempOperand), + /// An immediate value operand. Immediate(ImmediateOperand), } impl Operand { + /// Returns `true` if `self` is an [`Operand::Local`]. pub fn is_local(self) -> bool { matches!(self, Self::Local(_)) } + /// Returns `true` if `self` is an [`Operand::Temp`]. pub fn is_temp(self) -> bool { matches!(self, Self::Temp(_)) } + /// Returns `true` if `self` is an [`Operand::Immediate`]. pub fn is_immediate(self) -> bool { matches!(self, Self::Immediate(_)) } + /// Returns the type of the [`Operand`]. pub fn ty(self) -> ValType { match self { Self::Local(local_operand) => local_operand.ty(), @@ -78,47 +105,60 @@ impl Operand { } } +/// A local variable on the [`Stack`]. #[derive(Debug, Copy, Clone)] pub struct LocalOperand { + /// The index of the local variable. index: LocalIdx, + /// The type of the local variable. ty: ValType, } impl LocalOperand { + /// Returns the index of the local variable. pub fn index(self) -> LocalIdx { self.index } + /// Returns the type of the local variable. pub fn ty(self) -> ValType { self.ty } } +/// A temporary on the [`Stack`]. #[derive(Debug, Copy, Clone)] pub struct TempOperand { + /// The type of the temporary. ty: ValType, } impl TempOperand { + /// Returns the type of the temporary. pub fn ty(self) -> ValType { self.ty } } +/// An immediate value on the [`Stack`]. #[derive(Debug, Copy, Clone)] pub struct ImmediateOperand { + /// The value and type of the immediate value. val: TypedVal, } impl ImmediateOperand { + /// Returns the value (and type) of the immediate value. pub fn val(self) -> TypedVal { self.val } + /// Returns the type of the immediate value. pub fn ty(self) -> ValType { self.val.ty() } } +/// A local variable index. #[derive(Debug, Copy, Clone)] pub struct LocalIdx(usize); From c761cf96455576803e8c02ca51d97a12364c04fb Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Sat, 10 May 2025 12:23:41 +0200 Subject: [PATCH 003/343] rename ValueStack -> Stack --- crates/wasmi/src/engine/translator/stack2/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/stack2/mod.rs b/crates/wasmi/src/engine/translator/stack2/mod.rs index 539884e6fe..8f39f9486e 100644 --- a/crates/wasmi/src/engine/translator/stack2/mod.rs +++ b/crates/wasmi/src/engine/translator/stack2/mod.rs @@ -7,8 +7,8 @@ use alloc::vec::Vec; use core::num::NonZeroUsize; #[derive(Debug, Clone)] -pub struct ValueStack { /// The Wasm value stack during translation from Wasm to Wasmi bytecode. +pub struct Stack { /// The stack of operands. operands: Vec, /// All function local constants. From e209b4e298d3484d7214965b347b16555d13d537 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Sat, 10 May 2025 12:23:58 +0200 Subject: [PATCH 004/343] add missing docs to OperandIdx --- crates/wasmi/src/engine/translator/stack2/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/wasmi/src/engine/translator/stack2/mod.rs b/crates/wasmi/src/engine/translator/stack2/mod.rs index 8f39f9486e..6352f84ff7 100644 --- a/crates/wasmi/src/engine/translator/stack2/mod.rs +++ b/crates/wasmi/src/engine/translator/stack2/mod.rs @@ -23,6 +23,7 @@ pub struct Stack { max_stack_height: usize, } +/// A [`StackOperand`] or [`Operand`] index on the [`Stack`]. #[derive(Debug, Copy, Clone)] pub struct OperandIdx(NonZeroUsize); From b108917cbdcd640a83f8e7ed06d9ab5bc2d9e59f Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Sat, 10 May 2025 12:24:10 +0200 Subject: [PATCH 005/343] add Default impl to Stack --- crates/wasmi/src/engine/translator/stack2/consts.rs | 2 +- crates/wasmi/src/engine/translator/stack2/locals.rs | 2 +- crates/wasmi/src/engine/translator/stack2/mod.rs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/wasmi/src/engine/translator/stack2/consts.rs b/crates/wasmi/src/engine/translator/stack2/consts.rs index eafac51f16..312219e9ae 100644 --- a/crates/wasmi/src/engine/translator/stack2/consts.rs +++ b/crates/wasmi/src/engine/translator/stack2/consts.rs @@ -1,2 +1,2 @@ -#[derive(Debug, Clone)] +#[derive(Debug, Default, Clone)] pub struct ConstRegistry {} diff --git a/crates/wasmi/src/engine/translator/stack2/locals.rs b/crates/wasmi/src/engine/translator/stack2/locals.rs index 13866cd70a..d016601e20 100644 --- a/crates/wasmi/src/engine/translator/stack2/locals.rs +++ b/crates/wasmi/src/engine/translator/stack2/locals.rs @@ -1,7 +1,7 @@ use crate::core::ValType; use alloc::vec::Vec; -#[derive(Debug, Clone)] +#[derive(Debug, Default, Clone)] pub struct LocalsRegistry { groups: Vec, len_locals: usize, diff --git a/crates/wasmi/src/engine/translator/stack2/mod.rs b/crates/wasmi/src/engine/translator/stack2/mod.rs index 6352f84ff7..13495f46ce 100644 --- a/crates/wasmi/src/engine/translator/stack2/mod.rs +++ b/crates/wasmi/src/engine/translator/stack2/mod.rs @@ -6,8 +6,8 @@ use crate::core::{TypedVal, ValType}; use alloc::vec::Vec; use core::num::NonZeroUsize; -#[derive(Debug, Clone)] /// The Wasm value stack during translation from Wasm to Wasmi bytecode. +#[derive(Debug, Default, Clone)] pub struct Stack { /// The stack of operands. operands: Vec, From 5346abfe1835edb62913286a4b71601f8c425695 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Sat, 10 May 2025 12:24:19 +0200 Subject: [PATCH 006/343] add StackPhase to Stack --- .../wasmi/src/engine/translator/stack2/mod.rs | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/crates/wasmi/src/engine/translator/stack2/mod.rs b/crates/wasmi/src/engine/translator/stack2/mod.rs index 13495f46ce..7ea54a8038 100644 --- a/crates/wasmi/src/engine/translator/stack2/mod.rs +++ b/crates/wasmi/src/engine/translator/stack2/mod.rs @@ -21,6 +21,55 @@ pub struct Stack { last_local: Option, /// The maximum size of the [`Stack`]. max_stack_height: usize, + /// The current phase of the [`Stack`]. + phase: StackPhase, +} + +/// The current phase of the [`Stack`]. +#[derive(Debug, Default, Copy, Clone)] +pub enum StackPhase { + /// Phase that allows to define local variables. + #[default] + DefineLocals, + /// Phase that allows to manipulate the stack and allocate function local constants. + Translation, + /// Phase after finishing translation. + /// + /// In this phase state changes are no longer allowed. + /// Only resetting the [`Stack`] is allowed in order to restart the phase cycle. + Finish, +} + +impl StackPhase { + /// Resets the [`StackPhase`] to the [`StackPhase::DefineLocals`] phase. + pub fn reset(&mut self) { + *self = Self::default(); + } + + /// Ensures that the current phase is [`StackPhase::DefineLocals`]. + pub fn assert_define_locals(&self) { + assert!(matches!(self, Self::DefineLocals)); + } + + /// Turns the current phase into [`StackPhase::Translation`]. + /// + /// # Panics + /// + /// If the current phase is incompatible with this phase shift. + pub fn assert_translation(&mut self) { + assert!(matches!(self, Self::DefineLocals | Self::Translation)); + *self = Self::Translation; + } + + /// Turns the current phase into [`StackPhase::Finish`]. + /// + /// # Panics + /// + /// If the current phase is incompatible with this phase shift. + pub fn assert_finish(&mut self) { + assert!(matches!(self, Self::Translation | Self::Finish)); + *self = Self::Finish; + } } /// A [`StackOperand`] or [`Operand`] index on the [`Stack`]. From 7290092b6df96011ff5aab623c25e8927a976d4c Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Sun, 11 May 2025 21:01:22 +0200 Subject: [PATCH 007/343] apply rustfmt --- crates/wasmi/src/engine/translator/stack2/mod.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/wasmi/src/engine/translator/stack2/mod.rs b/crates/wasmi/src/engine/translator/stack2/mod.rs index 7ea54a8038..e9c490ddcc 100644 --- a/crates/wasmi/src/engine/translator/stack2/mod.rs +++ b/crates/wasmi/src/engine/translator/stack2/mod.rs @@ -34,7 +34,7 @@ pub enum StackPhase { /// Phase that allows to manipulate the stack and allocate function local constants. Translation, /// Phase after finishing translation. - /// + /// /// In this phase state changes are no longer allowed. /// Only resetting the [`Stack`] is allowed in order to restart the phase cycle. Finish, @@ -52,9 +52,9 @@ impl StackPhase { } /// Turns the current phase into [`StackPhase::Translation`]. - /// + /// /// # Panics - /// + /// /// If the current phase is incompatible with this phase shift. pub fn assert_translation(&mut self) { assert!(matches!(self, Self::DefineLocals | Self::Translation)); @@ -62,9 +62,9 @@ impl StackPhase { } /// Turns the current phase into [`StackPhase::Finish`]. - /// + /// /// # Panics - /// + /// /// If the current phase is incompatible with this phase shift. pub fn assert_finish(&mut self) { assert!(matches!(self, Self::Translation | Self::Finish)); @@ -92,7 +92,7 @@ impl From for OperandIdx { } /// An [`Operand`] on the [`Stack`]. -/// +/// /// This is the internal version of [`Operand`] with information that shall remain /// hidden to the outside. #[derive(Debug, Copy, Clone)] From 3b64f0e8255cbe0651c60c658e6c97d552fb0a65 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 12 May 2025 11:31:27 +0200 Subject: [PATCH 008/343] add LocalIdx type to locals.rs submodule --- crates/wasmi/src/engine/translator/stack2/locals.rs | 4 ++++ crates/wasmi/src/engine/translator/stack2/mod.rs | 9 ++++----- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/crates/wasmi/src/engine/translator/stack2/locals.rs b/crates/wasmi/src/engine/translator/stack2/locals.rs index d016601e20..3757bf979f 100644 --- a/crates/wasmi/src/engine/translator/stack2/locals.rs +++ b/crates/wasmi/src/engine/translator/stack2/locals.rs @@ -1,6 +1,10 @@ use crate::core::ValType; use alloc::vec::Vec; +/// A local variable index. +#[derive(Debug, Copy, Clone)] +pub struct LocalIdx(usize); + #[derive(Debug, Default, Clone)] pub struct LocalsRegistry { groups: Vec, diff --git a/crates/wasmi/src/engine/translator/stack2/mod.rs b/crates/wasmi/src/engine/translator/stack2/mod.rs index e9c490ddcc..ae1d00608b 100644 --- a/crates/wasmi/src/engine/translator/stack2/mod.rs +++ b/crates/wasmi/src/engine/translator/stack2/mod.rs @@ -1,8 +1,11 @@ mod consts; mod locals; -use self::{consts::ConstRegistry, locals::LocalsRegistry}; use crate::core::{TypedVal, ValType}; +use self::{ + consts::ConstRegistry, + locals::{LocalIdx, LocalsRegistry}, +}; use alloc::vec::Vec; use core::num::NonZeroUsize; @@ -208,7 +211,3 @@ impl ImmediateOperand { self.val.ty() } } - -/// A local variable index. -#[derive(Debug, Copy, Clone)] -pub struct LocalIdx(usize); From ede2b8d2d24f74a49e45e8d621a407d1e2b83ac3 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 12 May 2025 11:33:00 +0200 Subject: [PATCH 009/343] add OperandIdx field to each Operand enum variant --- .../wasmi/src/engine/translator/stack2/mod.rs | 50 +++++++++++++++---- 1 file changed, 40 insertions(+), 10 deletions(-) diff --git a/crates/wasmi/src/engine/translator/stack2/mod.rs b/crates/wasmi/src/engine/translator/stack2/mod.rs index ae1d00608b..aef413f9b0 100644 --- a/crates/wasmi/src/engine/translator/stack2/mod.rs +++ b/crates/wasmi/src/engine/translator/stack2/mod.rs @@ -103,7 +103,7 @@ enum StackOperand { /// A local variable. Local { /// The index of the local variable. - index: LocalIdx, + local_index: LocalIdx, /// The previous [`StackOperand::Local`] on the [`Stack`]. prev_local: Option, /// The next [`StackOperand::Local`] on the [`Stack`]. @@ -148,6 +148,15 @@ impl Operand { matches!(self, Self::Immediate(_)) } + /// Returns the [`OperandIdx`] of the [`Operand`]. + pub fn index(&self) -> OperandIdx { + match self { + Operand::Local(operand) => operand.operand_index(), + Operand::Temp(operand) => operand.operand_index(), + Operand::Immediate(operand) => operand.operand_index(), + } + } + /// Returns the type of the [`Operand`]. pub fn ty(self) -> ValType { match self { @@ -161,20 +170,27 @@ impl Operand { /// A local variable on the [`Stack`]. #[derive(Debug, Copy, Clone)] pub struct LocalOperand { + /// The index of the operand. + operand_index: OperandIdx, /// The index of the local variable. - index: LocalIdx, + local_index: LocalIdx, /// The type of the local variable. ty: ValType, } impl LocalOperand { - /// Returns the index of the local variable. - pub fn index(self) -> LocalIdx { - self.index + /// Returns the operand index of the [`LocalOperand`]. + pub fn operand_index(&self) -> OperandIdx { + self.operand_index } - /// Returns the type of the local variable. - pub fn ty(self) -> ValType { + /// Returns the index of the [`LocalOperand`]. + pub fn local_index(&self) -> LocalIdx { + self.local_index + } + + /// Returns the type of the [`LocalOperand`]. + pub fn ty(&self) -> ValType { self.ty } } @@ -182,12 +198,19 @@ impl LocalOperand { /// A temporary on the [`Stack`]. #[derive(Debug, Copy, Clone)] pub struct TempOperand { + /// The index of the operand. + operand_index: OperandIdx, /// The type of the temporary. ty: ValType, } impl TempOperand { - /// Returns the type of the temporary. + /// Returns the operand index of the [`TempOperand`]. + pub fn operand_index(&self) -> OperandIdx { + self.operand_index + } + + /// Returns the type of the [`TempOperand`]. pub fn ty(self) -> ValType { self.ty } @@ -196,17 +219,24 @@ impl TempOperand { /// An immediate value on the [`Stack`]. #[derive(Debug, Copy, Clone)] pub struct ImmediateOperand { + /// The index of the operand. + operand_index: OperandIdx, /// The value and type of the immediate value. val: TypedVal, } impl ImmediateOperand { - /// Returns the value (and type) of the immediate value. + /// Returns the operand index of the [`ImmediateOperand`]. + pub fn operand_index(&self) -> OperandIdx { + self.operand_index + } + + /// Returns the immediate value (and its type) of the [`ImmediateOperand`]. pub fn val(self) -> TypedVal { self.val } - /// Returns the type of the immediate value. + /// Returns the type of the [`ImmediateOperand`]. pub fn ty(self) -> ValType { self.val.ty() } From 6f2de524a2969c102c56ea47863dbe054f8d96d5 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 12 May 2025 11:33:33 +0200 Subject: [PATCH 010/343] add conversion from StackOperand -> Operand --- .../src/engine/translator/stack2/locals.rs | 6 ++++++ .../wasmi/src/engine/translator/stack2/mod.rs | 19 +++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/crates/wasmi/src/engine/translator/stack2/locals.rs b/crates/wasmi/src/engine/translator/stack2/locals.rs index 3757bf979f..417c5b4f08 100644 --- a/crates/wasmi/src/engine/translator/stack2/locals.rs +++ b/crates/wasmi/src/engine/translator/stack2/locals.rs @@ -11,6 +11,12 @@ pub struct LocalsRegistry { len_locals: usize, } +impl LocalsRegistry { + pub fn ty(&self, local_index: LocalIdx) -> Option { + todo!() + } +} + #[derive(Debug, Copy, Clone)] struct LocalGroup { start_idx: usize, diff --git a/crates/wasmi/src/engine/translator/stack2/mod.rs b/crates/wasmi/src/engine/translator/stack2/mod.rs index aef413f9b0..e03b8850c4 100644 --- a/crates/wasmi/src/engine/translator/stack2/mod.rs +++ b/crates/wasmi/src/engine/translator/stack2/mod.rs @@ -133,6 +133,25 @@ pub enum Operand { } impl Operand { + fn new(operand_index: OperandIdx, operand: StackOperand, locals: &LocalsRegistry) -> Self { + match operand { + StackOperand::Local { local_index, .. } => { + let Some(ty) = locals.ty(local_index) else { + panic!("failed to query type of local at: {local_index:?}"); + }; + Self::Local(LocalOperand { + operand_index, + local_index, + ty, + }) + } + StackOperand::Temp { ty } => Self::Temp(TempOperand { operand_index, ty }), + StackOperand::Immediate { val } => { + Self::Immediate(ImmediateOperand { operand_index, val }) + } + } + } + /// Returns `true` if `self` is an [`Operand::Local`]. pub fn is_local(self) -> bool { matches!(self, Self::Local(_)) From 78f6ce8937006c8af97b10c0c38707219a781dae Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 12 May 2025 11:33:44 +0200 Subject: [PATCH 011/343] add stub Stack base API --- .../wasmi/src/engine/translator/stack2/mod.rs | 52 ++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/stack2/mod.rs b/crates/wasmi/src/engine/translator/stack2/mod.rs index e03b8850c4..61421d0644 100644 --- a/crates/wasmi/src/engine/translator/stack2/mod.rs +++ b/crates/wasmi/src/engine/translator/stack2/mod.rs @@ -1,11 +1,15 @@ mod consts; mod locals; -use crate::core::{TypedVal, ValType}; use self::{ consts::ConstRegistry, locals::{LocalIdx, LocalsRegistry}, }; +use crate::{ + core::{TypedVal, UntypedVal, ValType}, + ir::Reg, + Error, +}; use alloc::vec::Vec; use core::num::NonZeroUsize; @@ -75,6 +79,52 @@ impl StackPhase { } } +impl Stack { + pub fn register_locals(&mut self, amount: u32) -> Result<(), Error> { + todo!() + } + + pub fn finish_register_locals(&mut self) { + todo!() + } + + pub fn push_local(&mut self, local_idx: u32) -> Result { + todo!() + } + + pub fn push_temp(&mut self, ty: ValType) -> Result { + todo!() + } + + pub fn push_immediate(&mut self, value: impl Into) -> Result { + todo!() + } + + pub fn pop(&mut self) -> Option { + let operand = self.operands.pop()?; + let index = OperandIdx::from(self.operands.len()); + Some(Operand::new(index, operand, &self.locals)) + } + + pub fn pop2(&mut self) -> Option<(Operand, Operand)> { + let [o1, o2] = self.pop_some::<2>()?; + Some((o1, o2)) + } + + pub fn pop3(&mut self) -> Option<(Operand, Operand, Operand)> { + let [o1, o2, o3] = self.pop_some::<3>()?; + Some((o1, o2, o3)) + } + + fn pop_some(&mut self) -> Option<[Operand; N]> { + todo!() + } + + pub fn operand_to_reg(&mut self, index: OperandIdx) -> Result { + todo!() + } +} + /// A [`StackOperand`] or [`Operand`] index on the [`Stack`]. #[derive(Debug, Copy, Clone)] pub struct OperandIdx(NonZeroUsize); From b04848074d89ee57fc1fa1e97680fea82c16ce27 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 12 May 2025 11:41:42 +0200 Subject: [PATCH 012/343] add RegisterSpace API --- .../wasmi/src/engine/translator/stack2/mod.rs | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/crates/wasmi/src/engine/translator/stack2/mod.rs b/crates/wasmi/src/engine/translator/stack2/mod.rs index 61421d0644..17af755169 100644 --- a/crates/wasmi/src/engine/translator/stack2/mod.rs +++ b/crates/wasmi/src/engine/translator/stack2/mod.rs @@ -123,6 +123,38 @@ impl Stack { pub fn operand_to_reg(&mut self, index: OperandIdx) -> Result { todo!() } + + pub fn reg_space(&self, reg: Reg) -> Option { + todo!() + } +} + +/// The [`RegisterSpace`] of a [`Reg`]. +#[derive(Debug, Copy, Clone)] +pub enum RegisterSpace { + /// Register referring to a local variable. + Local, + /// Register referring to a function local constant value. + Const, + /// Register referring to a temporary stack operand. + Temp, +} + +impl RegisterSpace { + /// Returns `true` if `self` is [`RegisterSpace::Local`]. + pub fn is_local(self) -> bool { + matches!(self, Self::Local) + } + + /// Returns `true` if `self` is [`RegisterSpace::Temp`]. + pub fn is_temp(self) -> bool { + matches!(self, Self::Temp) + } + + /// Returns `true` if `self` is [`RegisterSpace::Const`]. + pub fn is_const(self) -> bool { + matches!(self, Self::Const) + } } /// A [`StackOperand`] or [`Operand`] index on the [`Stack`]. From 89084760fc2618a64ccbfd5b951fbab62941c19c Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 12 May 2025 11:41:55 +0200 Subject: [PATCH 013/343] fix some warnings --- crates/wasmi/src/engine/translator/stack2/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/wasmi/src/engine/translator/stack2/mod.rs b/crates/wasmi/src/engine/translator/stack2/mod.rs index 17af755169..dc207d2a46 100644 --- a/crates/wasmi/src/engine/translator/stack2/mod.rs +++ b/crates/wasmi/src/engine/translator/stack2/mod.rs @@ -6,7 +6,7 @@ use self::{ locals::{LocalIdx, LocalsRegistry}, }; use crate::{ - core::{TypedVal, UntypedVal, ValType}, + core::{TypedVal, ValType}, ir::Reg, Error, }; @@ -96,7 +96,7 @@ impl Stack { todo!() } - pub fn push_immediate(&mut self, value: impl Into) -> Result { + pub fn push_immediate(&mut self, value: impl Into) -> Result { todo!() } From da4f399335f98723a9d06d80d47a59ff1ddd8cfa Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 12 May 2025 11:42:03 +0200 Subject: [PATCH 014/343] silence warnings for now --- crates/wasmi/src/engine/translator/stack2/mod.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/wasmi/src/engine/translator/stack2/mod.rs b/crates/wasmi/src/engine/translator/stack2/mod.rs index dc207d2a46..5c305cc0d9 100644 --- a/crates/wasmi/src/engine/translator/stack2/mod.rs +++ b/crates/wasmi/src/engine/translator/stack2/mod.rs @@ -1,3 +1,5 @@ +#![expect(unused_variables, dead_code)] + mod consts; mod locals; From 046ff43c4ed0ba06422fbaccb535fd80a43828f3 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 12 May 2025 11:52:34 +0200 Subject: [PATCH 015/343] add docs to Stack's stub API --- .../wasmi/src/engine/translator/stack2/mod.rs | 56 ++++++++++++++++++- 1 file changed, 54 insertions(+), 2 deletions(-) diff --git a/crates/wasmi/src/engine/translator/stack2/mod.rs b/crates/wasmi/src/engine/translator/stack2/mod.rs index 5c305cc0d9..b6f1970c11 100644 --- a/crates/wasmi/src/engine/translator/stack2/mod.rs +++ b/crates/wasmi/src/engine/translator/stack2/mod.rs @@ -82,50 +82,102 @@ impl StackPhase { } impl Stack { - pub fn register_locals(&mut self, amount: u32) -> Result<(), Error> { + /// Register `amount` local variables of common type `ty`. + /// + /// # Errors + /// + /// If too many local variables are being registered. + pub fn register_locals(&mut self, amount: u32, ty: ValType) -> Result<(), Error> { todo!() } - pub fn finish_register_locals(&mut self) { + /// Finish registration of local variables. + /// + /// # Errors + /// + /// If the current [`StackPhase`] is not [`StackPhase::DefineLocals`]. + pub fn finish_register_locals(&mut self) -> Result<(), Error> { todo!() } + /// Pushes a local variable with index `local_idx` to the [`Stack`]. + /// + /// # Errors + /// + /// - If the current [`StackPhase`] is not [`StackPhase::Translation`]. + /// - If too many operands have been pushed onto the [`Stack`]. + /// - If the local with `local_idx` does not exist. pub fn push_local(&mut self, local_idx: u32) -> Result { todo!() } + /// Pushes a temporary with type `ty` on the [`Stack`]. + /// + /// # Errors + /// + /// - If the current [`StackPhase`] is not [`StackPhase::Translation`]. + /// - If too many operands have been pushed onto the [`Stack`]. pub fn push_temp(&mut self, ty: ValType) -> Result { todo!() } + /// Pushes an immediate `value` on the [`Stack`]. + /// + /// # Errors + /// + /// - If the current [`StackPhase`] is not [`StackPhase::Translation`]. + /// - If too many operands have been pushed onto the [`Stack`]. pub fn push_immediate(&mut self, value: impl Into) -> Result { todo!() } + /// Pops the top-most [`Operand`] from the [`Stack`]. + /// + /// Returns `None` if the [`Stack`] is empty. pub fn pop(&mut self) -> Option { let operand = self.operands.pop()?; let index = OperandIdx::from(self.operands.len()); Some(Operand::new(index, operand, &self.locals)) } + /// Pops the two top-most [`Operand`] from the [`Stack`]. + /// + /// - Returns `None` if the [`Stack`] is empty. + /// - The last returned [`Operand`] is the top-most one. pub fn pop2(&mut self) -> Option<(Operand, Operand)> { let [o1, o2] = self.pop_some::<2>()?; Some((o1, o2)) } + /// Pops the three top-most [`Operand`] from the [`Stack`]. + /// + /// - Returns `None` if the [`Stack`] is empty. + /// - The last returned [`Operand`] is the top-most one. pub fn pop3(&mut self) -> Option<(Operand, Operand, Operand)> { let [o1, o2, o3] = self.pop_some::<3>()?; Some((o1, o2, o3)) } + /// Pops the top-most `N` [`Operand`]s from the [`Stack`]. + /// + /// - Returns `None` if the [`Stack`] is empty. + /// - The last returned [`Operand`] is the top-most one. fn pop_some(&mut self) -> Option<[Operand; N]> { todo!() } + /// Converts the [`Operand`] at `index` to a [`Reg`] if possible. + /// + /// # Errors + /// + /// If the translator ran out of [`Reg`]s for the function. pub fn operand_to_reg(&mut self, index: OperandIdx) -> Result { todo!() } + /// Returns the [`RegisterSpace`] of the [`Reg`]. + /// + /// Returns `None` if the [`Reg`] is unknown to the [`Stack`]. pub fn reg_space(&self, reg: Reg) -> Option { todo!() } From 24e527bdc06204d940cd22311124051d45d98c8c Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 12 May 2025 11:58:58 +0200 Subject: [PATCH 016/343] add Stack::peek API --- crates/wasmi/src/engine/translator/stack2/mod.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/crates/wasmi/src/engine/translator/stack2/mod.rs b/crates/wasmi/src/engine/translator/stack2/mod.rs index b6f1970c11..22e5caf76b 100644 --- a/crates/wasmi/src/engine/translator/stack2/mod.rs +++ b/crates/wasmi/src/engine/translator/stack2/mod.rs @@ -131,6 +131,15 @@ impl Stack { todo!() } + /// Peeks the top-most [`Operand`] on the [`Stack`]. + /// + /// Returns `None` if the [`Stack`] is empty. + pub fn peek(&self) -> Option { + let operand = self.operands.last().copied()?; + let index = OperandIdx::from(self.operands.len() - 1); + Some(Operand::new(index, operand, &self.locals)) + } + /// Pops the top-most [`Operand`] from the [`Stack`]. /// /// Returns `None` if the [`Stack`] is empty. From 30db48f3bcdff665a5b7b2132f195ad74ad0012e Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 12 May 2025 11:59:11 +0200 Subject: [PATCH 017/343] add some inline annotations --- crates/wasmi/src/engine/translator/stack2/mod.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/wasmi/src/engine/translator/stack2/mod.rs b/crates/wasmi/src/engine/translator/stack2/mod.rs index 22e5caf76b..f8ab3695ff 100644 --- a/crates/wasmi/src/engine/translator/stack2/mod.rs +++ b/crates/wasmi/src/engine/translator/stack2/mod.rs @@ -205,16 +205,19 @@ pub enum RegisterSpace { impl RegisterSpace { /// Returns `true` if `self` is [`RegisterSpace::Local`]. + #[inline] pub fn is_local(self) -> bool { matches!(self, Self::Local) } /// Returns `true` if `self` is [`RegisterSpace::Temp`]. + #[inline] pub fn is_temp(self) -> bool { matches!(self, Self::Temp) } /// Returns `true` if `self` is [`RegisterSpace::Const`]. + #[inline] pub fn is_const(self) -> bool { matches!(self, Self::Const) } From 7945270abc68af00714c3a713a160dcf1b3a33f3 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 12 May 2025 13:39:39 +0200 Subject: [PATCH 018/343] add Stack::alloc_const stub API --- crates/wasmi/src/engine/translator/stack2/mod.rs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/crates/wasmi/src/engine/translator/stack2/mod.rs b/crates/wasmi/src/engine/translator/stack2/mod.rs index f8ab3695ff..f93d184947 100644 --- a/crates/wasmi/src/engine/translator/stack2/mod.rs +++ b/crates/wasmi/src/engine/translator/stack2/mod.rs @@ -8,7 +8,7 @@ use self::{ locals::{LocalIdx, LocalsRegistry}, }; use crate::{ - core::{TypedVal, ValType}, + core::{TypedVal, UntypedVal, ValType}, ir::Reg, Error, }; @@ -132,7 +132,7 @@ impl Stack { } /// Peeks the top-most [`Operand`] on the [`Stack`]. - /// + /// /// Returns `None` if the [`Stack`] is empty. pub fn peek(&self) -> Option { let operand = self.operands.last().copied()?; @@ -190,6 +190,15 @@ impl Stack { pub fn reg_space(&self, reg: Reg) -> Option { todo!() } + + /// Allocates a function local constant `value`. + /// + /// # Errors + /// + /// If too many function local constants have been allocated already. + pub fn alloc_const(&mut self, value: impl Into) -> Result { + todo!() + } } /// The [`RegisterSpace`] of a [`Reg`]. From 616996f49af1844054561b4643d7fb12cd92c1af Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 12 May 2025 13:45:33 +0200 Subject: [PATCH 019/343] update locals docs --- crates/wasmi/src/engine/translator/stack2/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/stack2/mod.rs b/crates/wasmi/src/engine/translator/stack2/mod.rs index f93d184947..0385f0a54c 100644 --- a/crates/wasmi/src/engine/translator/stack2/mod.rs +++ b/crates/wasmi/src/engine/translator/stack2/mod.rs @@ -22,7 +22,7 @@ pub struct Stack { operands: Vec, /// All function local constants. consts: ConstRegistry, - /// All function parameters and locals and their types. + /// All function locals and their associated types. locals: LocalsRegistry, /// The index of the first [`StackOperand::Local`] on the [`Stack`]. first_local: Option, From cd34e33b49febcc3e665b71d61a3e0986f7c8c6d Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 12 May 2025 14:03:03 +0200 Subject: [PATCH 020/343] add ConstRegistry impl --- .../src/engine/translator/stack2/consts.rs | 122 +++++++++++++++++- .../wasmi/src/engine/translator/stack2/mod.rs | 7 +- 2 files changed, 127 insertions(+), 2 deletions(-) diff --git a/crates/wasmi/src/engine/translator/stack2/consts.rs b/crates/wasmi/src/engine/translator/stack2/consts.rs index 312219e9ae..10a7df6336 100644 --- a/crates/wasmi/src/engine/translator/stack2/consts.rs +++ b/crates/wasmi/src/engine/translator/stack2/consts.rs @@ -1,2 +1,122 @@ +use crate::{core::UntypedVal, engine::TranslationError, Error}; +use alloc::{ + collections::{btree_map, BTreeMap}, + vec::Vec, +}; +use core::slice::Iter as SliceIter; + +/// A pool of deduplicated function local constant values. +/// +/// - Those constant values are identified by their associated [`ConstIdx`]. +/// - All constant values are also deduplicated so that no duplicates +/// are stored in a [`ConstRegistry`]. This also means that deciding if two +/// [`ConstIdx`] values refer to the equal constant values can be efficiently +/// done by comparing the [`ConstIdx`]s without resolving to their +/// underlying constant values. #[derive(Debug, Default, Clone)] -pub struct ConstRegistry {} +pub struct ConstRegistry { + /// Mapping from constant [`UntypedVal`] values to [`ConstIdx`] indices. + const2idx: BTreeMap, + /// Mapping from [`ConstIdx`] indices to constant [`UntypedVal`] values. + idx2const: Vec, +} + +/// An index referring to a function local constant. +#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub struct ConstIdx(usize); + +impl ConstRegistry { + /// The maximum number of function local constants per function. + const MAX_LEN: usize = (1 << 16) - 1; + + /// Resets the [`ConstRegistry`] data structure. + pub fn reset(&mut self) { + self.const2idx.clear(); + self.idx2const.clear(); + } + + /// Returns `true` if no function local constants have been allocated. + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Returns the number of allocated function local constant values. + pub fn len(&self) -> usize { + self.idx2const.len() + } + + /// Allocates a new constant `value` on the [`ConstRegistry`] and returns its identifier. + /// + /// # Note + /// + /// If the constant `value` already exists in this [`ConstRegistry`] no new value is + /// allocated and the identifier of the existing constant `value` returned instead. + /// + /// # Errors + /// + /// If too many constant values have been allocated for this [`ConstRegistry`]. + pub fn alloc(&mut self, value: UntypedVal) -> Result { + let len = self.len(); + if len >= Self::MAX_LEN { + return Err(Error::from(TranslationError::TooManyFuncLocalConstValues)); + } + match self.const2idx.entry(value) { + btree_map::Entry::Occupied(entry) => Ok(*entry.get()), + btree_map::Entry::Vacant(entry) => { + let idx = ConstIdx(len); + entry.insert(idx); + self.idx2const.push(value); + Ok(idx) + } + } + } + + /// Returns the function local constant [`UntypedVal`] of the [`Reg`] if any. + pub fn get(&self, index: ConstIdx) -> Option { + self.idx2const.get(index.0).copied() + } + + /// Returns an iterator yielding all function local constant values of the [`ConstRegistry`]. + /// + /// # Note + /// + /// The function local constant values are yielded in their allocation order. + pub fn iter(&self) -> ConstRegistryIter { + ConstRegistryIter::new(self) + } +} + +/// Iterator yielding all allocated function local constant values. +pub struct ConstRegistryIter<'a> { + /// The underlying iterator. + iter: SliceIter<'a, UntypedVal>, +} + +impl<'a> ConstRegistryIter<'a> { + /// Creates a new [`ConstRegistryIter`] from the given slice of [`UntypedVal`]. + pub fn new(consts: &'a ConstRegistry) -> Self { + Self { + iter: consts.idx2const.as_slice().iter(), + } + } +} + +impl Iterator for ConstRegistryIter<'_> { + type Item = UntypedVal; + + fn next(&mut self) -> Option { + self.iter.next().copied() + } +} + +impl DoubleEndedIterator for ConstRegistryIter<'_> { + fn next_back(&mut self) -> Option { + self.iter.next_back().copied() + } +} + +impl ExactSizeIterator for ConstRegistryIter<'_> { + fn len(&self) -> usize { + self.iter.len() + } +} diff --git a/crates/wasmi/src/engine/translator/stack2/mod.rs b/crates/wasmi/src/engine/translator/stack2/mod.rs index 0385f0a54c..a3fb078dfe 100644 --- a/crates/wasmi/src/engine/translator/stack2/mod.rs +++ b/crates/wasmi/src/engine/translator/stack2/mod.rs @@ -4,7 +4,7 @@ mod consts; mod locals; use self::{ - consts::ConstRegistry, + consts::{ConstRegistry, ConstIdx}, locals::{LocalIdx, LocalsRegistry}, }; use crate::{ @@ -199,6 +199,11 @@ impl Stack { pub fn alloc_const(&mut self, value: impl Into) -> Result { todo!() } + + /// Returns the [`Reg`] associated to the [`ConstIdx`]. + pub fn const_idx_to_reg(&self, index: ConstIdx) -> Result { + todo!() + } } /// The [`RegisterSpace`] of a [`Reg`]. From b8d6d61cd49b8b2277f5c87a170db3755ba369be Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 12 May 2025 18:10:45 +0200 Subject: [PATCH 021/343] apply rustfmt --- crates/wasmi/src/engine/translator/stack2/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/stack2/mod.rs b/crates/wasmi/src/engine/translator/stack2/mod.rs index a3fb078dfe..4a477ac413 100644 --- a/crates/wasmi/src/engine/translator/stack2/mod.rs +++ b/crates/wasmi/src/engine/translator/stack2/mod.rs @@ -4,7 +4,7 @@ mod consts; mod locals; use self::{ - consts::{ConstRegistry, ConstIdx}, + consts::{ConstIdx, ConstRegistry}, locals::{LocalIdx, LocalsRegistry}, }; use crate::{ From e9990c413230e52e9096b75f412d8a26824eb15d Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 12 May 2025 18:10:59 +0200 Subject: [PATCH 022/343] implement most of the LocalsRegistry --- .../src/engine/translator/stack2/locals.rs | 80 ++++++++++++++++++- 1 file changed, 77 insertions(+), 3 deletions(-) diff --git a/crates/wasmi/src/engine/translator/stack2/locals.rs b/crates/wasmi/src/engine/translator/stack2/locals.rs index 417c5b4f08..fb59beb183 100644 --- a/crates/wasmi/src/engine/translator/stack2/locals.rs +++ b/crates/wasmi/src/engine/translator/stack2/locals.rs @@ -1,25 +1,99 @@ -use crate::core::ValType; +use crate::{core::ValType, Error}; use alloc::vec::Vec; /// A local variable index. #[derive(Debug, Copy, Clone)] pub struct LocalIdx(usize); +/// Stores definitions of locals. #[derive(Debug, Default, Clone)] pub struct LocalsRegistry { + /// Groups of local variables sharing a common type. groups: Vec, + /// The index of the first local variable in a group of more than 1 locals. + /// + /// # Note + /// + /// All local indices that are smaller than this can be queried faster. + first_group: Option, + /// The total number of registered locals. len_locals: usize, } impl LocalsRegistry { + /// Resets `self` for reuse. + pub fn reset(&mut self) { + self.groups.clear(); + self.first_group = None; + self.len_locals = 0; + } + + /// Registers `amount` of locals of type `ty` for `self`. + /// + /// # Errors + /// + /// If too many locals are registered. + pub fn register(&mut self, amount: usize, ty: ValType) -> Result<(), Error> { + if amount == 0 { + return Ok(()); + } + let first_local = self.len_locals; + if amount > 1 { + self.first_group.get_or_insert(first_local); + } + self.groups.push(LocalGroup { first_local, ty }); + self.len_locals += amount; + Ok(()) + } + + /// Returns the type of the `local_index` if any. + /// + /// Returns `None` if `local_index` does not refer to any local in `self`. pub fn ty(&self, local_index: LocalIdx) -> Option { + if let Some(first_group) = self.first_group { + if local_index.0 >= first_group { + return self.ty_slow(local_index); + } + } + self.ty_fast(local_index) + } + + /// Returns the [`ValType`] of the local at `local_index`. + /// + /// # Note + /// + /// This is the fast version used for locals with indices + /// smaller than the first local group. + #[inline] + fn ty_fast(&self, local_index: LocalIdx) -> Option { + self.groups.get(local_index.0).map(LocalGroup::ty) + } + + /// Returns the [`ValType`] of the local at `local_index`. + /// + /// # Note + /// + /// This is the slow version used for locals with indices + /// equal to or greater than the first local group. + #[cold] + fn ty_slow(&self, local_index: LocalIdx) -> Option { + // slow path using binary search todo!() } } +/// A local group of one or more locals sharing a common type. #[derive(Debug, Copy, Clone)] struct LocalGroup { - start_idx: usize, - len: usize, + /// The local index of the first local in the group. + first_local: usize, + /// The shared type of the locals in the local group. ty: ValType, } + +impl LocalGroup { + /// Returns the [`ValType`] of the [`LocalGroup`]. + fn ty(&self) -> ValType { + self.ty + } +} From 436ebc2cb8e7a7247da28657d8d964543d13cd1a Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 12 May 2025 18:15:15 +0200 Subject: [PATCH 023/343] fix broken doc link --- crates/wasmi/src/engine/translator/stack2/consts.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/stack2/consts.rs b/crates/wasmi/src/engine/translator/stack2/consts.rs index 10a7df6336..d2ca82438e 100644 --- a/crates/wasmi/src/engine/translator/stack2/consts.rs +++ b/crates/wasmi/src/engine/translator/stack2/consts.rs @@ -71,7 +71,7 @@ impl ConstRegistry { } } - /// Returns the function local constant [`UntypedVal`] of the [`Reg`] if any. + /// Returns the function local constant [`UntypedVal`] of the [`ConstIdx`] if any. pub fn get(&self, index: ConstIdx) -> Option { self.idx2const.get(index.0).copied() } From 221a0e9b5dde234dfd8fc4276671d37f7fa0f9b5 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 12 May 2025 18:15:57 +0200 Subject: [PATCH 024/343] swap consts and locals fields --- crates/wasmi/src/engine/translator/stack2/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/wasmi/src/engine/translator/stack2/mod.rs b/crates/wasmi/src/engine/translator/stack2/mod.rs index 4a477ac413..797544b5d9 100644 --- a/crates/wasmi/src/engine/translator/stack2/mod.rs +++ b/crates/wasmi/src/engine/translator/stack2/mod.rs @@ -20,10 +20,10 @@ use core::num::NonZeroUsize; pub struct Stack { /// The stack of operands. operands: Vec, - /// All function local constants. - consts: ConstRegistry, /// All function locals and their associated types. locals: LocalsRegistry, + /// All function local constants. + consts: ConstRegistry, /// The index of the first [`StackOperand::Local`] on the [`Stack`]. first_local: Option, /// The index of the last [`StackOperand::Local`] on the [`Stack`]. From 92d0d77ca99339a81ae85e7ee48a995ec748f5c0 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 12 May 2025 18:17:52 +0200 Subject: [PATCH 025/343] add Stack::reset --- crates/wasmi/src/engine/translator/stack2/mod.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/crates/wasmi/src/engine/translator/stack2/mod.rs b/crates/wasmi/src/engine/translator/stack2/mod.rs index 797544b5d9..9d38dc7cc2 100644 --- a/crates/wasmi/src/engine/translator/stack2/mod.rs +++ b/crates/wasmi/src/engine/translator/stack2/mod.rs @@ -82,6 +82,17 @@ impl StackPhase { } impl Stack { + /// Resets the [`Stack`] for reuse. + pub fn reset(&mut self) { + self.operands.clear(); + self.locals.reset(); + self.consts.reset(); + self.first_local = None; + self.last_local = None; + self.max_stack_height = 0; + self.phase = StackPhase::DefineLocals; + } + /// Register `amount` local variables of common type `ty`. /// /// # Errors From f6c934052453b6d9a6004972adf644b60f9c87ee Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 14 May 2025 15:23:10 +0200 Subject: [PATCH 026/343] use NonZero instead of NonZeroUsize --- crates/wasmi/src/engine/translator/stack2/mod.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/wasmi/src/engine/translator/stack2/mod.rs b/crates/wasmi/src/engine/translator/stack2/mod.rs index 9d38dc7cc2..936306f68b 100644 --- a/crates/wasmi/src/engine/translator/stack2/mod.rs +++ b/crates/wasmi/src/engine/translator/stack2/mod.rs @@ -13,7 +13,7 @@ use crate::{ Error, }; use alloc::vec::Vec; -use core::num::NonZeroUsize; +use core::num::NonZero; /// The Wasm value stack during translation from Wasm to Wasmi bytecode. #[derive(Debug, Default, Clone)] @@ -250,7 +250,7 @@ impl RegisterSpace { /// A [`StackOperand`] or [`Operand`] index on the [`Stack`]. #[derive(Debug, Copy, Clone)] -pub struct OperandIdx(NonZeroUsize); +pub struct OperandIdx(NonZero); impl From for usize { fn from(value: OperandIdx) -> Self { @@ -260,7 +260,7 @@ impl From for usize { impl From for OperandIdx { fn from(value: usize) -> Self { - let Some(operand_idx) = NonZeroUsize::new(value.wrapping_add(1)) else { + let Some(operand_idx) = NonZero::new(value.wrapping_add(1)) else { panic!("out of bounds `OperandIdx`: {value}") }; Self(operand_idx) From 8a4ffce68b1973d85494c656fababf8e5f070c5a Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 14 May 2025 15:23:22 +0200 Subject: [PATCH 027/343] improve LocalsRegistry and add tests --- .../src/engine/translator/stack2/locals.rs | 131 ++++++++++++++++-- 1 file changed, 123 insertions(+), 8 deletions(-) diff --git a/crates/wasmi/src/engine/translator/stack2/locals.rs b/crates/wasmi/src/engine/translator/stack2/locals.rs index fb59beb183..057b734b57 100644 --- a/crates/wasmi/src/engine/translator/stack2/locals.rs +++ b/crates/wasmi/src/engine/translator/stack2/locals.rs @@ -1,5 +1,7 @@ +use super::OperandIdx; use crate::{core::ValType, Error}; use alloc::vec::Vec; +use core::{cmp::Ordering, iter}; /// A local variable index. #[derive(Debug, Copy, Clone)] @@ -16,8 +18,8 @@ pub struct LocalsRegistry { /// /// All local indices that are smaller than this can be queried faster. first_group: Option, - /// The total number of registered locals. - len_locals: usize, + /// The first operand for the local on the stack if any. + first_operands: Vec>, } impl LocalsRegistry { @@ -25,7 +27,12 @@ impl LocalsRegistry { pub fn reset(&mut self) { self.groups.clear(); self.first_group = None; - self.len_locals = 0; + self.first_operands.clear(); + } + + /// Returns the number of registered local variables in `self`. + pub fn len(&self) -> usize { + self.first_operands.len() } /// Registers `amount` of locals of type `ty` for `self`. @@ -37,15 +44,60 @@ impl LocalsRegistry { if amount == 0 { return Ok(()); } - let first_local = self.len_locals; + let first_local = self.len(); if amount > 1 { self.first_group.get_or_insert(first_local); } - self.groups.push(LocalGroup { first_local, ty }); - self.len_locals += amount; + let last_local = first_local + amount; + self.groups.push(LocalGroup { + first_local, + last_local, + ty, + }); + self.first_operands.extend(iter::repeat_n(None, amount)); Ok(()) } + /// Returns the first operand for this local on the stack if any. + /// + /// # Panics + /// + /// If `index` is out of bounds. + fn first_operand(&self, index: LocalIdx) -> Option { + let Some(cell) = self.first_operands.get(index.0) else { + panic!("`first_operand`: out of bounds local index: {index:?}") + }; + *cell + } + + /// Takes the first operand for this local from the stack if any. + /// + /// # Panics + /// + /// If `index` is out of bounds. + fn take_first_operand(&mut self, index: LocalIdx) -> Option { + let Some(cell) = self.first_operands.get_mut(index.0) else { + panic!("`first_operand`: out of bounds local index: {index:?}") + }; + cell.take() + } + + /// Returns an exclusive reference to the first operand for this local on the stack. + /// + /// # Panics + /// + /// If `index` is out of bounds. + fn set_first_operand( + &mut self, + index: LocalIdx, + first_operand: OperandIdx, + ) -> Option { + let Some(cell) = self.first_operands.get_mut(index.0) else { + panic!("`first_operand`: out of bounds local index: {index:?}") + }; + cell.replace(first_operand) + } + /// Returns the type of the `local_index` if any. /// /// Returns `None` if `local_index` does not refer to any local in `self`. @@ -77,8 +129,27 @@ impl LocalsRegistry { /// equal to or greater than the first local group. #[cold] fn ty_slow(&self, local_index: LocalIdx) -> Option { - // slow path using binary search - todo!() + let Some(first_group) = self.first_group else { + unreachable!("cannot use `ty_slow` with `first_group` being `None`"); + }; + let groups = &self.groups[first_group..]; + if groups.is_empty() { + return None; + } + let local_index = local_index.0; + let index = groups + .binary_search_by(|group| { + if local_index < group.first_local { + Ordering::Greater + } else if local_index > group.last_local { + Ordering::Less + } else { + Ordering::Equal + } + }) + .ok()?; + let ty = groups[index].ty(); + Some(ty) } } @@ -87,6 +158,8 @@ impl LocalsRegistry { struct LocalGroup { /// The local index of the first local in the group. first_local: usize, + /// The local index right after the last local in the group. + last_local: usize, /// The shared type of the locals in the local group. ty: ValType, } @@ -97,3 +170,45 @@ impl LocalGroup { self.ty } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn ty_works() { + for locals_per_type in [1, 2, 10] { + let mut locals = LocalsRegistry::default(); + let tys = [ValType::I32, ValType::I64, ValType::F32, ValType::F64]; + for ty in tys { + locals.register(locals_per_type, ty).unwrap(); + } + assert_eq!(locals.len(), locals_per_type * tys.len()); + for i in 0..locals.len() { + assert_eq!(locals.ty(LocalIdx(i)), Some(tys[i / locals_per_type])); + } + } + } + + #[test] + fn locals_followed_by_groups() { + let mut locals = LocalsRegistry::default(); + let len_single = 10; + let len_groups = 10; + let locals_per_group = 100; + let len_locals = len_single + len_groups * locals_per_group; + for i in 0..len_single { + locals.register(1, ValType::I32).unwrap(); + } + for i in 0..len_groups { + locals.register(locals_per_group, ValType::I64).unwrap(); + } + for i in 0..len_locals { + let ty = match i < len_single { + true => ValType::I32, + false => ValType::I64, + }; + assert_eq!(locals.ty(LocalIdx(i)), Some(ty)); + } + } +} From c6723084f6708cfbdde6aa44921eb9e9ed82b454 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Thu, 15 May 2025 14:06:34 +0200 Subject: [PATCH 028/343] use u32 instead of usize in LocalsRegistry::register --- crates/wasmi/src/engine/translator/stack2/locals.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/crates/wasmi/src/engine/translator/stack2/locals.rs b/crates/wasmi/src/engine/translator/stack2/locals.rs index 057b734b57..c31a5c4654 100644 --- a/crates/wasmi/src/engine/translator/stack2/locals.rs +++ b/crates/wasmi/src/engine/translator/stack2/locals.rs @@ -40,10 +40,15 @@ impl LocalsRegistry { /// # Errors /// /// If too many locals are registered. - pub fn register(&mut self, amount: usize, ty: ValType) -> Result<(), Error> { + pub fn register(&mut self, amount: u32, ty: ValType) -> Result<(), Error> { if amount == 0 { return Ok(()); } + let Ok(amount) = usize::try_from(amount) else { + panic!( + "failed to register {amount} local variables of type {ty:?}: out of bounds `usize`" + ) + }; let first_local = self.len(); if amount > 1 { self.first_group.get_or_insert(first_local); @@ -183,6 +188,7 @@ mod tests { for ty in tys { locals.register(locals_per_type, ty).unwrap(); } + let locals_per_type = locals_per_type as usize; assert_eq!(locals.len(), locals_per_type * tys.len()); for i in 0..locals.len() { assert_eq!(locals.ty(LocalIdx(i)), Some(tys[i / locals_per_type])); @@ -208,7 +214,7 @@ mod tests { true => ValType::I32, false => ValType::I64, }; - assert_eq!(locals.ty(LocalIdx(i)), Some(ty)); + assert_eq!(locals.ty(LocalIdx(i as usize)), Some(ty)); } } } From 691d8f391ee34ace241410cdaeaa803623c97775 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Thu, 15 May 2025 14:06:44 +0200 Subject: [PATCH 029/343] refactor StackPhase --- .../wasmi/src/engine/translator/stack2/mod.rs | 35 +++++++++++++++---- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/crates/wasmi/src/engine/translator/stack2/mod.rs b/crates/wasmi/src/engine/translator/stack2/mod.rs index 936306f68b..88847eaf24 100644 --- a/crates/wasmi/src/engine/translator/stack2/mod.rs +++ b/crates/wasmi/src/engine/translator/stack2/mod.rs @@ -57,7 +57,7 @@ impl StackPhase { /// Ensures that the current phase is [`StackPhase::DefineLocals`]. pub fn assert_define_locals(&self) { - assert!(matches!(self, Self::DefineLocals)); + debug_assert!(matches!(self, Self::DefineLocals)); } /// Turns the current phase into [`StackPhase::Translation`]. @@ -65,18 +65,27 @@ impl StackPhase { /// # Panics /// /// If the current phase is incompatible with this phase shift. - pub fn assert_translation(&mut self) { - assert!(matches!(self, Self::DefineLocals | Self::Translation)); + pub fn tranlate(&mut self) { + assert!(matches!(self, Self::DefineLocals)); *self = Self::Translation; } + /// Turns the current phase into [`StackPhase::Translation`]. + /// + /// # Panics + /// + /// If the current phase is incompatible with this phase shift. + pub fn assert_translation(&self) { + debug_assert!(matches!(self, Self::Translation)); + } + /// Turns the current phase into [`StackPhase::Finish`]. /// /// # Panics /// /// If the current phase is incompatible with this phase shift. - pub fn assert_finish(&mut self) { - assert!(matches!(self, Self::Translation | Self::Finish)); + pub fn finish(&mut self) { + assert!(matches!(self, Self::Translation)); *self = Self::Finish; } } @@ -99,7 +108,8 @@ impl Stack { /// /// If too many local variables are being registered. pub fn register_locals(&mut self, amount: u32, ty: ValType) -> Result<(), Error> { - todo!() + self.phase.assert_define_locals(); + self.locals.register(amount, ty) } /// Finish registration of local variables. @@ -108,7 +118,8 @@ impl Stack { /// /// If the current [`StackPhase`] is not [`StackPhase::DefineLocals`]. pub fn finish_register_locals(&mut self) -> Result<(), Error> { - todo!() + self.phase.assert_translation(); + Ok(()) } /// Pushes a local variable with index `local_idx` to the [`Stack`]. @@ -119,6 +130,7 @@ impl Stack { /// - If too many operands have been pushed onto the [`Stack`]. /// - If the local with `local_idx` does not exist. pub fn push_local(&mut self, local_idx: u32) -> Result { + self.phase.assert_translation(); todo!() } @@ -129,6 +141,7 @@ impl Stack { /// - If the current [`StackPhase`] is not [`StackPhase::Translation`]. /// - If too many operands have been pushed onto the [`Stack`]. pub fn push_temp(&mut self, ty: ValType) -> Result { + self.phase.assert_translation(); todo!() } @@ -139,6 +152,7 @@ impl Stack { /// - If the current [`StackPhase`] is not [`StackPhase::Translation`]. /// - If too many operands have been pushed onto the [`Stack`]. pub fn push_immediate(&mut self, value: impl Into) -> Result { + self.phase.assert_translation(); todo!() } @@ -146,6 +160,7 @@ impl Stack { /// /// Returns `None` if the [`Stack`] is empty. pub fn peek(&self) -> Option { + self.phase.assert_translation(); let operand = self.operands.last().copied()?; let index = OperandIdx::from(self.operands.len() - 1); Some(Operand::new(index, operand, &self.locals)) @@ -155,6 +170,7 @@ impl Stack { /// /// Returns `None` if the [`Stack`] is empty. pub fn pop(&mut self) -> Option { + self.phase.assert_translation(); let operand = self.operands.pop()?; let index = OperandIdx::from(self.operands.len()); Some(Operand::new(index, operand, &self.locals)) @@ -183,6 +199,7 @@ impl Stack { /// - Returns `None` if the [`Stack`] is empty. /// - The last returned [`Operand`] is the top-most one. fn pop_some(&mut self) -> Option<[Operand; N]> { + self.phase.assert_translation(); todo!() } @@ -192,6 +209,7 @@ impl Stack { /// /// If the translator ran out of [`Reg`]s for the function. pub fn operand_to_reg(&mut self, index: OperandIdx) -> Result { + self.phase.assert_translation(); todo!() } @@ -199,6 +217,7 @@ impl Stack { /// /// Returns `None` if the [`Reg`] is unknown to the [`Stack`]. pub fn reg_space(&self, reg: Reg) -> Option { + self.phase.assert_translation(); todo!() } @@ -208,11 +227,13 @@ impl Stack { /// /// If too many function local constants have been allocated already. pub fn alloc_const(&mut self, value: impl Into) -> Result { + self.phase.assert_translation(); todo!() } /// Returns the [`Reg`] associated to the [`ConstIdx`]. pub fn const_idx_to_reg(&self, index: ConstIdx) -> Result { + self.phase.assert_translation(); todo!() } } From 30113c96d5e5ab77e0057dc2b4f6088fb30bc19f Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Thu, 15 May 2025 14:16:45 +0200 Subject: [PATCH 030/343] implement Stack::pop_some --- .../wasmi/src/engine/translator/stack2/mod.rs | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/crates/wasmi/src/engine/translator/stack2/mod.rs b/crates/wasmi/src/engine/translator/stack2/mod.rs index 88847eaf24..741f37d776 100644 --- a/crates/wasmi/src/engine/translator/stack2/mod.rs +++ b/crates/wasmi/src/engine/translator/stack2/mod.rs @@ -13,7 +13,7 @@ use crate::{ Error, }; use alloc::vec::Vec; -use core::num::NonZero; +use core::{array, num::NonZero}; /// The Wasm value stack during translation from Wasm to Wasmi bytecode. #[derive(Debug, Default, Clone)] @@ -156,6 +156,11 @@ impl Stack { todo!() } + /// Returns the current number of [`Operand`]s on the [`Stack`]. + pub fn len_operands(&self) -> usize { + self.operands.len() + } + /// Peeks the top-most [`Operand`] on the [`Stack`]. /// /// Returns `None` if the [`Stack`] is empty. @@ -200,7 +205,17 @@ impl Stack { /// - The last returned [`Operand`] is the top-most one. fn pop_some(&mut self) -> Option<[Operand; N]> { self.phase.assert_translation(); - todo!() + if N >= self.len_operands() { + return None; + } + let start = self.len_operands() - N; + let drained = self.operands.drain(start..); + let popped: [Operand; N] = array::from_fn(|i| { + let index = OperandIdx::from(start + i); + let operand = drained.as_slice()[i]; + Operand::new(index, operand, &self.locals) + }); + Some(popped) } /// Converts the [`Operand`] at `index` to a [`Reg`] if possible. From b3ce5a4268af5d208117f986d5903df165753777 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 19 May 2025 12:05:27 +0200 Subject: [PATCH 031/343] implement all stubs of new Stack --- .../src/engine/translator/stack2/consts.rs | 82 ++-- .../src/engine/translator/stack2/locals.rs | 74 +++- .../wasmi/src/engine/translator/stack2/mod.rs | 380 +++++++++++++++--- 3 files changed, 420 insertions(+), 116 deletions(-) diff --git a/crates/wasmi/src/engine/translator/stack2/consts.rs b/crates/wasmi/src/engine/translator/stack2/consts.rs index d2ca82438e..0ccaabeb70 100644 --- a/crates/wasmi/src/engine/translator/stack2/consts.rs +++ b/crates/wasmi/src/engine/translator/stack2/consts.rs @@ -1,48 +1,59 @@ -use crate::{core::UntypedVal, engine::TranslationError, Error}; +use crate::{core::UntypedVal, engine::TranslationError, ir::Reg, Error}; use alloc::{ collections::{btree_map, BTreeMap}, vec::Vec, }; -use core::slice::Iter as SliceIter; +use core::{iter::Rev, slice::Iter as SliceIter}; /// A pool of deduplicated function local constant values. /// -/// - Those constant values are identified by their associated [`ConstIdx`]. +/// - Those constant values are identified by their associated [`Reg`]. /// - All constant values are also deduplicated so that no duplicates /// are stored in a [`ConstRegistry`]. This also means that deciding if two -/// [`ConstIdx`] values refer to the equal constant values can be efficiently -/// done by comparing the [`ConstIdx`]s without resolving to their +/// [`Reg`] values refer to the equal constant values can be efficiently +/// done by comparing the [`Reg`] indices without resolving to their /// underlying constant values. -#[derive(Debug, Default, Clone)] +#[derive(Debug, Default)] pub struct ConstRegistry { - /// Mapping from constant [`UntypedVal`] values to [`ConstIdx`] indices. - const2idx: BTreeMap, - /// Mapping from [`ConstIdx`] indices to constant [`UntypedVal`] values. + /// Mapping from constant [`UntypedVal`] values to [`Reg`] indices. + const2idx: BTreeMap, + /// Mapping from [`Reg`] indices to constant [`UntypedVal`] values. idx2const: Vec, + /// The [`Reg`] index for the next allocated function local constant value. + next_idx: i16, } -/// An index referring to a function local constant. -#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord)] -pub struct ConstIdx(usize); - impl ConstRegistry { - /// The maximum number of function local constants per function. - const MAX_LEN: usize = (1 << 16) - 1; - /// Resets the [`ConstRegistry`] data structure. pub fn reset(&mut self) { self.const2idx.clear(); self.idx2const.clear(); + self.next_idx = Self::first_index(); } - /// Returns `true` if no function local constants have been allocated. - pub fn is_empty(&self) -> bool { - self.len() == 0 + /// The maximum index for [`Reg`] referring to function local constant values. + /// + /// # Note + /// + /// The maximum index is also the one to be assigned to the first allocated + /// function local constant value as indices are counting downwards. + fn first_index() -> i16 { + -1 + } + + /// The mininmum index for [`Reg`] referring to function local constant values. + /// + /// # Note + /// + /// This index is not assignable to a function local constant value and acts + /// as a bound to guard against overflowing the range of indices. + fn last_index() -> i16 { + i16::MIN } /// Returns the number of allocated function local constant values. - pub fn len(&self) -> usize { - self.idx2const.len() + pub fn len_consts(&self) -> u16 { + self.next_idx.abs_diff(Self::first_index()) } /// Allocates a new constant `value` on the [`ConstRegistry`] and returns its identifier. @@ -55,25 +66,29 @@ impl ConstRegistry { /// # Errors /// /// If too many constant values have been allocated for this [`ConstRegistry`]. - pub fn alloc(&mut self, value: UntypedVal) -> Result { - let len = self.len(); - if len >= Self::MAX_LEN { + pub fn alloc(&mut self, value: UntypedVal) -> Result { + if self.next_idx == Self::last_index() { return Err(Error::from(TranslationError::TooManyFuncLocalConstValues)); } match self.const2idx.entry(value) { btree_map::Entry::Occupied(entry) => Ok(*entry.get()), btree_map::Entry::Vacant(entry) => { - let idx = ConstIdx(len); - entry.insert(idx); + let register = Reg::from(self.next_idx); + self.next_idx -= 1; + entry.insert(register); self.idx2const.push(value); - Ok(idx) + Ok(register) } } } - /// Returns the function local constant [`UntypedVal`] of the [`ConstIdx`] if any. - pub fn get(&self, index: ConstIdx) -> Option { - self.idx2const.get(index.0).copied() + /// Returns the function local constant [`UntypedVal`] of the [`Reg`] if any. + pub fn get(&self, register: Reg) -> Option { + if !register.is_const() { + return None; + } + let index = i16::from(register).wrapping_add(1).unsigned_abs() as usize; + self.idx2const.get(index).copied() } /// Returns an iterator yielding all function local constant values of the [`ConstRegistry`]. @@ -89,14 +104,17 @@ impl ConstRegistry { /// Iterator yielding all allocated function local constant values. pub struct ConstRegistryIter<'a> { /// The underlying iterator. - iter: SliceIter<'a, UntypedVal>, + iter: Rev>, } impl<'a> ConstRegistryIter<'a> { /// Creates a new [`ConstRegistryIter`] from the given slice of [`UntypedVal`]. pub fn new(consts: &'a ConstRegistry) -> Self { + // Note: we need to revert the iteration since we allocate new + // function local constants in reverse order of their absolute + // vector indices in the function call frame during execution. Self { - iter: consts.idx2const.as_slice().iter(), + iter: consts.idx2const.as_slice().iter().rev(), } } } diff --git a/crates/wasmi/src/engine/translator/stack2/locals.rs b/crates/wasmi/src/engine/translator/stack2/locals.rs index c31a5c4654..47bd03db68 100644 --- a/crates/wasmi/src/engine/translator/stack2/locals.rs +++ b/crates/wasmi/src/engine/translator/stack2/locals.rs @@ -5,7 +5,19 @@ use core::{cmp::Ordering, iter}; /// A local variable index. #[derive(Debug, Copy, Clone)] -pub struct LocalIdx(usize); +pub struct LocalIdx(u32); + +impl From for LocalIdx { + fn from(index: u32) -> Self { + Self(index) + } +} + +impl From for u32 { + fn from(index: LocalIdx) -> Self { + index.0 + } +} /// Stores definitions of locals. #[derive(Debug, Default, Clone)] @@ -63,13 +75,23 @@ impl LocalsRegistry { Ok(()) } + /// Converts `index` into a `usize` value. + fn local_idx_to_index(index: LocalIdx) -> usize { + let index = u32::from(index); + let Ok(index) = usize::try_from(index) else { + panic!("out of bounds `LocalIdx`: {index}") + }; + index + } + /// Returns the first operand for this local on the stack if any. /// /// # Panics /// /// If `index` is out of bounds. - fn first_operand(&self, index: LocalIdx) -> Option { - let Some(cell) = self.first_operands.get(index.0) else { + pub fn first_operand(&self, index: LocalIdx) -> Option { + let index = Self::local_idx_to_index(index); + let Some(cell) = self.first_operands.get(index) else { panic!("`first_operand`: out of bounds local index: {index:?}") }; *cell @@ -80,39 +102,45 @@ impl LocalsRegistry { /// # Panics /// /// If `index` is out of bounds. - fn take_first_operand(&mut self, index: LocalIdx) -> Option { - let Some(cell) = self.first_operands.get_mut(index.0) else { + pub fn take_first_operand(&mut self, index: LocalIdx) -> Option { + let index = Self::local_idx_to_index(index); + let Some(cell) = self.first_operands.get_mut(index) else { panic!("`first_operand`: out of bounds local index: {index:?}") }; cell.take() } - /// Returns an exclusive reference to the first operand for this local on the stack. + /// Replaces the first operand for this local on the stack and returns the old one. /// /// # Panics /// /// If `index` is out of bounds. - fn set_first_operand( + pub fn replace_first_operand( &mut self, index: LocalIdx, - first_operand: OperandIdx, + first_operand: Option, ) -> Option { - let Some(cell) = self.first_operands.get_mut(index.0) else { + let index = Self::local_idx_to_index(index); + let Some(cell) = self.first_operands.get_mut(index) else { panic!("`first_operand`: out of bounds local index: {index:?}") }; - cell.replace(first_operand) + match first_operand { + Some(first_operand) => cell.replace(first_operand), + None => cell.take(), + } } - /// Returns the type of the `local_index` if any. + /// Returns the type of the `index` if any. /// - /// Returns `None` if `local_index` does not refer to any local in `self`. - pub fn ty(&self, local_index: LocalIdx) -> Option { + /// Returns `None` if `index` does not refer to any local in `self`. + pub fn ty(&self, index: LocalIdx) -> Option { + let index = Self::local_idx_to_index(index); if let Some(first_group) = self.first_group { - if local_index.0 >= first_group { - return self.ty_slow(local_index); + if index >= first_group { + return self.ty_slow(index); } } - self.ty_fast(local_index) + self.ty_fast(index) } /// Returns the [`ValType`] of the local at `local_index`. @@ -122,8 +150,8 @@ impl LocalsRegistry { /// This is the fast version used for locals with indices /// smaller than the first local group. #[inline] - fn ty_fast(&self, local_index: LocalIdx) -> Option { - self.groups.get(local_index.0).map(LocalGroup::ty) + fn ty_fast(&self, local_index: usize) -> Option { + self.groups.get(local_index).map(LocalGroup::ty) } /// Returns the [`ValType`] of the local at `local_index`. @@ -133,7 +161,7 @@ impl LocalsRegistry { /// This is the slow version used for locals with indices /// equal to or greater than the first local group. #[cold] - fn ty_slow(&self, local_index: LocalIdx) -> Option { + fn ty_slow(&self, local_index: usize) -> Option { let Some(first_group) = self.first_group else { unreachable!("cannot use `ty_slow` with `first_group` being `None`"); }; @@ -141,7 +169,6 @@ impl LocalsRegistry { if groups.is_empty() { return None; } - let local_index = local_index.0; let index = groups .binary_search_by(|group| { if local_index < group.first_local { @@ -191,7 +218,10 @@ mod tests { let locals_per_type = locals_per_type as usize; assert_eq!(locals.len(), locals_per_type * tys.len()); for i in 0..locals.len() { - assert_eq!(locals.ty(LocalIdx(i)), Some(tys[i / locals_per_type])); + assert_eq!( + locals.ty(LocalIdx(i as u32)), + Some(tys[i / locals_per_type]) + ); } } } @@ -214,7 +244,7 @@ mod tests { true => ValType::I32, false => ValType::I64, }; - assert_eq!(locals.ty(LocalIdx(i as usize)), Some(ty)); + assert_eq!(locals.ty(LocalIdx(i)), Some(ty)); } } } diff --git a/crates/wasmi/src/engine/translator/stack2/mod.rs b/crates/wasmi/src/engine/translator/stack2/mod.rs index 741f37d776..26dac7d842 100644 --- a/crates/wasmi/src/engine/translator/stack2/mod.rs +++ b/crates/wasmi/src/engine/translator/stack2/mod.rs @@ -4,19 +4,21 @@ mod consts; mod locals; use self::{ - consts::{ConstIdx, ConstRegistry}, + consts::ConstRegistry, locals::{LocalIdx, LocalsRegistry}, }; +use super::Instr; use crate::{ core::{TypedVal, UntypedVal, ValType}, + engine::TranslationError, ir::Reg, Error, }; use alloc::vec::Vec; -use core::{array, num::NonZero}; +use core::{array, mem, num::NonZero}; /// The Wasm value stack during translation from Wasm to Wasmi bytecode. -#[derive(Debug, Default, Clone)] +#[derive(Debug, Default)] pub struct Stack { /// The stack of operands. operands: Vec, @@ -25,10 +27,6 @@ pub struct Stack { /// All function local constants. consts: ConstRegistry, /// The index of the first [`StackOperand::Local`] on the [`Stack`]. - first_local: Option, - /// The index of the last [`StackOperand::Local`] on the [`Stack`]. - last_local: Option, - /// The maximum size of the [`Stack`]. max_stack_height: usize, /// The current phase of the [`Stack`]. phase: StackPhase, @@ -65,7 +63,7 @@ impl StackPhase { /// # Panics /// /// If the current phase is incompatible with this phase shift. - pub fn tranlate(&mut self) { + pub fn translation(&mut self) { assert!(matches!(self, Self::DefineLocals)); *self = Self::Translation; } @@ -76,7 +74,7 @@ impl StackPhase { /// /// If the current phase is incompatible with this phase shift. pub fn assert_translation(&self) { - debug_assert!(matches!(self, Self::Translation)); + debug_assert!(matches!(self, Self::Translation)) } /// Turns the current phase into [`StackPhase::Finish`]. @@ -85,7 +83,7 @@ impl StackPhase { /// /// If the current phase is incompatible with this phase shift. pub fn finish(&mut self) { - assert!(matches!(self, Self::Translation)); + debug_assert!(matches!(self, Self::Translation)); *self = Self::Finish; } } @@ -96,8 +94,6 @@ impl Stack { self.operands.clear(); self.locals.reset(); self.consts.reset(); - self.first_local = None; - self.last_local = None; self.max_stack_height = 0; self.phase = StackPhase::DefineLocals; } @@ -109,7 +105,8 @@ impl Stack { /// If too many local variables are being registered. pub fn register_locals(&mut self, amount: u32, ty: ValType) -> Result<(), Error> { self.phase.assert_define_locals(); - self.locals.register(amount, ty) + self.locals.register(amount, ty)?; + Ok(()) } /// Finish registration of local variables. @@ -118,10 +115,77 @@ impl Stack { /// /// If the current [`StackPhase`] is not [`StackPhase::DefineLocals`]. pub fn finish_register_locals(&mut self) -> Result<(), Error> { - self.phase.assert_translation(); + self.phase.translation(); Ok(()) } + /// Finish translation of the function body. + /// + /// # Errors + /// + /// If the current [`StackPhase`] is not [`StackPhase::Translation`]. + pub fn finish_translation(&mut self) -> Result<(), Error> { + self.phase.finish(); + Ok(()) + } + + /// Returns the current number of [`Operand`]s on the [`Stack`]. + pub fn height(&self) -> usize { + self.operands.len() + } + + /// Truncates `self` to the target `height`. + /// + /// All operands above `height` are dropped. + /// + /// # Panic + /// + /// If `height` is greater than the current height of `self`. + pub fn trunc(&mut self, height: usize) { + assert!(height <= self.height()); + self.operands.truncate(height); + } + + /// Updates the maximum stack height if needed. + fn update_max_stack_height(&mut self) { + self.max_stack_height = core::cmp::max(self.max_stack_height, self.height()); + } + + /// Returns the [`OperandIdx`] of the next pushed operand. + fn next_operand_idx(&self) -> OperandIdx { + OperandIdx::from(self.operands.len()) + } + + /// Updates the `prev_local` of the [`StackOperand::Local`] at `local_index` to `prev_index`. + /// + /// # Panics + /// + /// - If `local_index` does not refer to a [`StackOperand::Local`]. + /// - If `local_index` is out of bounds of the operand stack. + fn update_prev_local(&mut self, local_index: OperandIdx, prev_index: Option) { + match self.operands.get_mut(usize::from(local_index)) { + Some(StackOperand::Local { prev_local, .. }) => { + *prev_local = prev_index; + } + operand => panic!("expected `StackOperand::Local` but found: {operand:?}"), + } + } + + /// Updates the `next_local` of the [`StackOperand::Local`] at `local_index` to `prev_index`. + /// + /// # Panics + /// + /// - If `local_index` does not refer to a [`StackOperand::Local`]. + /// - If `local_index` is out of bounds of the operand stack. + fn update_next_local(&mut self, local_index: OperandIdx, prev_index: Option) { + match self.operands.get_mut(usize::from(local_index)) { + Some(StackOperand::Local { next_local, .. }) => { + *next_local = prev_index; + } + operand => panic!("expected `StackOperand::Local` but found: {operand:?}"), + } + } + /// Pushes a local variable with index `local_idx` to the [`Stack`]. /// /// # Errors @@ -129,9 +193,22 @@ impl Stack { /// - If the current [`StackPhase`] is not [`StackPhase::Translation`]. /// - If too many operands have been pushed onto the [`Stack`]. /// - If the local with `local_idx` does not exist. - pub fn push_local(&mut self, local_idx: u32) -> Result { + pub fn push_local(&mut self, local_index: LocalIdx) -> Result { self.phase.assert_translation(); - todo!() + let operand_index = self.next_operand_idx(); + let next_local = self + .locals + .replace_first_operand(local_index, Some(operand_index)); + if let Some(next_local) = next_local { + self.update_prev_local(next_local, Some(operand_index)); + } + self.operands.push(StackOperand::Local { + local_index, + prev_local: None, + next_local, + }); + self.update_max_stack_height(); + Ok(operand_index) } /// Pushes a temporary with type `ty` on the [`Stack`]. @@ -140,9 +217,12 @@ impl Stack { /// /// - If the current [`StackPhase`] is not [`StackPhase::Translation`]. /// - If too many operands have been pushed onto the [`Stack`]. - pub fn push_temp(&mut self, ty: ValType) -> Result { + pub fn push_temp(&mut self, ty: ValType, instr: Option) -> Result { self.phase.assert_translation(); - todo!() + let operand_index = self.next_operand_idx(); + self.operands.push(StackOperand::Temp { ty, instr }); + self.update_max_stack_height(); + Ok(operand_index) } /// Pushes an immediate `value` on the [`Stack`]. @@ -153,12 +233,11 @@ impl Stack { /// - If too many operands have been pushed onto the [`Stack`]. pub fn push_immediate(&mut self, value: impl Into) -> Result { self.phase.assert_translation(); - todo!() - } - - /// Returns the current number of [`Operand`]s on the [`Stack`]. - pub fn len_operands(&self) -> usize { - self.operands.len() + let operand_index = self.next_operand_idx(); + self.operands + .push(StackOperand::Immediate { val: value.into() }); + self.update_max_stack_height(); + Ok(operand_index) } /// Peeks the top-most [`Operand`] on the [`Stack`]. @@ -205,10 +284,10 @@ impl Stack { /// - The last returned [`Operand`] is the top-most one. fn pop_some(&mut self) -> Option<[Operand; N]> { self.phase.assert_translation(); - if N >= self.len_operands() { + if N >= self.height() { return None; } - let start = self.len_operands() - N; + let start = self.height() - N; let drained = self.operands.drain(start..); let popped: [Operand; N] = array::from_fn(|i| { let index = OperandIdx::from(start + i); @@ -218,22 +297,119 @@ impl Stack { Some(popped) } - /// Converts the [`Operand`] at `index` to a [`Reg`] if possible. + /// Returns the [`RegSpace`] of the [`Reg`]. /// - /// # Errors + /// Returns `None` if the [`Reg`] is unknown to the [`Stack`]. + #[must_use] + pub fn stack_space(&self, reg: Reg) -> RegSpace { + self.phase.assert_translation(); + let index = i16::from(reg); + if index.is_negative() { + return RegSpace::Const; + } + let index = index as u16; + if usize::from(index) < self.locals.len() { + return RegSpace::Local; + } + RegSpace::Temp + } + + /// Preserve all locals on the [`Stack`] that refer to `local_index`. + /// + /// This is done by converting those locals to [`StackOperand::Temp`] and yielding them. /// - /// If the translator ran out of [`Reg`]s for the function. - pub fn operand_to_reg(&mut self, index: OperandIdx) -> Result { + /// # Note + /// + /// The users must fully consume all items yielded by the returned iterator in order + /// for the local preservation to take full effect. + /// + /// # Panics + /// + /// If the local at `local_index` is out of bounds. + #[must_use] + pub fn preserve_locals(&mut self, local_index: LocalIdx) -> PreservedLocalsIter { + let Some(ty) = self.locals.ty(local_index) else { + panic!("out of bounds local at: {local_index:?}") + }; + let index = self.locals.first_operand(local_index); + PreservedLocalsIter { + stack: self, + index, + ty, + } + } + + /// Converts and returns the [`StackOperand`] at `depth` into a [`StackOperand::Temp`]. + /// + /// # Note + /// + /// Returns `None` if operand at `depth` is [`StackOperand::Temp`] already. + /// + /// # Panics + /// + /// - If `depth` is out of bounds for the [`Stack`] of operands. + #[must_use] + pub fn operand_to_temp(&mut self, depth: usize) -> Option { self.phase.assert_translation(); - todo!() + let len = self.height(); + if depth >= len { + panic!( + "out of bounds access: tried to access `Stack` with length {len} at depth {depth}" + ); + } + let index = len - depth - 1; + let operand_index = OperandIdx::from(index); + let operand = match self.operands[index] { + StackOperand::Local { + local_index, + prev_local, + next_local, + } => { + if prev_local.is_none() { + // Note: if `prev_local` is `None` then this local is the first + // in the linked list of locals and must be updated. + debug_assert_eq!(self.locals.first_operand(local_index), Some(operand_index)); + self.locals.replace_first_operand(local_index, next_local); + } + if let Some(prev_local) = prev_local { + self.update_next_local(prev_local, next_local); + } + if let Some(next_local) = next_local { + self.update_prev_local(next_local, prev_local); + } + Operand::local(operand_index, local_index, &self.locals) + } + StackOperand::Immediate { val } => Operand::immediate(operand_index, val), + StackOperand::Temp { ty, instr } => return None, + }; + self.operands[index] = StackOperand::Temp { + ty: operand.ty(), + instr: None, + }; + Some(operand) } - /// Returns the [`RegisterSpace`] of the [`Reg`]. + /// Converts the [`Operand`] at `index` to a [`Reg`] if possible. /// - /// Returns `None` if the [`Reg`] is unknown to the [`Stack`]. - pub fn reg_space(&self, reg: Reg) -> Option { + /// # Panics + /// + /// If the `index` is out of bounds. + #[must_use] + pub fn operand_to_reg(&mut self, depth: usize) -> Result { self.phase.assert_translation(); - todo!() + let len = self.height(); + if depth >= len { + panic!( + "out of bounds access: tried to access `Stack` with length {len} at depth {depth}" + ); + } + let index = len - depth - 1; + let operand = self.operands[index]; + match operand { + StackOperand::Local { local_index, .. } => self.local_to_reg(local_index), + StackOperand::Temp { .. } => self.temp_to_reg(OperandIdx::from(index)), + StackOperand::Immediate { val } => self.const_to_reg(val), + } } /// Allocates a function local constant `value`. @@ -241,43 +417,102 @@ impl Stack { /// # Errors /// /// If too many function local constants have been allocated already. - pub fn alloc_const(&mut self, value: impl Into) -> Result { + #[must_use] + pub fn const_to_reg(&mut self, value: impl Into) -> Result { + self.phase.assert_translation(); + self.consts.alloc(value.into()) + } + + /// Converts the local `index` into the associated [`Reg`]. + /// + /// # Errors + /// + /// If `index` cannot be converted into a [`Reg`]. + #[must_use] + fn local_to_reg(&self, index: LocalIdx) -> Result { self.phase.assert_translation(); - todo!() + let Ok(index) = i16::try_from(u32::from(index)) else { + return Err(Error::from(TranslationError::AllocatedTooManyRegisters)); + }; + Ok(Reg::from(index)) } - /// Returns the [`Reg`] associated to the [`ConstIdx`]. - pub fn const_idx_to_reg(&self, index: ConstIdx) -> Result { + /// Converts the [`Operand::Temp`] `index` into the associated [`Reg`]. + /// + /// # Errors + /// + /// If `index` cannot be converted into a [`StackSlot`]. + #[must_use] + pub fn temp_to_reg(&self, index: OperandIdx) -> Result { self.phase.assert_translation(); - todo!() + let index = usize::from(index); + debug_assert!(matches!(&self.operands[index], StackOperand::Temp { .. },)); + let Some(index) = index.checked_add(self.locals.len()) else { + return Err(Error::from(TranslationError::AllocatedTooManyRegisters)); + }; + let Ok(index) = i16::try_from(index) else { + return Err(Error::from(TranslationError::AllocatedTooManyRegisters)); + }; + Ok(Reg::from(index)) + } +} + +/// Iterator yielding preserved local indices while preserving them. +#[derive(Debug)] +pub struct PreservedLocalsIter<'a> { + /// The underlying operand stack. + stack: &'a mut Stack, + /// The current operand index of the next preserved local if any. + index: Option, + /// Type of local at preserved `local_index`. + ty: ValType, +} + +impl Iterator for PreservedLocalsIter<'_> { + type Item = OperandIdx; + + fn next(&mut self) -> Option { + let index = self.index?; + let operand = mem::replace( + &mut self.stack.operands[usize::from(index)], + StackOperand::Temp { + ty: self.ty, + instr: None, + }, + ); + self.index = match operand { + StackOperand::Local { next_local, .. } => next_local, + operand => panic!("expected `StackOperand::Local` but found: {operand:?}"), + }; + Some(index) } } -/// The [`RegisterSpace`] of a [`Reg`]. +/// The [`RegSpace`] of a [`Reg`]. #[derive(Debug, Copy, Clone)] -pub enum RegisterSpace { - /// Register referring to a local variable. +pub enum RegSpace { + /// Stack slot referring to a local variable. Local, - /// Register referring to a function local constant value. + /// Stack slot referring to a function local constant value. Const, - /// Register referring to a temporary stack operand. + /// Stack slot referring to a temporary stack operand. Temp, } -impl RegisterSpace { - /// Returns `true` if `self` is [`RegisterSpace::Local`]. +impl RegSpace { + /// Returns `true` if `self` is [`RegSpace::Local`]. #[inline] pub fn is_local(self) -> bool { matches!(self, Self::Local) } - /// Returns `true` if `self` is [`RegisterSpace::Temp`]. + /// Returns `true` if `self` is [`RegSpace::Temp`]. #[inline] pub fn is_temp(self) -> bool { matches!(self, Self::Temp) } - /// Returns `true` if `self` is [`RegisterSpace::Const`]. + /// Returns `true` if `self` is [`RegSpace::Const`]. #[inline] pub fn is_const(self) -> bool { matches!(self, Self::Const) @@ -285,7 +520,7 @@ impl RegisterSpace { } /// A [`StackOperand`] or [`Operand`] index on the [`Stack`]. -#[derive(Debug, Copy, Clone)] +#[derive(Debug, Copy, Clone, PartialEq, Eq)] pub struct OperandIdx(NonZero); impl From for usize { @@ -322,6 +557,8 @@ enum StackOperand { Temp { /// The type of the temporary value. ty: ValType, + /// The instruction which has this [`StackOperand`] as result if any. + instr: Option, }, /// An immediate value on the [`Stack`]. Immediate { @@ -345,22 +582,39 @@ impl Operand { fn new(operand_index: OperandIdx, operand: StackOperand, locals: &LocalsRegistry) -> Self { match operand { StackOperand::Local { local_index, .. } => { - let Some(ty) = locals.ty(local_index) else { - panic!("failed to query type of local at: {local_index:?}"); - }; - Self::Local(LocalOperand { - operand_index, - local_index, - ty, - }) - } - StackOperand::Temp { ty } => Self::Temp(TempOperand { operand_index, ty }), - StackOperand::Immediate { val } => { - Self::Immediate(ImmediateOperand { operand_index, val }) + Self::local(operand_index, local_index, locals) } + StackOperand::Temp { ty, instr } => Self::temp(operand_index, ty, instr), + StackOperand::Immediate { val } => Self::immediate(operand_index, val), } } + /// Creates a local [`Operand`]. + fn local(operand_index: OperandIdx, local_index: LocalIdx, locals: &LocalsRegistry) -> Self { + let Some(ty) = locals.ty(local_index) else { + panic!("failed to query type of local at: {local_index:?}"); + }; + Self::Local(LocalOperand { + operand_index, + local_index, + ty, + }) + } + + /// Creates a temporary [`Operand`]. + fn temp(operand_index: OperandIdx, ty: ValType, instr: Option) -> Self { + Self::Temp(TempOperand { + operand_index, + ty, + instr, + }) + } + + /// Creates an immediate [`Operand`]. + fn immediate(operand_index: OperandIdx, val: TypedVal) -> Self { + Self::Immediate(ImmediateOperand { operand_index, val }) + } + /// Returns `true` if `self` is an [`Operand::Local`]. pub fn is_local(self) -> bool { matches!(self, Self::Local(_)) @@ -430,6 +684,8 @@ pub struct TempOperand { operand_index: OperandIdx, /// The type of the temporary. ty: ValType, + /// The instruction which created this [`TempOperand`] as its result. + instr: Option, } impl TempOperand { From b1ecd6eba2df5ec3265f53e009580e4e44e99e1b Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Tue, 20 May 2025 11:47:29 +0200 Subject: [PATCH 032/343] remove unnecessary #[must_use] attributes --- crates/wasmi/src/engine/translator/stack2/mod.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/crates/wasmi/src/engine/translator/stack2/mod.rs b/crates/wasmi/src/engine/translator/stack2/mod.rs index 26dac7d842..6cc16804d1 100644 --- a/crates/wasmi/src/engine/translator/stack2/mod.rs +++ b/crates/wasmi/src/engine/translator/stack2/mod.rs @@ -394,7 +394,6 @@ impl Stack { /// # Panics /// /// If the `index` is out of bounds. - #[must_use] pub fn operand_to_reg(&mut self, depth: usize) -> Result { self.phase.assert_translation(); let len = self.height(); @@ -417,7 +416,6 @@ impl Stack { /// # Errors /// /// If too many function local constants have been allocated already. - #[must_use] pub fn const_to_reg(&mut self, value: impl Into) -> Result { self.phase.assert_translation(); self.consts.alloc(value.into()) @@ -428,7 +426,6 @@ impl Stack { /// # Errors /// /// If `index` cannot be converted into a [`Reg`]. - #[must_use] fn local_to_reg(&self, index: LocalIdx) -> Result { self.phase.assert_translation(); let Ok(index) = i16::try_from(u32::from(index)) else { @@ -442,7 +439,6 @@ impl Stack { /// # Errors /// /// If `index` cannot be converted into a [`StackSlot`]. - #[must_use] pub fn temp_to_reg(&self, index: OperandIdx) -> Result { self.phase.assert_translation(); let index = usize::from(index); From c8476793c8f6ddfe79f5a84c87ece66eebe912ef Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Tue, 20 May 2025 11:51:47 +0200 Subject: [PATCH 033/343] optimize LocalsRegistry perf and memory usage --- crates/wasmi/src/engine/translator/error.rs | 5 + .../src/engine/translator/stack2/locals.rs | 178 ++++++++---------- 2 files changed, 86 insertions(+), 97 deletions(-) diff --git a/crates/wasmi/src/engine/translator/error.rs b/crates/wasmi/src/engine/translator/error.rs index ca7598706d..b3ed065bec 100644 --- a/crates/wasmi/src/engine/translator/error.rs +++ b/crates/wasmi/src/engine/translator/error.rs @@ -30,6 +30,8 @@ pub enum TranslationError { TooManyFunctionResults, /// Tried to define a function with too many function parameters. TooManyFunctionParams, + /// Tried to define a function with too many local variables. + TooManyLocalVariables, /// The function failed to compiled lazily. LazyCompilationFailed, } @@ -99,6 +101,9 @@ impl Display for TranslationError { Self::TooManyFunctionParams => { write!(f, "encountered function with too many function parameters") } + Self::TooManyLocalVariables => { + write!(f, "encountered function with too many local variables") + } Self::LazyCompilationFailed => { write!( f, diff --git a/crates/wasmi/src/engine/translator/stack2/locals.rs b/crates/wasmi/src/engine/translator/stack2/locals.rs index 47bd03db68..44b91326cb 100644 --- a/crates/wasmi/src/engine/translator/stack2/locals.rs +++ b/crates/wasmi/src/engine/translator/stack2/locals.rs @@ -1,7 +1,7 @@ use super::OperandIdx; -use crate::{core::ValType, Error}; +use crate::{core::ValType, engine::TranslationError, Error}; use alloc::vec::Vec; -use core::{cmp::Ordering, iter}; +use core::{cmp, iter}; /// A local variable index. #[derive(Debug, Copy, Clone)] @@ -22,14 +22,10 @@ impl From for u32 { /// Stores definitions of locals. #[derive(Debug, Default, Clone)] pub struct LocalsRegistry { - /// Groups of local variables sharing a common type. - groups: Vec, - /// The index of the first local variable in a group of more than 1 locals. - /// - /// # Note - /// - /// All local indices that are smaller than this can be queried faster. - first_group: Option, + /// The types of the first defined local variables. + tys_first: Vec, + /// The types of the remaining defined local variables. + tys_remaining: Vec, /// The first operand for the local on the stack if any. first_operands: Vec>, } @@ -37,8 +33,8 @@ pub struct LocalsRegistry { impl LocalsRegistry { /// Resets `self` for reuse. pub fn reset(&mut self) { - self.groups.clear(); - self.first_group = None; + self.tys_first.clear(); + self.tys_remaining.clear(); self.first_operands.clear(); } @@ -47,6 +43,12 @@ impl LocalsRegistry { self.first_operands.len() } + /// The maximum number of local variables per function. + const LOCAL_VARIABLES_MAX: usize = 30_000; + + /// The maximum number of local variables in the fast `tys_first` vector. + const FIRST_TYS_MAX: usize = 100; + /// Registers `amount` of locals of type `ty` for `self`. /// /// # Errors @@ -61,16 +63,18 @@ impl LocalsRegistry { "failed to register {amount} local variables of type {ty:?}: out of bounds `usize`" ) }; - let first_local = self.len(); - if amount > 1 { - self.first_group.get_or_insert(first_local); + if self.len().saturating_add(amount) > Self::LOCAL_VARIABLES_MAX { + return Err(Error::from(TranslationError::TooManyFunctionParams)); + } + let vacant_first = Self::FIRST_TYS_MAX.saturating_sub(self.tys_first.len()); + let push_to_first = cmp::min(vacant_first, amount); + self.tys_first.extend(iter::repeat_n(ty, push_to_first)); + let remaining_amount = amount - push_to_first; + let remaining_index = (self.len() + amount - 1) as u32; + if remaining_amount > 0 { + self.tys_remaining + .push(LocalGroup::new(remaining_index, ty)); } - let last_local = first_local + amount; - self.groups.push(LocalGroup { - first_local, - last_local, - ty, - }); self.first_operands.extend(iter::repeat_n(None, amount)); Ok(()) } @@ -90,11 +94,7 @@ impl LocalsRegistry { /// /// If `index` is out of bounds. pub fn first_operand(&self, index: LocalIdx) -> Option { - let index = Self::local_idx_to_index(index); - let Some(cell) = self.first_operands.get(index) else { - panic!("`first_operand`: out of bounds local index: {index:?}") - }; - *cell + self.first_operands[Self::local_idx_to_index(index)] } /// Takes the first operand for this local from the stack if any. @@ -104,9 +104,7 @@ impl LocalsRegistry { /// If `index` is out of bounds. pub fn take_first_operand(&mut self, index: LocalIdx) -> Option { let index = Self::local_idx_to_index(index); - let Some(cell) = self.first_operands.get_mut(index) else { - panic!("`first_operand`: out of bounds local index: {index:?}") - }; + let cell = &mut self.first_operands[index]; cell.take() } @@ -121,67 +119,37 @@ impl LocalsRegistry { first_operand: Option, ) -> Option { let index = Self::local_idx_to_index(index); - let Some(cell) = self.first_operands.get_mut(index) else { - panic!("`first_operand`: out of bounds local index: {index:?}") - }; + let cell = &mut self.first_operands[index]; match first_operand { Some(first_operand) => cell.replace(first_operand), None => cell.take(), } } - /// Returns the type of the `index` if any. - /// - /// Returns `None` if `index` does not refer to any local in `self`. + /// Returns the type of the local variable at `index` if any. pub fn ty(&self, index: LocalIdx) -> Option { - let index = Self::local_idx_to_index(index); - if let Some(first_group) = self.first_group { - if index >= first_group { - return self.ty_slow(index); - } + let index_sz = Self::local_idx_to_index(index); + match self.tys_first.get(index_sz) { + Some(ty) => Some(*ty), + None => self.ty_slow(index), } - self.ty_fast(index) } - /// Returns the [`ValType`] of the local at `local_index`. + /// Returns the type of the local variable at `index` if any. /// - /// # Note - /// - /// This is the fast version used for locals with indices - /// smaller than the first local group. - #[inline] - fn ty_fast(&self, local_index: usize) -> Option { - self.groups.get(local_index).map(LocalGroup::ty) - } - - /// Returns the [`ValType`] of the local at `local_index`. - /// - /// # Note - /// - /// This is the slow version used for locals with indices - /// equal to or greater than the first local group. + /// This is the slow-path for local variables that have been stored in the `remaining` buffer. #[cold] - fn ty_slow(&self, local_index: usize) -> Option { - let Some(first_group) = self.first_group else { - unreachable!("cannot use `ty_slow` with `first_group` being `None`"); - }; - let groups = &self.groups[first_group..]; - if groups.is_empty() { + fn ty_slow(&self, index: LocalIdx) -> Option { + if self.tys_remaining.is_empty() { return None; } - let index = groups - .binary_search_by(|group| { - if local_index < group.first_local { - Ordering::Greater - } else if local_index > group.last_local { - Ordering::Less - } else { - Ordering::Equal - } - }) - .ok()?; - let ty = groups[index].ty(); - Some(ty) + match self + .tys_remaining + .binary_search_by_key(&index.0, LocalGroup::max_index) + { + Err(i) if i == self.tys_remaining.len() => None, + Ok(i) | Err(i) => Some(self.tys_remaining[i].ty()), + } } } @@ -189,14 +157,22 @@ impl LocalsRegistry { #[derive(Debug, Copy, Clone)] struct LocalGroup { /// The local index of the first local in the group. - first_local: usize, - /// The local index right after the last local in the group. - last_local: usize, + max_index: u32, /// The shared type of the locals in the local group. ty: ValType, } impl LocalGroup { + /// Creates a new [`LocalGroup`]. + fn new(max_index: u32, ty: ValType) -> Self { + Self { max_index, ty } + } + + /// Returns the maximum index of the local variables in the [`LocalGroup`]. + fn max_index(&self) -> u32 { + self.max_index + } + /// Returns the [`ValType`] of the [`LocalGroup`]. fn ty(&self) -> ValType { self.ty @@ -209,8 +185,9 @@ mod tests { #[test] fn ty_works() { - for locals_per_type in [1, 2, 10] { - let mut locals = LocalsRegistry::default(); + let mut locals = LocalsRegistry::default(); + for locals_per_type in [1, 2, 10, 100] { + locals.reset(); let tys = [ValType::I32, ValType::I64, ValType::F32, ValType::F64]; for ty in tys { locals.register(locals_per_type, ty).unwrap(); @@ -229,22 +206,29 @@ mod tests { #[test] fn locals_followed_by_groups() { let mut locals = LocalsRegistry::default(); - let len_single = 10; - let len_groups = 10; - let locals_per_group = 100; - let len_locals = len_single + len_groups * locals_per_group; - for i in 0..len_single { - locals.register(1, ValType::I32).unwrap(); - } - for i in 0..len_groups { - locals.register(locals_per_group, ValType::I64).unwrap(); - } - for i in 0..len_locals { - let ty = match i < len_single { - true => ValType::I32, - false => ValType::I64, - }; - assert_eq!(locals.ty(LocalIdx(i)), Some(ty)); + let len_single = [1, 10, 100]; + let len_groups = [1, 10, 100]; + let locals_per_group = [10, 100]; + for len_single in len_single { + for len_groups in len_groups { + for locals_per_group in locals_per_group { + locals.reset(); + let len_locals = len_single + (len_groups * locals_per_group); + for i in 0..len_single { + locals.register(1, ValType::I32).unwrap(); + } + for i in 0..len_groups { + locals.register(locals_per_group, ValType::I64).unwrap(); + } + for i in 0..len_locals { + let ty = match i < len_single { + true => ValType::I32, + false => ValType::I64, + }; + assert_eq!(locals.ty(LocalIdx(i)), Some(ty)); + } + } + } } } } From 95647ab33354e09b8ebfbe4e92387305f69212b4 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Tue, 20 May 2025 11:53:45 +0200 Subject: [PATCH 034/343] add missing wrapping_sub --- crates/wasmi/src/engine/translator/stack2/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/stack2/mod.rs b/crates/wasmi/src/engine/translator/stack2/mod.rs index 6cc16804d1..395b8d72bb 100644 --- a/crates/wasmi/src/engine/translator/stack2/mod.rs +++ b/crates/wasmi/src/engine/translator/stack2/mod.rs @@ -521,7 +521,7 @@ pub struct OperandIdx(NonZero); impl From for usize { fn from(value: OperandIdx) -> Self { - value.0.get() + value.0.get().wrapping_sub(1) } } From 122c6a2bd02f8f68e20685a492f89b414e3a7271 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Tue, 20 May 2025 11:59:01 +0200 Subject: [PATCH 035/343] enable unused_variables warning again --- crates/wasmi/src/engine/translator/stack2/locals.rs | 4 ++-- crates/wasmi/src/engine/translator/stack2/mod.rs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/wasmi/src/engine/translator/stack2/locals.rs b/crates/wasmi/src/engine/translator/stack2/locals.rs index 44b91326cb..e174f5cd23 100644 --- a/crates/wasmi/src/engine/translator/stack2/locals.rs +++ b/crates/wasmi/src/engine/translator/stack2/locals.rs @@ -214,10 +214,10 @@ mod tests { for locals_per_group in locals_per_group { locals.reset(); let len_locals = len_single + (len_groups * locals_per_group); - for i in 0..len_single { + for _ in 0..len_single { locals.register(1, ValType::I32).unwrap(); } - for i in 0..len_groups { + for _ in 0..len_groups { locals.register(locals_per_group, ValType::I64).unwrap(); } for i in 0..len_locals { diff --git a/crates/wasmi/src/engine/translator/stack2/mod.rs b/crates/wasmi/src/engine/translator/stack2/mod.rs index 395b8d72bb..fb788629b2 100644 --- a/crates/wasmi/src/engine/translator/stack2/mod.rs +++ b/crates/wasmi/src/engine/translator/stack2/mod.rs @@ -1,4 +1,4 @@ -#![expect(unused_variables, dead_code)] +#![expect(dead_code)] mod consts; mod locals; @@ -380,7 +380,7 @@ impl Stack { Operand::local(operand_index, local_index, &self.locals) } StackOperand::Immediate { val } => Operand::immediate(operand_index, val), - StackOperand::Temp { ty, instr } => return None, + StackOperand::Temp { .. } => return None, }; self.operands[index] = StackOperand::Temp { ty: operand.ty(), From 47a532749906b471ff1d26730d329e0726c1474d Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Tue, 20 May 2025 12:06:10 +0200 Subject: [PATCH 036/343] move Operand definitions into its own submodule --- .../wasmi/src/engine/translator/stack2/mod.rs | 161 +---------------- .../src/engine/translator/stack2/operand.rs | 167 ++++++++++++++++++ 2 files changed, 169 insertions(+), 159 deletions(-) create mode 100644 crates/wasmi/src/engine/translator/stack2/operand.rs diff --git a/crates/wasmi/src/engine/translator/stack2/mod.rs b/crates/wasmi/src/engine/translator/stack2/mod.rs index fb788629b2..212066ffee 100644 --- a/crates/wasmi/src/engine/translator/stack2/mod.rs +++ b/crates/wasmi/src/engine/translator/stack2/mod.rs @@ -2,7 +2,9 @@ mod consts; mod locals; +mod operand; +pub use self::operand::Operand; use self::{ consts::ConstRegistry, locals::{LocalIdx, LocalsRegistry}, @@ -562,162 +564,3 @@ enum StackOperand { val: TypedVal, }, } - -/// An operand on the [`Stack`]. -#[derive(Debug, Copy, Clone)] -pub enum Operand { - /// A local variable operand. - Local(LocalOperand), - /// A temporary operand. - Temp(TempOperand), - /// An immediate value operand. - Immediate(ImmediateOperand), -} - -impl Operand { - fn new(operand_index: OperandIdx, operand: StackOperand, locals: &LocalsRegistry) -> Self { - match operand { - StackOperand::Local { local_index, .. } => { - Self::local(operand_index, local_index, locals) - } - StackOperand::Temp { ty, instr } => Self::temp(operand_index, ty, instr), - StackOperand::Immediate { val } => Self::immediate(operand_index, val), - } - } - - /// Creates a local [`Operand`]. - fn local(operand_index: OperandIdx, local_index: LocalIdx, locals: &LocalsRegistry) -> Self { - let Some(ty) = locals.ty(local_index) else { - panic!("failed to query type of local at: {local_index:?}"); - }; - Self::Local(LocalOperand { - operand_index, - local_index, - ty, - }) - } - - /// Creates a temporary [`Operand`]. - fn temp(operand_index: OperandIdx, ty: ValType, instr: Option) -> Self { - Self::Temp(TempOperand { - operand_index, - ty, - instr, - }) - } - - /// Creates an immediate [`Operand`]. - fn immediate(operand_index: OperandIdx, val: TypedVal) -> Self { - Self::Immediate(ImmediateOperand { operand_index, val }) - } - - /// Returns `true` if `self` is an [`Operand::Local`]. - pub fn is_local(self) -> bool { - matches!(self, Self::Local(_)) - } - - /// Returns `true` if `self` is an [`Operand::Temp`]. - pub fn is_temp(self) -> bool { - matches!(self, Self::Temp(_)) - } - - /// Returns `true` if `self` is an [`Operand::Immediate`]. - pub fn is_immediate(self) -> bool { - matches!(self, Self::Immediate(_)) - } - - /// Returns the [`OperandIdx`] of the [`Operand`]. - pub fn index(&self) -> OperandIdx { - match self { - Operand::Local(operand) => operand.operand_index(), - Operand::Temp(operand) => operand.operand_index(), - Operand::Immediate(operand) => operand.operand_index(), - } - } - - /// Returns the type of the [`Operand`]. - pub fn ty(self) -> ValType { - match self { - Self::Local(local_operand) => local_operand.ty(), - Self::Temp(temp_operand) => temp_operand.ty(), - Self::Immediate(immediate_operand) => immediate_operand.ty(), - } - } -} - -/// A local variable on the [`Stack`]. -#[derive(Debug, Copy, Clone)] -pub struct LocalOperand { - /// The index of the operand. - operand_index: OperandIdx, - /// The index of the local variable. - local_index: LocalIdx, - /// The type of the local variable. - ty: ValType, -} - -impl LocalOperand { - /// Returns the operand index of the [`LocalOperand`]. - pub fn operand_index(&self) -> OperandIdx { - self.operand_index - } - - /// Returns the index of the [`LocalOperand`]. - pub fn local_index(&self) -> LocalIdx { - self.local_index - } - - /// Returns the type of the [`LocalOperand`]. - pub fn ty(&self) -> ValType { - self.ty - } -} - -/// A temporary on the [`Stack`]. -#[derive(Debug, Copy, Clone)] -pub struct TempOperand { - /// The index of the operand. - operand_index: OperandIdx, - /// The type of the temporary. - ty: ValType, - /// The instruction which created this [`TempOperand`] as its result. - instr: Option, -} - -impl TempOperand { - /// Returns the operand index of the [`TempOperand`]. - pub fn operand_index(&self) -> OperandIdx { - self.operand_index - } - - /// Returns the type of the [`TempOperand`]. - pub fn ty(self) -> ValType { - self.ty - } -} - -/// An immediate value on the [`Stack`]. -#[derive(Debug, Copy, Clone)] -pub struct ImmediateOperand { - /// The index of the operand. - operand_index: OperandIdx, - /// The value and type of the immediate value. - val: TypedVal, -} - -impl ImmediateOperand { - /// Returns the operand index of the [`ImmediateOperand`]. - pub fn operand_index(&self) -> OperandIdx { - self.operand_index - } - - /// Returns the immediate value (and its type) of the [`ImmediateOperand`]. - pub fn val(self) -> TypedVal { - self.val - } - - /// Returns the type of the [`ImmediateOperand`]. - pub fn ty(self) -> ValType { - self.val.ty() - } -} diff --git a/crates/wasmi/src/engine/translator/stack2/operand.rs b/crates/wasmi/src/engine/translator/stack2/operand.rs new file mode 100644 index 0000000000..57aa46bd1b --- /dev/null +++ b/crates/wasmi/src/engine/translator/stack2/operand.rs @@ -0,0 +1,167 @@ +use super::{LocalIdx, LocalsRegistry, OperandIdx, StackOperand}; +use crate::{ + core::{TypedVal, ValType}, + engine::Instr, +}; + +/// An operand on the [`Stack`]. +#[derive(Debug, Copy, Clone)] +pub enum Operand { + /// A local variable operand. + Local(LocalOperand), + /// A temporary operand. + Temp(TempOperand), + /// An immediate value operand. + Immediate(ImmediateOperand), +} + +impl Operand { + /// Creates a new [`Operand`] from the given [`StackOperand`] and its [`OperandIdx`]. + pub(super) fn new(index: OperandIdx, operand: StackOperand, locals: &LocalsRegistry) -> Self { + match operand { + StackOperand::Local { local_index, .. } => Self::local(index, local_index, locals), + StackOperand::Temp { ty, instr } => Self::temp(index, ty, instr), + StackOperand::Immediate { val } => Self::immediate(index, val), + } + } + + /// Creates a local [`Operand`]. + pub(super) fn local( + operand_index: OperandIdx, + local_index: LocalIdx, + locals: &LocalsRegistry, + ) -> Self { + let Some(ty) = locals.ty(local_index) else { + panic!("failed to query type of local at: {local_index:?}"); + }; + Self::Local(LocalOperand { + operand_index, + local_index, + ty, + }) + } + + /// Creates a temporary [`Operand`]. + pub(super) fn temp(operand_index: OperandIdx, ty: ValType, instr: Option) -> Self { + Self::Temp(TempOperand { + operand_index, + ty, + instr, + }) + } + + /// Creates an immediate [`Operand`]. + pub(super) fn immediate(operand_index: OperandIdx, val: TypedVal) -> Self { + Self::Immediate(ImmediateOperand { operand_index, val }) + } + + /// Returns `true` if `self` is an [`Operand::Local`]. + pub fn is_local(self) -> bool { + matches!(self, Self::Local(_)) + } + + /// Returns `true` if `self` is an [`Operand::Temp`]. + pub fn is_temp(self) -> bool { + matches!(self, Self::Temp(_)) + } + + /// Returns `true` if `self` is an [`Operand::Immediate`]. + pub fn is_immediate(self) -> bool { + matches!(self, Self::Immediate(_)) + } + + /// Returns the [`OperandIdx`] of the [`Operand`]. + pub fn index(&self) -> OperandIdx { + match self { + Operand::Local(operand) => operand.operand_index(), + Operand::Temp(operand) => operand.operand_index(), + Operand::Immediate(operand) => operand.operand_index(), + } + } + + /// Returns the type of the [`Operand`]. + pub fn ty(self) -> ValType { + match self { + Self::Local(local_operand) => local_operand.ty(), + Self::Temp(temp_operand) => temp_operand.ty(), + Self::Immediate(immediate_operand) => immediate_operand.ty(), + } + } +} + +/// A local variable on the [`Stack`]. +#[derive(Debug, Copy, Clone)] +pub struct LocalOperand { + /// The index of the operand. + operand_index: OperandIdx, + /// The index of the local variable. + local_index: LocalIdx, + /// The type of the local variable. + ty: ValType, +} + +impl LocalOperand { + /// Returns the operand index of the [`LocalOperand`]. + pub fn operand_index(&self) -> OperandIdx { + self.operand_index + } + + /// Returns the index of the [`LocalOperand`]. + pub fn local_index(&self) -> LocalIdx { + self.local_index + } + + /// Returns the type of the [`LocalOperand`]. + pub fn ty(&self) -> ValType { + self.ty + } +} + +/// A temporary on the [`Stack`]. +#[derive(Debug, Copy, Clone)] +pub struct TempOperand { + /// The index of the operand. + operand_index: OperandIdx, + /// The type of the temporary. + ty: ValType, + /// The instruction which created this [`TempOperand`] as its result. + instr: Option, +} + +impl TempOperand { + /// Returns the operand index of the [`TempOperand`]. + pub fn operand_index(&self) -> OperandIdx { + self.operand_index + } + + /// Returns the type of the [`TempOperand`]. + pub fn ty(self) -> ValType { + self.ty + } +} + +/// An immediate value on the [`Stack`]. +#[derive(Debug, Copy, Clone)] +pub struct ImmediateOperand { + /// The index of the operand. + operand_index: OperandIdx, + /// The value and type of the immediate value. + val: TypedVal, +} + +impl ImmediateOperand { + /// Returns the operand index of the [`ImmediateOperand`]. + pub fn operand_index(&self) -> OperandIdx { + self.operand_index + } + + /// Returns the immediate value (and its type) of the [`ImmediateOperand`]. + pub fn val(self) -> TypedVal { + self.val + } + + /// Returns the type of the [`ImmediateOperand`]. + pub fn ty(self) -> ValType { + self.val.ty() + } +} From d13ccad65c72b4ae4b8d4b881e44e31dec7d59d9 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Tue, 20 May 2025 12:06:45 +0200 Subject: [PATCH 037/343] change imports --- crates/wasmi/src/engine/translator/stack2/mod.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/wasmi/src/engine/translator/stack2/mod.rs b/crates/wasmi/src/engine/translator/stack2/mod.rs index 212066ffee..d150b522ff 100644 --- a/crates/wasmi/src/engine/translator/stack2/mod.rs +++ b/crates/wasmi/src/engine/translator/stack2/mod.rs @@ -9,10 +9,9 @@ use self::{ consts::ConstRegistry, locals::{LocalIdx, LocalsRegistry}, }; -use super::Instr; use crate::{ core::{TypedVal, UntypedVal, ValType}, - engine::TranslationError, + engine::{Instr, TranslationError}, ir::Reg, Error, }; From be74240ba98aa56c8670d733d71ea1bfe844ecd3 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Tue, 20 May 2025 12:11:42 +0200 Subject: [PATCH 038/343] rename max_stack_height -> max_height, add getter, update docs --- .../wasmi/src/engine/translator/stack2/mod.rs | 21 +++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/crates/wasmi/src/engine/translator/stack2/mod.rs b/crates/wasmi/src/engine/translator/stack2/mod.rs index d150b522ff..345925b048 100644 --- a/crates/wasmi/src/engine/translator/stack2/mod.rs +++ b/crates/wasmi/src/engine/translator/stack2/mod.rs @@ -28,7 +28,7 @@ pub struct Stack { /// All function local constants. consts: ConstRegistry, /// The index of the first [`StackOperand::Local`] on the [`Stack`]. - max_stack_height: usize, + max_height: usize, /// The current phase of the [`Stack`]. phase: StackPhase, } @@ -95,7 +95,7 @@ impl Stack { self.operands.clear(); self.locals.reset(); self.consts.reset(); - self.max_stack_height = 0; + self.max_height = 0; self.phase = StackPhase::DefineLocals; } @@ -130,11 +130,24 @@ impl Stack { Ok(()) } - /// Returns the current number of [`Operand`]s on the [`Stack`]. + /// Returns the current height of the [`Stack`]. + /// + /// # Note + /// + /// The height is equal to the number of [`Operand`]s on the [`Stack`]. pub fn height(&self) -> usize { self.operands.len() } + /// Returns the maximum height of the [`Stack`]. + /// + /// # Note + /// + /// The height is equal to the number of [`Operand`]s on the [`Stack`]. + pub fn max_height(&self) -> usize { + self.max_height + } + /// Truncates `self` to the target `height`. /// /// All operands above `height` are dropped. @@ -149,7 +162,7 @@ impl Stack { /// Updates the maximum stack height if needed. fn update_max_stack_height(&mut self) { - self.max_stack_height = core::cmp::max(self.max_stack_height, self.height()); + self.max_height = core::cmp::max(self.max_height, self.height()); } /// Returns the [`OperandIdx`] of the next pushed operand. From 919c5c7c1706f3ba9e1746dbc774716a75a73469 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Tue, 20 May 2025 12:12:32 +0200 Subject: [PATCH 039/343] fix broken doc links --- crates/wasmi/src/engine/translator/stack2/mod.rs | 2 +- crates/wasmi/src/engine/translator/stack2/operand.rs | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/stack2/mod.rs b/crates/wasmi/src/engine/translator/stack2/mod.rs index 345925b048..ecf68509e4 100644 --- a/crates/wasmi/src/engine/translator/stack2/mod.rs +++ b/crates/wasmi/src/engine/translator/stack2/mod.rs @@ -452,7 +452,7 @@ impl Stack { /// /// # Errors /// - /// If `index` cannot be converted into a [`StackSlot`]. + /// If `index` cannot be converted into a [`Reg`]. pub fn temp_to_reg(&self, index: OperandIdx) -> Result { self.phase.assert_translation(); let index = usize::from(index); diff --git a/crates/wasmi/src/engine/translator/stack2/operand.rs b/crates/wasmi/src/engine/translator/stack2/operand.rs index 57aa46bd1b..a706a7a1e8 100644 --- a/crates/wasmi/src/engine/translator/stack2/operand.rs +++ b/crates/wasmi/src/engine/translator/stack2/operand.rs @@ -4,6 +4,9 @@ use crate::{ engine::Instr, }; +#[cfg(doc)] +use super::Stack; + /// An operand on the [`Stack`]. #[derive(Debug, Copy, Clone)] pub enum Operand { From 52572360d7eb542678956fdbb26dd5685c07df6e Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Tue, 20 May 2025 12:16:29 +0200 Subject: [PATCH 040/343] remove StackPhase Not really needed anymore without preservation registers. --- .../wasmi/src/engine/translator/stack2/mod.rs | 83 ------------------- 1 file changed, 83 deletions(-) diff --git a/crates/wasmi/src/engine/translator/stack2/mod.rs b/crates/wasmi/src/engine/translator/stack2/mod.rs index ecf68509e4..1224c66855 100644 --- a/crates/wasmi/src/engine/translator/stack2/mod.rs +++ b/crates/wasmi/src/engine/translator/stack2/mod.rs @@ -29,64 +29,6 @@ pub struct Stack { consts: ConstRegistry, /// The index of the first [`StackOperand::Local`] on the [`Stack`]. max_height: usize, - /// The current phase of the [`Stack`]. - phase: StackPhase, -} - -/// The current phase of the [`Stack`]. -#[derive(Debug, Default, Copy, Clone)] -pub enum StackPhase { - /// Phase that allows to define local variables. - #[default] - DefineLocals, - /// Phase that allows to manipulate the stack and allocate function local constants. - Translation, - /// Phase after finishing translation. - /// - /// In this phase state changes are no longer allowed. - /// Only resetting the [`Stack`] is allowed in order to restart the phase cycle. - Finish, -} - -impl StackPhase { - /// Resets the [`StackPhase`] to the [`StackPhase::DefineLocals`] phase. - pub fn reset(&mut self) { - *self = Self::default(); - } - - /// Ensures that the current phase is [`StackPhase::DefineLocals`]. - pub fn assert_define_locals(&self) { - debug_assert!(matches!(self, Self::DefineLocals)); - } - - /// Turns the current phase into [`StackPhase::Translation`]. - /// - /// # Panics - /// - /// If the current phase is incompatible with this phase shift. - pub fn translation(&mut self) { - assert!(matches!(self, Self::DefineLocals)); - *self = Self::Translation; - } - - /// Turns the current phase into [`StackPhase::Translation`]. - /// - /// # Panics - /// - /// If the current phase is incompatible with this phase shift. - pub fn assert_translation(&self) { - debug_assert!(matches!(self, Self::Translation)) - } - - /// Turns the current phase into [`StackPhase::Finish`]. - /// - /// # Panics - /// - /// If the current phase is incompatible with this phase shift. - pub fn finish(&mut self) { - debug_assert!(matches!(self, Self::Translation)); - *self = Self::Finish; - } } impl Stack { @@ -96,7 +38,6 @@ impl Stack { self.locals.reset(); self.consts.reset(); self.max_height = 0; - self.phase = StackPhase::DefineLocals; } /// Register `amount` local variables of common type `ty`. @@ -105,28 +46,16 @@ impl Stack { /// /// If too many local variables are being registered. pub fn register_locals(&mut self, amount: u32, ty: ValType) -> Result<(), Error> { - self.phase.assert_define_locals(); self.locals.register(amount, ty)?; Ok(()) } - /// Finish registration of local variables. - /// - /// # Errors - /// - /// If the current [`StackPhase`] is not [`StackPhase::DefineLocals`]. - pub fn finish_register_locals(&mut self) -> Result<(), Error> { - self.phase.translation(); - Ok(()) - } - /// Finish translation of the function body. /// /// # Errors /// /// If the current [`StackPhase`] is not [`StackPhase::Translation`]. pub fn finish_translation(&mut self) -> Result<(), Error> { - self.phase.finish(); Ok(()) } @@ -208,7 +137,6 @@ impl Stack { /// - If too many operands have been pushed onto the [`Stack`]. /// - If the local with `local_idx` does not exist. pub fn push_local(&mut self, local_index: LocalIdx) -> Result { - self.phase.assert_translation(); let operand_index = self.next_operand_idx(); let next_local = self .locals @@ -232,7 +160,6 @@ impl Stack { /// - If the current [`StackPhase`] is not [`StackPhase::Translation`]. /// - If too many operands have been pushed onto the [`Stack`]. pub fn push_temp(&mut self, ty: ValType, instr: Option) -> Result { - self.phase.assert_translation(); let operand_index = self.next_operand_idx(); self.operands.push(StackOperand::Temp { ty, instr }); self.update_max_stack_height(); @@ -246,7 +173,6 @@ impl Stack { /// - If the current [`StackPhase`] is not [`StackPhase::Translation`]. /// - If too many operands have been pushed onto the [`Stack`]. pub fn push_immediate(&mut self, value: impl Into) -> Result { - self.phase.assert_translation(); let operand_index = self.next_operand_idx(); self.operands .push(StackOperand::Immediate { val: value.into() }); @@ -258,7 +184,6 @@ impl Stack { /// /// Returns `None` if the [`Stack`] is empty. pub fn peek(&self) -> Option { - self.phase.assert_translation(); let operand = self.operands.last().copied()?; let index = OperandIdx::from(self.operands.len() - 1); Some(Operand::new(index, operand, &self.locals)) @@ -268,7 +193,6 @@ impl Stack { /// /// Returns `None` if the [`Stack`] is empty. pub fn pop(&mut self) -> Option { - self.phase.assert_translation(); let operand = self.operands.pop()?; let index = OperandIdx::from(self.operands.len()); Some(Operand::new(index, operand, &self.locals)) @@ -297,7 +221,6 @@ impl Stack { /// - Returns `None` if the [`Stack`] is empty. /// - The last returned [`Operand`] is the top-most one. fn pop_some(&mut self) -> Option<[Operand; N]> { - self.phase.assert_translation(); if N >= self.height() { return None; } @@ -316,7 +239,6 @@ impl Stack { /// Returns `None` if the [`Reg`] is unknown to the [`Stack`]. #[must_use] pub fn stack_space(&self, reg: Reg) -> RegSpace { - self.phase.assert_translation(); let index = i16::from(reg); if index.is_negative() { return RegSpace::Const; @@ -364,7 +286,6 @@ impl Stack { /// - If `depth` is out of bounds for the [`Stack`] of operands. #[must_use] pub fn operand_to_temp(&mut self, depth: usize) -> Option { - self.phase.assert_translation(); let len = self.height(); if depth >= len { panic!( @@ -409,7 +330,6 @@ impl Stack { /// /// If the `index` is out of bounds. pub fn operand_to_reg(&mut self, depth: usize) -> Result { - self.phase.assert_translation(); let len = self.height(); if depth >= len { panic!( @@ -431,7 +351,6 @@ impl Stack { /// /// If too many function local constants have been allocated already. pub fn const_to_reg(&mut self, value: impl Into) -> Result { - self.phase.assert_translation(); self.consts.alloc(value.into()) } @@ -441,7 +360,6 @@ impl Stack { /// /// If `index` cannot be converted into a [`Reg`]. fn local_to_reg(&self, index: LocalIdx) -> Result { - self.phase.assert_translation(); let Ok(index) = i16::try_from(u32::from(index)) else { return Err(Error::from(TranslationError::AllocatedTooManyRegisters)); }; @@ -454,7 +372,6 @@ impl Stack { /// /// If `index` cannot be converted into a [`Reg`]. pub fn temp_to_reg(&self, index: OperandIdx) -> Result { - self.phase.assert_translation(); let index = usize::from(index); debug_assert!(matches!(&self.operands[index], StackOperand::Temp { .. },)); let Some(index) = index.checked_add(self.locals.len()) else { From 3484ea59e7898d70e811e6306b34cba9195c9b3c Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Tue, 20 May 2025 12:17:29 +0200 Subject: [PATCH 041/343] remove Stack::finish_translation method --- crates/wasmi/src/engine/translator/stack2/mod.rs | 9 --------- 1 file changed, 9 deletions(-) diff --git a/crates/wasmi/src/engine/translator/stack2/mod.rs b/crates/wasmi/src/engine/translator/stack2/mod.rs index 1224c66855..e989a94d8d 100644 --- a/crates/wasmi/src/engine/translator/stack2/mod.rs +++ b/crates/wasmi/src/engine/translator/stack2/mod.rs @@ -50,15 +50,6 @@ impl Stack { Ok(()) } - /// Finish translation of the function body. - /// - /// # Errors - /// - /// If the current [`StackPhase`] is not [`StackPhase::Translation`]. - pub fn finish_translation(&mut self) -> Result<(), Error> { - Ok(()) - } - /// Returns the current height of the [`Stack`]. /// /// # Note From f3aa40445ca270c588e70f8c58620d6f25e75958 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Tue, 20 May 2025 12:17:59 +0200 Subject: [PATCH 042/343] update #Error docs --- crates/wasmi/src/engine/translator/stack2/mod.rs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/crates/wasmi/src/engine/translator/stack2/mod.rs b/crates/wasmi/src/engine/translator/stack2/mod.rs index e989a94d8d..51d6b9460d 100644 --- a/crates/wasmi/src/engine/translator/stack2/mod.rs +++ b/crates/wasmi/src/engine/translator/stack2/mod.rs @@ -124,7 +124,6 @@ impl Stack { /// /// # Errors /// - /// - If the current [`StackPhase`] is not [`StackPhase::Translation`]. /// - If too many operands have been pushed onto the [`Stack`]. /// - If the local with `local_idx` does not exist. pub fn push_local(&mut self, local_index: LocalIdx) -> Result { @@ -148,8 +147,7 @@ impl Stack { /// /// # Errors /// - /// - If the current [`StackPhase`] is not [`StackPhase::Translation`]. - /// - If too many operands have been pushed onto the [`Stack`]. + /// If too many operands have been pushed onto the [`Stack`]. pub fn push_temp(&mut self, ty: ValType, instr: Option) -> Result { let operand_index = self.next_operand_idx(); self.operands.push(StackOperand::Temp { ty, instr }); @@ -161,8 +159,7 @@ impl Stack { /// /// # Errors /// - /// - If the current [`StackPhase`] is not [`StackPhase::Translation`]. - /// - If too many operands have been pushed onto the [`Stack`]. + /// If too many operands have been pushed onto the [`Stack`]. pub fn push_immediate(&mut self, value: impl Into) -> Result { let operand_index = self.next_operand_idx(); self.operands From 066549d7553548a1ad22ba32bfbbb032a5ff0b35 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Tue, 20 May 2025 12:19:31 +0200 Subject: [PATCH 043/343] add TempOperand::instr getter --- crates/wasmi/src/engine/translator/stack2/operand.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/stack2/operand.rs b/crates/wasmi/src/engine/translator/stack2/operand.rs index a706a7a1e8..c453e4420c 100644 --- a/crates/wasmi/src/engine/translator/stack2/operand.rs +++ b/crates/wasmi/src/engine/translator/stack2/operand.rs @@ -127,7 +127,7 @@ pub struct TempOperand { operand_index: OperandIdx, /// The type of the temporary. ty: ValType, - /// The instruction which created this [`TempOperand`] as its result. + /// The instruction which created this [`TempOperand`] as its result if any. instr: Option, } @@ -141,6 +141,11 @@ impl TempOperand { pub fn ty(self) -> ValType { self.ty } + + /// Returns the instruction whcih created this [`TempOperand`] as its result if any. + pub fn instr(&self) -> Option { + self.instr + } } /// An immediate value on the [`Stack`]. From 212f3991d168c0d3428c2efd1fdde21cf0f10a28 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Tue, 20 May 2025 12:21:05 +0200 Subject: [PATCH 044/343] use &self for Operand getters --- .../wasmi/src/engine/translator/stack2/operand.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/crates/wasmi/src/engine/translator/stack2/operand.rs b/crates/wasmi/src/engine/translator/stack2/operand.rs index c453e4420c..41e70fc11a 100644 --- a/crates/wasmi/src/engine/translator/stack2/operand.rs +++ b/crates/wasmi/src/engine/translator/stack2/operand.rs @@ -59,17 +59,17 @@ impl Operand { } /// Returns `true` if `self` is an [`Operand::Local`]. - pub fn is_local(self) -> bool { + pub fn is_local(&self) -> bool { matches!(self, Self::Local(_)) } /// Returns `true` if `self` is an [`Operand::Temp`]. - pub fn is_temp(self) -> bool { + pub fn is_temp(&self) -> bool { matches!(self, Self::Temp(_)) } /// Returns `true` if `self` is an [`Operand::Immediate`]. - pub fn is_immediate(self) -> bool { + pub fn is_immediate(&self) -> bool { matches!(self, Self::Immediate(_)) } @@ -83,7 +83,7 @@ impl Operand { } /// Returns the type of the [`Operand`]. - pub fn ty(self) -> ValType { + pub fn ty(&self) -> ValType { match self { Self::Local(local_operand) => local_operand.ty(), Self::Temp(temp_operand) => temp_operand.ty(), @@ -138,7 +138,7 @@ impl TempOperand { } /// Returns the type of the [`TempOperand`]. - pub fn ty(self) -> ValType { + pub fn ty(&self) -> ValType { self.ty } @@ -164,12 +164,12 @@ impl ImmediateOperand { } /// Returns the immediate value (and its type) of the [`ImmediateOperand`]. - pub fn val(self) -> TypedVal { + pub fn val(&self) -> TypedVal { self.val } /// Returns the type of the [`ImmediateOperand`]. - pub fn ty(self) -> ValType { + pub fn ty(&self) -> ValType { self.val.ty() } } From 7719a00bfe13af2bb658101a6ae4f60726747a66 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Tue, 20 May 2025 12:26:41 +0200 Subject: [PATCH 045/343] make LocalsRegistry::ty panic instead of returning Option An out of bounds LocalIdx should be considered a bug. --- .../src/engine/translator/stack2/locals.rs | 19 +++++++++++-------- .../wasmi/src/engine/translator/stack2/mod.rs | 4 +--- .../src/engine/translator/stack2/operand.rs | 4 +--- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/crates/wasmi/src/engine/translator/stack2/locals.rs b/crates/wasmi/src/engine/translator/stack2/locals.rs index e174f5cd23..708e5a3fa2 100644 --- a/crates/wasmi/src/engine/translator/stack2/locals.rs +++ b/crates/wasmi/src/engine/translator/stack2/locals.rs @@ -127,11 +127,17 @@ impl LocalsRegistry { } /// Returns the type of the local variable at `index` if any. - pub fn ty(&self, index: LocalIdx) -> Option { + /// + /// # Panics + /// + /// If `index` is out of bounds and does not refer to a local in `self`. + pub fn ty(&self, index: LocalIdx) -> ValType { let index_sz = Self::local_idx_to_index(index); match self.tys_first.get(index_sz) { - Some(ty) => Some(*ty), - None => self.ty_slow(index), + Some(ty) => *ty, + None => self + .ty_slow(index) + .unwrap_or_else(|| panic!("out of bounds local index: {index:?}")), } } @@ -195,10 +201,7 @@ mod tests { let locals_per_type = locals_per_type as usize; assert_eq!(locals.len(), locals_per_type * tys.len()); for i in 0..locals.len() { - assert_eq!( - locals.ty(LocalIdx(i as u32)), - Some(tys[i / locals_per_type]) - ); + assert_eq!(locals.ty(LocalIdx(i as u32)), tys[i / locals_per_type]); } } } @@ -225,7 +228,7 @@ mod tests { true => ValType::I32, false => ValType::I64, }; - assert_eq!(locals.ty(LocalIdx(i)), Some(ty)); + assert_eq!(locals.ty(LocalIdx(i)), ty); } } } diff --git a/crates/wasmi/src/engine/translator/stack2/mod.rs b/crates/wasmi/src/engine/translator/stack2/mod.rs index 51d6b9460d..b6504b7b67 100644 --- a/crates/wasmi/src/engine/translator/stack2/mod.rs +++ b/crates/wasmi/src/engine/translator/stack2/mod.rs @@ -252,9 +252,7 @@ impl Stack { /// If the local at `local_index` is out of bounds. #[must_use] pub fn preserve_locals(&mut self, local_index: LocalIdx) -> PreservedLocalsIter { - let Some(ty) = self.locals.ty(local_index) else { - panic!("out of bounds local at: {local_index:?}") - }; + let ty = self.locals.ty(local_index); let index = self.locals.first_operand(local_index); PreservedLocalsIter { stack: self, diff --git a/crates/wasmi/src/engine/translator/stack2/operand.rs b/crates/wasmi/src/engine/translator/stack2/operand.rs index 41e70fc11a..3121b4afb8 100644 --- a/crates/wasmi/src/engine/translator/stack2/operand.rs +++ b/crates/wasmi/src/engine/translator/stack2/operand.rs @@ -34,9 +34,7 @@ impl Operand { local_index: LocalIdx, locals: &LocalsRegistry, ) -> Self { - let Some(ty) = locals.ty(local_index) else { - panic!("failed to query type of local at: {local_index:?}"); - }; + let ty = locals.ty(local_index); Self::Local(LocalOperand { operand_index, local_index, From 8c4756b250a876cbfb9d68895473ccb524bcbf38 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 21 May 2025 12:51:09 +0200 Subject: [PATCH 046/343] PreservedLocalsIter: use StackOperand slice instead of Stack --- crates/wasmi/src/engine/translator/stack2/mod.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/crates/wasmi/src/engine/translator/stack2/mod.rs b/crates/wasmi/src/engine/translator/stack2/mod.rs index b6504b7b67..1a72b4ebc3 100644 --- a/crates/wasmi/src/engine/translator/stack2/mod.rs +++ b/crates/wasmi/src/engine/translator/stack2/mod.rs @@ -254,8 +254,9 @@ impl Stack { pub fn preserve_locals(&mut self, local_index: LocalIdx) -> PreservedLocalsIter { let ty = self.locals.ty(local_index); let index = self.locals.first_operand(local_index); + let operands = &mut self.operands[..]; PreservedLocalsIter { - stack: self, + operands, index, ty, } @@ -372,9 +373,9 @@ impl Stack { /// Iterator yielding preserved local indices while preserving them. #[derive(Debug)] -pub struct PreservedLocalsIter<'a> { +pub struct PreservedLocalsIter<'stack> { /// The underlying operand stack. - stack: &'a mut Stack, + operands: &'stack mut [StackOperand], /// The current operand index of the next preserved local if any. index: Option, /// Type of local at preserved `local_index`. @@ -387,7 +388,7 @@ impl Iterator for PreservedLocalsIter<'_> { fn next(&mut self) -> Option { let index = self.index?; let operand = mem::replace( - &mut self.stack.operands[usize::from(index)], + &mut self.operands[usize::from(index)], StackOperand::Temp { ty: self.ty, instr: None, From 0408ce1b64b56a682cfe959a5b1a70b183590d67 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 21 May 2025 12:53:31 +0200 Subject: [PATCH 047/343] use replace_first_operand in Stack::preserve_locals --- crates/wasmi/src/engine/translator/stack2/locals.rs | 11 ----------- crates/wasmi/src/engine/translator/stack2/mod.rs | 2 +- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/crates/wasmi/src/engine/translator/stack2/locals.rs b/crates/wasmi/src/engine/translator/stack2/locals.rs index 708e5a3fa2..dd072fe272 100644 --- a/crates/wasmi/src/engine/translator/stack2/locals.rs +++ b/crates/wasmi/src/engine/translator/stack2/locals.rs @@ -97,17 +97,6 @@ impl LocalsRegistry { self.first_operands[Self::local_idx_to_index(index)] } - /// Takes the first operand for this local from the stack if any. - /// - /// # Panics - /// - /// If `index` is out of bounds. - pub fn take_first_operand(&mut self, index: LocalIdx) -> Option { - let index = Self::local_idx_to_index(index); - let cell = &mut self.first_operands[index]; - cell.take() - } - /// Replaces the first operand for this local on the stack and returns the old one. /// /// # Panics diff --git a/crates/wasmi/src/engine/translator/stack2/mod.rs b/crates/wasmi/src/engine/translator/stack2/mod.rs index 1a72b4ebc3..eaa41b49d9 100644 --- a/crates/wasmi/src/engine/translator/stack2/mod.rs +++ b/crates/wasmi/src/engine/translator/stack2/mod.rs @@ -253,7 +253,7 @@ impl Stack { #[must_use] pub fn preserve_locals(&mut self, local_index: LocalIdx) -> PreservedLocalsIter { let ty = self.locals.ty(local_index); - let index = self.locals.first_operand(local_index); + let index = self.locals.replace_first_operand(local_index, None); let operands = &mut self.operands[..]; PreservedLocalsIter { operands, From 40fd65928d2fdee9477341e082e1647c6ec84776 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Fri, 23 May 2025 14:44:18 +0200 Subject: [PATCH 048/343] restructure into new translator2 sub-module --- .../src/engine/translator/translator2/mod.rs | 88 ++ .../engine/translator/translator2/simd/mod.rs | 3 + .../translator/translator2/simd/visit.rs | 1028 +++++++++++++++++ .../{stack2 => translator2/stack}/consts.rs | 0 .../{stack2 => translator2/stack}/locals.rs | 0 .../{stack2 => translator2/stack}/mod.rs | 2 - .../{stack2 => translator2/stack}/operand.rs | 0 .../engine/translator/translator2/visit.rs | 894 ++++++++++++++ 8 files changed, 2013 insertions(+), 2 deletions(-) create mode 100644 crates/wasmi/src/engine/translator/translator2/mod.rs create mode 100644 crates/wasmi/src/engine/translator/translator2/simd/mod.rs create mode 100644 crates/wasmi/src/engine/translator/translator2/simd/visit.rs rename crates/wasmi/src/engine/translator/{stack2 => translator2/stack}/consts.rs (100%) rename crates/wasmi/src/engine/translator/{stack2 => translator2/stack}/locals.rs (100%) rename crates/wasmi/src/engine/translator/{stack2 => translator2/stack}/mod.rs (99%) rename crates/wasmi/src/engine/translator/{stack2 => translator2/stack}/operand.rs (100%) create mode 100644 crates/wasmi/src/engine/translator/translator2/visit.rs diff --git a/crates/wasmi/src/engine/translator/translator2/mod.rs b/crates/wasmi/src/engine/translator/translator2/mod.rs new file mode 100644 index 0000000000..7c392d4b24 --- /dev/null +++ b/crates/wasmi/src/engine/translator/translator2/mod.rs @@ -0,0 +1,88 @@ +#![expect(dead_code)] + +#[cfg(feature = "simd")] +mod simd; +mod stack; +mod visit; + +use self::stack::{Operand, OperandIdx, Stack}; +use crate::{ + core::FuelCostsProvider, + engine::{translator::WasmTranslator, CompiledFuncEntity}, + module::{FuncIdx, ModuleHeader}, + Engine, + Error, +}; +use wasmparser::WasmFeatures; + +/// Type concerned with translating from Wasm bytecode to Wasmi bytecode. +#[derive(Debug)] +pub struct FuncTranslator { + /// The reference to the Wasm module function under construction. + func: FuncIdx, + /// The engine for which the function is compiled. + /// + /// # Note + /// + /// Technically this is not needed since the information is redundant given via + /// the `module` field. However, this acts like a faster access since `module` + /// only holds a weak reference to the engine. + engine: Engine, + /// The immutable Wasmi module resources. + module: ModuleHeader, + /// This represents the reachability of the currently translated code. + /// + /// - `true`: The currently translated code is reachable. + /// - `false`: The currently translated code is unreachable and can be skipped. + /// + /// # Note + /// + /// Visiting the Wasm `Else` or `End` control flow operator resets + /// reachability to `true` again. + reachable: bool, + /// Fuel costs for fuel metering. + /// + /// `None` if fuel metering is disabled. + fuel_costs: Option, + /// The reusable data structures of the [`FuncTranslator`]. + alloc: FuncTranslatorAllocations, +} + +#[derive(Debug, Default)] +pub struct FuncTranslatorAllocations { + stack: Stack, +} + +impl WasmTranslator<'_> for FuncTranslator { + type Allocations = FuncTranslatorAllocations; + + fn setup(&mut self, _bytes: &[u8]) -> Result { + Ok(false) + } + + #[inline] + fn features(&self) -> WasmFeatures { + self.engine.config().wasm_features() + } + + fn translate_locals( + &mut self, + _amount: u32, + _value_type: wasmparser::ValType, + ) -> Result<(), Error> { + todo!() + } + + fn finish_translate_locals(&mut self) -> Result<(), Error> { + todo!() + } + + fn update_pos(&mut self, _pos: usize) {} + + fn finish( + self, + _finalize: impl FnOnce(CompiledFuncEntity), + ) -> Result { + todo!() + } +} diff --git a/crates/wasmi/src/engine/translator/translator2/simd/mod.rs b/crates/wasmi/src/engine/translator/translator2/simd/mod.rs new file mode 100644 index 0000000000..10f88caa8a --- /dev/null +++ b/crates/wasmi/src/engine/translator/translator2/simd/mod.rs @@ -0,0 +1,3 @@ +use super::FuncTranslator; + +mod visit; diff --git a/crates/wasmi/src/engine/translator/translator2/simd/visit.rs b/crates/wasmi/src/engine/translator/translator2/simd/visit.rs new file mode 100644 index 0000000000..da8023629a --- /dev/null +++ b/crates/wasmi/src/engine/translator/translator2/simd/visit.rs @@ -0,0 +1,1028 @@ +use super::FuncTranslator; +use wasmparser::VisitSimdOperator; + +impl VisitSimdOperator<'_> for FuncTranslator { + fn visit_v128_load(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + todo!() + } + + fn visit_v128_load8x8_s(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + todo!() + } + + fn visit_v128_load8x8_u(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + todo!() + } + + fn visit_v128_load16x4_s(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + todo!() + } + + fn visit_v128_load16x4_u(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + todo!() + } + + fn visit_v128_load32x2_s(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + todo!() + } + + fn visit_v128_load32x2_u(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + todo!() + } + + fn visit_v128_load8_splat(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + todo!() + } + + fn visit_v128_load16_splat(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + todo!() + } + + fn visit_v128_load32_splat(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + todo!() + } + + fn visit_v128_load64_splat(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + todo!() + } + + fn visit_v128_load32_zero(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + todo!() + } + + fn visit_v128_load64_zero(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + todo!() + } + + fn visit_v128_store(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + todo!() + } + + fn visit_v128_load8_lane(&mut self, memarg: wasmparser::MemArg, lane: u8) -> Self::Output { + todo!() + } + + fn visit_v128_load16_lane(&mut self, memarg: wasmparser::MemArg, lane: u8) -> Self::Output { + todo!() + } + + fn visit_v128_load32_lane(&mut self, memarg: wasmparser::MemArg, lane: u8) -> Self::Output { + todo!() + } + + fn visit_v128_load64_lane(&mut self, memarg: wasmparser::MemArg, lane: u8) -> Self::Output { + todo!() + } + + fn visit_v128_store8_lane(&mut self, memarg: wasmparser::MemArg, lane: u8) -> Self::Output { + todo!() + } + + fn visit_v128_store16_lane(&mut self, memarg: wasmparser::MemArg, lane: u8) -> Self::Output { + todo!() + } + + fn visit_v128_store32_lane(&mut self, memarg: wasmparser::MemArg, lane: u8) -> Self::Output { + todo!() + } + + fn visit_v128_store64_lane(&mut self, memarg: wasmparser::MemArg, lane: u8) -> Self::Output { + todo!() + } + + fn visit_v128_const(&mut self, value: wasmparser::V128) -> Self::Output { + todo!() + } + + fn visit_i8x16_shuffle(&mut self, lanes: [u8; 16]) -> Self::Output { + todo!() + } + + fn visit_i8x16_extract_lane_s(&mut self, lane: u8) -> Self::Output { + todo!() + } + + fn visit_i8x16_extract_lane_u(&mut self, lane: u8) -> Self::Output { + todo!() + } + + fn visit_i8x16_replace_lane(&mut self, lane: u8) -> Self::Output { + todo!() + } + + fn visit_i16x8_extract_lane_s(&mut self, lane: u8) -> Self::Output { + todo!() + } + + fn visit_i16x8_extract_lane_u(&mut self, lane: u8) -> Self::Output { + todo!() + } + + fn visit_i16x8_replace_lane(&mut self, lane: u8) -> Self::Output { + todo!() + } + + fn visit_i32x4_extract_lane(&mut self, lane: u8) -> Self::Output { + todo!() + } + + fn visit_i32x4_replace_lane(&mut self, lane: u8) -> Self::Output { + todo!() + } + + fn visit_i64x2_extract_lane(&mut self, lane: u8) -> Self::Output { + todo!() + } + + fn visit_i64x2_replace_lane(&mut self, lane: u8) -> Self::Output { + todo!() + } + + fn visit_f32x4_extract_lane(&mut self, lane: u8) -> Self::Output { + todo!() + } + + fn visit_f32x4_replace_lane(&mut self, lane: u8) -> Self::Output { + todo!() + } + + fn visit_f64x2_extract_lane(&mut self, lane: u8) -> Self::Output { + todo!() + } + + fn visit_f64x2_replace_lane(&mut self, lane: u8) -> Self::Output { + todo!() + } + + fn visit_i8x16_swizzle(&mut self) -> Self::Output { + todo!() + } + + fn visit_i8x16_splat(&mut self) -> Self::Output { + todo!() + } + + fn visit_i16x8_splat(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32x4_splat(&mut self) -> Self::Output { + todo!() + } + + fn visit_i64x2_splat(&mut self) -> Self::Output { + todo!() + } + + fn visit_f32x4_splat(&mut self) -> Self::Output { + todo!() + } + + fn visit_f64x2_splat(&mut self) -> Self::Output { + todo!() + } + + fn visit_i8x16_eq(&mut self) -> Self::Output { + todo!() + } + + fn visit_i8x16_ne(&mut self) -> Self::Output { + todo!() + } + + fn visit_i8x16_lt_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i8x16_lt_u(&mut self) -> Self::Output { + todo!() + } + + fn visit_i8x16_gt_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i8x16_gt_u(&mut self) -> Self::Output { + todo!() + } + + fn visit_i8x16_le_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i8x16_le_u(&mut self) -> Self::Output { + todo!() + } + + fn visit_i8x16_ge_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i8x16_ge_u(&mut self) -> Self::Output { + todo!() + } + + fn visit_i16x8_eq(&mut self) -> Self::Output { + todo!() + } + + fn visit_i16x8_ne(&mut self) -> Self::Output { + todo!() + } + + fn visit_i16x8_lt_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i16x8_lt_u(&mut self) -> Self::Output { + todo!() + } + + fn visit_i16x8_gt_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i16x8_gt_u(&mut self) -> Self::Output { + todo!() + } + + fn visit_i16x8_le_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i16x8_le_u(&mut self) -> Self::Output { + todo!() + } + + fn visit_i16x8_ge_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i16x8_ge_u(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32x4_eq(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32x4_ne(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32x4_lt_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32x4_lt_u(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32x4_gt_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32x4_gt_u(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32x4_le_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32x4_le_u(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32x4_ge_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32x4_ge_u(&mut self) -> Self::Output { + todo!() + } + + fn visit_i64x2_eq(&mut self) -> Self::Output { + todo!() + } + + fn visit_i64x2_ne(&mut self) -> Self::Output { + todo!() + } + + fn visit_i64x2_lt_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i64x2_gt_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i64x2_le_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i64x2_ge_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_f32x4_eq(&mut self) -> Self::Output { + todo!() + } + + fn visit_f32x4_ne(&mut self) -> Self::Output { + todo!() + } + + fn visit_f32x4_lt(&mut self) -> Self::Output { + todo!() + } + + fn visit_f32x4_gt(&mut self) -> Self::Output { + todo!() + } + + fn visit_f32x4_le(&mut self) -> Self::Output { + todo!() + } + + fn visit_f32x4_ge(&mut self) -> Self::Output { + todo!() + } + + fn visit_f64x2_eq(&mut self) -> Self::Output { + todo!() + } + + fn visit_f64x2_ne(&mut self) -> Self::Output { + todo!() + } + + fn visit_f64x2_lt(&mut self) -> Self::Output { + todo!() + } + + fn visit_f64x2_gt(&mut self) -> Self::Output { + todo!() + } + + fn visit_f64x2_le(&mut self) -> Self::Output { + todo!() + } + + fn visit_f64x2_ge(&mut self) -> Self::Output { + todo!() + } + + fn visit_v128_not(&mut self) -> Self::Output { + todo!() + } + + fn visit_v128_and(&mut self) -> Self::Output { + todo!() + } + + fn visit_v128_andnot(&mut self) -> Self::Output { + todo!() + } + + fn visit_v128_or(&mut self) -> Self::Output { + todo!() + } + + fn visit_v128_xor(&mut self) -> Self::Output { + todo!() + } + + fn visit_v128_bitselect(&mut self) -> Self::Output { + todo!() + } + + fn visit_v128_any_true(&mut self) -> Self::Output { + todo!() + } + + fn visit_i8x16_abs(&mut self) -> Self::Output { + todo!() + } + + fn visit_i8x16_neg(&mut self) -> Self::Output { + todo!() + } + + fn visit_i8x16_popcnt(&mut self) -> Self::Output { + todo!() + } + + fn visit_i8x16_all_true(&mut self) -> Self::Output { + todo!() + } + + fn visit_i8x16_bitmask(&mut self) -> Self::Output { + todo!() + } + + fn visit_i8x16_narrow_i16x8_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i8x16_narrow_i16x8_u(&mut self) -> Self::Output { + todo!() + } + + fn visit_i8x16_shl(&mut self) -> Self::Output { + todo!() + } + + fn visit_i8x16_shr_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i8x16_shr_u(&mut self) -> Self::Output { + todo!() + } + + fn visit_i8x16_add(&mut self) -> Self::Output { + todo!() + } + + fn visit_i8x16_add_sat_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i8x16_add_sat_u(&mut self) -> Self::Output { + todo!() + } + + fn visit_i8x16_sub(&mut self) -> Self::Output { + todo!() + } + + fn visit_i8x16_sub_sat_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i8x16_sub_sat_u(&mut self) -> Self::Output { + todo!() + } + + fn visit_i8x16_min_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i8x16_min_u(&mut self) -> Self::Output { + todo!() + } + + fn visit_i8x16_max_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i8x16_max_u(&mut self) -> Self::Output { + todo!() + } + + fn visit_i8x16_avgr_u(&mut self) -> Self::Output { + todo!() + } + + fn visit_i16x8_extadd_pairwise_i8x16_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i16x8_extadd_pairwise_i8x16_u(&mut self) -> Self::Output { + todo!() + } + + fn visit_i16x8_abs(&mut self) -> Self::Output { + todo!() + } + + fn visit_i16x8_neg(&mut self) -> Self::Output { + todo!() + } + + fn visit_i16x8_q15mulr_sat_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i16x8_all_true(&mut self) -> Self::Output { + todo!() + } + + fn visit_i16x8_bitmask(&mut self) -> Self::Output { + todo!() + } + + fn visit_i16x8_narrow_i32x4_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i16x8_narrow_i32x4_u(&mut self) -> Self::Output { + todo!() + } + + fn visit_i16x8_extend_low_i8x16_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i16x8_extend_high_i8x16_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i16x8_extend_low_i8x16_u(&mut self) -> Self::Output { + todo!() + } + + fn visit_i16x8_extend_high_i8x16_u(&mut self) -> Self::Output { + todo!() + } + + fn visit_i16x8_shl(&mut self) -> Self::Output { + todo!() + } + + fn visit_i16x8_shr_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i16x8_shr_u(&mut self) -> Self::Output { + todo!() + } + + fn visit_i16x8_add(&mut self) -> Self::Output { + todo!() + } + + fn visit_i16x8_add_sat_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i16x8_add_sat_u(&mut self) -> Self::Output { + todo!() + } + + fn visit_i16x8_sub(&mut self) -> Self::Output { + todo!() + } + + fn visit_i16x8_sub_sat_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i16x8_sub_sat_u(&mut self) -> Self::Output { + todo!() + } + + fn visit_i16x8_mul(&mut self) -> Self::Output { + todo!() + } + + fn visit_i16x8_min_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i16x8_min_u(&mut self) -> Self::Output { + todo!() + } + + fn visit_i16x8_max_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i16x8_max_u(&mut self) -> Self::Output { + todo!() + } + + fn visit_i16x8_avgr_u(&mut self) -> Self::Output { + todo!() + } + + fn visit_i16x8_extmul_low_i8x16_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i16x8_extmul_high_i8x16_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i16x8_extmul_low_i8x16_u(&mut self) -> Self::Output { + todo!() + } + + fn visit_i16x8_extmul_high_i8x16_u(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32x4_extadd_pairwise_i16x8_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32x4_extadd_pairwise_i16x8_u(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32x4_abs(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32x4_neg(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32x4_all_true(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32x4_bitmask(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32x4_extend_low_i16x8_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32x4_extend_high_i16x8_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32x4_extend_low_i16x8_u(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32x4_extend_high_i16x8_u(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32x4_shl(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32x4_shr_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32x4_shr_u(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32x4_add(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32x4_sub(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32x4_mul(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32x4_min_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32x4_min_u(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32x4_max_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32x4_max_u(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32x4_dot_i16x8_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32x4_extmul_low_i16x8_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32x4_extmul_high_i16x8_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32x4_extmul_low_i16x8_u(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32x4_extmul_high_i16x8_u(&mut self) -> Self::Output { + todo!() + } + + fn visit_i64x2_abs(&mut self) -> Self::Output { + todo!() + } + + fn visit_i64x2_neg(&mut self) -> Self::Output { + todo!() + } + + fn visit_i64x2_all_true(&mut self) -> Self::Output { + todo!() + } + + fn visit_i64x2_bitmask(&mut self) -> Self::Output { + todo!() + } + + fn visit_i64x2_extend_low_i32x4_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i64x2_extend_high_i32x4_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i64x2_extend_low_i32x4_u(&mut self) -> Self::Output { + todo!() + } + + fn visit_i64x2_extend_high_i32x4_u(&mut self) -> Self::Output { + todo!() + } + + fn visit_i64x2_shl(&mut self) -> Self::Output { + todo!() + } + + fn visit_i64x2_shr_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i64x2_shr_u(&mut self) -> Self::Output { + todo!() + } + + fn visit_i64x2_add(&mut self) -> Self::Output { + todo!() + } + + fn visit_i64x2_sub(&mut self) -> Self::Output { + todo!() + } + + fn visit_i64x2_mul(&mut self) -> Self::Output { + todo!() + } + + fn visit_i64x2_extmul_low_i32x4_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i64x2_extmul_high_i32x4_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i64x2_extmul_low_i32x4_u(&mut self) -> Self::Output { + todo!() + } + + fn visit_i64x2_extmul_high_i32x4_u(&mut self) -> Self::Output { + todo!() + } + + fn visit_f32x4_ceil(&mut self) -> Self::Output { + todo!() + } + + fn visit_f32x4_floor(&mut self) -> Self::Output { + todo!() + } + + fn visit_f32x4_trunc(&mut self) -> Self::Output { + todo!() + } + + fn visit_f32x4_nearest(&mut self) -> Self::Output { + todo!() + } + + fn visit_f32x4_abs(&mut self) -> Self::Output { + todo!() + } + + fn visit_f32x4_neg(&mut self) -> Self::Output { + todo!() + } + + fn visit_f32x4_sqrt(&mut self) -> Self::Output { + todo!() + } + + fn visit_f32x4_add(&mut self) -> Self::Output { + todo!() + } + + fn visit_f32x4_sub(&mut self) -> Self::Output { + todo!() + } + + fn visit_f32x4_mul(&mut self) -> Self::Output { + todo!() + } + + fn visit_f32x4_div(&mut self) -> Self::Output { + todo!() + } + + fn visit_f32x4_min(&mut self) -> Self::Output { + todo!() + } + + fn visit_f32x4_max(&mut self) -> Self::Output { + todo!() + } + + fn visit_f32x4_pmin(&mut self) -> Self::Output { + todo!() + } + + fn visit_f32x4_pmax(&mut self) -> Self::Output { + todo!() + } + + fn visit_f64x2_ceil(&mut self) -> Self::Output { + todo!() + } + + fn visit_f64x2_floor(&mut self) -> Self::Output { + todo!() + } + + fn visit_f64x2_trunc(&mut self) -> Self::Output { + todo!() + } + + fn visit_f64x2_nearest(&mut self) -> Self::Output { + todo!() + } + + fn visit_f64x2_abs(&mut self) -> Self::Output { + todo!() + } + + fn visit_f64x2_neg(&mut self) -> Self::Output { + todo!() + } + + fn visit_f64x2_sqrt(&mut self) -> Self::Output { + todo!() + } + + fn visit_f64x2_add(&mut self) -> Self::Output { + todo!() + } + + fn visit_f64x2_sub(&mut self) -> Self::Output { + todo!() + } + + fn visit_f64x2_mul(&mut self) -> Self::Output { + todo!() + } + + fn visit_f64x2_div(&mut self) -> Self::Output { + todo!() + } + + fn visit_f64x2_min(&mut self) -> Self::Output { + todo!() + } + + fn visit_f64x2_max(&mut self) -> Self::Output { + todo!() + } + + fn visit_f64x2_pmin(&mut self) -> Self::Output { + todo!() + } + + fn visit_f64x2_pmax(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32x4_trunc_sat_f32x4_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32x4_trunc_sat_f32x4_u(&mut self) -> Self::Output { + todo!() + } + + fn visit_f32x4_convert_i32x4_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_f32x4_convert_i32x4_u(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32x4_trunc_sat_f64x2_s_zero(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32x4_trunc_sat_f64x2_u_zero(&mut self) -> Self::Output { + todo!() + } + + fn visit_f64x2_convert_low_i32x4_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_f64x2_convert_low_i32x4_u(&mut self) -> Self::Output { + todo!() + } + + fn visit_f32x4_demote_f64x2_zero(&mut self) -> Self::Output { + todo!() + } + + fn visit_f64x2_promote_low_f32x4(&mut self) -> Self::Output { + todo!() + } + + fn visit_i8x16_relaxed_swizzle(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32x4_relaxed_trunc_f32x4_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32x4_relaxed_trunc_f32x4_u(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32x4_relaxed_trunc_f64x2_s_zero(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32x4_relaxed_trunc_f64x2_u_zero(&mut self) -> Self::Output { + todo!() + } + + fn visit_f32x4_relaxed_madd(&mut self) -> Self::Output { + todo!() + } + + fn visit_f32x4_relaxed_nmadd(&mut self) -> Self::Output { + todo!() + } + + fn visit_f64x2_relaxed_madd(&mut self) -> Self::Output { + todo!() + } + + fn visit_f64x2_relaxed_nmadd(&mut self) -> Self::Output { + todo!() + } + + fn visit_i8x16_relaxed_laneselect(&mut self) -> Self::Output { + todo!() + } + + fn visit_i16x8_relaxed_laneselect(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32x4_relaxed_laneselect(&mut self) -> Self::Output { + todo!() + } + + fn visit_i64x2_relaxed_laneselect(&mut self) -> Self::Output { + todo!() + } + + fn visit_f32x4_relaxed_min(&mut self) -> Self::Output { + todo!() + } + + fn visit_f32x4_relaxed_max(&mut self) -> Self::Output { + todo!() + } + + fn visit_f64x2_relaxed_min(&mut self) -> Self::Output { + todo!() + } + + fn visit_f64x2_relaxed_max(&mut self) -> Self::Output { + todo!() + } + + fn visit_i16x8_relaxed_q15mulr_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i16x8_relaxed_dot_i8x16_i7x16_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32x4_relaxed_dot_i8x16_i7x16_add_s(&mut self) -> Self::Output { + todo!() + } +} diff --git a/crates/wasmi/src/engine/translator/stack2/consts.rs b/crates/wasmi/src/engine/translator/translator2/stack/consts.rs similarity index 100% rename from crates/wasmi/src/engine/translator/stack2/consts.rs rename to crates/wasmi/src/engine/translator/translator2/stack/consts.rs diff --git a/crates/wasmi/src/engine/translator/stack2/locals.rs b/crates/wasmi/src/engine/translator/translator2/stack/locals.rs similarity index 100% rename from crates/wasmi/src/engine/translator/stack2/locals.rs rename to crates/wasmi/src/engine/translator/translator2/stack/locals.rs diff --git a/crates/wasmi/src/engine/translator/stack2/mod.rs b/crates/wasmi/src/engine/translator/translator2/stack/mod.rs similarity index 99% rename from crates/wasmi/src/engine/translator/stack2/mod.rs rename to crates/wasmi/src/engine/translator/translator2/stack/mod.rs index eaa41b49d9..3f6694731c 100644 --- a/crates/wasmi/src/engine/translator/stack2/mod.rs +++ b/crates/wasmi/src/engine/translator/translator2/stack/mod.rs @@ -1,5 +1,3 @@ -#![expect(dead_code)] - mod consts; mod locals; mod operand; diff --git a/crates/wasmi/src/engine/translator/stack2/operand.rs b/crates/wasmi/src/engine/translator/translator2/stack/operand.rs similarity index 100% rename from crates/wasmi/src/engine/translator/stack2/operand.rs rename to crates/wasmi/src/engine/translator/translator2/stack/operand.rs diff --git a/crates/wasmi/src/engine/translator/translator2/visit.rs b/crates/wasmi/src/engine/translator/translator2/visit.rs new file mode 100644 index 0000000000..4fc412c150 --- /dev/null +++ b/crates/wasmi/src/engine/translator/translator2/visit.rs @@ -0,0 +1,894 @@ +use super::FuncTranslator; +use crate::Error; +use wasmparser::VisitOperator; + +macro_rules! impl_visit_operator { + ( @mvp $($rest:tt)* ) => { + impl_visit_operator!(@@skipped $($rest)*); + }; + ( @sign_extension $($rest:tt)* ) => { + impl_visit_operator!(@@skipped $($rest)*); + }; + ( @saturating_float_to_int $($rest:tt)* ) => { + impl_visit_operator!(@@skipped $($rest)*); + }; + ( @bulk_memory $($rest:tt)* ) => { + impl_visit_operator!(@@skipped $($rest)*); + }; + ( @reference_types $($rest:tt)* ) => { + impl_visit_operator!(@@skipped $($rest)*); + }; + ( @tail_call $($rest:tt)* ) => { + impl_visit_operator!(@@skipped $($rest)*); + }; + ( @wide_arithmetic $($rest:tt)* ) => { + impl_visit_operator!(@@skipped $($rest)*); + }; + ( @@skipped $op:ident $({ $($arg:ident: $argty:ty),* })? => $visit:ident $_ann:tt $($rest:tt)* ) => { + // We skip Wasm operators that we already implement manually. + impl_visit_operator!($($rest)*); + }; + ( @$proposal:ident $op:ident $({ $($arg:ident: $argty:ty),* })? => $visit:ident $_ann:tt $($rest:tt)* ) => { + // Wildcard match arm for all the other (yet) unsupported Wasm proposals. + fn $visit(&mut self $($(, $arg: $argty)*)?) -> Self::Output { + self.translate_unsupported_operator(stringify!($op)) + } + impl_visit_operator!($($rest)*); + }; + () => {}; +} + +impl FuncTranslator { + /// Called when translating an unsupported Wasm operator. + /// + /// # Note + /// + /// We panic instead of returning an error because unsupported Wasm + /// errors should have been filtered out by the validation procedure + /// already, therefore encountering an unsupported Wasm operator + /// in the function translation procedure can be considered a bug. + pub fn translate_unsupported_operator(&self, name: &str) -> Result<(), Error> { + panic!("tried to translate an unsupported Wasm operator: {name}") + } +} + +impl<'a> VisitOperator<'a> for FuncTranslator { + type Output = Result<(), Error>; + + #[cfg(feature = "simd")] + fn simd_visitor( + &mut self, + ) -> Option<&mut dyn wasmparser::VisitSimdOperator<'a, Output = Self::Output>> { + Some(self) + } + + wasmparser::for_each_visit_operator!(impl_visit_operator); + + fn visit_unreachable(&mut self) -> Self::Output { + todo!() + } + + fn visit_nop(&mut self) -> Self::Output { + todo!() + } + + fn visit_block(&mut self, blockty: wasmparser::BlockType) -> Self::Output { + todo!() + } + + fn visit_loop(&mut self, blockty: wasmparser::BlockType) -> Self::Output { + todo!() + } + + fn visit_if(&mut self, blockty: wasmparser::BlockType) -> Self::Output { + todo!() + } + + fn visit_else(&mut self) -> Self::Output { + todo!() + } + + fn visit_end(&mut self) -> Self::Output { + todo!() + } + + fn visit_br(&mut self, relative_depth: u32) -> Self::Output { + todo!() + } + + fn visit_br_if(&mut self, relative_depth: u32) -> Self::Output { + todo!() + } + + fn visit_br_table(&mut self, targets: wasmparser::BrTable<'a>) -> Self::Output { + todo!() + } + + fn visit_return(&mut self) -> Self::Output { + todo!() + } + + fn visit_call(&mut self, function_index: u32) -> Self::Output { + todo!() + } + + fn visit_call_indirect(&mut self, type_index: u32, table_index: u32) -> Self::Output { + todo!() + } + + fn visit_drop(&mut self) -> Self::Output { + todo!() + } + + fn visit_select(&mut self) -> Self::Output { + todo!() + } + + fn visit_local_get(&mut self, local_index: u32) -> Self::Output { + todo!() + } + + fn visit_local_set(&mut self, local_index: u32) -> Self::Output { + todo!() + } + + fn visit_local_tee(&mut self, local_index: u32) -> Self::Output { + todo!() + } + + fn visit_global_get(&mut self, global_index: u32) -> Self::Output { + todo!() + } + + fn visit_global_set(&mut self, global_index: u32) -> Self::Output { + todo!() + } + + fn visit_i32_load(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + todo!() + } + + fn visit_i64_load(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + todo!() + } + + fn visit_f32_load(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + todo!() + } + + fn visit_f64_load(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + todo!() + } + + fn visit_i32_load8_s(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + todo!() + } + + fn visit_i32_load8_u(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + todo!() + } + + fn visit_i32_load16_s(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + todo!() + } + + fn visit_i32_load16_u(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + todo!() + } + + fn visit_i64_load8_s(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + todo!() + } + + fn visit_i64_load8_u(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + todo!() + } + + fn visit_i64_load16_s(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + todo!() + } + + fn visit_i64_load16_u(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + todo!() + } + + fn visit_i64_load32_s(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + todo!() + } + + fn visit_i64_load32_u(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + todo!() + } + + fn visit_i32_store(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + todo!() + } + + fn visit_i64_store(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + todo!() + } + + fn visit_f32_store(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + todo!() + } + + fn visit_f64_store(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + todo!() + } + + fn visit_i32_store8(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + todo!() + } + + fn visit_i32_store16(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + todo!() + } + + fn visit_i64_store8(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + todo!() + } + + fn visit_i64_store16(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + todo!() + } + + fn visit_i64_store32(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + todo!() + } + + fn visit_memory_size(&mut self, mem: u32) -> Self::Output { + todo!() + } + + fn visit_memory_grow(&mut self, mem: u32) -> Self::Output { + todo!() + } + + fn visit_i32_const(&mut self, value: i32) -> Self::Output { + todo!() + } + + fn visit_i64_const(&mut self, value: i64) -> Self::Output { + todo!() + } + + fn visit_f32_const(&mut self, value: wasmparser::Ieee32) -> Self::Output { + todo!() + } + + fn visit_f64_const(&mut self, value: wasmparser::Ieee64) -> Self::Output { + todo!() + } + + fn visit_i32_eqz(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32_eq(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32_ne(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32_lt_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32_lt_u(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32_gt_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32_gt_u(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32_le_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32_le_u(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32_ge_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32_ge_u(&mut self) -> Self::Output { + todo!() + } + + fn visit_i64_eqz(&mut self) -> Self::Output { + todo!() + } + + fn visit_i64_eq(&mut self) -> Self::Output { + todo!() + } + + fn visit_i64_ne(&mut self) -> Self::Output { + todo!() + } + + fn visit_i64_lt_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i64_lt_u(&mut self) -> Self::Output { + todo!() + } + + fn visit_i64_gt_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i64_gt_u(&mut self) -> Self::Output { + todo!() + } + + fn visit_i64_le_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i64_le_u(&mut self) -> Self::Output { + todo!() + } + + fn visit_i64_ge_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i64_ge_u(&mut self) -> Self::Output { + todo!() + } + + fn visit_f32_eq(&mut self) -> Self::Output { + todo!() + } + + fn visit_f32_ne(&mut self) -> Self::Output { + todo!() + } + + fn visit_f32_lt(&mut self) -> Self::Output { + todo!() + } + + fn visit_f32_gt(&mut self) -> Self::Output { + todo!() + } + + fn visit_f32_le(&mut self) -> Self::Output { + todo!() + } + + fn visit_f32_ge(&mut self) -> Self::Output { + todo!() + } + + fn visit_f64_eq(&mut self) -> Self::Output { + todo!() + } + + fn visit_f64_ne(&mut self) -> Self::Output { + todo!() + } + + fn visit_f64_lt(&mut self) -> Self::Output { + todo!() + } + + fn visit_f64_gt(&mut self) -> Self::Output { + todo!() + } + + fn visit_f64_le(&mut self) -> Self::Output { + todo!() + } + + fn visit_f64_ge(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32_clz(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32_ctz(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32_popcnt(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32_add(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32_sub(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32_mul(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32_div_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32_div_u(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32_rem_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32_rem_u(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32_and(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32_or(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32_xor(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32_shl(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32_shr_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32_shr_u(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32_rotl(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32_rotr(&mut self) -> Self::Output { + todo!() + } + + fn visit_i64_clz(&mut self) -> Self::Output { + todo!() + } + + fn visit_i64_ctz(&mut self) -> Self::Output { + todo!() + } + + fn visit_i64_popcnt(&mut self) -> Self::Output { + todo!() + } + + fn visit_i64_add(&mut self) -> Self::Output { + todo!() + } + + fn visit_i64_sub(&mut self) -> Self::Output { + todo!() + } + + fn visit_i64_mul(&mut self) -> Self::Output { + todo!() + } + + fn visit_i64_div_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i64_div_u(&mut self) -> Self::Output { + todo!() + } + + fn visit_i64_rem_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i64_rem_u(&mut self) -> Self::Output { + todo!() + } + + fn visit_i64_and(&mut self) -> Self::Output { + todo!() + } + + fn visit_i64_or(&mut self) -> Self::Output { + todo!() + } + + fn visit_i64_xor(&mut self) -> Self::Output { + todo!() + } + + fn visit_i64_shl(&mut self) -> Self::Output { + todo!() + } + + fn visit_i64_shr_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i64_shr_u(&mut self) -> Self::Output { + todo!() + } + + fn visit_i64_rotl(&mut self) -> Self::Output { + todo!() + } + + fn visit_i64_rotr(&mut self) -> Self::Output { + todo!() + } + + fn visit_f32_abs(&mut self) -> Self::Output { + todo!() + } + + fn visit_f32_neg(&mut self) -> Self::Output { + todo!() + } + + fn visit_f32_ceil(&mut self) -> Self::Output { + todo!() + } + + fn visit_f32_floor(&mut self) -> Self::Output { + todo!() + } + + fn visit_f32_trunc(&mut self) -> Self::Output { + todo!() + } + + fn visit_f32_nearest(&mut self) -> Self::Output { + todo!() + } + + fn visit_f32_sqrt(&mut self) -> Self::Output { + todo!() + } + + fn visit_f32_add(&mut self) -> Self::Output { + todo!() + } + + fn visit_f32_sub(&mut self) -> Self::Output { + todo!() + } + + fn visit_f32_mul(&mut self) -> Self::Output { + todo!() + } + + fn visit_f32_div(&mut self) -> Self::Output { + todo!() + } + + fn visit_f32_min(&mut self) -> Self::Output { + todo!() + } + + fn visit_f32_max(&mut self) -> Self::Output { + todo!() + } + + fn visit_f32_copysign(&mut self) -> Self::Output { + todo!() + } + + fn visit_f64_abs(&mut self) -> Self::Output { + todo!() + } + + fn visit_f64_neg(&mut self) -> Self::Output { + todo!() + } + + fn visit_f64_ceil(&mut self) -> Self::Output { + todo!() + } + + fn visit_f64_floor(&mut self) -> Self::Output { + todo!() + } + + fn visit_f64_trunc(&mut self) -> Self::Output { + todo!() + } + + fn visit_f64_nearest(&mut self) -> Self::Output { + todo!() + } + + fn visit_f64_sqrt(&mut self) -> Self::Output { + todo!() + } + + fn visit_f64_add(&mut self) -> Self::Output { + todo!() + } + + fn visit_f64_sub(&mut self) -> Self::Output { + todo!() + } + + fn visit_f64_mul(&mut self) -> Self::Output { + todo!() + } + + fn visit_f64_div(&mut self) -> Self::Output { + todo!() + } + + fn visit_f64_min(&mut self) -> Self::Output { + todo!() + } + + fn visit_f64_max(&mut self) -> Self::Output { + todo!() + } + + fn visit_f64_copysign(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32_wrap_i64(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32_trunc_f32_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32_trunc_f32_u(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32_trunc_f64_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32_trunc_f64_u(&mut self) -> Self::Output { + todo!() + } + + fn visit_i64_extend_i32_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i64_extend_i32_u(&mut self) -> Self::Output { + todo!() + } + + fn visit_i64_trunc_f32_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i64_trunc_f32_u(&mut self) -> Self::Output { + todo!() + } + + fn visit_i64_trunc_f64_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i64_trunc_f64_u(&mut self) -> Self::Output { + todo!() + } + + fn visit_f32_convert_i32_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_f32_convert_i32_u(&mut self) -> Self::Output { + todo!() + } + + fn visit_f32_convert_i64_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_f32_convert_i64_u(&mut self) -> Self::Output { + todo!() + } + + fn visit_f32_demote_f64(&mut self) -> Self::Output { + todo!() + } + + fn visit_f64_convert_i32_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_f64_convert_i32_u(&mut self) -> Self::Output { + todo!() + } + + fn visit_f64_convert_i64_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_f64_convert_i64_u(&mut self) -> Self::Output { + todo!() + } + + fn visit_f64_promote_f32(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32_reinterpret_f32(&mut self) -> Self::Output { + todo!() + } + + fn visit_i64_reinterpret_f64(&mut self) -> Self::Output { + todo!() + } + + fn visit_f32_reinterpret_i32(&mut self) -> Self::Output { + todo!() + } + + fn visit_f64_reinterpret_i64(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32_extend8_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32_extend16_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i64_extend8_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i64_extend16_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i64_extend32_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32_trunc_sat_f32_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32_trunc_sat_f32_u(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32_trunc_sat_f64_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i32_trunc_sat_f64_u(&mut self) -> Self::Output { + todo!() + } + + fn visit_i64_trunc_sat_f32_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i64_trunc_sat_f32_u(&mut self) -> Self::Output { + todo!() + } + + fn visit_i64_trunc_sat_f64_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i64_trunc_sat_f64_u(&mut self) -> Self::Output { + todo!() + } + + fn visit_memory_init(&mut self, data_index: u32, mem: u32) -> Self::Output { + todo!() + } + + fn visit_data_drop(&mut self, data_index: u32) -> Self::Output { + todo!() + } + + fn visit_memory_copy(&mut self, dst_mem: u32, src_mem: u32) -> Self::Output { + todo!() + } + + fn visit_memory_fill(&mut self, mem: u32) -> Self::Output { + todo!() + } + + fn visit_table_init(&mut self, elem_index: u32, table: u32) -> Self::Output { + todo!() + } + + fn visit_elem_drop(&mut self, elem_index: u32) -> Self::Output { + todo!() + } + + fn visit_table_copy(&mut self, dst_table: u32, src_table: u32) -> Self::Output { + todo!() + } + + fn visit_typed_select(&mut self, ty: wasmparser::ValType) -> Self::Output { + todo!() + } + + fn visit_ref_null(&mut self, hty: wasmparser::HeapType) -> Self::Output { + todo!() + } + + fn visit_ref_is_null(&mut self) -> Self::Output { + todo!() + } + + fn visit_ref_func(&mut self, function_index: u32) -> Self::Output { + todo!() + } + + fn visit_table_fill(&mut self, table: u32) -> Self::Output { + todo!() + } + + fn visit_table_get(&mut self, table: u32) -> Self::Output { + todo!() + } + + fn visit_table_set(&mut self, table: u32) -> Self::Output { + todo!() + } + + fn visit_table_grow(&mut self, table: u32) -> Self::Output { + todo!() + } + + fn visit_table_size(&mut self, table: u32) -> Self::Output { + todo!() + } + + fn visit_return_call(&mut self, function_index: u32) -> Self::Output { + todo!() + } + + fn visit_return_call_indirect(&mut self, type_index: u32, table_index: u32) -> Self::Output { + todo!() + } + + fn visit_i64_add128(&mut self) -> Self::Output { + todo!() + } + + fn visit_i64_sub128(&mut self) -> Self::Output { + todo!() + } + + fn visit_i64_mul_wide_s(&mut self) -> Self::Output { + todo!() + } + + fn visit_i64_mul_wide_u(&mut self) -> Self::Output { + todo!() + } +} From f8040ba45cc43f67c46a5e1613df4b4eff701bb7 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Fri, 23 May 2025 14:45:49 +0200 Subject: [PATCH 049/343] expect (silence) more warnings --- crates/wasmi/src/engine/translator/translator2/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/translator2/mod.rs b/crates/wasmi/src/engine/translator/translator2/mod.rs index 7c392d4b24..316ae5f1ff 100644 --- a/crates/wasmi/src/engine/translator/translator2/mod.rs +++ b/crates/wasmi/src/engine/translator/translator2/mod.rs @@ -1,4 +1,4 @@ -#![expect(dead_code)] +#![expect(dead_code, unused_imports, unused_variables)] #[cfg(feature = "simd")] mod simd; From 84ff8918c7b5b9e0f80ff13bc9ab9bf13a5b4bbd Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Fri, 23 May 2025 15:20:24 +0200 Subject: [PATCH 050/343] update docs --- crates/wasmi/src/engine/translator/translator2/stack/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/translator2/stack/mod.rs b/crates/wasmi/src/engine/translator/translator2/stack/mod.rs index 3f6694731c..24bbf0ae26 100644 --- a/crates/wasmi/src/engine/translator/translator2/stack/mod.rs +++ b/crates/wasmi/src/engine/translator/translator2/stack/mod.rs @@ -25,7 +25,7 @@ pub struct Stack { locals: LocalsRegistry, /// All function local constants. consts: ConstRegistry, - /// The index of the first [`StackOperand::Local`] on the [`Stack`]. + /// The maximim number of operands on the [`Stack`] at the same time. max_height: usize, } From 8ea517eb333eb42dc4d7acd8b3d4f1379225dc7d Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Sat, 24 May 2025 11:40:02 +0200 Subject: [PATCH 051/343] add Reset trait and impls --- .../src/engine/translator/translator2/stack/consts.rs | 8 +++++--- .../src/engine/translator/translator2/stack/locals.rs | 9 +++++---- .../wasmi/src/engine/translator/translator2/stack/mod.rs | 6 ++++++ 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/crates/wasmi/src/engine/translator/translator2/stack/consts.rs b/crates/wasmi/src/engine/translator/translator2/stack/consts.rs index 0ccaabeb70..d25862b09e 100644 --- a/crates/wasmi/src/engine/translator/translator2/stack/consts.rs +++ b/crates/wasmi/src/engine/translator/translator2/stack/consts.rs @@ -1,3 +1,4 @@ +use super::Reset; use crate::{core::UntypedVal, engine::TranslationError, ir::Reg, Error}; use alloc::{ collections::{btree_map, BTreeMap}, @@ -23,14 +24,15 @@ pub struct ConstRegistry { next_idx: i16, } -impl ConstRegistry { - /// Resets the [`ConstRegistry`] data structure. - pub fn reset(&mut self) { +impl Reset for ConstRegistry { + fn reset(&mut self) { self.const2idx.clear(); self.idx2const.clear(); self.next_idx = Self::first_index(); } +} +impl ConstRegistry { /// The maximum index for [`Reg`] referring to function local constant values. /// /// # Note diff --git a/crates/wasmi/src/engine/translator/translator2/stack/locals.rs b/crates/wasmi/src/engine/translator/translator2/stack/locals.rs index dd072fe272..97c4b126a7 100644 --- a/crates/wasmi/src/engine/translator/translator2/stack/locals.rs +++ b/crates/wasmi/src/engine/translator/translator2/stack/locals.rs @@ -1,4 +1,4 @@ -use super::OperandIdx; +use super::{OperandIdx, Reset}; use crate::{core::ValType, engine::TranslationError, Error}; use alloc::vec::Vec; use core::{cmp, iter}; @@ -30,14 +30,15 @@ pub struct LocalsRegistry { first_operands: Vec>, } -impl LocalsRegistry { - /// Resets `self` for reuse. - pub fn reset(&mut self) { +impl Reset for LocalsRegistry { + fn reset(&mut self) { self.tys_first.clear(); self.tys_remaining.clear(); self.first_operands.clear(); } +} +impl LocalsRegistry { /// Returns the number of registered local variables in `self`. pub fn len(&self) -> usize { self.first_operands.len() diff --git a/crates/wasmi/src/engine/translator/translator2/stack/mod.rs b/crates/wasmi/src/engine/translator/translator2/stack/mod.rs index 24bbf0ae26..af3dd925b1 100644 --- a/crates/wasmi/src/engine/translator/translator2/stack/mod.rs +++ b/crates/wasmi/src/engine/translator/translator2/stack/mod.rs @@ -16,6 +16,12 @@ use crate::{ use alloc::vec::Vec; use core::{array, mem, num::NonZero}; +/// Implemented by types that can be reset for reuse. +trait Reset { + /// Resets `self` for reuse. + fn reset(&mut self); +} + /// The Wasm value stack during translation from Wasm to Wasmi bytecode. #[derive(Debug, Default)] pub struct Stack { From aee36fd8734a4ef65b64d557d2b139119f303e0d Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Sat, 24 May 2025 11:40:29 +0200 Subject: [PATCH 052/343] refactor Stack internals --- .../translator/translator2/stack/mod.rs | 101 +++++++++++------- 1 file changed, 61 insertions(+), 40 deletions(-) diff --git a/crates/wasmi/src/engine/translator/translator2/stack/mod.rs b/crates/wasmi/src/engine/translator/translator2/stack/mod.rs index af3dd925b1..89769dbac6 100644 --- a/crates/wasmi/src/engine/translator/translator2/stack/mod.rs +++ b/crates/wasmi/src/engine/translator/translator2/stack/mod.rs @@ -25,7 +25,7 @@ trait Reset { /// The Wasm value stack during translation from Wasm to Wasmi bytecode. #[derive(Debug, Default)] pub struct Stack { - /// The stack of operands. + /// The Wasm value stack. operands: Vec, /// All function locals and their associated types. locals: LocalsRegistry, @@ -90,10 +90,15 @@ impl Stack { } /// Returns the [`OperandIdx`] of the next pushed operand. - fn next_operand_idx(&self) -> OperandIdx { + fn next_operand_index(&self) -> OperandIdx { OperandIdx::from(self.operands.len()) } + /// Returns the [`OperandIdx`] of the operand at `depth`. + fn operand_index(&self, depth: usize) -> OperandIdx { + OperandIdx::from(self.height() - depth - 1) + } + /// Updates the `prev_local` of the [`StackOperand::Local`] at `local_index` to `prev_index`. /// /// # Panics @@ -101,8 +106,8 @@ impl Stack { /// - If `local_index` does not refer to a [`StackOperand::Local`]. /// - If `local_index` is out of bounds of the operand stack. fn update_prev_local(&mut self, local_index: OperandIdx, prev_index: Option) { - match self.operands.get_mut(usize::from(local_index)) { - Some(StackOperand::Local { prev_local, .. }) => { + match self.get_mut_at(local_index) { + StackOperand::Local { prev_local, .. } => { *prev_local = prev_index; } operand => panic!("expected `StackOperand::Local` but found: {operand:?}"), @@ -116,8 +121,8 @@ impl Stack { /// - If `local_index` does not refer to a [`StackOperand::Local`]. /// - If `local_index` is out of bounds of the operand stack. fn update_next_local(&mut self, local_index: OperandIdx, prev_index: Option) { - match self.operands.get_mut(usize::from(local_index)) { - Some(StackOperand::Local { next_local, .. }) => { + match self.get_mut_at(local_index) { + StackOperand::Local { next_local, .. } => { *next_local = prev_index; } operand => panic!("expected `StackOperand::Local` but found: {operand:?}"), @@ -131,7 +136,7 @@ impl Stack { /// - If too many operands have been pushed onto the [`Stack`]. /// - If the local with `local_idx` does not exist. pub fn push_local(&mut self, local_index: LocalIdx) -> Result { - let operand_index = self.next_operand_idx(); + let operand_index = self.next_operand_index(); let next_local = self .locals .replace_first_operand(local_index, Some(operand_index)); @@ -153,7 +158,7 @@ impl Stack { /// /// If too many operands have been pushed onto the [`Stack`]. pub fn push_temp(&mut self, ty: ValType, instr: Option) -> Result { - let operand_index = self.next_operand_idx(); + let operand_index = self.next_operand_index(); self.operands.push(StackOperand::Temp { ty, instr }); self.update_max_stack_height(); Ok(operand_index) @@ -165,7 +170,7 @@ impl Stack { /// /// If too many operands have been pushed onto the [`Stack`]. pub fn push_immediate(&mut self, value: impl Into) -> Result { - let operand_index = self.next_operand_idx(); + let operand_index = self.next_operand_index(); self.operands .push(StackOperand::Immediate { val: value.into() }); self.update_max_stack_height(); @@ -175,10 +180,10 @@ impl Stack { /// Peeks the top-most [`Operand`] on the [`Stack`]. /// /// Returns `None` if the [`Stack`] is empty. - pub fn peek(&self) -> Option { - let operand = self.operands.last().copied()?; - let index = OperandIdx::from(self.operands.len() - 1); - Some(Operand::new(index, operand, &self.locals)) + pub fn peek(&self) -> Operand { + let index = self.operand_index(0); + let operand = self.get_at(index); + Operand::new(index, operand, &self.locals) } /// Pops the top-most [`Operand`] from the [`Stack`]. @@ -266,6 +271,33 @@ impl Stack { } } + /// Returns the [`StackOperand`] at `index`. + /// + /// # Panics + /// + /// If `depth` is out of bounds for `self`. + fn get_at(&self, index: OperandIdx) -> StackOperand { + self.operands[usize::from(index)] + } + + /// Returns an exlusive reference to the [`StackOperand`] at `index`. + /// + /// # Panics + /// + /// If `depth` is out of bounds for `self`. + fn get_mut_at(&mut self, index: OperandIdx) -> &mut StackOperand { + &mut self.operands[usize::from(index)] + } + + /// Sets the [`StackOperand`] at `index` to `operand`. + /// + /// # Panics + /// + /// If `depth` is out of bounds for `self`. + fn set_at(&mut self, index: OperandIdx, operand: StackOperand) { + self.operands[usize::from(index)] = operand; + } + /// Converts and returns the [`StackOperand`] at `depth` into a [`StackOperand::Temp`]. /// /// # Note @@ -277,15 +309,8 @@ impl Stack { /// - If `depth` is out of bounds for the [`Stack`] of operands. #[must_use] pub fn operand_to_temp(&mut self, depth: usize) -> Option { - let len = self.height(); - if depth >= len { - panic!( - "out of bounds access: tried to access `Stack` with length {len} at depth {depth}" - ); - } - let index = len - depth - 1; - let operand_index = OperandIdx::from(index); - let operand = match self.operands[index] { + let index = self.operand_index(depth); + let operand = match self.get_at(index) { StackOperand::Local { local_index, prev_local, @@ -294,7 +319,7 @@ impl Stack { if prev_local.is_none() { // Note: if `prev_local` is `None` then this local is the first // in the linked list of locals and must be updated. - debug_assert_eq!(self.locals.first_operand(local_index), Some(operand_index)); + debug_assert_eq!(self.locals.first_operand(local_index), Some(index)); self.locals.replace_first_operand(local_index, next_local); } if let Some(prev_local) = prev_local { @@ -303,15 +328,18 @@ impl Stack { if let Some(next_local) = next_local { self.update_prev_local(next_local, prev_local); } - Operand::local(operand_index, local_index, &self.locals) + Operand::local(index, local_index, &self.locals) } - StackOperand::Immediate { val } => Operand::immediate(operand_index, val), + StackOperand::Immediate { val } => Operand::immediate(index, val), StackOperand::Temp { .. } => return None, }; - self.operands[index] = StackOperand::Temp { - ty: operand.ty(), - instr: None, - }; + self.set_at( + index, + StackOperand::Temp { + ty: operand.ty(), + instr: None, + }, + ); Some(operand) } @@ -321,17 +349,10 @@ impl Stack { /// /// If the `index` is out of bounds. pub fn operand_to_reg(&mut self, depth: usize) -> Result { - let len = self.height(); - if depth >= len { - panic!( - "out of bounds access: tried to access `Stack` with length {len} at depth {depth}" - ); - } - let index = len - depth - 1; - let operand = self.operands[index]; - match operand { + let index = self.operand_index(depth); + match self.get_at(index) { StackOperand::Local { local_index, .. } => self.local_to_reg(local_index), - StackOperand::Temp { .. } => self.temp_to_reg(OperandIdx::from(index)), + StackOperand::Temp { .. } => self.temp_to_reg(index), StackOperand::Immediate { val } => self.const_to_reg(val), } } @@ -350,7 +371,7 @@ impl Stack { /// # Errors /// /// If `index` cannot be converted into a [`Reg`]. - fn local_to_reg(&self, index: LocalIdx) -> Result { + pub fn local_to_reg(&self, index: LocalIdx) -> Result { let Ok(index) = i16::try_from(u32::from(index)) else { return Err(Error::from(TranslationError::AllocatedTooManyRegisters)); }; From 7693b4bc8c2909484df3f47d6dac637d5d9eecb9 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Sat, 24 May 2025 11:41:01 +0200 Subject: [PATCH 053/343] add ControlStack to Stack internals --- .../translator/translator2/stack/control.rs | 537 ++++++++++++++++++ .../translator/translator2/stack/mod.rs | 16 + 2 files changed, 553 insertions(+) create mode 100644 crates/wasmi/src/engine/translator/translator2/stack/control.rs diff --git a/crates/wasmi/src/engine/translator/translator2/stack/control.rs b/crates/wasmi/src/engine/translator/translator2/stack/control.rs new file mode 100644 index 0000000000..fe0e227b36 --- /dev/null +++ b/crates/wasmi/src/engine/translator/translator2/stack/control.rs @@ -0,0 +1,537 @@ +use super::{Reset, StackOperand}; +use crate::engine::{ + translator::{BlockType, LabelRef}, + Instr, +}; +use alloc::vec::{Drain, Vec}; + +#[cfg(doc)] +use crate::ir::Instruction; + +/// The Wasm control stack. +#[derive(Debug, Default)] +pub struct ControlStack { + /// The stack of control frames. + frames: Vec, + /// Special operand stack to memorize operands for `else` control frames. + else_operands: ElseOperands, +} + +/// Duplicated operands for Wasm `else` control frames. +#[derive(Debug, Default)] +pub struct ElseOperands { + /// The end indices of each `else` operands. + ends: Vec, + /// All operands of all allocated `else` control frames. + providers: Vec, +} + +impl Reset for ElseOperands { + fn reset(&mut self) { + self.ends.clear(); + self.providers.clear(); + } +} + +impl ElseOperands { + /// Pushes operands for a new Wasm `else` control frame. + pub fn push(&mut self, operands: &[StackOperand]) { + self.providers.extend(operands.iter().copied()); + let end = self.providers.len(); + let index = self.ends.len(); + self.ends.push(end); + } + + /// Pops the top-most Wasm `else` operands from `self` and returns it. + pub fn pop(&mut self) -> Option> { + let end = self.ends.pop()?; + let start = self.ends.last().copied().unwrap_or(0); + Some(self.providers.drain(start..end)) + } +} + +impl Reset for ControlStack { + fn reset(&mut self) { + self.frames.clear(); + self.else_operands.reset(); + } +} + +impl ControlStack { + /// Returns the height of the [`ControlStack`]. + pub fn height(&self) -> usize { + self.frames.len() + } + + /// Pushes a new unreachable Wasm control frame onto the [`ControlStack`]. + pub fn push_unreachable(&mut self, ty: BlockType, height: u32, kind: UnreachableControlFrame) { + self.frames + .push(ControlFrame::new_unreachable(ty, height, kind)) + } + + /// Pushes a new Wasm `block` onto the [`ControlStack`]. + pub fn push_block( + &mut self, + ty: BlockType, + height: u32, + label: LabelRef, + consume_fuel: Option, + ) { + self.frames + .push(ControlFrame::new_block(ty, height, label, consume_fuel)) + } + + /// Pushes a new Wasm `loop` onto the [`ControlStack`]. + pub fn push_loop( + &mut self, + ty: BlockType, + height: u32, + label: LabelRef, + consume_fuel: Option, + ) { + self.frames + .push(ControlFrame::new_loop(ty, height, label, consume_fuel)) + } + + /// Pushes a new Wasm `if` onto the [`ControlStack`]. + pub fn push_if( + &mut self, + ty: BlockType, + height: u32, + label: LabelRef, + consume_fuel: Option, + reachability: IfReachability, + else_operands: &[StackOperand], + ) { + self.frames.push(ControlFrame::new_if( + ty, + height, + label, + consume_fuel, + reachability, + )); + self.else_operands.push(else_operands); + } + + /// Pushes a new Wasm `else` onto the [`ControlStack`]. + /// + /// Returns iterator yielding the memorized `else` operands. + pub fn push_else( + &mut self, + ty: BlockType, + height: u32, + label: LabelRef, + consume_fuel: Option, + reachability: ElseReachability, + is_end_of_then_reachable: bool, + ) -> Drain { + self.frames.push(ControlFrame::new_else( + ty, + height, + label, + consume_fuel, + reachability, + is_end_of_then_reachable, + )); + self.else_operands + .pop() + .unwrap_or_else(|| panic!("missing operands for `else` control frame")) + } + + /// Returns a shared reference to the [`ControlFrame`] at `depth` if any. + pub fn get(&self, depth: u32) -> Option<&ControlFrame> { + self.frames.iter().rev().nth(depth as usize) + } + + /// Returns an exclusive reference to the [`ControlFrame`] at `depth` if any. + pub fn get_mut(&mut self, depth: u32) -> Option<&mut ControlFrame> { + self.frames.iter_mut().rev().nth(depth as usize) + } +} + +/// An acquired branch target. +#[derive(Debug)] +pub enum AcquiredTarget<'a> { + /// The branch targets the function enclosing `block` and therefore is a `return`. + Return(&'a mut ControlFrame), + /// The branch targets a regular [`ControlFrame`]. + Branch(&'a mut ControlFrame), +} + +impl<'a> AcquiredTarget<'a> { + /// Returns an exclusive reference to the [`ControlFrame`] of the [`AcquiredTarget`]. + pub fn control_frame(&'a mut self) -> &'a mut ControlFrame { + match self { + Self::Return(frame) => frame, + Self::Branch(frame) => frame, + } + } +} + +impl ControlStack { + /// Acquires the target [`ControlFrame`] at the given relative `depth`. + pub fn acquire_target(&mut self, depth: u32) -> AcquiredTarget { + let is_root = self.is_root(depth); + let height = self.height(); + let Some(frame) = self.get_mut(depth) else { + panic!("out of bounds `depth` (={depth}) for `ControlStack` with a height of {height}") + }; + if is_root { + AcquiredTarget::Return(frame) + } else { + AcquiredTarget::Branch(frame) + } + } + + /// Returns `true` if `depth` points to the first control flow frame. + fn is_root(&self, depth: u32) -> bool { + if self.frames.is_empty() { + return false; + } + depth as usize == self.height() - 1 + } +} + +/// A Wasm control frame. +#[derive(Debug)] +pub struct ControlFrame { + /// The block type of the [`ControlFrame`]. + ty: BlockType, + /// The value stack height upon entering the [`ControlFrame`]. + height: u32, + /// The number of branches to the [`ControlFrame`]. + len_branches: usize, + /// The [`ControlFrame`]'s [`Instruction::ConsumeFuel`] if fuel metering is enabled. + /// + /// # Note + /// + /// This is `Some` if fuel metering is enabled and `None` otherwise. + consume_fuel: Option, + /// The kind of [`ControlFrame`] with associated data. + kind: ControlFrameKind, +} + +impl ControlFrame { + /// Creates a new unreachable [`ControlFrame`] of `kind`. + pub fn new_unreachable(ty: BlockType, height: u32, kind: UnreachableControlFrame) -> Self { + Self { + ty, + height, + len_branches: 0, + consume_fuel: None, + kind: ControlFrameKind::Unreachable(kind), + } + } + + /// Creates a new Wasm `block` [`ControlFrame`]. + pub fn new_block( + ty: BlockType, + height: u32, + label: LabelRef, + consume_fuel: Option, + ) -> Self { + Self { + ty, + height, + len_branches: 0, + consume_fuel, + kind: ControlFrameKind::Block(BlockControlFrame { label }), + } + } + + /// Creates a new Wasm `loop` [`ControlFrame`]. + pub fn new_loop( + ty: BlockType, + height: u32, + label: LabelRef, + consume_fuel: Option, + ) -> Self { + Self { + ty, + height, + len_branches: 0, + consume_fuel, + kind: ControlFrameKind::Loop(LoopControlFrame { label }), + } + } + + /// Creates a new Wasm `if` [`ControlFrame`]. + pub fn new_if( + ty: BlockType, + height: u32, + label: LabelRef, + consume_fuel: Option, + reachability: IfReachability, + ) -> Self { + Self { + ty, + height, + len_branches: 0, + consume_fuel, + kind: ControlFrameKind::If(IfControlFrame { + label, + reachability, + }), + } + } + + /// Creates a new Wasm `else` [`ControlFrame`]. + pub fn new_else( + ty: BlockType, + height: u32, + label: LabelRef, + consume_fuel: Option, + reachability: ElseReachability, + is_end_of_then_reachable: bool, + ) -> Self { + Self { + ty, + height, + len_branches: 0, + consume_fuel, + kind: ControlFrameKind::Else(ElseControlFrame { + label, + reachability, + is_end_of_then_reachable, + }), + } + } + + /// Returns the stack height of the [`ControlFrame`]. + pub fn height(&self) -> u32 { + self.height + } + + /// Returns the [`BlockType`] of the [`ControlFrame`]. + pub fn block_type(&self) -> BlockType { + self.ty + } + + /// Returns `true` if at least one branch targets the [`ControlFrame`]. + pub fn is_branched_to(&self) -> bool { + self.len_branches() >= 1 + } + + /// Returns the number of branches to the [`ControlFrame`]. + fn len_branches(&self) -> usize { + self.len_branches + } + + /// Bumps the number of branches to the [`ControlFrame`] by 1. + fn bump_branches(&mut self) { + self.len_branches += 1; + } + + /// Returns a reference to the [`Instruction::ConsumeFuel`] of the [`ControlFrame`] if any. + /// + /// Returns `None` if fuel metering is disabled. + pub fn consume_fuel_instr(&self) -> Option { + self.consume_fuel + } +} + +/// A Wasm control frame kind. +#[derive(Debug)] +pub enum ControlFrameKind { + /// A Wasm `block` control frame. + Block(BlockControlFrame), + /// A Wasm `loop` control frame. + Loop(LoopControlFrame), + /// A Wasm `if` control frame, including `then`. + If(IfControlFrame), + /// A Wasm `else` control frame, as part of `if`. + Else(ElseControlFrame), + /// A generic unreachable Wasm control frame. + Unreachable(UnreachableControlFrame), +} + +/// A Wasm `block` control frame. +#[derive(Debug)] +pub struct BlockControlFrame { + /// The label used to branch to the [`BlockControlFrame`]. + label: LabelRef, +} + +impl BlockControlFrame { + /// Returns the branch label of the [`BlockControlFrame`]. + pub fn label(&self) -> LabelRef { + self.label + } +} + +/// A Wasm `loop` control frame. +#[derive(Debug)] +pub struct LoopControlFrame { + /// The label used to branch to the [`LoopControlFrame`]. + label: LabelRef, +} + +impl LoopControlFrame { + /// Returns the branch label of the [`LoopControlFrame`]. + pub fn label(&self) -> LabelRef { + self.label + } +} + +/// A Wasm `if` control frame including `then`. +#[derive(Debug)] +pub struct IfControlFrame { + /// The label used to branch to the [`IfControlFrame`]. + label: LabelRef, + /// The reachability of the `then` and `else` blocks. + reachability: IfReachability, +} + +impl IfControlFrame { + /// Returns the branch label of the [`IfControlFrame`]. + pub fn label(&self) -> LabelRef { + self.label + } + + /// Returns the label to the `else` branch of the [`IfControlFrame`]. + pub fn else_label(&self) -> Option { + match self.reachability { + IfReachability::Both { else_label } => Some(else_label), + IfReachability::OnlyThen | IfReachability::OnlyElse => None, + } + } + + /// Returns `true` if the `then` branch is reachable. + /// + /// # Note + /// + /// The `then` branch is unreachable if the `if` condition is a constant `false` value. + pub fn is_then_reachable(&self) -> bool { + match self.reachability { + IfReachability::Both { .. } | IfReachability::OnlyThen => true, + IfReachability::OnlyElse => false, + } + } + + /// Returns `true` if the `else` branch is reachable. + /// + /// # Note + /// + /// The `else` branch is unreachable if the `if` condition is a constant `true` value. + pub fn is_else_reachable(&self) -> bool { + match self.reachability { + IfReachability::Both { .. } | IfReachability::OnlyElse => true, + IfReachability::OnlyThen => false, + } + } +} + +/// The reachability of the `if` control flow frame. +#[derive(Debug, Copy, Clone)] +pub enum IfReachability { + /// Both, `then` and `else` blocks of the `if` are reachable. + /// + /// # Note + /// + /// This variant does not mean that necessarily both `then` and `else` + /// blocks do exist and are non-empty. The `then` block might still be + /// empty and the `then` block might still be missing. + Both { else_label: LabelRef }, + /// Only the `then` block of the `if` is reachable. + /// + /// # Note + /// + /// This case happens only in case the `if` has a `true` constant condition. + OnlyThen, + /// Only the `else` block of the `if` is reachable. + /// + /// # Note + /// + /// This case happens only in case the `if` has a `false` constant condition. + OnlyElse, +} + +/// A Wasm `else` control frame part of Wasm `if`. +#[derive(Debug)] +pub struct ElseControlFrame { + /// The label used to branch to the [`ElseControlFrame`]. + label: LabelRef, + /// The reachability of the `then` and `else` blocks. + reachability: ElseReachability, + /// Is `true` if code is reachable when entering the `else` block. + /// + /// # Note + /// + /// This means that the end of the `then` block was reachable. + is_end_of_then_reachable: bool, +} + +/// The reachability of the `else` control flow frame. +#[derive(Debug, Copy, Clone)] +pub enum ElseReachability { + /// Both, `then` and `else` blocks of the `if` are reachable. + /// + /// # Note + /// + /// This variant does not mean that necessarily both `then` and `else` + /// blocks do exist and are non-empty. The `then` block might still be + /// empty and the `then` block might still be missing. + Both, + /// Only the `then` block of the `if` is reachable. + /// + /// # Note + /// + /// This case happens only in case the `if` has a `true` constant condition. + OnlyThen, + /// Only the `else` block of the `if` is reachable. + /// + /// # Note + /// + /// This case happens only in case the `if` has a `false` constant condition. + OnlyElse, +} + +impl ElseControlFrame { + /// Returns the branch label of the [`IfControlFrame`]. + pub fn label(&self) -> LabelRef { + self.label + } + + /// Returns `true` if the `then` branch is reachable. + /// + /// # Note + /// + /// The `then` branch is unreachable if the `if` condition is a constant `false` value. + pub fn is_then_reachable(&self) -> bool { + match self.reachability { + ElseReachability::Both { .. } | ElseReachability::OnlyThen => true, + ElseReachability::OnlyElse => false, + } + } + + /// Returns `true` if the `else` branch is reachable. + /// + /// # Note + /// + /// The `else` branch is unreachable if the `if` condition is a constant `true` value. + pub fn is_else_reachable(&self) -> bool { + match self.reachability { + ElseReachability::Both { .. } | ElseReachability::OnlyElse => true, + ElseReachability::OnlyThen => false, + } + } + + /// Returns `true` if the end of the `then` branch is reachable. + #[track_caller] + pub fn is_end_of_then_reachable(&self) -> bool { + self.is_end_of_then_reachable + } +} + +/// A generic unreachable Wasm control frame. +#[derive(Debug, Copy, Clone)] +pub enum UnreachableControlFrame { + /// An unreachable Wasm `block` control frame. + Block, + /// An unreachable Wasm `loop` control frame. + Loop, + /// An unreachable Wasm `if` control frame. + If, + /// An unreachable Wasm `else` control frame. + Else, +} diff --git a/crates/wasmi/src/engine/translator/translator2/stack/mod.rs b/crates/wasmi/src/engine/translator/translator2/stack/mod.rs index 89769dbac6..b3964eebf3 100644 --- a/crates/wasmi/src/engine/translator/translator2/stack/mod.rs +++ b/crates/wasmi/src/engine/translator/translator2/stack/mod.rs @@ -1,10 +1,23 @@ mod consts; +mod control; mod locals; mod operand; pub use self::operand::Operand; use self::{ consts::ConstRegistry, + control::{ + BlockControlFrame, + ControlFrame, + ControlFrameKind, + ControlStack, + ElseControlFrame, + ElseReachability, + IfControlFrame, + IfReachability, + LoopControlFrame, + UnreachableControlFrame, + }, locals::{LocalIdx, LocalsRegistry}, }; use crate::{ @@ -31,6 +44,8 @@ pub struct Stack { locals: LocalsRegistry, /// All function local constants. consts: ConstRegistry, + /// The Wasm control stack. + controls: ControlStack, /// The maximim number of operands on the [`Stack`] at the same time. max_height: usize, } @@ -41,6 +56,7 @@ impl Stack { self.operands.clear(); self.locals.reset(); self.consts.reset(); + self.controls.reset(); self.max_height = 0; } From 10e234fa6294149b5ce75e2a8ded8a6d8cc9f5f4 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 26 May 2025 10:21:38 +0200 Subject: [PATCH 054/343] carve out StackLayout from Stack --- .../translator/translator2/stack/layout.rs | 115 ++++++++++++++++++ .../translator/translator2/stack/mod.rs | 99 ++------------- 2 files changed, 124 insertions(+), 90 deletions(-) create mode 100644 crates/wasmi/src/engine/translator/translator2/stack/layout.rs diff --git a/crates/wasmi/src/engine/translator/translator2/stack/layout.rs b/crates/wasmi/src/engine/translator/translator2/stack/layout.rs new file mode 100644 index 0000000000..790549da07 --- /dev/null +++ b/crates/wasmi/src/engine/translator/translator2/stack/layout.rs @@ -0,0 +1,115 @@ +use super::{ConstRegistry, LocalIdx, OperandIdx, Reset}; +use crate::{core::UntypedVal, engine::TranslationError, ir::Reg, Error}; + +#[cfg(doc)] +use super::Stack; + +/// The layout of the [`Stack`]. +#[derive(Debug, Default)] +pub struct StackLayout { + /// The number of locals registered to the function. + len_locals: usize, + /// All function local constants. + consts: ConstRegistry, +} + +impl Reset for StackLayout { + fn reset(&mut self) { + self.len_locals = 0; + self.consts.reset(); + } +} + +impl StackLayout { + /// Returns the [`StackSpace`] of the [`Reg`]. + /// + /// Returns `None` if the [`Reg`] is unknown to the [`Stack`]. + #[must_use] + pub fn stack_space(&self, reg: Reg) -> StackSpace { + let index = i16::from(reg); + if index.is_negative() { + return StackSpace::Const; + } + let index = index as u16; + if usize::from(index) < self.len_locals { + return StackSpace::Local; + } + StackSpace::Temp + } + + /// Converts the local `index` into the associated [`Reg`]. + /// + /// # Errors + /// + /// If `index` cannot be converted into a [`Reg`]. + pub fn local_to_reg(&self, index: LocalIdx) -> Result { + debug_assert!( + (u32::from(index) as usize) < self.len_locals, + "out of bounds local operand index: {index:?}" + ); + let Ok(index) = i16::try_from(u32::from(index)) else { + return Err(Error::from(TranslationError::AllocatedTooManyRegisters)); + }; + Ok(Reg::from(index)) + } + + /// Converts the operand `index` into the associated [`Reg`]. + /// + /// # Errors + /// + /// If `index` cannot be converted into a [`Reg`]. + pub fn temp_to_reg(&self, index: OperandIdx) -> Result { + debug_assert!( + usize::from(index) >= self.len_locals, + "index must refer to a temporary operand" + ); + let index = usize::from(index); + let Some(index) = index.checked_add(self.len_locals) else { + return Err(Error::from(TranslationError::AllocatedTooManyRegisters)); + }; + let Ok(index) = i16::try_from(index) else { + return Err(Error::from(TranslationError::AllocatedTooManyRegisters)); + }; + Ok(Reg::from(index)) + } + + /// Allocates a function local constant `value`. + /// + /// # Errors + /// + /// If too many function local constants have been allocated already. + pub fn const_to_reg(&mut self, value: impl Into) -> Result { + self.consts.alloc(value.into()) + } +} + +/// The [`StackSpace`] of a [`Reg`]. +#[derive(Debug, Copy, Clone)] +pub enum StackSpace { + /// Stack slot referring to a local variable. + Local, + /// Stack slot referring to a function local constant value. + Const, + /// Stack slot referring to a temporary stack operand. + Temp, +} + +impl StackSpace { + /// Returns `true` if `self` is [`StackSpace::Local`]. + #[inline] + pub fn is_local(self) -> bool { + matches!(self, Self::Local) + } + + /// Returns `true` if `self` is [`StackSpace::Temp`]. + #[inline] + pub fn is_temp(self) -> bool { + matches!(self, Self::Temp) + } + + /// Returns `true` if `self` is [`StackSpace::Const`]. + #[inline] + pub fn is_const(self) -> bool { + matches!(self, Self::Const) + } +} diff --git a/crates/wasmi/src/engine/translator/translator2/stack/mod.rs b/crates/wasmi/src/engine/translator/translator2/stack/mod.rs index b3964eebf3..55aa5f762a 100644 --- a/crates/wasmi/src/engine/translator/translator2/stack/mod.rs +++ b/crates/wasmi/src/engine/translator/translator2/stack/mod.rs @@ -1,9 +1,9 @@ mod consts; mod control; +mod layout; mod locals; mod operand; -pub use self::operand::Operand; use self::{ consts::ConstRegistry, control::{ @@ -20,6 +20,10 @@ use self::{ }, locals::{LocalIdx, LocalsRegistry}, }; +pub use self::{ + layout::{StackLayout, StackSpace}, + operand::Operand, +}; use crate::{ core::{TypedVal, UntypedVal, ValType}, engine::{Instr, TranslationError}, @@ -247,22 +251,6 @@ impl Stack { Some(popped) } - /// Returns the [`RegSpace`] of the [`Reg`]. - /// - /// Returns `None` if the [`Reg`] is unknown to the [`Stack`]. - #[must_use] - pub fn stack_space(&self, reg: Reg) -> RegSpace { - let index = i16::from(reg); - if index.is_negative() { - return RegSpace::Const; - } - let index = index as u16; - if usize::from(index) < self.locals.len() { - return RegSpace::Local; - } - RegSpace::Temp - } - /// Preserve all locals on the [`Stack`] that refer to `local_index`. /// /// This is done by converting those locals to [`StackOperand::Temp`] and yielding them. @@ -364,52 +352,14 @@ impl Stack { /// # Panics /// /// If the `index` is out of bounds. - pub fn operand_to_reg(&mut self, depth: usize) -> Result { + pub fn operand_to_reg(&mut self, depth: usize, layout: &mut StackLayout) -> Result { let index = self.operand_index(depth); match self.get_at(index) { - StackOperand::Local { local_index, .. } => self.local_to_reg(local_index), - StackOperand::Temp { .. } => self.temp_to_reg(index), - StackOperand::Immediate { val } => self.const_to_reg(val), + StackOperand::Local { local_index, .. } => layout.local_to_reg(local_index), + StackOperand::Temp { .. } => layout.temp_to_reg(index), + StackOperand::Immediate { val } => layout.const_to_reg(val), } } - - /// Allocates a function local constant `value`. - /// - /// # Errors - /// - /// If too many function local constants have been allocated already. - pub fn const_to_reg(&mut self, value: impl Into) -> Result { - self.consts.alloc(value.into()) - } - - /// Converts the local `index` into the associated [`Reg`]. - /// - /// # Errors - /// - /// If `index` cannot be converted into a [`Reg`]. - pub fn local_to_reg(&self, index: LocalIdx) -> Result { - let Ok(index) = i16::try_from(u32::from(index)) else { - return Err(Error::from(TranslationError::AllocatedTooManyRegisters)); - }; - Ok(Reg::from(index)) - } - - /// Converts the [`Operand::Temp`] `index` into the associated [`Reg`]. - /// - /// # Errors - /// - /// If `index` cannot be converted into a [`Reg`]. - pub fn temp_to_reg(&self, index: OperandIdx) -> Result { - let index = usize::from(index); - debug_assert!(matches!(&self.operands[index], StackOperand::Temp { .. },)); - let Some(index) = index.checked_add(self.locals.len()) else { - return Err(Error::from(TranslationError::AllocatedTooManyRegisters)); - }; - let Ok(index) = i16::try_from(index) else { - return Err(Error::from(TranslationError::AllocatedTooManyRegisters)); - }; - Ok(Reg::from(index)) - } } /// Iterator yielding preserved local indices while preserving them. @@ -443,37 +393,6 @@ impl Iterator for PreservedLocalsIter<'_> { } } -/// The [`RegSpace`] of a [`Reg`]. -#[derive(Debug, Copy, Clone)] -pub enum RegSpace { - /// Stack slot referring to a local variable. - Local, - /// Stack slot referring to a function local constant value. - Const, - /// Stack slot referring to a temporary stack operand. - Temp, -} - -impl RegSpace { - /// Returns `true` if `self` is [`RegSpace::Local`]. - #[inline] - pub fn is_local(self) -> bool { - matches!(self, Self::Local) - } - - /// Returns `true` if `self` is [`RegSpace::Temp`]. - #[inline] - pub fn is_temp(self) -> bool { - matches!(self, Self::Temp) - } - - /// Returns `true` if `self` is [`RegSpace::Const`]. - #[inline] - pub fn is_const(self) -> bool { - matches!(self, Self::Const) - } -} - /// A [`StackOperand`] or [`Operand`] index on the [`Stack`]. #[derive(Debug, Copy, Clone, PartialEq, Eq)] pub struct OperandIdx(NonZero); From 54beeb713300e5e04c06ec3e0dda389a013fe5dc Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 26 May 2025 10:21:52 +0200 Subject: [PATCH 055/343] add stack and layout fields to FuncTranslator --- .../src/engine/translator/translator2/mod.rs | 56 ++++++++++++++++++- 1 file changed, 53 insertions(+), 3 deletions(-) diff --git a/crates/wasmi/src/engine/translator/translator2/mod.rs b/crates/wasmi/src/engine/translator/translator2/mod.rs index 316ae5f1ff..2244e148be 100644 --- a/crates/wasmi/src/engine/translator/translator2/mod.rs +++ b/crates/wasmi/src/engine/translator/translator2/mod.rs @@ -5,7 +5,7 @@ mod simd; mod stack; mod visit; -use self::stack::{Operand, OperandIdx, Stack}; +use self::stack::{Operand, OperandIdx, Stack, StackLayout}; use crate::{ core::FuelCostsProvider, engine::{translator::WasmTranslator, CompiledFuncEntity}, @@ -44,13 +44,19 @@ pub struct FuncTranslator { /// /// `None` if fuel metering is disabled. fuel_costs: Option, - /// The reusable data structures of the [`FuncTranslator`]. - alloc: FuncTranslatorAllocations, + /// Wasm value and control stack. + stack: Stack, + /// Wasm layout to map stack slots to Wasmi registers. + layout: StackLayout, } +/// Heap allocated data structured used by the [`FuncTranslator`]. #[derive(Debug, Default)] pub struct FuncTranslatorAllocations { + /// Wasm value and control stack. stack: Stack, + /// Wasm layout to map stack slots to Wasmi registers. + layout: StackLayout, } impl WasmTranslator<'_> for FuncTranslator { @@ -86,3 +92,47 @@ impl WasmTranslator<'_> for FuncTranslator { todo!() } } + +impl FuncTranslator { + /// Creates a new [`FuncTranslator`]. + pub fn new( + func: FuncIdx, + module: ModuleHeader, + alloc: FuncTranslatorAllocations, + ) -> Result { + let Some(engine) = module.engine().upgrade() else { + panic!( + "cannot compile function since engine does no longer exist: {:?}", + module.engine() + ) + }; + let config = engine.config(); + let fuel_costs = config + .get_consume_fuel() + .then(|| config.fuel_costs()) + .cloned(); + let FuncTranslatorAllocations { stack, layout } = alloc; + Ok(Self { + func, + engine, + module, + reachable: true, + fuel_costs, + stack, + layout, + }) + } + + /// Consumes `self` and returns the underlying reusable [`FuncTranslatorAllocations`]. + fn into_allocations(self) -> FuncTranslatorAllocations { + FuncTranslatorAllocations { + stack: self.stack, + layout: self.layout, + } + } + + /// Returns the [`Engine`] for which the function is compiled. + fn engine(&self) -> &Engine { + &self.engine + } +} From f193ff1cd3b120c7b7202c624ccb1e7b78900169 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 26 May 2025 10:37:39 +0200 Subject: [PATCH 056/343] rename lifetime --- .../src/engine/translator/translator2/stack/control.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/wasmi/src/engine/translator/translator2/stack/control.rs b/crates/wasmi/src/engine/translator/translator2/stack/control.rs index fe0e227b36..972ab08a00 100644 --- a/crates/wasmi/src/engine/translator/translator2/stack/control.rs +++ b/crates/wasmi/src/engine/translator/translator2/stack/control.rs @@ -151,16 +151,16 @@ impl ControlStack { /// An acquired branch target. #[derive(Debug)] -pub enum AcquiredTarget<'a> { +pub enum AcquiredTarget<'stack> { /// The branch targets the function enclosing `block` and therefore is a `return`. - Return(&'a mut ControlFrame), + Return(&'stack mut ControlFrame), /// The branch targets a regular [`ControlFrame`]. - Branch(&'a mut ControlFrame), + Branch(&'stack mut ControlFrame), } -impl<'a> AcquiredTarget<'a> { +impl<'stack> AcquiredTarget<'stack> { /// Returns an exclusive reference to the [`ControlFrame`] of the [`AcquiredTarget`]. - pub fn control_frame(&'a mut self) -> &'a mut ControlFrame { + pub fn control_frame(&'stack mut self) -> &'stack mut ControlFrame { match self { Self::Return(frame) => frame, Self::Branch(frame) => frame, From 29b954c1ef0a3578616c855078fbef402c44bc4f Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 26 May 2025 14:05:41 +0200 Subject: [PATCH 057/343] use usize instead of u32 for ControlFrame height --- .../translator/translator2/stack/control.rs | 29 +++++++++++-------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/crates/wasmi/src/engine/translator/translator2/stack/control.rs b/crates/wasmi/src/engine/translator/translator2/stack/control.rs index 972ab08a00..d75cc74771 100644 --- a/crates/wasmi/src/engine/translator/translator2/stack/control.rs +++ b/crates/wasmi/src/engine/translator/translator2/stack/control.rs @@ -64,7 +64,12 @@ impl ControlStack { } /// Pushes a new unreachable Wasm control frame onto the [`ControlStack`]. - pub fn push_unreachable(&mut self, ty: BlockType, height: u32, kind: UnreachableControlFrame) { + pub fn push_unreachable( + &mut self, + ty: BlockType, + height: usize, + kind: UnreachableControlFrame, + ) { self.frames .push(ControlFrame::new_unreachable(ty, height, kind)) } @@ -73,7 +78,7 @@ impl ControlStack { pub fn push_block( &mut self, ty: BlockType, - height: u32, + height: usize, label: LabelRef, consume_fuel: Option, ) { @@ -85,7 +90,7 @@ impl ControlStack { pub fn push_loop( &mut self, ty: BlockType, - height: u32, + height: usize, label: LabelRef, consume_fuel: Option, ) { @@ -97,7 +102,7 @@ impl ControlStack { pub fn push_if( &mut self, ty: BlockType, - height: u32, + height: usize, label: LabelRef, consume_fuel: Option, reachability: IfReachability, @@ -119,7 +124,7 @@ impl ControlStack { pub fn push_else( &mut self, ty: BlockType, - height: u32, + height: usize, label: LabelRef, consume_fuel: Option, reachability: ElseReachability, @@ -198,7 +203,7 @@ pub struct ControlFrame { /// The block type of the [`ControlFrame`]. ty: BlockType, /// The value stack height upon entering the [`ControlFrame`]. - height: u32, + height: usize, /// The number of branches to the [`ControlFrame`]. len_branches: usize, /// The [`ControlFrame`]'s [`Instruction::ConsumeFuel`] if fuel metering is enabled. @@ -213,7 +218,7 @@ pub struct ControlFrame { impl ControlFrame { /// Creates a new unreachable [`ControlFrame`] of `kind`. - pub fn new_unreachable(ty: BlockType, height: u32, kind: UnreachableControlFrame) -> Self { + pub fn new_unreachable(ty: BlockType, height: usize, kind: UnreachableControlFrame) -> Self { Self { ty, height, @@ -226,7 +231,7 @@ impl ControlFrame { /// Creates a new Wasm `block` [`ControlFrame`]. pub fn new_block( ty: BlockType, - height: u32, + height: usize, label: LabelRef, consume_fuel: Option, ) -> Self { @@ -242,7 +247,7 @@ impl ControlFrame { /// Creates a new Wasm `loop` [`ControlFrame`]. pub fn new_loop( ty: BlockType, - height: u32, + height: usize, label: LabelRef, consume_fuel: Option, ) -> Self { @@ -258,7 +263,7 @@ impl ControlFrame { /// Creates a new Wasm `if` [`ControlFrame`]. pub fn new_if( ty: BlockType, - height: u32, + height: usize, label: LabelRef, consume_fuel: Option, reachability: IfReachability, @@ -278,7 +283,7 @@ impl ControlFrame { /// Creates a new Wasm `else` [`ControlFrame`]. pub fn new_else( ty: BlockType, - height: u32, + height: usize, label: LabelRef, consume_fuel: Option, reachability: ElseReachability, @@ -298,7 +303,7 @@ impl ControlFrame { } /// Returns the stack height of the [`ControlFrame`]. - pub fn height(&self) -> u32 { + pub fn height(&self) -> usize { self.height } From b313ff0506279c737e32e093f748de5f7bf228e3 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 26 May 2025 14:06:05 +0200 Subject: [PATCH 058/343] return direct references in ControlStack::get[_mut] --- .../translator/translator2/stack/control.rs | 26 ++++++++++++++----- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/crates/wasmi/src/engine/translator/translator2/stack/control.rs b/crates/wasmi/src/engine/translator/translator2/stack/control.rs index d75cc74771..48a089eeb1 100644 --- a/crates/wasmi/src/engine/translator/translator2/stack/control.rs +++ b/crates/wasmi/src/engine/translator/translator2/stack/control.rs @@ -144,13 +144,27 @@ impl ControlStack { } /// Returns a shared reference to the [`ControlFrame`] at `depth` if any. - pub fn get(&self, depth: u32) -> Option<&ControlFrame> { - self.frames.iter().rev().nth(depth as usize) + pub fn get(&self, depth: u32) -> &ControlFrame { + let height = self.height(); + self.frames + .iter() + .rev() + .nth(depth as usize) + .unwrap_or_else(|| { + panic!("out of bounds control frame at depth (={depth}) for stack of height (={height})") + }) } /// Returns an exclusive reference to the [`ControlFrame`] at `depth` if any. - pub fn get_mut(&mut self, depth: u32) -> Option<&mut ControlFrame> { - self.frames.iter_mut().rev().nth(depth as usize) + pub fn get_mut(&mut self, depth: u32) -> &mut ControlFrame { + let height = self.height(); + self.frames + .iter_mut() + .rev() + .nth(depth as usize) + .unwrap_or_else(|| { + panic!("out of bounds control frame at depth (={depth}) for stack of height (={height})") + }) } } @@ -178,9 +192,7 @@ impl ControlStack { pub fn acquire_target(&mut self, depth: u32) -> AcquiredTarget { let is_root = self.is_root(depth); let height = self.height(); - let Some(frame) = self.get_mut(depth) else { - panic!("out of bounds `depth` (={depth}) for `ControlStack` with a height of {height}") - }; + let frame = self.get_mut(depth); if is_root { AcquiredTarget::Return(frame) } else { From 38e4659b6438cef88c1b59435ca2366f7fbada35 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 26 May 2025 14:10:25 +0200 Subject: [PATCH 059/343] add control stack API to Stack --- .../translator/translator2/stack/control.rs | 5 + .../translator/translator2/stack/mod.rs | 171 +++++++++++++++++- 2 files changed, 175 insertions(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/translator2/stack/control.rs b/crates/wasmi/src/engine/translator/translator2/stack/control.rs index 48a089eeb1..98b0dec654 100644 --- a/crates/wasmi/src/engine/translator/translator2/stack/control.rs +++ b/crates/wasmi/src/engine/translator/translator2/stack/control.rs @@ -143,6 +143,11 @@ impl ControlStack { .unwrap_or_else(|| panic!("missing operands for `else` control frame")) } + /// Pops the top-most [`ControlFrame`] and returns it if any. + pub fn pop(&mut self) -> Option { + self.frames.pop() + } + /// Returns a shared reference to the [`ControlFrame`] at `depth` if any. pub fn get(&self, depth: u32) -> &ControlFrame { let height = self.height(); diff --git a/crates/wasmi/src/engine/translator/translator2/stack/mod.rs b/crates/wasmi/src/engine/translator/translator2/stack/mod.rs index 55aa5f762a..b3101c3318 100644 --- a/crates/wasmi/src/engine/translator/translator2/stack/mod.rs +++ b/crates/wasmi/src/engine/translator/translator2/stack/mod.rs @@ -26,8 +26,13 @@ pub use self::{ }; use crate::{ core::{TypedVal, UntypedVal, ValType}, - engine::{Instr, TranslationError}, + engine::{ + translator::{BlockType, LabelRef}, + Instr, + TranslationError, + }, ir::Reg, + Engine, Error, }; use alloc::vec::Vec; @@ -42,6 +47,8 @@ trait Reset { /// The Wasm value stack during translation from Wasm to Wasmi bytecode. #[derive(Debug, Default)] pub struct Stack { + /// The underlying [`Engine`]. + engine: Engine, /// The Wasm value stack. operands: Vec, /// All function locals and their associated types. @@ -149,6 +156,168 @@ impl Stack { } } + /// Pushes a Wasm `block` onto the [`Stack`]. + /// + /// # Note + /// + /// If `consume_fuel` is `None` and fuel metering is enabled this will infer + /// the [`Instruction::ConsumeFuel`] from the last control frame on the [`Stack`]. + /// + /// # Errors + /// + /// If the stack height exceeds the maximum height. + pub fn push_block( + &mut self, + ty: BlockType, + label: LabelRef, + consume_fuel: Option, + ) -> Result<(), Error> { + let len_params = usize::from(ty.len_params(&self.engine)); + let block_height = self.height() - len_params; + let fuel_metering = self.engine.config().get_consume_fuel(); + let consume_fuel = match consume_fuel { + None if fuel_metering => { + let consume_instr = self + .controls + .get(0) + .consume_fuel_instr() + .expect("control frame must have consume instructions"); + Some(consume_instr) + } + consume_fuel => consume_fuel, + }; + self.controls + .push_block(ty, block_height, label, consume_fuel); + Ok(()) + } + + /// Pushes a Wasm `loop` onto the [`Stack`]. + /// + /// # Note + /// + /// Calls `f` for every non [`StackOperand::Temp`] operand on the [`Stack`] + /// that is also a parameter to the pushed Wasm `loop` control frame. + /// + /// # Panics (debug) + /// + /// If `consume_fuel` is `None` and fuel metering is enabled. + /// + /// # Errors + /// + /// If the stack height exceeds the maximum height. + pub fn push_loop( + &mut self, + ty: BlockType, + label: LabelRef, + consume_fuel: Option, + mut f: impl FnMut(Operand) -> Result<(), Error>, + ) -> Result<(), Error> { + debug_assert!(self.engine.config().get_consume_fuel() == consume_fuel.is_some()); + let len_params = usize::from(ty.len_params(&self.engine)); + let block_height = self.height() - len_params; + for depth in 0..block_height { + if let Some(operand) = self.operand_to_temp(depth) { + f(operand)?; + } + } + self.controls + .push_loop(ty, block_height, label, consume_fuel); + Ok(()) + } + + /// Pushes a Wasm `if` onto the [`Stack`]. + /// + /// # Panics (debug) + /// + /// If `consume_fuel` is `None` and fuel metering is enabled. + /// + /// # Errors + /// + /// If the stack height exceeds the maximum height. + pub fn push_if( + &mut self, + ty: BlockType, + label: LabelRef, + reachability: IfReachability, + consume_fuel: Option, + ) -> Result<(), Error> { + debug_assert!(self.engine.config().get_consume_fuel() == consume_fuel.is_some()); + let len_params = usize::from(ty.len_params(&self.engine)); + let block_height = self.height() - len_params; + let else_operands = self.operands.get(block_height..).unwrap_or(&[]); + debug_assert!(len_params == else_operands.len()); + self.controls.push_if( + ty, + block_height, + label, + consume_fuel, + reachability, + else_operands, + ); + Ok(()) + } + + /// Pushes a Wasm `else` onto the [`Stack`]. + /// + /// # Panics (debug) + /// + /// If `consume_fuel` is `None` and fuel metering is enabled. + /// + /// # Errors + /// + /// If the stack height exceeds the maximum height. + pub fn push_else( + &mut self, + ty: BlockType, + label: LabelRef, + reachability: ElseReachability, + is_end_of_then_reachable: bool, + consume_fuel: Option, + ) -> Result<(), Error> { + debug_assert!(self.engine.config().get_consume_fuel() == consume_fuel.is_some()); + let len_params = usize::from(ty.len_params(&self.engine)); + let block_height = self.height() - len_params; + let else_operands = self.controls.push_else( + ty, + block_height, + label, + consume_fuel, + reachability, + is_end_of_then_reachable, + ); + for operand in else_operands { + // TODO: push `operand` to stack, resolve borrow issues + } + Ok(()) + } + + /// Pushes an unreachable Wasm control onto the [`Stack`]. + /// + /// # Errors + /// + /// If the stack height exceeds the maximum height. + pub fn push_unreachable( + &mut self, + ty: BlockType, + kind: UnreachableControlFrame, + ) -> Result<(), Error> { + let len_params = usize::from(ty.len_params(&self.engine)); + let block_height = self.height() - len_params; + self.controls.push_unreachable(ty, block_height, kind); + Ok(()) + } + + /// Pops the top-most control frame from the control stack and returns it. + /// + /// # Panics + /// + /// If the control stack is empty. + pub fn pop_control( + &mut self, + ) -> ControlFrame { + self.controls.pop().unwrap_or_else(|| panic!("tried to pop control from empty control stack")) + } + /// Pushes a local variable with index `local_idx` to the [`Stack`]. /// /// # Errors From 7ed22cfa908eff465b210e5fbed6818ab7949a15 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 26 May 2025 14:10:32 +0200 Subject: [PATCH 060/343] apply rustfmt --- .../src/engine/translator/translator2/stack/mod.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/wasmi/src/engine/translator/translator2/stack/mod.rs b/crates/wasmi/src/engine/translator/translator2/stack/mod.rs index b3101c3318..44a3192a47 100644 --- a/crates/wasmi/src/engine/translator/translator2/stack/mod.rs +++ b/crates/wasmi/src/engine/translator/translator2/stack/mod.rs @@ -308,14 +308,14 @@ impl Stack { } /// Pops the top-most control frame from the control stack and returns it. - /// + /// /// # Panics - /// + /// /// If the control stack is empty. - pub fn pop_control( - &mut self, - ) -> ControlFrame { - self.controls.pop().unwrap_or_else(|| panic!("tried to pop control from empty control stack")) + pub fn pop_control(&mut self) -> ControlFrame { + self.controls + .pop() + .unwrap_or_else(|| panic!("tried to pop control from empty control stack")) } /// Pushes a local variable with index `local_idx` to the [`Stack`]. From d0d1c098673e7f499f72e072f8ab3487dbc76965 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 26 May 2025 14:11:42 +0200 Subject: [PATCH 061/343] remove unused ConstsRegistry from Stack --- crates/wasmi/src/engine/translator/translator2/stack/mod.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/crates/wasmi/src/engine/translator/translator2/stack/mod.rs b/crates/wasmi/src/engine/translator/translator2/stack/mod.rs index 44a3192a47..9dd025d3b6 100644 --- a/crates/wasmi/src/engine/translator/translator2/stack/mod.rs +++ b/crates/wasmi/src/engine/translator/translator2/stack/mod.rs @@ -53,8 +53,6 @@ pub struct Stack { operands: Vec, /// All function locals and their associated types. locals: LocalsRegistry, - /// All function local constants. - consts: ConstRegistry, /// The Wasm control stack. controls: ControlStack, /// The maximim number of operands on the [`Stack`] at the same time. @@ -66,7 +64,6 @@ impl Stack { pub fn reset(&mut self) { self.operands.clear(); self.locals.reset(); - self.consts.reset(); self.controls.reset(); self.max_height = 0; } From 6266dc62a466853a43591496b69f8261ba1e617c Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Tue, 27 May 2025 11:50:08 +0200 Subject: [PATCH 062/343] refactor Stack to separate control and value stacks This resolved a bunch of borrow issues and cleaned up the overall design. --- .../translator/translator2/stack/control.rs | 14 +- .../translator/translator2/stack/mod.rs | 325 +++----------- .../translator/translator2/stack/operands.rs | 405 ++++++++++++++++++ 3 files changed, 469 insertions(+), 275 deletions(-) create mode 100644 crates/wasmi/src/engine/translator/translator2/stack/operands.rs diff --git a/crates/wasmi/src/engine/translator/translator2/stack/control.rs b/crates/wasmi/src/engine/translator/translator2/stack/control.rs index 98b0dec654..5ec1818b24 100644 --- a/crates/wasmi/src/engine/translator/translator2/stack/control.rs +++ b/crates/wasmi/src/engine/translator/translator2/stack/control.rs @@ -1,4 +1,4 @@ -use super::{Reset, StackOperand}; +use super::{Operand, Reset}; use crate::engine::{ translator::{BlockType, LabelRef}, Instr, @@ -23,7 +23,7 @@ pub struct ElseOperands { /// The end indices of each `else` operands. ends: Vec, /// All operands of all allocated `else` control frames. - providers: Vec, + providers: Vec, } impl Reset for ElseOperands { @@ -35,15 +35,15 @@ impl Reset for ElseOperands { impl ElseOperands { /// Pushes operands for a new Wasm `else` control frame. - pub fn push(&mut self, operands: &[StackOperand]) { - self.providers.extend(operands.iter().copied()); + pub fn push(&mut self, operands: impl IntoIterator) { + self.providers.extend(operands); let end = self.providers.len(); let index = self.ends.len(); self.ends.push(end); } /// Pops the top-most Wasm `else` operands from `self` and returns it. - pub fn pop(&mut self) -> Option> { + pub fn pop(&mut self) -> Option> { let end = self.ends.pop()?; let start = self.ends.last().copied().unwrap_or(0); Some(self.providers.drain(start..end)) @@ -106,7 +106,7 @@ impl ControlStack { label: LabelRef, consume_fuel: Option, reachability: IfReachability, - else_operands: &[StackOperand], + else_operands: impl IntoIterator, ) { self.frames.push(ControlFrame::new_if( ty, @@ -129,7 +129,7 @@ impl ControlStack { consume_fuel: Option, reachability: ElseReachability, is_end_of_then_reachable: bool, - ) -> Drain { + ) -> Drain { self.frames.push(ControlFrame::new_else( ty, height, diff --git a/crates/wasmi/src/engine/translator/translator2/stack/mod.rs b/crates/wasmi/src/engine/translator/translator2/stack/mod.rs index 9dd025d3b6..e69e39ba8d 100644 --- a/crates/wasmi/src/engine/translator/translator2/stack/mod.rs +++ b/crates/wasmi/src/engine/translator/translator2/stack/mod.rs @@ -3,6 +3,7 @@ mod control; mod layout; mod locals; mod operand; +mod operands; use self::{ consts::ConstRegistry, @@ -19,10 +20,12 @@ use self::{ UnreachableControlFrame, }, locals::{LocalIdx, LocalsRegistry}, + operands::{OperandStack, StackOperand}, }; pub use self::{ layout::{StackLayout, StackSpace}, operand::Operand, + operands::{OperandIdx, PreservedLocalsIter}, }; use crate::{ core::{TypedVal, UntypedVal, ValType}, @@ -50,22 +53,16 @@ pub struct Stack { /// The underlying [`Engine`]. engine: Engine, /// The Wasm value stack. - operands: Vec, - /// All function locals and their associated types. - locals: LocalsRegistry, + operands: OperandStack, /// The Wasm control stack. controls: ControlStack, - /// The maximim number of operands on the [`Stack`] at the same time. - max_height: usize, } impl Stack { /// Resets the [`Stack`] for reuse. pub fn reset(&mut self) { - self.operands.clear(); - self.locals.reset(); + self.operands.reset(); self.controls.reset(); - self.max_height = 0; } /// Register `amount` local variables of common type `ty`. @@ -74,8 +71,7 @@ impl Stack { /// /// If too many local variables are being registered. pub fn register_locals(&mut self, amount: u32, ty: ValType) -> Result<(), Error> { - self.locals.register(amount, ty)?; - Ok(()) + self.operands.register_locals(amount, ty) } /// Returns the current height of the [`Stack`]. @@ -84,7 +80,7 @@ impl Stack { /// /// The height is equal to the number of [`Operand`]s on the [`Stack`]. pub fn height(&self) -> usize { - self.operands.len() + self.operands.height() } /// Returns the maximum height of the [`Stack`]. @@ -93,7 +89,7 @@ impl Stack { /// /// The height is equal to the number of [`Operand`]s on the [`Stack`]. pub fn max_height(&self) -> usize { - self.max_height + self.operands.max_height() } /// Truncates `self` to the target `height`. @@ -104,53 +100,7 @@ impl Stack { /// /// If `height` is greater than the current height of `self`. pub fn trunc(&mut self, height: usize) { - assert!(height <= self.height()); - self.operands.truncate(height); - } - - /// Updates the maximum stack height if needed. - fn update_max_stack_height(&mut self) { - self.max_height = core::cmp::max(self.max_height, self.height()); - } - - /// Returns the [`OperandIdx`] of the next pushed operand. - fn next_operand_index(&self) -> OperandIdx { - OperandIdx::from(self.operands.len()) - } - - /// Returns the [`OperandIdx`] of the operand at `depth`. - fn operand_index(&self, depth: usize) -> OperandIdx { - OperandIdx::from(self.height() - depth - 1) - } - - /// Updates the `prev_local` of the [`StackOperand::Local`] at `local_index` to `prev_index`. - /// - /// # Panics - /// - /// - If `local_index` does not refer to a [`StackOperand::Local`]. - /// - If `local_index` is out of bounds of the operand stack. - fn update_prev_local(&mut self, local_index: OperandIdx, prev_index: Option) { - match self.get_mut_at(local_index) { - StackOperand::Local { prev_local, .. } => { - *prev_local = prev_index; - } - operand => panic!("expected `StackOperand::Local` but found: {operand:?}"), - } - } - - /// Updates the `next_local` of the [`StackOperand::Local`] at `local_index` to `prev_index`. - /// - /// # Panics - /// - /// - If `local_index` does not refer to a [`StackOperand::Local`]. - /// - If `local_index` is out of bounds of the operand stack. - fn update_next_local(&mut self, local_index: OperandIdx, prev_index: Option) { - match self.get_mut_at(local_index) { - StackOperand::Local { next_local, .. } => { - *next_local = prev_index; - } - operand => panic!("expected `StackOperand::Local` but found: {operand:?}"), - } + self.operands.trunc(height); } /// Pushes a Wasm `block` onto the [`Stack`]. @@ -241,7 +191,7 @@ impl Stack { debug_assert!(self.engine.config().get_consume_fuel() == consume_fuel.is_some()); let len_params = usize::from(ty.len_params(&self.engine)); let block_height = self.height() - len_params; - let else_operands = self.operands.get(block_height..).unwrap_or(&[]); + let else_operands = self.operands.peek(len_params); debug_assert!(len_params == else_operands.len()); self.controls.push_if( ty, @@ -283,7 +233,17 @@ impl Stack { is_end_of_then_reachable, ); for operand in else_operands { - // TODO: push `operand` to stack, resolve borrow issues + match operand { + Operand::Local(op) => { + self.operands.push_local(op.local_index())?; + } + Operand::Temp(op) => { + self.operands.push_temp(op.ty(), op.instr())?; + } + Operand::Immediate(op) => { + self.operands.push_immediate(op.val())?; + } + } } Ok(()) } @@ -322,20 +282,7 @@ impl Stack { /// - If too many operands have been pushed onto the [`Stack`]. /// - If the local with `local_idx` does not exist. pub fn push_local(&mut self, local_index: LocalIdx) -> Result { - let operand_index = self.next_operand_index(); - let next_local = self - .locals - .replace_first_operand(local_index, Some(operand_index)); - if let Some(next_local) = next_local { - self.update_prev_local(next_local, Some(operand_index)); - } - self.operands.push(StackOperand::Local { - local_index, - prev_local: None, - next_local, - }); - self.update_max_stack_height(); - Ok(operand_index) + self.operands.push_local(local_index) } /// Pushes a temporary with type `ty` on the [`Stack`]. @@ -344,10 +291,7 @@ impl Stack { /// /// If too many operands have been pushed onto the [`Stack`]. pub fn push_temp(&mut self, ty: ValType, instr: Option) -> Result { - let operand_index = self.next_operand_index(); - self.operands.push(StackOperand::Temp { ty, instr }); - self.update_max_stack_height(); - Ok(operand_index) + self.operands.push_temp(ty, instr) } /// Pushes an immediate `value` on the [`Stack`]. @@ -356,67 +300,56 @@ impl Stack { /// /// If too many operands have been pushed onto the [`Stack`]. pub fn push_immediate(&mut self, value: impl Into) -> Result { - let operand_index = self.next_operand_index(); - self.operands - .push(StackOperand::Immediate { val: value.into() }); - self.update_max_stack_height(); - Ok(operand_index) + self.operands.push_immediate(value) } /// Peeks the top-most [`Operand`] on the [`Stack`]. /// /// Returns `None` if the [`Stack`] is empty. pub fn peek(&self) -> Operand { - let index = self.operand_index(0); - let operand = self.get_at(index); - Operand::new(index, operand, &self.locals) + self.operands.get(0) } /// Pops the top-most [`Operand`] from the [`Stack`]. /// - /// Returns `None` if the [`Stack`] is empty. - pub fn pop(&mut self) -> Option { - let operand = self.operands.pop()?; - let index = OperandIdx::from(self.operands.len()); - Some(Operand::new(index, operand, &self.locals)) + /// # Panics + /// + /// If `self` is empty. + pub fn pop(&mut self) -> Operand { + self.operands.pop() } /// Pops the two top-most [`Operand`] from the [`Stack`]. /// - /// - Returns `None` if the [`Stack`] is empty. - /// - The last returned [`Operand`] is the top-most one. + /// # Note + /// + /// The last returned [`Operand`] is the top-most one. + /// + /// # Panics + /// + /// If `self` does not contain enough operands to pop. pub fn pop2(&mut self) -> Option<(Operand, Operand)> { - let [o1, o2] = self.pop_some::<2>()?; + let o2 = self.pop(); + let o1 = self.pop(); Some((o1, o2)) } - /// Pops the three top-most [`Operand`] from the [`Stack`]. + /// Pops the two top-most [`Operand`] from the [`Stack`]. + /// + /// # Note + /// + /// The last returned [`Operand`] is the top-most one. /// - /// - Returns `None` if the [`Stack`] is empty. - /// - The last returned [`Operand`] is the top-most one. + /// # Panics + /// + /// If `self` does not contain enough operands to pop. pub fn pop3(&mut self) -> Option<(Operand, Operand, Operand)> { - let [o1, o2, o3] = self.pop_some::<3>()?; + let o3 = self.pop(); + let o2 = self.pop(); + let o1 = self.pop(); Some((o1, o2, o3)) } - /// Pops the top-most `N` [`Operand`]s from the [`Stack`]. - /// - /// - Returns `None` if the [`Stack`] is empty. - /// - The last returned [`Operand`] is the top-most one. - fn pop_some(&mut self) -> Option<[Operand; N]> { - if N >= self.height() { - return None; - } - let start = self.height() - N; - let drained = self.operands.drain(start..); - let popped: [Operand; N] = array::from_fn(|i| { - let index = OperandIdx::from(start + i); - let operand = drained.as_slice()[i]; - Operand::new(index, operand, &self.locals) - }); - Some(popped) - } - /// Preserve all locals on the [`Stack`] that refer to `local_index`. /// /// This is done by converting those locals to [`StackOperand::Temp`] and yielding them. @@ -431,41 +364,7 @@ impl Stack { /// If the local at `local_index` is out of bounds. #[must_use] pub fn preserve_locals(&mut self, local_index: LocalIdx) -> PreservedLocalsIter { - let ty = self.locals.ty(local_index); - let index = self.locals.replace_first_operand(local_index, None); - let operands = &mut self.operands[..]; - PreservedLocalsIter { - operands, - index, - ty, - } - } - - /// Returns the [`StackOperand`] at `index`. - /// - /// # Panics - /// - /// If `depth` is out of bounds for `self`. - fn get_at(&self, index: OperandIdx) -> StackOperand { - self.operands[usize::from(index)] - } - - /// Returns an exlusive reference to the [`StackOperand`] at `index`. - /// - /// # Panics - /// - /// If `depth` is out of bounds for `self`. - fn get_mut_at(&mut self, index: OperandIdx) -> &mut StackOperand { - &mut self.operands[usize::from(index)] - } - - /// Sets the [`StackOperand`] at `index` to `operand`. - /// - /// # Panics - /// - /// If `depth` is out of bounds for `self`. - fn set_at(&mut self, index: OperandIdx, operand: StackOperand) { - self.operands[usize::from(index)] = operand; + self.operands.preserve_locals(local_index) } /// Converts and returns the [`StackOperand`] at `depth` into a [`StackOperand::Temp`]. @@ -479,38 +378,7 @@ impl Stack { /// - If `depth` is out of bounds for the [`Stack`] of operands. #[must_use] pub fn operand_to_temp(&mut self, depth: usize) -> Option { - let index = self.operand_index(depth); - let operand = match self.get_at(index) { - StackOperand::Local { - local_index, - prev_local, - next_local, - } => { - if prev_local.is_none() { - // Note: if `prev_local` is `None` then this local is the first - // in the linked list of locals and must be updated. - debug_assert_eq!(self.locals.first_operand(local_index), Some(index)); - self.locals.replace_first_operand(local_index, next_local); - } - if let Some(prev_local) = prev_local { - self.update_next_local(prev_local, next_local); - } - if let Some(next_local) = next_local { - self.update_prev_local(next_local, prev_local); - } - Operand::local(index, local_index, &self.locals) - } - StackOperand::Immediate { val } => Operand::immediate(index, val), - StackOperand::Temp { .. } => return None, - }; - self.set_at( - index, - StackOperand::Temp { - ty: operand.ty(), - instr: None, - }, - ); - Some(operand) + self.operands.operand_to_temp(depth) } /// Converts the [`Operand`] at `index` to a [`Reg`] if possible. @@ -519,90 +387,11 @@ impl Stack { /// /// If the `index` is out of bounds. pub fn operand_to_reg(&mut self, depth: usize, layout: &mut StackLayout) -> Result { - let index = self.operand_index(depth); - match self.get_at(index) { - StackOperand::Local { local_index, .. } => layout.local_to_reg(local_index), - StackOperand::Temp { .. } => layout.temp_to_reg(index), - StackOperand::Immediate { val } => layout.const_to_reg(val), + let operand = self.operands.get(depth); + match operand { + Operand::Local(op) => layout.local_to_reg(op.local_index()), + Operand::Temp(_) => layout.temp_to_reg(operand.index()), + Operand::Immediate(op) => layout.const_to_reg(op.val()), } } } - -/// Iterator yielding preserved local indices while preserving them. -#[derive(Debug)] -pub struct PreservedLocalsIter<'stack> { - /// The underlying operand stack. - operands: &'stack mut [StackOperand], - /// The current operand index of the next preserved local if any. - index: Option, - /// Type of local at preserved `local_index`. - ty: ValType, -} - -impl Iterator for PreservedLocalsIter<'_> { - type Item = OperandIdx; - - fn next(&mut self) -> Option { - let index = self.index?; - let operand = mem::replace( - &mut self.operands[usize::from(index)], - StackOperand::Temp { - ty: self.ty, - instr: None, - }, - ); - self.index = match operand { - StackOperand::Local { next_local, .. } => next_local, - operand => panic!("expected `StackOperand::Local` but found: {operand:?}"), - }; - Some(index) - } -} - -/// A [`StackOperand`] or [`Operand`] index on the [`Stack`]. -#[derive(Debug, Copy, Clone, PartialEq, Eq)] -pub struct OperandIdx(NonZero); - -impl From for usize { - fn from(value: OperandIdx) -> Self { - value.0.get().wrapping_sub(1) - } -} - -impl From for OperandIdx { - fn from(value: usize) -> Self { - let Some(operand_idx) = NonZero::new(value.wrapping_add(1)) else { - panic!("out of bounds `OperandIdx`: {value}") - }; - Self(operand_idx) - } -} - -/// An [`Operand`] on the [`Stack`]. -/// -/// This is the internal version of [`Operand`] with information that shall remain -/// hidden to the outside. -#[derive(Debug, Copy, Clone)] -enum StackOperand { - /// A local variable. - Local { - /// The index of the local variable. - local_index: LocalIdx, - /// The previous [`StackOperand::Local`] on the [`Stack`]. - prev_local: Option, - /// The next [`StackOperand::Local`] on the [`Stack`]. - next_local: Option, - }, - /// A temporary value on the [`Stack`]. - Temp { - /// The type of the temporary value. - ty: ValType, - /// The instruction which has this [`StackOperand`] as result if any. - instr: Option, - }, - /// An immediate value on the [`Stack`]. - Immediate { - /// The value (and type) of the immediate value. - val: TypedVal, - }, -} diff --git a/crates/wasmi/src/engine/translator/translator2/stack/operands.rs b/crates/wasmi/src/engine/translator/translator2/stack/operands.rs new file mode 100644 index 0000000000..0dd70e7d71 --- /dev/null +++ b/crates/wasmi/src/engine/translator/translator2/stack/operands.rs @@ -0,0 +1,405 @@ +use super::{LocalIdx, LocalsRegistry, Operand, Reset}; +use crate::{ + core::{TypedVal, ValType}, + engine::Instr, + Error, +}; +use alloc::vec::Vec; +use core::{array, mem, num::NonZero, slice}; + +/// A [`StackOperand`] or [`Operand`] index on the [`Stack`]. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub struct OperandIdx(NonZero); + +impl From for usize { + fn from(value: OperandIdx) -> Self { + value.0.get().wrapping_sub(1) + } +} + +impl From for OperandIdx { + fn from(value: usize) -> Self { + let Some(operand_idx) = NonZero::new(value.wrapping_add(1)) else { + panic!("out of bounds `OperandIdx`: {value}") + }; + Self(operand_idx) + } +} + +/// An [`Operand`] on the [`Stack`]. +/// +/// This is the internal version of [`Operand`] with information that shall remain +/// hidden to the outside. +#[derive(Debug, Copy, Clone)] +pub enum StackOperand { + /// A local variable. + Local { + /// The index of the local variable. + local_index: LocalIdx, + /// The previous [`StackOperand::Local`] on the [`Stack`]. + prev_local: Option, + /// The next [`StackOperand::Local`] on the [`Stack`]. + next_local: Option, + }, + /// A temporary value on the [`Stack`]. + Temp { + /// The type of the temporary value. + ty: ValType, + /// The instruction which has this [`StackOperand`] as result if any. + instr: Option, + }, + /// An immediate value on the [`Stack`]. + Immediate { + /// The value (and type) of the immediate value. + val: TypedVal, + }, +} + +/// The Wasm operand (or value) stack. +#[derive(Debug, Default)] +pub struct OperandStack { + /// The current set of operands on the [`OperandStack`]. + operands: Vec, + /// All function locals and their associated types. + /// + /// Used to query types of locals and their first local on the [`OperandStack`]. + locals: LocalsRegistry, + /// The maximum height of the [`OperandStack`]. + max_height: usize, +} + +impl Reset for OperandStack { + fn reset(&mut self) { + self.operands.clear(); + self.locals.reset(); + self.max_height = 0; + } +} + +impl OperandStack { + /// Register `amount` local variables of common type `ty`. + /// + /// # Errors + /// + /// If too many local variables are being registered. + pub fn register_locals(&mut self, amount: u32, ty: ValType) -> Result<(), Error> { + self.locals.register(amount, ty)?; + Ok(()) + } + + /// Returns the current height of `self` + /// + /// # Note + /// + /// The height is equal to the number of [`StackOperand`]s on `self`. + pub fn height(&self) -> usize { + self.operands.len() + } + + /// Returns the maximum height of `self`. + /// + /// # Note + /// + /// The height is equal to the number of [`Operand`]s on `self`. + pub fn max_height(&self) -> usize { + self.max_height + } + + /// Truncates `self` to the target `height`. + /// + /// All operands above `height` are dropped. + /// + /// # Panic + /// + /// If `height` is greater than the current height of `self`. + pub fn trunc(&mut self, height: usize) { + assert!(height <= self.height()); + self.operands.truncate(height); + } + + /// Updates the maximum stack height if needed. + fn update_max_stack_height(&mut self) { + self.max_height = core::cmp::max(self.max_height, self.height()); + } + + /// Returns the [`OperandIdx`] of the next pushed operand. + fn next_index(&self) -> OperandIdx { + OperandIdx::from(self.operands.len()) + } + + /// Returns the [`OperandIdx`] of the operand at `depth`. + fn depth_to_index(&self, depth: usize) -> OperandIdx { + OperandIdx::from(self.height() - depth - 1) + } + + /// Pushes a local variable with index `local_idx` to the [`OperandStack`]. + /// + /// # Errors + /// + /// - If too many operands have been pushed onto the [`OperandStack`]. + /// - If the local with `local_idx` does not exist. + pub fn push_local(&mut self, local_index: LocalIdx) -> Result { + let operand_index = self.next_index(); + let next_local = self + .locals + .replace_first_operand(local_index, Some(operand_index)); + if let Some(next_local) = next_local { + self.update_prev_local(next_local, Some(operand_index)); + } + self.operands.push(StackOperand::Local { + local_index, + prev_local: None, + next_local, + }); + self.update_max_stack_height(); + Ok(operand_index) + } + + /// Pushes a temporary with type `ty` on the [`OperandStack`]. + /// + /// # Errors + /// + /// If too many operands have been pushed onto the [`OperandStack`]. + pub fn push_temp(&mut self, ty: ValType, instr: Option) -> Result { + let idx = self.next_index(); + self.operands.push(StackOperand::Temp { ty, instr }); + self.update_max_stack_height(); + Ok(idx) + } + + /// Pushes an immediate `value` on the [`OperandStack`]. + /// + /// # Errors + /// + /// If too many operands have been pushed onto the [`OperandStack`]. + pub fn push_immediate(&mut self, value: impl Into) -> Result { + let idx = self.next_index(); + self.operands + .push(StackOperand::Immediate { val: value.into() }); + self.update_max_stack_height(); + Ok(idx) + } + + /// Returns an iterator that yields all [`Operand`]s up to `depth`. + /// + /// # Panics + /// + /// If `depth` is out of bounds for `self`. + pub fn peek(&self, depth: usize) -> PeekedOperands { + let index = self.depth_to_index(depth); + let operands = &self.operands[usize::from(index)..]; + PeekedOperands { + index: usize::from(index), + operands: operands.iter(), + locals: &self.locals, + } + } + + /// Pops the top-most [`StackOperand`] from `self` if any. + /// + /// # Panics + /// + /// If `self` is empty. + pub fn pop(&mut self) -> Operand { + let Some(operand) = self.operands.pop() else { + panic!("tried to pop operand from empty stack"); + }; + let index = self.next_index(); + self.unlink_local(operand); + Operand::new(index, operand, &self.locals) + } + + /// Returns the [`Operand`] at `depth`. + /// + /// # Panics + /// + /// If `depth` is out of bounds for `self`. + pub fn get(&self, depth: usize) -> Operand { + let index = self.depth_to_index(depth); + let operand = self.get_at(index); + Operand::new(index, operand, &self.locals) + } + + /// Returns the [`StackOperand`] at `index`. + /// + /// # Panics + /// + /// If `index` is out of bounds for `self`. + fn get_at(&self, index: OperandIdx) -> StackOperand { + self.operands[usize::from(index)] + } + + /// Converts and returns the [`StackOperand`] at `depth` into a [`Operand::Temp`]. + /// + /// # Note + /// + /// Returns `None` if operand at `depth` is [`Operand::Temp`] already. + /// + /// # Panics + /// + /// If `depth` is out of bounds for `self`. + #[must_use] + pub fn operand_to_temp(&mut self, depth: usize) -> Option { + let index = self.depth_to_index(depth); + let operand = self.operand_to_temp_at(index)?; + Some(Operand::new(index, operand, &self.locals)) + } + + /// Converts and returns the [`StackOperand`] at `index` into a [`StackOperand::Temp`]. + /// + /// # Note + /// + /// Returns `None` if operand at `index` is [`StackOperand::Temp`] already. + /// + /// # Panics + /// + /// If `index` is out of bounds for `self`. + #[must_use] + fn operand_to_temp_at(&mut self, index: OperandIdx) -> Option { + let operand = self.get_at(index); + let ty = match operand { + StackOperand::Temp { ty, instr } => return None, + StackOperand::Immediate { val } => val.ty(), + StackOperand::Local { + local_index, + next_local, + prev_local, + } => self.locals.ty(local_index), + }; + self.unlink_local(operand); + self.operands[usize::from(index)] = operand; + Some(operand) + } + + /// Preserve all locals on the [`OperandStack`] that refer to `local_index`. + /// + /// This is done by converting those locals to [`StackOperand::Temp`] and yielding them. + /// + /// # Note + /// + /// The users must fully consume all items yielded by the returned iterator in order + /// for the local preservation to take full effect. + /// + /// # Panics + /// + /// If the local at `local_index` is out of bounds. + #[must_use] + pub fn preserve_locals(&mut self, local_index: LocalIdx) -> PreservedLocalsIter { + let ty = self.locals.ty(local_index); + let index = self.locals.replace_first_operand(local_index, None); + PreservedLocalsIter { + operands: self, + index, + ty, + } + } + + /// Unlinks the [`StackOperand::Local`] `operand` at `index` from `self`. + /// + /// Does nothing if `operand` is not a [`StackOperand::Local`]. + fn unlink_local(&mut self, operand: StackOperand) { + let StackOperand::Local { + local_index, + prev_local, + next_local, + } = operand + else { + return; + }; + if prev_local.is_none() { + self.locals.replace_first_operand(local_index, next_local); + } + if let Some(prev_local) = prev_local { + self.update_next_local(prev_local, next_local); + } + if let Some(next_local) = next_local { + self.update_prev_local(next_local, prev_local); + } + } + + /// Updates the `prev_local` of the [`StackOperand::Local`] at `local_index` to `prev_index`. + /// + /// # Panics + /// + /// - If `local_index` does not refer to a [`StackOperand::Local`]. + /// - If `local_index` is out of bounds of the operand stack. + fn update_prev_local(&mut self, local_index: OperandIdx, prev_index: Option) { + match &mut self.operands[usize::from(local_index)] { + StackOperand::Local { prev_local, .. } => { + *prev_local = prev_index; + } + operand => panic!("expected `StackOperand::Local` but found: {operand:?}"), + } + } + + /// Updates the `next_local` of the [`StackOperand::Local`] at `local_index` to `prev_index`. + /// + /// # Panics + /// + /// - If `local_index` does not refer to a [`StackOperand::Local`]. + /// - If `local_index` is out of bounds of the operand stack. + fn update_next_local(&mut self, local_index: OperandIdx, prev_index: Option) { + match &mut self.operands[usize::from(local_index)] { + StackOperand::Local { next_local, .. } => { + *next_local = prev_index; + } + operand => panic!("expected `StackOperand::Local` but found: {operand:?}"), + } + } +} + +/// Iterator yielding preserved local indices while preserving them. +#[derive(Debug)] +pub struct PreservedLocalsIter<'stack> { + /// The underlying operand stack. + operands: &'stack mut OperandStack, + /// The current operand index of the next preserved local if any. + index: Option, + /// Type of local at preserved `local_index`. + ty: ValType, +} + +impl Iterator for PreservedLocalsIter<'_> { + type Item = OperandIdx; + + fn next(&mut self) -> Option { + let index = self.index?; + let operand = self + .operands + .operand_to_temp_at(index) + .expect("local operand in linked list"); + self.index = match operand { + StackOperand::Local { next_local, .. } => next_local, + op => panic!("expected `StackOperand::Local` but found: {op:?}"), + }; + Some(index) + } +} + +/// Iterator yielding peeked stack operators. +#[derive(Debug)] +pub struct PeekedOperands<'stack> { + /// The index of the next yielded operand. + index: usize, + /// The iterator of peeked stack operands. + operands: slice::Iter<'stack, StackOperand>, + /// Used to query types of local operands. + locals: &'stack LocalsRegistry, +} + +impl Iterator for PeekedOperands<'_> { + type Item = Operand; + + fn next(&mut self) -> Option { + let operand = self.operands.next().copied()?; + let index = OperandIdx::from(self.index); + self.index += 1; + Some(Operand::new(index, operand, &self.locals)) + } +} + +impl ExactSizeIterator for PeekedOperands<'_> { + fn len(&self) -> usize { + self.operands.len() + } +} From 38a0c66388373754c0049dddb2037783fc16c5f3 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Tue, 27 May 2025 11:56:03 +0200 Subject: [PATCH 063/343] move StackLayout to its own module --- .../translator2/{stack => layout}/consts.rs | 0 .../{stack/layout.rs => layout/mod.rs} | 5 +++- .../src/engine/translator/translator2/mod.rs | 8 +++++- .../translator/translator2/stack/mod.rs | 28 ++----------------- .../engine/translator/translator2/utils.rs | 5 ++++ 5 files changed, 19 insertions(+), 27 deletions(-) rename crates/wasmi/src/engine/translator/translator2/{stack => layout}/consts.rs (100%) rename crates/wasmi/src/engine/translator/translator2/{stack/layout.rs => layout/mod.rs} (97%) create mode 100644 crates/wasmi/src/engine/translator/translator2/utils.rs diff --git a/crates/wasmi/src/engine/translator/translator2/stack/consts.rs b/crates/wasmi/src/engine/translator/translator2/layout/consts.rs similarity index 100% rename from crates/wasmi/src/engine/translator/translator2/stack/consts.rs rename to crates/wasmi/src/engine/translator/translator2/layout/consts.rs diff --git a/crates/wasmi/src/engine/translator/translator2/stack/layout.rs b/crates/wasmi/src/engine/translator/translator2/layout/mod.rs similarity index 97% rename from crates/wasmi/src/engine/translator/translator2/stack/layout.rs rename to crates/wasmi/src/engine/translator/translator2/layout/mod.rs index 790549da07..99bfab5a19 100644 --- a/crates/wasmi/src/engine/translator/translator2/stack/layout.rs +++ b/crates/wasmi/src/engine/translator/translator2/layout/mod.rs @@ -1,4 +1,7 @@ -use super::{ConstRegistry, LocalIdx, OperandIdx, Reset}; +mod consts; + +use self::consts::ConstRegistry; +use super::{LocalIdx, OperandIdx, Reset}; use crate::{core::UntypedVal, engine::TranslationError, ir::Reg, Error}; #[cfg(doc)] diff --git a/crates/wasmi/src/engine/translator/translator2/mod.rs b/crates/wasmi/src/engine/translator/translator2/mod.rs index 2244e148be..73faae1db0 100644 --- a/crates/wasmi/src/engine/translator/translator2/mod.rs +++ b/crates/wasmi/src/engine/translator/translator2/mod.rs @@ -1,11 +1,17 @@ #![expect(dead_code, unused_imports, unused_variables)] +mod layout; #[cfg(feature = "simd")] mod simd; mod stack; +mod utils; mod visit; -use self::stack::{Operand, OperandIdx, Stack, StackLayout}; +use self::{ + layout::{StackLayout, StackSpace}, + stack::{LocalIdx, Operand, OperandIdx, Stack}, + utils::Reset, +}; use crate::{ core::FuelCostsProvider, engine::{translator::WasmTranslator, CompiledFuncEntity}, diff --git a/crates/wasmi/src/engine/translator/translator2/stack/mod.rs b/crates/wasmi/src/engine/translator/translator2/stack/mod.rs index e69e39ba8d..911b95e591 100644 --- a/crates/wasmi/src/engine/translator/translator2/stack/mod.rs +++ b/crates/wasmi/src/engine/translator/translator2/stack/mod.rs @@ -1,12 +1,9 @@ -mod consts; mod control; -mod layout; mod locals; mod operand; mod operands; use self::{ - consts::ConstRegistry, control::{ BlockControlFrame, ControlFrame, @@ -19,14 +16,15 @@ use self::{ LoopControlFrame, UnreachableControlFrame, }, - locals::{LocalIdx, LocalsRegistry}, + locals::LocalsRegistry, operands::{OperandStack, StackOperand}, }; pub use self::{ - layout::{StackLayout, StackSpace}, + locals::LocalIdx, operand::Operand, operands::{OperandIdx, PreservedLocalsIter}, }; +use super::Reset; use crate::{ core::{TypedVal, UntypedVal, ValType}, engine::{ @@ -41,12 +39,6 @@ use crate::{ use alloc::vec::Vec; use core::{array, mem, num::NonZero}; -/// Implemented by types that can be reset for reuse. -trait Reset { - /// Resets `self` for reuse. - fn reset(&mut self); -} - /// The Wasm value stack during translation from Wasm to Wasmi bytecode. #[derive(Debug, Default)] pub struct Stack { @@ -380,18 +372,4 @@ impl Stack { pub fn operand_to_temp(&mut self, depth: usize) -> Option { self.operands.operand_to_temp(depth) } - - /// Converts the [`Operand`] at `index` to a [`Reg`] if possible. - /// - /// # Panics - /// - /// If the `index` is out of bounds. - pub fn operand_to_reg(&mut self, depth: usize, layout: &mut StackLayout) -> Result { - let operand = self.operands.get(depth); - match operand { - Operand::Local(op) => layout.local_to_reg(op.local_index()), - Operand::Temp(_) => layout.temp_to_reg(operand.index()), - Operand::Immediate(op) => layout.const_to_reg(op.val()), - } - } } diff --git a/crates/wasmi/src/engine/translator/translator2/utils.rs b/crates/wasmi/src/engine/translator/translator2/utils.rs new file mode 100644 index 0000000000..e3db31c5d3 --- /dev/null +++ b/crates/wasmi/src/engine/translator/translator2/utils.rs @@ -0,0 +1,5 @@ +/// Implemented by types that can be reset for reuse. +pub trait Reset { + /// Resets `self` for reuse. + fn reset(&mut self); +} From 51c6dde1a4bbf50f9f1679dc887acaf3b8833d64 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Tue, 27 May 2025 11:57:20 +0200 Subject: [PATCH 064/343] make Stack::peek more powerful --- .../src/engine/translator/translator2/stack/mod.rs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/crates/wasmi/src/engine/translator/translator2/stack/mod.rs b/crates/wasmi/src/engine/translator/translator2/stack/mod.rs index 911b95e591..5dd15bb1fc 100644 --- a/crates/wasmi/src/engine/translator/translator2/stack/mod.rs +++ b/crates/wasmi/src/engine/translator/translator2/stack/mod.rs @@ -295,11 +295,17 @@ impl Stack { self.operands.push_immediate(value) } - /// Peeks the top-most [`Operand`] on the [`Stack`]. + /// Peeks the [`Operand`] at `depth`. /// - /// Returns `None` if the [`Stack`] is empty. - pub fn peek(&self) -> Operand { - self.operands.get(0) + /// # Note + /// + /// A depth of 0 peeks the top-most [`Operand`] on `self`. + /// + /// # Panics + /// + /// If `depth` is out of bounds for `self`. + pub fn peek(&self, depth: usize) -> Operand { + self.operands.get(depth) } /// Pops the top-most [`Operand`] from the [`Stack`]. From 157fae6fdb69b5f1d67385fbafd9daea58f7d9d1 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Tue, 27 May 2025 11:59:15 +0200 Subject: [PATCH 065/343] use `usize` for `depth` params --- .../translator/translator2/stack/control.rs | 34 ++++++++----------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/crates/wasmi/src/engine/translator/translator2/stack/control.rs b/crates/wasmi/src/engine/translator/translator2/stack/control.rs index 5ec1818b24..9c8a1ac273 100644 --- a/crates/wasmi/src/engine/translator/translator2/stack/control.rs +++ b/crates/wasmi/src/engine/translator/translator2/stack/control.rs @@ -149,27 +149,23 @@ impl ControlStack { } /// Returns a shared reference to the [`ControlFrame`] at `depth` if any. - pub fn get(&self, depth: u32) -> &ControlFrame { + pub fn get(&self, depth: usize) -> &ControlFrame { let height = self.height(); - self.frames - .iter() - .rev() - .nth(depth as usize) - .unwrap_or_else(|| { - panic!("out of bounds control frame at depth (={depth}) for stack of height (={height})") - }) + self.frames.iter().rev().nth(depth).unwrap_or_else(|| { + panic!( + "out of bounds control frame at depth (={depth}) for stack of height (={height})" + ) + }) } /// Returns an exclusive reference to the [`ControlFrame`] at `depth` if any. - pub fn get_mut(&mut self, depth: u32) -> &mut ControlFrame { + pub fn get_mut(&mut self, depth: usize) -> &mut ControlFrame { let height = self.height(); - self.frames - .iter_mut() - .rev() - .nth(depth as usize) - .unwrap_or_else(|| { - panic!("out of bounds control frame at depth (={depth}) for stack of height (={height})") - }) + self.frames.iter_mut().rev().nth(depth).unwrap_or_else(|| { + panic!( + "out of bounds control frame at depth (={depth}) for stack of height (={height})" + ) + }) } } @@ -194,7 +190,7 @@ impl<'stack> AcquiredTarget<'stack> { impl ControlStack { /// Acquires the target [`ControlFrame`] at the given relative `depth`. - pub fn acquire_target(&mut self, depth: u32) -> AcquiredTarget { + pub fn acquire_target(&mut self, depth: usize) -> AcquiredTarget { let is_root = self.is_root(depth); let height = self.height(); let frame = self.get_mut(depth); @@ -206,11 +202,11 @@ impl ControlStack { } /// Returns `true` if `depth` points to the first control flow frame. - fn is_root(&self, depth: u32) -> bool { + fn is_root(&self, depth: usize) -> bool { if self.frames.is_empty() { return false; } - depth as usize == self.height() - 1 + depth == self.height() - 1 } } From ece412ce53a9818680e4cc7549b0ee2cc5bc91e3 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Tue, 27 May 2025 12:00:02 +0200 Subject: [PATCH 066/343] apply clippy suggestions --- .../wasmi/src/engine/translator/translator2/stack/control.rs | 4 ++-- .../wasmi/src/engine/translator/translator2/stack/operands.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/wasmi/src/engine/translator/translator2/stack/control.rs b/crates/wasmi/src/engine/translator/translator2/stack/control.rs index 9c8a1ac273..6a08bfe8cf 100644 --- a/crates/wasmi/src/engine/translator/translator2/stack/control.rs +++ b/crates/wasmi/src/engine/translator/translator2/stack/control.rs @@ -517,7 +517,7 @@ impl ElseControlFrame { /// The `then` branch is unreachable if the `if` condition is a constant `false` value. pub fn is_then_reachable(&self) -> bool { match self.reachability { - ElseReachability::Both { .. } | ElseReachability::OnlyThen => true, + ElseReachability::Both | ElseReachability::OnlyThen => true, ElseReachability::OnlyElse => false, } } @@ -529,7 +529,7 @@ impl ElseControlFrame { /// The `else` branch is unreachable if the `if` condition is a constant `true` value. pub fn is_else_reachable(&self) -> bool { match self.reachability { - ElseReachability::Both { .. } | ElseReachability::OnlyElse => true, + ElseReachability::Both | ElseReachability::OnlyElse => true, ElseReachability::OnlyThen => false, } } diff --git a/crates/wasmi/src/engine/translator/translator2/stack/operands.rs b/crates/wasmi/src/engine/translator/translator2/stack/operands.rs index 0dd70e7d71..d7d172ff13 100644 --- a/crates/wasmi/src/engine/translator/translator2/stack/operands.rs +++ b/crates/wasmi/src/engine/translator/translator2/stack/operands.rs @@ -394,7 +394,7 @@ impl Iterator for PeekedOperands<'_> { let operand = self.operands.next().copied()?; let index = OperandIdx::from(self.index); self.index += 1; - Some(Operand::new(index, operand, &self.locals)) + Some(Operand::new(index, operand, self.locals)) } } From f5c1232b5156ac16873f121a6307d7f0c4caf19f Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Tue, 27 May 2025 12:05:50 +0200 Subject: [PATCH 067/343] implement FuncTranslator::register_locals --- .../engine/translator/translator2/layout/mod.rs | 17 ++++++++++++++++- .../src/engine/translator/translator2/mod.rs | 11 +++++++---- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/crates/wasmi/src/engine/translator/translator2/layout/mod.rs b/crates/wasmi/src/engine/translator/translator2/layout/mod.rs index 99bfab5a19..52a206a471 100644 --- a/crates/wasmi/src/engine/translator/translator2/layout/mod.rs +++ b/crates/wasmi/src/engine/translator/translator2/layout/mod.rs @@ -2,7 +2,12 @@ mod consts; use self::consts::ConstRegistry; use super::{LocalIdx, OperandIdx, Reset}; -use crate::{core::UntypedVal, engine::TranslationError, ir::Reg, Error}; +use crate::{ + core::{UntypedVal, ValType}, + engine::TranslationError, + ir::Reg, + Error, +}; #[cfg(doc)] use super::Stack; @@ -24,6 +29,16 @@ impl Reset for StackLayout { } impl StackLayout { + /// Register `amount` local variables of common type `ty`. + /// + /// # Errors + /// + /// If too many local variables are being registered. + pub fn register_locals(&mut self, amount: u32, ty: ValType) -> Result<(), Error> { + self.len_locals += amount as usize; + Ok(()) + } + /// Returns the [`StackSpace`] of the [`Reg`]. /// /// Returns `None` if the [`Reg`] is unknown to the [`Stack`]. diff --git a/crates/wasmi/src/engine/translator/translator2/mod.rs b/crates/wasmi/src/engine/translator/translator2/mod.rs index 73faae1db0..81de215bfe 100644 --- a/crates/wasmi/src/engine/translator/translator2/mod.rs +++ b/crates/wasmi/src/engine/translator/translator2/mod.rs @@ -15,7 +15,7 @@ use self::{ use crate::{ core::FuelCostsProvider, engine::{translator::WasmTranslator, CompiledFuncEntity}, - module::{FuncIdx, ModuleHeader}, + module::{FuncIdx, ModuleHeader, WasmiValueType}, Engine, Error, }; @@ -79,10 +79,13 @@ impl WasmTranslator<'_> for FuncTranslator { fn translate_locals( &mut self, - _amount: u32, - _value_type: wasmparser::ValType, + amount: u32, + value_type: wasmparser::ValType, ) -> Result<(), Error> { - todo!() + let ty = WasmiValueType::from(value_type).into_inner(); + self.stack.register_locals(amount, ty)?; + self.layout.register_locals(amount, ty)?; + Ok(()) } fn finish_translate_locals(&mut self) -> Result<(), Error> { From 9139d6e7ecaf486a27a2952d81fe8fd815f10daf Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Tue, 27 May 2025 12:13:03 +0200 Subject: [PATCH 068/343] add is_fuel_metering_enabled convenience func --- .../engine/translator/translator2/stack/mod.rs | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/crates/wasmi/src/engine/translator/translator2/stack/mod.rs b/crates/wasmi/src/engine/translator/translator2/stack/mod.rs index 5dd15bb1fc..958d8bc606 100644 --- a/crates/wasmi/src/engine/translator/translator2/stack/mod.rs +++ b/crates/wasmi/src/engine/translator/translator2/stack/mod.rs @@ -95,11 +95,16 @@ impl Stack { self.operands.trunc(height); } + /// Returns `true` is fuel metering is enabled for the associated [`Engine`]. + fn is_fuel_metering_enabled(&self) -> bool { + self.engine.config().get_consume_fuel() + } + /// Pushes a Wasm `block` onto the [`Stack`]. /// /// # Note /// - /// If `consume_fuel` is `None` and fuel metering is enabled this will infer + /// If `consume_fuel` is `None` and fuel metering is enabled this will inherit /// the [`Instruction::ConsumeFuel`] from the last control frame on the [`Stack`]. /// /// # Errors @@ -113,7 +118,7 @@ impl Stack { ) -> Result<(), Error> { let len_params = usize::from(ty.len_params(&self.engine)); let block_height = self.height() - len_params; - let fuel_metering = self.engine.config().get_consume_fuel(); + let fuel_metering = self.is_fuel_metering_enabled(); let consume_fuel = match consume_fuel { None if fuel_metering => { let consume_instr = self @@ -151,7 +156,7 @@ impl Stack { consume_fuel: Option, mut f: impl FnMut(Operand) -> Result<(), Error>, ) -> Result<(), Error> { - debug_assert!(self.engine.config().get_consume_fuel() == consume_fuel.is_some()); + debug_assert!(self.is_fuel_metering_enabled() == consume_fuel.is_some()); let len_params = usize::from(ty.len_params(&self.engine)); let block_height = self.height() - len_params; for depth in 0..block_height { @@ -180,7 +185,7 @@ impl Stack { reachability: IfReachability, consume_fuel: Option, ) -> Result<(), Error> { - debug_assert!(self.engine.config().get_consume_fuel() == consume_fuel.is_some()); + debug_assert!(self.is_fuel_metering_enabled() == consume_fuel.is_some()); let len_params = usize::from(ty.len_params(&self.engine)); let block_height = self.height() - len_params; let else_operands = self.operands.peek(len_params); @@ -213,7 +218,7 @@ impl Stack { is_end_of_then_reachable: bool, consume_fuel: Option, ) -> Result<(), Error> { - debug_assert!(self.engine.config().get_consume_fuel() == consume_fuel.is_some()); + debug_assert!(self.is_fuel_metering_enabled() == consume_fuel.is_some()); let len_params = usize::from(ty.len_params(&self.engine)); let block_height = self.height() - len_params; let else_operands = self.controls.push_else( From cde60634e2811f01756c187f95352d1dea588265 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Tue, 27 May 2025 12:13:15 +0200 Subject: [PATCH 069/343] impl finish_translate_locals as no-op --- crates/wasmi/src/engine/translator/translator2/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/translator2/mod.rs b/crates/wasmi/src/engine/translator/translator2/mod.rs index 81de215bfe..cbb577e020 100644 --- a/crates/wasmi/src/engine/translator/translator2/mod.rs +++ b/crates/wasmi/src/engine/translator/translator2/mod.rs @@ -89,7 +89,7 @@ impl WasmTranslator<'_> for FuncTranslator { } fn finish_translate_locals(&mut self) -> Result<(), Error> { - todo!() + Ok(()) } fn update_pos(&mut self, _pos: usize) {} From 11ee9684d1a16963a5ce99abbc8e4c3952083883 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Tue, 27 May 2025 12:25:58 +0200 Subject: [PATCH 070/343] fix doc links --- .../src/engine/translator/translator2/stack/mod.rs | 3 +++ .../engine/translator/translator2/stack/operands.rs | 12 ++++++------ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/crates/wasmi/src/engine/translator/translator2/stack/mod.rs b/crates/wasmi/src/engine/translator/translator2/stack/mod.rs index 958d8bc606..cb816be3c6 100644 --- a/crates/wasmi/src/engine/translator/translator2/stack/mod.rs +++ b/crates/wasmi/src/engine/translator/translator2/stack/mod.rs @@ -39,6 +39,9 @@ use crate::{ use alloc::vec::Vec; use core::{array, mem, num::NonZero}; +#[cfg(doc)] +use crate::ir::Instruction; + /// The Wasm value stack during translation from Wasm to Wasmi bytecode. #[derive(Debug, Default)] pub struct Stack { diff --git a/crates/wasmi/src/engine/translator/translator2/stack/operands.rs b/crates/wasmi/src/engine/translator/translator2/stack/operands.rs index d7d172ff13..b58f74e224 100644 --- a/crates/wasmi/src/engine/translator/translator2/stack/operands.rs +++ b/crates/wasmi/src/engine/translator/translator2/stack/operands.rs @@ -7,7 +7,7 @@ use crate::{ use alloc::vec::Vec; use core::{array, mem, num::NonZero, slice}; -/// A [`StackOperand`] or [`Operand`] index on the [`Stack`]. +/// A [`StackOperand`] or [`Operand`] index on the [`OperandStack`]. #[derive(Debug, Copy, Clone, PartialEq, Eq)] pub struct OperandIdx(NonZero); @@ -26,7 +26,7 @@ impl From for OperandIdx { } } -/// An [`Operand`] on the [`Stack`]. +/// An [`Operand`] on the [`OperandStack`]. /// /// This is the internal version of [`Operand`] with information that shall remain /// hidden to the outside. @@ -36,19 +36,19 @@ pub enum StackOperand { Local { /// The index of the local variable. local_index: LocalIdx, - /// The previous [`StackOperand::Local`] on the [`Stack`]. + /// The previous [`StackOperand::Local`] on the [`OperandStack`]. prev_local: Option, - /// The next [`StackOperand::Local`] on the [`Stack`]. + /// The next [`StackOperand::Local`] on the [`OperandStack`]. next_local: Option, }, - /// A temporary value on the [`Stack`]. + /// A temporary value on the [`OperandStack`]. Temp { /// The type of the temporary value. ty: ValType, /// The instruction which has this [`StackOperand`] as result if any. instr: Option, }, - /// An immediate value on the [`Stack`]. + /// An immediate value on the [`OperandStack`]. Immediate { /// The value (and type) of the immediate value. val: TypedVal, From e4139bc2725783eff43b8d710f5a954ee16ecfc3 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Tue, 27 May 2025 15:02:15 +0200 Subject: [PATCH 071/343] no longer return Option from Stack::pop{2,3} --- .../wasmi/src/engine/translator/translator2/stack/mod.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/wasmi/src/engine/translator/translator2/stack/mod.rs b/crates/wasmi/src/engine/translator/translator2/stack/mod.rs index cb816be3c6..05845def65 100644 --- a/crates/wasmi/src/engine/translator/translator2/stack/mod.rs +++ b/crates/wasmi/src/engine/translator/translator2/stack/mod.rs @@ -334,10 +334,10 @@ impl Stack { /// # Panics /// /// If `self` does not contain enough operands to pop. - pub fn pop2(&mut self) -> Option<(Operand, Operand)> { + pub fn pop2(&mut self) -> (Operand, Operand) { let o2 = self.pop(); let o1 = self.pop(); - Some((o1, o2)) + (o1, o2) } /// Pops the two top-most [`Operand`] from the [`Stack`]. @@ -349,11 +349,11 @@ impl Stack { /// # Panics /// /// If `self` does not contain enough operands to pop. - pub fn pop3(&mut self) -> Option<(Operand, Operand, Operand)> { + pub fn pop3(&mut self) -> (Operand, Operand, Operand) { let o3 = self.pop(); let o2 = self.pop(); let o1 = self.pop(); - Some((o1, o2, o3)) + (o1, o2, o3) } /// Preserve all locals on the [`Stack`] that refer to `local_index`. From 9400ccbf28e193f55d78dd26b4e9ab2ea63f2b50 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Tue, 27 May 2025 15:05:08 +0200 Subject: [PATCH 072/343] change StackOperand doc links to Operand --- .../wasmi/src/engine/translator/translator2/stack/mod.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/wasmi/src/engine/translator/translator2/stack/mod.rs b/crates/wasmi/src/engine/translator/translator2/stack/mod.rs index 05845def65..fddb43b1aa 100644 --- a/crates/wasmi/src/engine/translator/translator2/stack/mod.rs +++ b/crates/wasmi/src/engine/translator/translator2/stack/mod.rs @@ -142,7 +142,7 @@ impl Stack { /// /// # Note /// - /// Calls `f` for every non [`StackOperand::Temp`] operand on the [`Stack`] + /// Calls `f` for every non [`Operand::Temp`] operand on the [`Stack`] /// that is also a parameter to the pushed Wasm `loop` control frame. /// /// # Panics (debug) @@ -358,7 +358,7 @@ impl Stack { /// Preserve all locals on the [`Stack`] that refer to `local_index`. /// - /// This is done by converting those locals to [`StackOperand::Temp`] and yielding them. + /// This is done by converting those locals to [`Operand::Temp`] and yielding them. /// /// # Note /// @@ -373,11 +373,11 @@ impl Stack { self.operands.preserve_locals(local_index) } - /// Converts and returns the [`StackOperand`] at `depth` into a [`StackOperand::Temp`]. + /// Converts and returns the [`StackOperand`] at `depth` into a [`Operand::Temp`]. /// /// # Note /// - /// Returns `None` if operand at `depth` is [`StackOperand::Temp`] already. + /// Returns `None` if operand at `depth` is [`Operand::Temp`] already. /// /// # Panics /// From 62d6d5a3e3790a7bb9d487a1944db44daf3f5461 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Tue, 27 May 2025 15:05:53 +0200 Subject: [PATCH 073/343] cleanup panic doc --- crates/wasmi/src/engine/translator/translator2/stack/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/translator2/stack/mod.rs b/crates/wasmi/src/engine/translator/translator2/stack/mod.rs index fddb43b1aa..ef9e410c20 100644 --- a/crates/wasmi/src/engine/translator/translator2/stack/mod.rs +++ b/crates/wasmi/src/engine/translator/translator2/stack/mod.rs @@ -381,7 +381,7 @@ impl Stack { /// /// # Panics /// - /// - If `depth` is out of bounds for the [`Stack`] of operands. + /// If `depth` is out of bounds for the [`Stack`] of operands. #[must_use] pub fn operand_to_temp(&mut self, depth: usize) -> Option { self.operands.operand_to_temp(depth) From 56dca33ec5ba6e5ac5213e5c6cc3d805a03eddea Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 28 May 2025 11:51:56 +0200 Subject: [PATCH 074/343] add LabelRegistry to new FuncTranslator --- .../src/engine/translator/translator2/mod.rs | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/crates/wasmi/src/engine/translator/translator2/mod.rs b/crates/wasmi/src/engine/translator/translator2/mod.rs index cbb577e020..38ce2eb3fb 100644 --- a/crates/wasmi/src/engine/translator/translator2/mod.rs +++ b/crates/wasmi/src/engine/translator/translator2/mod.rs @@ -14,7 +14,10 @@ use self::{ }; use crate::{ core::FuelCostsProvider, - engine::{translator::WasmTranslator, CompiledFuncEntity}, + engine::{ + translator::{LabelRegistry, WasmTranslator}, + CompiledFuncEntity, + }, module::{FuncIdx, ModuleHeader, WasmiValueType}, Engine, Error, @@ -54,6 +57,8 @@ pub struct FuncTranslator { stack: Stack, /// Wasm layout to map stack slots to Wasmi registers. layout: StackLayout, + /// Registers and pins labels and tracks their users. + labels: LabelRegistry, } /// Heap allocated data structured used by the [`FuncTranslator`]. @@ -63,6 +68,8 @@ pub struct FuncTranslatorAllocations { stack: Stack, /// Wasm layout to map stack slots to Wasmi registers. layout: StackLayout, + /// Registers and pins labels and tracks their users. + labels: LabelRegistry, } impl WasmTranslator<'_> for FuncTranslator { @@ -120,7 +127,11 @@ impl FuncTranslator { .get_consume_fuel() .then(|| config.fuel_costs()) .cloned(); - let FuncTranslatorAllocations { stack, layout } = alloc; + let FuncTranslatorAllocations { + stack, + layout, + labels, + } = alloc; Ok(Self { func, engine, @@ -129,6 +140,7 @@ impl FuncTranslator { fuel_costs, stack, layout, + labels, }) } @@ -137,6 +149,7 @@ impl FuncTranslator { FuncTranslatorAllocations { stack: self.stack, layout: self.layout, + labels: self.labels, } } From 037f055c5b1e69e4dedc0b7b471948fd0d5fe31f Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Fri, 30 May 2025 11:32:50 +0200 Subject: [PATCH 075/343] add basic instruction encoder --- .../engine/translator/translator2/instrs.rs | 50 +++++++++++++++++++ .../src/engine/translator/translator2/mod.rs | 10 +++- 2 files changed, 59 insertions(+), 1 deletion(-) create mode 100644 crates/wasmi/src/engine/translator/translator2/instrs.rs diff --git a/crates/wasmi/src/engine/translator/translator2/instrs.rs b/crates/wasmi/src/engine/translator/translator2/instrs.rs new file mode 100644 index 0000000000..f0b70aa286 --- /dev/null +++ b/crates/wasmi/src/engine/translator/translator2/instrs.rs @@ -0,0 +1,50 @@ +use super::{Instr, Reset}; +use crate::ir::Instruction; +use alloc::vec::Vec; + +/// Creates and encodes the list of [`Instruction`]s for a function. +#[derive(Debug, Default)] +pub struct InstrEncoder { + /// The list of constructed instructions and their parameters. + instrs: Vec, +} + +impl Reset for InstrEncoder { + fn reset(&mut self) { + self.instrs.clear(); + } +} + +impl InstrEncoder { + /// Returns the next [`Instr`]. + #[must_use] + fn next_instr(&self) -> Instr { + Instr::from_usize(self.instrs.len()) + } + + /// Pushes an [`Instruction`] to the [`InstrEncoder`]. + /// + /// Returns an [`Instr`] that refers to the pushed [`Instruction`]. + #[must_use] + pub fn push_instr(&mut self, instruction: Instruction) -> Instr { + let instr = self.next_instr(); + self.instrs.push(instruction); + instr + } + + /// Pushes an [`Instruction`] parameter to the [`InstrEncoder`]. + /// + /// The parameter is associated to the last pushed [`Instruction`]. + pub fn push_param(&mut self, instruction: Instruction) { + self.instrs.push(instruction); + } + + /// Returns a shared reference to the [`Instruction`] associated to [`Instr`]. + /// + /// # Panics + /// + /// If `instr` is out of bounds for `self`. + pub fn get(&self, instr: Instr) -> &Instruction { + &self.instrs[instr.into_usize()] + } +} diff --git a/crates/wasmi/src/engine/translator/translator2/mod.rs b/crates/wasmi/src/engine/translator/translator2/mod.rs index 38ce2eb3fb..518a94ff4c 100644 --- a/crates/wasmi/src/engine/translator/translator2/mod.rs +++ b/crates/wasmi/src/engine/translator/translator2/mod.rs @@ -1,5 +1,6 @@ #![expect(dead_code, unused_imports, unused_variables)] +mod instrs; mod layout; #[cfg(feature = "simd")] mod simd; @@ -8,6 +9,7 @@ mod utils; mod visit; use self::{ + instrs::InstrEncoder, layout::{StackLayout, StackSpace}, stack::{LocalIdx, Operand, OperandIdx, Stack}, utils::Reset, @@ -15,7 +17,7 @@ use self::{ use crate::{ core::FuelCostsProvider, engine::{ - translator::{LabelRegistry, WasmTranslator}, + translator::{Instr, LabelRegistry, WasmTranslator}, CompiledFuncEntity, }, module::{FuncIdx, ModuleHeader, WasmiValueType}, @@ -59,6 +61,8 @@ pub struct FuncTranslator { layout: StackLayout, /// Registers and pins labels and tracks their users. labels: LabelRegistry, + /// Constructs and encodes function instructions. + instrs: InstrEncoder, } /// Heap allocated data structured used by the [`FuncTranslator`]. @@ -70,6 +74,8 @@ pub struct FuncTranslatorAllocations { layout: StackLayout, /// Registers and pins labels and tracks their users. labels: LabelRegistry, + /// Constructs and encodes function instructions. + instrs: InstrEncoder, } impl WasmTranslator<'_> for FuncTranslator { @@ -141,6 +147,7 @@ impl FuncTranslator { stack, layout, labels, + instrs, }) } @@ -150,6 +157,7 @@ impl FuncTranslator { stack: self.stack, layout: self.layout, labels: self.labels, + instrs: self.instrs, } } From 1b6d43763a22027dd23b34e6f046f601a300880a Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Fri, 30 May 2025 11:33:03 +0200 Subject: [PATCH 076/343] add Reset for FuncTranslatorAllocations --- .../src/engine/translator/translator2/mod.rs | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/translator2/mod.rs b/crates/wasmi/src/engine/translator/translator2/mod.rs index 518a94ff4c..61890851f7 100644 --- a/crates/wasmi/src/engine/translator/translator2/mod.rs +++ b/crates/wasmi/src/engine/translator/translator2/mod.rs @@ -78,6 +78,15 @@ pub struct FuncTranslatorAllocations { instrs: InstrEncoder, } +impl Reset for FuncTranslatorAllocations { + fn reset(&mut self) { + self.stack.reset(); + self.layout.reset(); + self.labels.reset(); + self.instrs.reset(); + } +} + impl WasmTranslator<'_> for FuncTranslator { type Allocations = FuncTranslatorAllocations; @@ -137,7 +146,12 @@ impl FuncTranslator { stack, layout, labels, - } = alloc; + instrs, + } = { + let mut alloc = alloc; + alloc.reset(); + alloc + }; Ok(Self { func, engine, From 6dd1f783a8c2b8270e3cab61dd480d09e56001fe Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Sat, 31 May 2025 20:30:49 +0200 Subject: [PATCH 077/343] initialize function body enclosing block --- .../src/engine/translator/translator2/mod.rs | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/crates/wasmi/src/engine/translator/translator2/mod.rs b/crates/wasmi/src/engine/translator/translator2/mod.rs index 61890851f7..77d09f8f3d 100644 --- a/crates/wasmi/src/engine/translator/translator2/mod.rs +++ b/crates/wasmi/src/engine/translator/translator2/mod.rs @@ -1,11 +1,12 @@ #![expect(dead_code, unused_imports, unused_variables)] +#[macro_use] +mod utils; mod instrs; mod layout; #[cfg(feature = "simd")] mod simd; mod stack; -mod utils; mod visit; use self::{ @@ -18,8 +19,10 @@ use crate::{ core::FuelCostsProvider, engine::{ translator::{Instr, LabelRegistry, WasmTranslator}, + BlockType, CompiledFuncEntity, }, + ir::Instruction, module::{FuncIdx, ModuleHeader, WasmiValueType}, Engine, Error, @@ -143,15 +146,22 @@ impl FuncTranslator { .then(|| config.fuel_costs()) .cloned(); let FuncTranslatorAllocations { - stack, + mut stack, layout, - labels, - instrs, + mut labels, + mut instrs, } = { let mut alloc = alloc; alloc.reset(); alloc }; + let func_ty = module.get_type_of_func(func); + let block_ty = BlockType::func_type(func_ty); + let end_label = labels.new_label(); + let consume_fuel = fuel_costs + .as_ref() + .map(|_| instrs.push_instr(Instruction::consume_fuel(1))); + stack.push_block(block_ty, end_label, consume_fuel)?; Ok(Self { func, engine, From e891a30fe756ae067c42ded45ec1ae882939dcc0 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Sat, 31 May 2025 20:30:57 +0200 Subject: [PATCH 078/343] add bail_unreachable utility macro --- .../src/engine/translator/translator2/utils.rs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/crates/wasmi/src/engine/translator/translator2/utils.rs b/crates/wasmi/src/engine/translator/translator2/utils.rs index e3db31c5d3..d3092898c0 100644 --- a/crates/wasmi/src/engine/translator/translator2/utils.rs +++ b/crates/wasmi/src/engine/translator/translator2/utils.rs @@ -1,3 +1,20 @@ +/// Bail out early in case the current code is unreachable. +/// +/// # Note +/// +/// - This should be prepended to most Wasm operator translation procedures. +/// - If we are in unreachable code most Wasm translation is skipped. Only +/// certain control flow operators such as `End` are going through the +/// translation process. In particular the `End` operator may end unreachable +/// code blocks. +macro_rules! bail_unreachable { + ($this:ident) => {{ + if !$this.reachable { + return Ok(()); + } + }}; +} + /// Implemented by types that can be reset for reuse. pub trait Reset { /// Resets `self` for reuse. From 8b95be1451823599b2ac4672b2ce5b09aa596ff0 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Sat, 31 May 2025 21:39:55 +0200 Subject: [PATCH 079/343] refactor initialization --- .../src/engine/translator/translator2/mod.rs | 56 +++++++++++++------ .../engine/translator/translator2/utils.rs | 10 +++- 2 files changed, 48 insertions(+), 18 deletions(-) diff --git a/crates/wasmi/src/engine/translator/translator2/mod.rs b/crates/wasmi/src/engine/translator/translator2/mod.rs index 77d09f8f3d..a06c8abb7f 100644 --- a/crates/wasmi/src/engine/translator/translator2/mod.rs +++ b/crates/wasmi/src/engine/translator/translator2/mod.rs @@ -26,6 +26,7 @@ use crate::{ module::{FuncIdx, ModuleHeader, WasmiValueType}, Engine, Error, + FuncType, }; use wasmparser::WasmFeatures; @@ -146,23 +147,12 @@ impl FuncTranslator { .then(|| config.fuel_costs()) .cloned(); let FuncTranslatorAllocations { - mut stack, + stack, layout, - mut labels, - mut instrs, - } = { - let mut alloc = alloc; - alloc.reset(); - alloc - }; - let func_ty = module.get_type_of_func(func); - let block_ty = BlockType::func_type(func_ty); - let end_label = labels.new_label(); - let consume_fuel = fuel_costs - .as_ref() - .map(|_| instrs.push_instr(Instruction::consume_fuel(1))); - stack.push_block(block_ty, end_label, consume_fuel)?; - Ok(Self { + labels, + instrs, + } = alloc.into_reset(); + let mut translator = Self { func, engine, module, @@ -172,7 +162,39 @@ impl FuncTranslator { layout, labels, instrs, - }) + }; + translator.init_func_body_block()?; + translator.init_func_params()?; + Ok(translator) + } + + /// Initializes the function body enclosing control block. + fn init_func_body_block(&mut self) -> Result<(), Error> { + let func_ty = self.module.get_type_of_func(self.func); + let block_ty = BlockType::func_type(func_ty); + let end_label = self.labels.new_label(); + let consume_fuel = self + .fuel_costs + .as_ref() + .map(|_| self.instrs.push_instr(Instruction::consume_fuel(1))); + self.stack.push_block(block_ty, end_label, consume_fuel)?; + Ok(()) + } + + /// Initializes the function's parameters. + fn init_func_params(&mut self) -> Result<(), Error> { + for ty in self.func_type().params() { + self.stack.register_locals(1, *ty)?; + self.layout.register_locals(1, *ty)?; + } + Ok(()) + } + + /// Returns the [`FuncType`] of the function that is currently translated. + fn func_type(&self) -> FuncType { + let dedup_func_type = self.module.get_type_of_func(self.func); + self.engine() + .resolve_func_type(dedup_func_type, Clone::clone) } /// Consumes `self` and returns the underlying reusable [`FuncTranslatorAllocations`]. diff --git a/crates/wasmi/src/engine/translator/translator2/utils.rs b/crates/wasmi/src/engine/translator/translator2/utils.rs index d3092898c0..8c3b47040e 100644 --- a/crates/wasmi/src/engine/translator/translator2/utils.rs +++ b/crates/wasmi/src/engine/translator/translator2/utils.rs @@ -16,7 +16,15 @@ macro_rules! bail_unreachable { } /// Implemented by types that can be reset for reuse. -pub trait Reset { +pub trait Reset: Sized { /// Resets `self` for reuse. fn reset(&mut self); + + /// Returns `self` in resetted state. + #[must_use] + fn into_reset(self) -> Self { + let mut this = self; + this.reset(); + this + } } From 06f9db63828a31093b21e981b9ac88037c959232 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Tue, 3 Jun 2025 12:31:38 +0200 Subject: [PATCH 080/343] implement WasmTranslator::finish for new translator --- .../engine/translator/translator2/instrs.rs | 34 ++++++++++++++++++- .../translator/translator2/layout/mod.rs | 11 +++++- .../src/engine/translator/translator2/mod.rs | 15 ++++++-- 3 files changed, 55 insertions(+), 5 deletions(-) diff --git a/crates/wasmi/src/engine/translator/translator2/instrs.rs b/crates/wasmi/src/engine/translator/translator2/instrs.rs index f0b70aa286..be7c0e7b10 100644 --- a/crates/wasmi/src/engine/translator/translator2/instrs.rs +++ b/crates/wasmi/src/engine/translator/translator2/instrs.rs @@ -1,6 +1,6 @@ use super::{Instr, Reset}; use crate::ir::Instruction; -use alloc::vec::Vec; +use alloc::vec::{self, Vec}; /// Creates and encodes the list of [`Instruction`]s for a function. #[derive(Debug, Default)] @@ -47,4 +47,36 @@ impl InstrEncoder { pub fn get(&self, instr: Instr) -> &Instruction { &self.instrs[instr.into_usize()] } + + /// Returns an iterator yielding all [`Instruction`]s of the [`InstrEncoder`]. + /// + /// # Note + /// + /// The [`InstrEncoder`] will be empty after this operation. + pub fn drain(&mut self) -> InstrEncoderIter { + InstrEncoderIter { + iter: self.instrs.drain(..), + } + } +} + +/// Iterator yielding all [`Instruction`]s of the [`InstrEncoder`]. +#[derive(Debug)] +pub struct InstrEncoderIter<'a> { + /// The underlying iterator. + iter: vec::Drain<'a, Instruction>, +} + +impl<'a> Iterator for InstrEncoderIter<'a> { + type Item = Instruction; + + fn next(&mut self) -> Option { + self.iter.next() + } +} + +impl ExactSizeIterator for InstrEncoderIter<'_> { + fn len(&self) -> usize { + self.iter.len() + } } diff --git a/crates/wasmi/src/engine/translator/translator2/layout/mod.rs b/crates/wasmi/src/engine/translator/translator2/layout/mod.rs index 52a206a471..09093deab2 100644 --- a/crates/wasmi/src/engine/translator/translator2/layout/mod.rs +++ b/crates/wasmi/src/engine/translator/translator2/layout/mod.rs @@ -1,6 +1,6 @@ mod consts; -use self::consts::ConstRegistry; +use self::consts::{ConstRegistry, ConstRegistryIter}; use super::{LocalIdx, OperandIdx, Reset}; use crate::{ core::{UntypedVal, ValType}, @@ -99,6 +99,15 @@ impl StackLayout { pub fn const_to_reg(&mut self, value: impl Into) -> Result { self.consts.alloc(value.into()) } + + /// Returns an iterator yielding all function local constants. + /// + /// # Note + /// + /// The function local constant are yielded in reverse order of allocation. + pub fn consts(&self) -> ConstRegistryIter { + self.consts.iter() + } } /// The [`StackSpace`] of a [`Reg`]. diff --git a/crates/wasmi/src/engine/translator/translator2/mod.rs b/crates/wasmi/src/engine/translator/translator2/mod.rs index a06c8abb7f..923d1fa778 100644 --- a/crates/wasmi/src/engine/translator/translator2/mod.rs +++ b/crates/wasmi/src/engine/translator/translator2/mod.rs @@ -21,6 +21,7 @@ use crate::{ translator::{Instr, LabelRegistry, WasmTranslator}, BlockType, CompiledFuncEntity, + TranslationError, }, ir::Instruction, module::{FuncIdx, ModuleHeader, WasmiValueType}, @@ -121,10 +122,18 @@ impl WasmTranslator<'_> for FuncTranslator { fn update_pos(&mut self, _pos: usize) {} fn finish( - self, - _finalize: impl FnOnce(CompiledFuncEntity), + mut self, + finalize: impl FnOnce(CompiledFuncEntity), ) -> Result { - todo!() + let Ok(max_height) = u16::try_from(self.stack.max_height()) else { + return Err(Error::from(TranslationError::AllocatedTooManyRegisters)); + }; + finalize(CompiledFuncEntity::new( + max_height, + self.instrs.drain(), + self.layout.consts(), + )); + Ok(self.into_allocations()) } } From d4405afc5c0cbe448ae86640d1033f262267f5dc Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Thu, 5 Jun 2025 12:48:39 +0200 Subject: [PATCH 081/343] post-rebase cleanup --- crates/wasmi/src/engine/translator/mod.rs | 2 ++ .../wasmi/src/engine/translator/translator2/stack/control.rs | 4 ++-- crates/wasmi/src/engine/translator/translator2/stack/mod.rs | 4 ++-- .../wasmi/src/engine/translator/translator2/stack/operand.rs | 2 +- .../wasmi/src/engine/translator/translator2/stack/operands.rs | 2 +- 5 files changed, 8 insertions(+), 6 deletions(-) diff --git a/crates/wasmi/src/engine/translator/mod.rs b/crates/wasmi/src/engine/translator/mod.rs index a3f04e4584..6a09f34ee2 100644 --- a/crates/wasmi/src/engine/translator/mod.rs +++ b/crates/wasmi/src/engine/translator/mod.rs @@ -6,6 +6,7 @@ mod error; mod func; mod labels; mod relink_result; +mod translator2; mod utils; mod visit_register; @@ -19,6 +20,7 @@ pub use self::{ driver::FuncTranslationDriver, error::TranslationError, func::{FuncTranslator, FuncTranslatorAllocations}, + labels::{LabelRef, LabelRegistry}, }; use super::code_map::CompiledFuncEntity; use crate::{ diff --git a/crates/wasmi/src/engine/translator/translator2/stack/control.rs b/crates/wasmi/src/engine/translator/translator2/stack/control.rs index 6a08bfe8cf..564b9b3b76 100644 --- a/crates/wasmi/src/engine/translator/translator2/stack/control.rs +++ b/crates/wasmi/src/engine/translator/translator2/stack/control.rs @@ -1,7 +1,7 @@ use super::{Operand, Reset}; use crate::engine::{ - translator::{BlockType, LabelRef}, - Instr, + translator::{Instr, LabelRef}, + BlockType, }; use alloc::vec::{Drain, Vec}; diff --git a/crates/wasmi/src/engine/translator/translator2/stack/mod.rs b/crates/wasmi/src/engine/translator/translator2/stack/mod.rs index ef9e410c20..57990fe179 100644 --- a/crates/wasmi/src/engine/translator/translator2/stack/mod.rs +++ b/crates/wasmi/src/engine/translator/translator2/stack/mod.rs @@ -28,8 +28,8 @@ use super::Reset; use crate::{ core::{TypedVal, UntypedVal, ValType}, engine::{ - translator::{BlockType, LabelRef}, - Instr, + translator::{Instr, LabelRef}, + BlockType, TranslationError, }, ir::Reg, diff --git a/crates/wasmi/src/engine/translator/translator2/stack/operand.rs b/crates/wasmi/src/engine/translator/translator2/stack/operand.rs index 3121b4afb8..e147f1bde8 100644 --- a/crates/wasmi/src/engine/translator/translator2/stack/operand.rs +++ b/crates/wasmi/src/engine/translator/translator2/stack/operand.rs @@ -1,7 +1,7 @@ use super::{LocalIdx, LocalsRegistry, OperandIdx, StackOperand}; use crate::{ core::{TypedVal, ValType}, - engine::Instr, + engine::translator::Instr, }; #[cfg(doc)] diff --git a/crates/wasmi/src/engine/translator/translator2/stack/operands.rs b/crates/wasmi/src/engine/translator/translator2/stack/operands.rs index b58f74e224..71e6be7176 100644 --- a/crates/wasmi/src/engine/translator/translator2/stack/operands.rs +++ b/crates/wasmi/src/engine/translator/translator2/stack/operands.rs @@ -1,7 +1,7 @@ use super::{LocalIdx, LocalsRegistry, Operand, Reset}; use crate::{ core::{TypedVal, ValType}, - engine::Instr, + engine::translator::Instr, Error, }; use alloc::vec::Vec; From 9be11733897069938e1993b86f72e764406ef37b Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Thu, 5 Jun 2025 12:51:51 +0200 Subject: [PATCH 082/343] move stack-based FuncTranslator into func2 submodule --- .../src/engine/translator/{translator2 => func2}/instrs.rs | 0 .../engine/translator/{translator2 => func2}/layout/consts.rs | 0 .../src/engine/translator/{translator2 => func2}/layout/mod.rs | 0 .../wasmi/src/engine/translator/{translator2 => func2}/mod.rs | 0 .../src/engine/translator/{translator2 => func2}/simd/mod.rs | 0 .../src/engine/translator/{translator2 => func2}/simd/visit.rs | 0 .../engine/translator/{translator2 => func2}/stack/control.rs | 0 .../engine/translator/{translator2 => func2}/stack/locals.rs | 0 .../src/engine/translator/{translator2 => func2}/stack/mod.rs | 0 .../engine/translator/{translator2 => func2}/stack/operand.rs | 0 .../engine/translator/{translator2 => func2}/stack/operands.rs | 0 .../wasmi/src/engine/translator/{translator2 => func2}/utils.rs | 0 .../wasmi/src/engine/translator/{translator2 => func2}/visit.rs | 0 crates/wasmi/src/engine/translator/mod.rs | 2 +- 14 files changed, 1 insertion(+), 1 deletion(-) rename crates/wasmi/src/engine/translator/{translator2 => func2}/instrs.rs (100%) rename crates/wasmi/src/engine/translator/{translator2 => func2}/layout/consts.rs (100%) rename crates/wasmi/src/engine/translator/{translator2 => func2}/layout/mod.rs (100%) rename crates/wasmi/src/engine/translator/{translator2 => func2}/mod.rs (100%) rename crates/wasmi/src/engine/translator/{translator2 => func2}/simd/mod.rs (100%) rename crates/wasmi/src/engine/translator/{translator2 => func2}/simd/visit.rs (100%) rename crates/wasmi/src/engine/translator/{translator2 => func2}/stack/control.rs (100%) rename crates/wasmi/src/engine/translator/{translator2 => func2}/stack/locals.rs (100%) rename crates/wasmi/src/engine/translator/{translator2 => func2}/stack/mod.rs (100%) rename crates/wasmi/src/engine/translator/{translator2 => func2}/stack/operand.rs (100%) rename crates/wasmi/src/engine/translator/{translator2 => func2}/stack/operands.rs (100%) rename crates/wasmi/src/engine/translator/{translator2 => func2}/utils.rs (100%) rename crates/wasmi/src/engine/translator/{translator2 => func2}/visit.rs (100%) diff --git a/crates/wasmi/src/engine/translator/translator2/instrs.rs b/crates/wasmi/src/engine/translator/func2/instrs.rs similarity index 100% rename from crates/wasmi/src/engine/translator/translator2/instrs.rs rename to crates/wasmi/src/engine/translator/func2/instrs.rs diff --git a/crates/wasmi/src/engine/translator/translator2/layout/consts.rs b/crates/wasmi/src/engine/translator/func2/layout/consts.rs similarity index 100% rename from crates/wasmi/src/engine/translator/translator2/layout/consts.rs rename to crates/wasmi/src/engine/translator/func2/layout/consts.rs diff --git a/crates/wasmi/src/engine/translator/translator2/layout/mod.rs b/crates/wasmi/src/engine/translator/func2/layout/mod.rs similarity index 100% rename from crates/wasmi/src/engine/translator/translator2/layout/mod.rs rename to crates/wasmi/src/engine/translator/func2/layout/mod.rs diff --git a/crates/wasmi/src/engine/translator/translator2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs similarity index 100% rename from crates/wasmi/src/engine/translator/translator2/mod.rs rename to crates/wasmi/src/engine/translator/func2/mod.rs diff --git a/crates/wasmi/src/engine/translator/translator2/simd/mod.rs b/crates/wasmi/src/engine/translator/func2/simd/mod.rs similarity index 100% rename from crates/wasmi/src/engine/translator/translator2/simd/mod.rs rename to crates/wasmi/src/engine/translator/func2/simd/mod.rs diff --git a/crates/wasmi/src/engine/translator/translator2/simd/visit.rs b/crates/wasmi/src/engine/translator/func2/simd/visit.rs similarity index 100% rename from crates/wasmi/src/engine/translator/translator2/simd/visit.rs rename to crates/wasmi/src/engine/translator/func2/simd/visit.rs diff --git a/crates/wasmi/src/engine/translator/translator2/stack/control.rs b/crates/wasmi/src/engine/translator/func2/stack/control.rs similarity index 100% rename from crates/wasmi/src/engine/translator/translator2/stack/control.rs rename to crates/wasmi/src/engine/translator/func2/stack/control.rs diff --git a/crates/wasmi/src/engine/translator/translator2/stack/locals.rs b/crates/wasmi/src/engine/translator/func2/stack/locals.rs similarity index 100% rename from crates/wasmi/src/engine/translator/translator2/stack/locals.rs rename to crates/wasmi/src/engine/translator/func2/stack/locals.rs diff --git a/crates/wasmi/src/engine/translator/translator2/stack/mod.rs b/crates/wasmi/src/engine/translator/func2/stack/mod.rs similarity index 100% rename from crates/wasmi/src/engine/translator/translator2/stack/mod.rs rename to crates/wasmi/src/engine/translator/func2/stack/mod.rs diff --git a/crates/wasmi/src/engine/translator/translator2/stack/operand.rs b/crates/wasmi/src/engine/translator/func2/stack/operand.rs similarity index 100% rename from crates/wasmi/src/engine/translator/translator2/stack/operand.rs rename to crates/wasmi/src/engine/translator/func2/stack/operand.rs diff --git a/crates/wasmi/src/engine/translator/translator2/stack/operands.rs b/crates/wasmi/src/engine/translator/func2/stack/operands.rs similarity index 100% rename from crates/wasmi/src/engine/translator/translator2/stack/operands.rs rename to crates/wasmi/src/engine/translator/func2/stack/operands.rs diff --git a/crates/wasmi/src/engine/translator/translator2/utils.rs b/crates/wasmi/src/engine/translator/func2/utils.rs similarity index 100% rename from crates/wasmi/src/engine/translator/translator2/utils.rs rename to crates/wasmi/src/engine/translator/func2/utils.rs diff --git a/crates/wasmi/src/engine/translator/translator2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs similarity index 100% rename from crates/wasmi/src/engine/translator/translator2/visit.rs rename to crates/wasmi/src/engine/translator/func2/visit.rs diff --git a/crates/wasmi/src/engine/translator/mod.rs b/crates/wasmi/src/engine/translator/mod.rs index 6a09f34ee2..8aa68e260e 100644 --- a/crates/wasmi/src/engine/translator/mod.rs +++ b/crates/wasmi/src/engine/translator/mod.rs @@ -4,9 +4,9 @@ mod comparator; mod driver; mod error; mod func; +mod func2; mod labels; mod relink_result; -mod translator2; mod utils; mod visit_register; From 436b7203e8c98ad3528d4fe65881891bc94d3b85 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Thu, 5 Jun 2025 13:03:12 +0200 Subject: [PATCH 083/343] add experimental-translator feature to wasmi crate This allows to quickly switch between the old and the new FuncTranslator backends. --- crates/wasmi/Cargo.toml | 1 + crates/wasmi/src/engine/translator/func2/mod.rs | 2 +- crates/wasmi/src/engine/translator/mod.rs | 8 +++++++- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/crates/wasmi/Cargo.toml b/crates/wasmi/Cargo.toml index eab447b783..09f68083ea 100644 --- a/crates/wasmi/Cargo.toml +++ b/crates/wasmi/Cargo.toml @@ -56,6 +56,7 @@ prefer-btree-collections = [ ] wat = ["dep:wat", "std"] simd = ["wasmi_core/simd", "wasmi_ir/simd", "wasmparser/simd"] +experimental-translator = [] # Enables extra checks performed during Wasmi bytecode execution. # diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index 923d1fa778..f92d0a01ef 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -1,4 +1,4 @@ -#![expect(dead_code, unused_imports, unused_variables)] +#![expect(dead_code, unused_imports, unused_variables, unused_macros)] #[macro_use] mod utils; diff --git a/crates/wasmi/src/engine/translator/mod.rs b/crates/wasmi/src/engine/translator/mod.rs index 8aa68e260e..4657476fd3 100644 --- a/crates/wasmi/src/engine/translator/mod.rs +++ b/crates/wasmi/src/engine/translator/mod.rs @@ -16,10 +16,16 @@ mod tests; #[cfg(doc)] use crate::Engine; +#[cfg(not(feature = "experimental-translator"))] +pub use self::func::{FuncTranslator, FuncTranslatorAllocations}; + +#[cfg(feature = "experimental-translator")] +pub use self::func2::{FuncTranslator, FuncTranslatorAllocations}; + pub use self::{ driver::FuncTranslationDriver, error::TranslationError, - func::{FuncTranslator, FuncTranslatorAllocations}, + utils::Instr, labels::{LabelRef, LabelRegistry}, }; use super::code_map::CompiledFuncEntity; From 368d0201c08edab2ca1ccb2aedba7ef3bd28ae05 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Thu, 5 Jun 2025 14:07:16 +0200 Subject: [PATCH 084/343] add StackAllocations and fix Stack init bug The bug was the removed Stack::default created a new Engine and did not properly (use the FuncTranslator engine which invalidated engine-local indices such as DedupFuncType. --- .../wasmi/src/engine/translator/func2/mod.rs | 7 ++-- .../src/engine/translator/func2/stack/mod.rs | 32 ++++++++++++++++++- 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index f92d0a01ef..f42202d6af 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -12,7 +12,7 @@ mod visit; use self::{ instrs::InstrEncoder, layout::{StackLayout, StackSpace}, - stack::{LocalIdx, Operand, OperandIdx, Stack}, + stack::{LocalIdx, Operand, OperandIdx, Stack, StackAllocations}, utils::Reset, }; use crate::{ @@ -74,7 +74,7 @@ pub struct FuncTranslator { #[derive(Debug, Default)] pub struct FuncTranslatorAllocations { /// Wasm value and control stack. - stack: Stack, + stack: StackAllocations, /// Wasm layout to map stack slots to Wasmi registers. layout: StackLayout, /// Registers and pins labels and tracks their users. @@ -161,6 +161,7 @@ impl FuncTranslator { labels, instrs, } = alloc.into_reset(); + let stack = Stack::new(&engine, stack); let mut translator = Self { func, engine, @@ -209,7 +210,7 @@ impl FuncTranslator { /// Consumes `self` and returns the underlying reusable [`FuncTranslatorAllocations`]. fn into_allocations(self) -> FuncTranslatorAllocations { FuncTranslatorAllocations { - stack: self.stack, + stack: self.stack.into_allocations(), layout: self.layout, labels: self.labels, instrs: self.instrs, diff --git a/crates/wasmi/src/engine/translator/func2/stack/mod.rs b/crates/wasmi/src/engine/translator/func2/stack/mod.rs index 57990fe179..7445c69c10 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/mod.rs @@ -43,7 +43,7 @@ use core::{array, mem, num::NonZero}; use crate::ir::Instruction; /// The Wasm value stack during translation from Wasm to Wasmi bytecode. -#[derive(Debug, Default)] +#[derive(Debug)] pub struct Stack { /// The underlying [`Engine`]. engine: Engine, @@ -53,7 +53,37 @@ pub struct Stack { controls: ControlStack, } +/// Reusable heap allocations for the [`Stack`]. +#[derive(Debug, Default)] +pub struct StackAllocations { + /// The Wasm value stack. + operands: OperandStack, + /// The Wasm control stack. + controls: ControlStack, +} + +impl Reset for StackAllocations { + fn reset(&mut self) { + self.operands.reset(); + self.controls.reset(); + } +} + impl Stack { + /// Creates a new empty [`Stack`] from the given `engine`. + pub fn new(engine: &Engine, alloc: StackAllocations) -> Self { + Self { + engine: engine.clone(), + operands: alloc.operands, + controls: alloc.controls, + } + } + + /// Returns the reusable [`StackAllocations`] of `self`. + pub fn into_allocations(self) -> StackAllocations { + StackAllocations { operands: self.operands, controls: self.controls } + } + /// Resets the [`Stack`] for reuse. pub fn reset(&mut self) { self.operands.reset(); From b795a7f562292b0f53ecbeb3465232522edf1348 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Thu, 5 Jun 2025 14:19:04 +0200 Subject: [PATCH 085/343] apply rustfmt --- crates/wasmi/src/engine/translator/func2/stack/mod.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/func2/stack/mod.rs b/crates/wasmi/src/engine/translator/func2/stack/mod.rs index 7445c69c10..406a9e539f 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/mod.rs @@ -81,7 +81,10 @@ impl Stack { /// Returns the reusable [`StackAllocations`] of `self`. pub fn into_allocations(self) -> StackAllocations { - StackAllocations { operands: self.operands, controls: self.controls } + StackAllocations { + operands: self.operands, + controls: self.controls, + } } /// Resets the [`Stack`] for reuse. From 0d47c17e40dbd589a5f9e3933076dfd860e838d0 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Thu, 5 Jun 2025 14:19:50 +0200 Subject: [PATCH 086/343] add Stack::push_func_block This is meant to only push the function body enclosing Wasm block to the Stack. All other Wasm blocks are meant to be pushed via the existing Stack::push_block method. --- .../wasmi/src/engine/translator/func2/mod.rs | 3 +- .../engine/translator/func2/stack/control.rs | 5 +++ .../src/engine/translator/func2/stack/mod.rs | 33 +++++++++++++++---- 3 files changed, 33 insertions(+), 8 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index f42202d6af..9086c98f57 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -187,7 +187,8 @@ impl FuncTranslator { .fuel_costs .as_ref() .map(|_| self.instrs.push_instr(Instruction::consume_fuel(1))); - self.stack.push_block(block_ty, end_label, consume_fuel)?; + self.stack + .push_func_block(block_ty, end_label, consume_fuel)?; Ok(()) } diff --git a/crates/wasmi/src/engine/translator/func2/stack/control.rs b/crates/wasmi/src/engine/translator/func2/stack/control.rs index 564b9b3b76..db22bac59e 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/control.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/control.rs @@ -58,6 +58,11 @@ impl Reset for ControlStack { } impl ControlStack { + /// Returns `true` if `self` is empty. + pub fn is_empty(&self) -> bool { + self.height() == 0 + } + /// Returns the height of the [`ControlStack`]. pub fn height(&self) -> usize { self.frames.len() diff --git a/crates/wasmi/src/engine/translator/func2/stack/mod.rs b/crates/wasmi/src/engine/translator/func2/stack/mod.rs index 406a9e539f..3522167a28 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/mod.rs @@ -136,27 +136,44 @@ impl Stack { self.engine.config().get_consume_fuel() } - /// Pushes a Wasm `block` onto the [`Stack`]. + /// Pushes the function enclosing Wasm `block` onto the [`Stack`]. /// /// # Note /// - /// If `consume_fuel` is `None` and fuel metering is enabled this will inherit - /// the [`Instruction::ConsumeFuel`] from the last control frame on the [`Stack`]. + /// - If `consume_fuel` is `None` fuel metering is expected to be disabled. + /// - If `consume_fuel` is `Some` fuel metering is expected to be enabled. /// /// # Errors /// /// If the stack height exceeds the maximum height. - pub fn push_block( + pub fn push_func_block( &mut self, ty: BlockType, label: LabelRef, consume_fuel: Option, ) -> Result<(), Error> { + debug_assert!(self.controls.is_empty()); + debug_assert!(self.is_fuel_metering_enabled() == consume_fuel.is_some()); + self.controls.push_block(ty, 0, label, consume_fuel); + Ok(()) + } + + /// Pushes a Wasm `block` onto the [`Stack`]. + /// + /// # Note + /// + /// This inherits the `consume_fuel` [`Instr`] from the parent [`ControlFrame`]. + /// + /// # Errors + /// + /// If the stack height exceeds the maximum height. + pub fn push_block(&mut self, ty: BlockType, label: LabelRef) -> Result<(), Error> { + debug_assert!(!self.controls.is_empty()); let len_params = usize::from(ty.len_params(&self.engine)); let block_height = self.height() - len_params; let fuel_metering = self.is_fuel_metering_enabled(); - let consume_fuel = match consume_fuel { - None if fuel_metering => { + let consume_fuel = match fuel_metering { + true => { let consume_instr = self .controls .get(0) @@ -164,7 +181,7 @@ impl Stack { .expect("control frame must have consume instructions"); Some(consume_instr) } - consume_fuel => consume_fuel, + false => None, }; self.controls .push_block(ty, block_height, label, consume_fuel); @@ -192,6 +209,7 @@ impl Stack { consume_fuel: Option, mut f: impl FnMut(Operand) -> Result<(), Error>, ) -> Result<(), Error> { + debug_assert!(!self.controls.is_empty()); debug_assert!(self.is_fuel_metering_enabled() == consume_fuel.is_some()); let len_params = usize::from(ty.len_params(&self.engine)); let block_height = self.height() - len_params; @@ -221,6 +239,7 @@ impl Stack { reachability: IfReachability, consume_fuel: Option, ) -> Result<(), Error> { + debug_assert!(!self.controls.is_empty()); debug_assert!(self.is_fuel_metering_enabled() == consume_fuel.is_some()); let len_params = usize::from(ty.len_params(&self.engine)); let block_height = self.height() - len_params; From 5adeadd1987aeb22636e626630911dd2b1e44ec9 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Thu, 5 Jun 2025 16:56:12 +0200 Subject: [PATCH 087/343] remove invalid debug assert It is not possible to assert that the given index refers to an Operand::Temp this way. For this we'd need to include this API into Stack but that has other (negative) consequences, so we likely won't. Another option is to take TempOperand instead of OperandIdx. --- crates/wasmi/src/engine/translator/func2/layout/mod.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/layout/mod.rs b/crates/wasmi/src/engine/translator/func2/layout/mod.rs index 09093deab2..a8db0c1169 100644 --- a/crates/wasmi/src/engine/translator/func2/layout/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/layout/mod.rs @@ -77,10 +77,6 @@ impl StackLayout { /// /// If `index` cannot be converted into a [`Reg`]. pub fn temp_to_reg(&self, index: OperandIdx) -> Result { - debug_assert!( - usize::from(index) >= self.len_locals, - "index must refer to a temporary operand" - ); let index = usize::from(index); let Some(index) = index.checked_add(self.len_locals) else { return Err(Error::from(TranslationError::AllocatedTooManyRegisters)); From d425d98d5c13651145dd6fa5bf5466f98b9e1b50 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Thu, 5 Jun 2025 16:57:42 +0200 Subject: [PATCH 088/343] add Layout::operand_to_reg utility method --- .../src/engine/translator/func2/layout/mod.rs | 23 ++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/func2/layout/mod.rs b/crates/wasmi/src/engine/translator/func2/layout/mod.rs index a8db0c1169..d14a73ecd3 100644 --- a/crates/wasmi/src/engine/translator/func2/layout/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/layout/mod.rs @@ -1,7 +1,7 @@ mod consts; use self::consts::{ConstRegistry, ConstRegistryIter}; -use super::{LocalIdx, OperandIdx, Reset}; +use super::{LocalIdx, Operand, OperandIdx, Reset}; use crate::{ core::{UntypedVal, ValType}, engine::TranslationError, @@ -55,6 +55,27 @@ impl StackLayout { StackSpace::Temp } + /// Converts the `operand` into the associated [`Reg`]. + /// + /// # Note + /// + /// Forwards to one of + /// + /// - [`StackLayout::local_to_reg`] + /// - [`StackLayout::temp_to_reg`] + /// - [`StackLayout::const_to_reg`] + /// + /// # Errors + /// + /// If the forwarded method returned an error. + pub fn operand_to_reg(&mut self, operand: Operand) -> Result { + match operand { + Operand::Local(operand) => self.local_to_reg(operand.local_index()), + Operand::Temp(operand) => self.temp_to_reg(operand.operand_index()), + Operand::Immediate(operand) => self.const_to_reg(operand.val()), + } + } + /// Converts the local `index` into the associated [`Reg`]. /// /// # Errors From 7aafcfca1afc91e7ed5c5ebdf542116bd7c075df Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Thu, 5 Jun 2025 16:58:04 +0200 Subject: [PATCH 089/343] implement some visit methods --- .../src/engine/translator/func2/instrs.rs | 2 +- .../wasmi/src/engine/translator/func2/mod.rs | 53 +++++++++++++++++-- .../src/engine/translator/func2/visit.rs | 18 +++++-- 3 files changed, 64 insertions(+), 9 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/instrs.rs b/crates/wasmi/src/engine/translator/func2/instrs.rs index be7c0e7b10..a0237d70ea 100644 --- a/crates/wasmi/src/engine/translator/func2/instrs.rs +++ b/crates/wasmi/src/engine/translator/func2/instrs.rs @@ -18,7 +18,7 @@ impl Reset for InstrEncoder { impl InstrEncoder { /// Returns the next [`Instr`]. #[must_use] - fn next_instr(&self) -> Instr { + pub fn next_instr(&self) -> Instr { Instr::from_usize(self.instrs.len()) } diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index 9086c98f57..4139bce22a 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -1,4 +1,4 @@ -#![expect(dead_code, unused_imports, unused_variables, unused_macros)] +#![expect(dead_code, unused_imports, unused_variables)] #[macro_use] mod utils; @@ -16,14 +16,14 @@ use self::{ utils::Reset, }; use crate::{ - core::FuelCostsProvider, + core::{FuelCostsProvider, Typed, TypedVal}, engine::{ translator::{Instr, LabelRegistry, WasmTranslator}, BlockType, CompiledFuncEntity, TranslationError, }, - ir::Instruction, + ir::{Const16, Instruction, Reg}, module::{FuncIdx, ModuleHeader, WasmiValueType}, Engine, Error, @@ -222,4 +222,51 @@ impl FuncTranslator { fn engine(&self) -> &Engine { &self.engine } + + /// Translates a commutative binary Wasm operator to Wasmi bytecode. + fn translate_binary_commutative( + &mut self, + make_instr: fn(result: Reg, lhs: Reg, rhs: Reg) -> Instruction, + make_instr_imm16: fn(result: Reg, lhs: Reg, rhs: Const16) -> Instruction, + consteval: fn(T, T) -> R, + ) -> Result<(), Error> + where + T: Copy + From + TryInto>, + R: Into + Typed, + { + bail_unreachable!(self); + match self.stack.pop2() { + (Operand::Immediate(lhs), Operand::Immediate(rhs)) => { + let value = consteval(lhs.val().into(), rhs.val().into()); + self.stack.push_immediate(value)?; + return Ok(()); + } + (val, Operand::Immediate(imm)) | (Operand::Immediate(imm), val) => { + let lhs = self.layout.operand_to_reg(val)?; + let iidx = self.instrs.next_instr(); + let result = self + .layout + .temp_to_reg(self.stack.push_temp(::TY, Some(iidx))?)?; + let instr = match T::from(imm.val()).try_into() { + Ok(rhs) => make_instr_imm16(result, lhs, rhs), + Err(_) => { + let rhs = self.layout.const_to_reg(imm.val())?; + make_instr(result, lhs, rhs) + } + }; + assert_eq!(self.instrs.push_instr(instr), iidx); + Ok(()) + } + (lhs, rhs) => { + let lhs = self.layout.operand_to_reg(lhs)?; + let rhs = self.layout.operand_to_reg(rhs)?; + let iidx = self.instrs.next_instr(); + let result = self + .layout + .temp_to_reg(self.stack.push_temp(::TY, Some(iidx))?)?; + assert_eq!(self.instrs.push_instr(make_instr(result, lhs, rhs)), iidx); + Ok(()) + } + } + } } diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index 4fc412c150..ec27a172b4 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -1,5 +1,5 @@ -use super::FuncTranslator; -use crate::Error; +use super::{FuncTranslator, LocalIdx}; +use crate::{core::wasm, ir::Instruction, Error}; use wasmparser::VisitOperator; macro_rules! impl_visit_operator { @@ -125,7 +125,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { } fn visit_local_get(&mut self, local_index: u32) -> Self::Output { - todo!() + bail_unreachable!(self); + self.stack.push_local(LocalIdx::from(local_index))?; + Ok(()) } fn visit_local_set(&mut self, local_index: u32) -> Self::Output { @@ -245,7 +247,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { } fn visit_i32_const(&mut self, value: i32) -> Self::Output { - todo!() + bail_unreachable!(self); + self.stack.push_immediate(value)?; + Ok(()) } fn visit_i64_const(&mut self, value: i64) -> Self::Output { @@ -409,7 +413,11 @@ impl<'a> VisitOperator<'a> for FuncTranslator { } fn visit_i32_add(&mut self) -> Self::Output { - todo!() + self.translate_binary_commutative::( + Instruction::i32_add, + Instruction::i32_add_imm16, + wasm::i32_add, + ) } fn visit_i32_sub(&mut self) -> Self::Output { From a23eb276bab34bcbcd6df2ef2a50854ced1209d0 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Fri, 6 Jun 2025 16:59:36 +0200 Subject: [PATCH 090/343] apply clippy suggestion --- crates/wasmi/src/engine/translator/func2/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index 4139bce22a..4d0111ce44 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -239,7 +239,7 @@ impl FuncTranslator { (Operand::Immediate(lhs), Operand::Immediate(rhs)) => { let value = consteval(lhs.val().into(), rhs.val().into()); self.stack.push_immediate(value)?; - return Ok(()); + Ok(()) } (val, Operand::Immediate(imm)) | (Operand::Immediate(imm), val) => { let lhs = self.layout.operand_to_reg(val)?; From cc8c0fa4fd2bd6f6cc52120eac74676a8cff565c Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Fri, 6 Jun 2025 19:02:57 +0200 Subject: [PATCH 091/343] refactor ControlFrame --- .../engine/translator/func2/stack/control.rs | 392 +++++++++++------- .../src/engine/translator/func2/stack/mod.rs | 19 +- 2 files changed, 252 insertions(+), 159 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/stack/control.rs b/crates/wasmi/src/engine/translator/func2/stack/control.rs index db22bac59e..81eb7d74bd 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/control.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/control.rs @@ -69,14 +69,8 @@ impl ControlStack { } /// Pushes a new unreachable Wasm control frame onto the [`ControlStack`]. - pub fn push_unreachable( - &mut self, - ty: BlockType, - height: usize, - kind: UnreachableControlFrame, - ) { - self.frames - .push(ControlFrame::new_unreachable(ty, height, kind)) + pub fn push_unreachable(&mut self, kind: UnreachableControlFrame) { + self.frames.push(ControlFrame::from(kind)) } /// Pushes a new Wasm `block` onto the [`ControlStack`]. @@ -87,8 +81,13 @@ impl ControlStack { label: LabelRef, consume_fuel: Option, ) { - self.frames - .push(ControlFrame::new_block(ty, height, label, consume_fuel)) + self.frames.push(ControlFrame::from(BlockControlFrame { + ty, + height, + is_branched_to: false, + consume_fuel, + label, + })) } /// Pushes a new Wasm `loop` onto the [`ControlStack`]. @@ -99,8 +98,13 @@ impl ControlStack { label: LabelRef, consume_fuel: Option, ) { - self.frames - .push(ControlFrame::new_loop(ty, height, label, consume_fuel)) + self.frames.push(ControlFrame::from(LoopControlFrame { + ty, + height, + is_branched_to: false, + consume_fuel, + label, + })) } /// Pushes a new Wasm `if` onto the [`ControlStack`]. @@ -113,13 +117,14 @@ impl ControlStack { reachability: IfReachability, else_operands: impl IntoIterator, ) { - self.frames.push(ControlFrame::new_if( + self.frames.push(ControlFrame::from(IfControlFrame { ty, height, - label, + is_branched_to: false, consume_fuel, + label, reachability, - )); + })); self.else_operands.push(else_operands); } @@ -135,14 +140,15 @@ impl ControlStack { reachability: ElseReachability, is_end_of_then_reachable: bool, ) -> Drain { - self.frames.push(ControlFrame::new_else( + self.frames.push(ControlFrame::from(ElseControlFrame { ty, height, - label, + is_branched_to: false, consume_fuel, + label, reachability, is_end_of_then_reachable, - )); + })); self.else_operands .pop() .unwrap_or_else(|| panic!("missing operands for `else` control frame")) @@ -217,165 +223,141 @@ impl ControlStack { /// A Wasm control frame. #[derive(Debug)] -pub struct ControlFrame { - /// The block type of the [`ControlFrame`]. - ty: BlockType, - /// The value stack height upon entering the [`ControlFrame`]. - height: usize, - /// The number of branches to the [`ControlFrame`]. - len_branches: usize, - /// The [`ControlFrame`]'s [`Instruction::ConsumeFuel`] if fuel metering is enabled. - /// - /// # Note - /// - /// This is `Some` if fuel metering is enabled and `None` otherwise. - consume_fuel: Option, - /// The kind of [`ControlFrame`] with associated data. - kind: ControlFrameKind, +pub enum ControlFrame { + /// A Wasm `block` control frame. + Block(BlockControlFrame), + /// A Wasm `loop` control frame. + Loop(LoopControlFrame), + /// A Wasm `if` control frame. + If(IfControlFrame), + /// A Wasm `else` control frame. + Else(ElseControlFrame), + /// A generic unreachable control frame. + Unreachable(UnreachableControlFrame), } -impl ControlFrame { - /// Creates a new unreachable [`ControlFrame`] of `kind`. - pub fn new_unreachable(ty: BlockType, height: usize, kind: UnreachableControlFrame) -> Self { - Self { - ty, - height, - len_branches: 0, - consume_fuel: None, - kind: ControlFrameKind::Unreachable(kind), - } - } - - /// Creates a new Wasm `block` [`ControlFrame`]. - pub fn new_block( - ty: BlockType, - height: usize, - label: LabelRef, - consume_fuel: Option, - ) -> Self { - Self { - ty, - height, - len_branches: 0, - consume_fuel, - kind: ControlFrameKind::Block(BlockControlFrame { label }), - } - } - - /// Creates a new Wasm `loop` [`ControlFrame`]. - pub fn new_loop( - ty: BlockType, - height: usize, - label: LabelRef, - consume_fuel: Option, - ) -> Self { - Self { - ty, - height, - len_branches: 0, - consume_fuel, - kind: ControlFrameKind::Loop(LoopControlFrame { label }), - } +impl From for ControlFrame { + fn from(frame: BlockControlFrame) -> Self { + Self::Block(frame) } +} - /// Creates a new Wasm `if` [`ControlFrame`]. - pub fn new_if( - ty: BlockType, - height: usize, - label: LabelRef, - consume_fuel: Option, - reachability: IfReachability, - ) -> Self { - Self { - ty, - height, - len_branches: 0, - consume_fuel, - kind: ControlFrameKind::If(IfControlFrame { - label, - reachability, - }), - } +impl From for ControlFrame { + fn from(frame: LoopControlFrame) -> Self { + Self::Loop(frame) } +} - /// Creates a new Wasm `else` [`ControlFrame`]. - pub fn new_else( - ty: BlockType, - height: usize, - label: LabelRef, - consume_fuel: Option, - reachability: ElseReachability, - is_end_of_then_reachable: bool, - ) -> Self { - Self { - ty, - height, - len_branches: 0, - consume_fuel, - kind: ControlFrameKind::Else(ElseControlFrame { - label, - reachability, - is_end_of_then_reachable, - }), - } +impl From for ControlFrame { + fn from(frame: IfControlFrame) -> Self { + Self::If(frame) } +} - /// Returns the stack height of the [`ControlFrame`]. - pub fn height(&self) -> usize { - self.height +impl From for ControlFrame { + fn from(frame: ElseControlFrame) -> Self { + Self::Else(frame) } +} - /// Returns the [`BlockType`] of the [`ControlFrame`]. - pub fn block_type(&self) -> BlockType { - self.ty +impl From for ControlFrame { + fn from(frame: UnreachableControlFrame) -> Self { + Self::Unreachable(frame) } +} +impl ControlFrame { /// Returns `true` if at least one branch targets the [`ControlFrame`]. pub fn is_branched_to(&self) -> bool { - self.len_branches() >= 1 - } - - /// Returns the number of branches to the [`ControlFrame`]. - fn len_branches(&self) -> usize { - self.len_branches + match self { + ControlFrame::Block(frame) => frame.is_branched_to(), + ControlFrame::Loop(frame) => frame.is_branched_to(), + ControlFrame::If(frame) => frame.is_branched_to(), + ControlFrame::Else(frame) => frame.is_branched_to(), + ControlFrame::Unreachable(frame) => { + panic!( + "invalid query for unreachable control frame: `ControlFrame::is_branched_to`" + ) + } + } } - /// Bumps the number of branches to the [`ControlFrame`] by 1. - fn bump_branches(&mut self) { - self.len_branches += 1; + /// Makes the [`ControlFrame`] aware that there is a branch to it. + pub fn branch_to(&mut self) { + match self { + ControlFrame::Block(frame) => frame.branch_to(), + ControlFrame::Loop(frame) => frame.branch_to(), + ControlFrame::If(frame) => frame.branch_to(), + ControlFrame::Else(frame) => frame.branch_to(), + ControlFrame::Unreachable(frame) => { + panic!("invalid query for unreachable control frame: `ControlFrame::branch_to`") + } + } } /// Returns a reference to the [`Instruction::ConsumeFuel`] of the [`ControlFrame`] if any. /// /// Returns `None` if fuel metering is disabled. pub fn consume_fuel_instr(&self) -> Option { - self.consume_fuel + match self { + ControlFrame::Block(frame) => frame.consume_fuel_instr(), + ControlFrame::Loop(frame) => frame.consume_fuel_instr(), + ControlFrame::If(frame) => frame.consume_fuel_instr(), + ControlFrame::Else(frame) => frame.consume_fuel_instr(), + ControlFrame::Unreachable(frame) => { + panic!("invalid query for unreachable control frame: `ControlFrame::consume_fuel_instr`") + } + } } } -/// A Wasm control frame kind. -#[derive(Debug)] -pub enum ControlFrameKind { - /// A Wasm `block` control frame. - Block(BlockControlFrame), - /// A Wasm `loop` control frame. - Loop(LoopControlFrame), - /// A Wasm `if` control frame, including `then`. - If(IfControlFrame), - /// A Wasm `else` control frame, as part of `if`. - Else(ElseControlFrame), - /// A generic unreachable Wasm control frame. - Unreachable(UnreachableControlFrame), -} - /// A Wasm `block` control frame. #[derive(Debug)] pub struct BlockControlFrame { + /// The block type of the [`BlockControlFrame`]. + ty: BlockType, + /// The value stack height upon entering the [`BlockControlFrame`]. + height: usize, + /// This is `true` if there is at least one branch to this [`BlockControlFrame`]. + is_branched_to: bool, + /// The [`BlockControlFrame`]'s [`Instruction::ConsumeFuel`] if fuel metering is enabled. + /// + /// # Note + /// + /// This is `Some` if fuel metering is enabled and `None` otherwise. + consume_fuel: Option, /// The label used to branch to the [`BlockControlFrame`]. label: LabelRef, } impl BlockControlFrame { + /// Returns the [`BlockType`] of the [`BlockControlFrame`]. + pub fn ty(&self) -> BlockType { + self.ty + } + + /// Returns the height of the [`BlockControlFrame`]. + pub fn height(&self) -> usize { + self.height + } + + /// Returns `true` if there are branches to this [`BlockControlFrame`]. + pub fn is_branched_to(&self) -> bool { + self.is_branched_to + } + + /// Makes the [`BlockControlFrame`] aware that there is a branch to it. + pub fn branch_to(&mut self) { + self.is_branched_to = true; + } + + /// Returns a reference to the [`Instruction::ConsumeFuel`] of the [`BlockControlFrame`] if any. + /// + /// Returns `None` if fuel metering is disabled. + pub fn consume_fuel_instr(&self) -> Option { + self.consume_fuel + } + /// Returns the branch label of the [`BlockControlFrame`]. pub fn label(&self) -> LabelRef { self.label @@ -385,20 +367,71 @@ impl BlockControlFrame { /// A Wasm `loop` control frame. #[derive(Debug)] pub struct LoopControlFrame { + /// The block type of the [`LoopControlFrame`]. + ty: BlockType, + /// The value stack height upon entering the [`LoopControlFrame`]. + height: usize, + /// This is `true` if there is at least one branch to this [`LoopControlFrame`]. + is_branched_to: bool, + /// The [`LoopControlFrame`]'s [`Instruction::ConsumeFuel`] if fuel metering is enabled. + /// + /// # Note + /// + /// This is `Some` if fuel metering is enabled and `None` otherwise. + consume_fuel: Option, /// The label used to branch to the [`LoopControlFrame`]. label: LabelRef, } impl LoopControlFrame { + /// Returns the [`BlockType`] of the [`BlockControlFrame`]. + pub fn ty(&self) -> BlockType { + self.ty + } + + /// Returns the height of the [`LoopControlFrame`]. + pub fn height(&self) -> usize { + self.height + } + + /// Returns `true` if there are branches to this [`LoopControlFrame`]. + pub fn is_branched_to(&self) -> bool { + self.is_branched_to + } + + /// Makes the [`LoopControlFrame`] aware that there is a branch to it. + pub fn branch_to(&mut self) { + self.is_branched_to = true; + } + + /// Returns a reference to the [`Instruction::ConsumeFuel`] of the [`LoopControlFrame`] if any. + /// + /// Returns `None` if fuel metering is disabled. + pub fn consume_fuel_instr(&self) -> Option { + self.consume_fuel + } + /// Returns the branch label of the [`LoopControlFrame`]. pub fn label(&self) -> LabelRef { self.label } } -/// A Wasm `if` control frame including `then`. +/// A Wasm `if` control frame including its `then` part. #[derive(Debug)] pub struct IfControlFrame { + /// The block type of the [`LoopControlFrame`]. + ty: BlockType, + /// The value stack height upon entering the [`LoopControlFrame`]. + height: usize, + /// This is `true` if there is at least one branch to this [`LoopControlFrame`]. + is_branched_to: bool, + /// The [`LoopControlFrame`]'s [`Instruction::ConsumeFuel`] if fuel metering is enabled. + /// + /// # Note + /// + /// This is `Some` if fuel metering is enabled and `None` otherwise. + consume_fuel: Option, /// The label used to branch to the [`IfControlFrame`]. label: LabelRef, /// The reachability of the `then` and `else` blocks. @@ -406,6 +439,33 @@ pub struct IfControlFrame { } impl IfControlFrame { + /// Returns the [`BlockType`] of the [`IfControlFrame`]. + pub fn ty(&self) -> BlockType { + self.ty + } + + /// Returns the height of the [`IfControlFrame`]. + pub fn height(&self) -> usize { + self.height + } + + /// Returns `true` if there are branches to this [`IfControlFrame`]. + pub fn is_branched_to(&self) -> bool { + self.is_branched_to + } + + /// Makes the [`IfControlFrame`] aware that there is a branch to it. + pub fn branch_to(&mut self) { + self.is_branched_to = true; + } + + /// Returns a reference to the [`Instruction::ConsumeFuel`] of the [`IfControlFrame`] if any. + /// + /// Returns `None` if fuel metering is disabled. + pub fn consume_fuel_instr(&self) -> Option { + self.consume_fuel + } + /// Returns the branch label of the [`IfControlFrame`]. pub fn label(&self) -> LabelRef { self.label @@ -472,6 +532,18 @@ pub enum IfReachability { /// A Wasm `else` control frame part of Wasm `if`. #[derive(Debug)] pub struct ElseControlFrame { + /// The block type of the [`LoopControlFrame`]. + ty: BlockType, + /// The value stack height upon entering the [`LoopControlFrame`]. + height: usize, + /// This is `true` if there is at least one branch to this [`LoopControlFrame`]. + is_branched_to: bool, + /// The [`LoopControlFrame`]'s [`Instruction::ConsumeFuel`] if fuel metering is enabled. + /// + /// # Note + /// + /// This is `Some` if fuel metering is enabled and `None` otherwise. + consume_fuel: Option, /// The label used to branch to the [`ElseControlFrame`]. label: LabelRef, /// The reachability of the `then` and `else` blocks. @@ -510,7 +582,34 @@ pub enum ElseReachability { } impl ElseControlFrame { - /// Returns the branch label of the [`IfControlFrame`]. + /// Returns the [`BlockType`] of the [`ElseControlFrame`]. + pub fn ty(&self) -> BlockType { + self.ty + } + + /// Returns the height of the [`ElseControlFrame`]. + pub fn height(&self) -> usize { + self.height + } + + /// Returns `true` if there are branches to this [`ElseControlFrame`]. + pub fn is_branched_to(&self) -> bool { + self.is_branched_to + } + + /// Makes the [`ElseControlFrame`] aware that there is a branch to it. + pub fn branch_to(&mut self) { + self.is_branched_to = true; + } + + /// Returns a reference to the [`Instruction::ConsumeFuel`] of the [`ElseControlFrame`] if any. + /// + /// Returns `None` if fuel metering is disabled. + pub fn consume_fuel_instr(&self) -> Option { + self.consume_fuel + } + + /// Returns the branch label of the [`ElseControlFrame`]. pub fn label(&self) -> LabelRef { self.label } @@ -540,7 +639,6 @@ impl ElseControlFrame { } /// Returns `true` if the end of the `then` branch is reachable. - #[track_caller] pub fn is_end_of_then_reachable(&self) -> bool { self.is_end_of_then_reachable } diff --git a/crates/wasmi/src/engine/translator/func2/stack/mod.rs b/crates/wasmi/src/engine/translator/func2/stack/mod.rs index 3522167a28..42099e171c 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/mod.rs @@ -3,27 +3,24 @@ mod locals; mod operand; mod operands; -use self::{ +pub use self::{ control::{ BlockControlFrame, ControlFrame, - ControlFrameKind, - ControlStack, ElseControlFrame, - ElseReachability, IfControlFrame, - IfReachability, LoopControlFrame, UnreachableControlFrame, }, - locals::LocalsRegistry, - operands::{OperandStack, StackOperand}, -}; -pub use self::{ locals::LocalIdx, operand::Operand, operands::{OperandIdx, PreservedLocalsIter}, }; +use self::{ + control::{ControlStack, ElseReachability, IfReachability}, + locals::LocalsRegistry, + operands::{OperandStack, StackOperand}, +}; use super::Reset; use crate::{ core::{TypedVal, UntypedVal, ValType}, @@ -310,9 +307,7 @@ impl Stack { ty: BlockType, kind: UnreachableControlFrame, ) -> Result<(), Error> { - let len_params = usize::from(ty.len_params(&self.engine)); - let block_height = self.height() - len_params; - self.controls.push_unreachable(ty, block_height, kind); + self.controls.push_unreachable(kind); Ok(()) } From 773e4f1466430a91667caf91633105549c95ce18 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Fri, 6 Jun 2025 19:03:10 +0200 Subject: [PATCH 092/343] implemented visit_end as stub --- .../wasmi/src/engine/translator/func2/mod.rs | 41 ++++++++++++++++++- .../src/engine/translator/func2/visit.rs | 10 ++++- 2 files changed, 47 insertions(+), 4 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index 4d0111ce44..faffcff891 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -12,13 +12,25 @@ mod visit; use self::{ instrs::InstrEncoder, layout::{StackLayout, StackSpace}, - stack::{LocalIdx, Operand, OperandIdx, Stack, StackAllocations}, + stack::{ + BlockControlFrame, + ControlFrame, + ElseControlFrame, + IfControlFrame, + LocalIdx, + LoopControlFrame, + Operand, + OperandIdx, + Stack, + StackAllocations, + UnreachableControlFrame, + }, utils::Reset, }; use crate::{ core::{FuelCostsProvider, Typed, TypedVal}, engine::{ - translator::{Instr, LabelRegistry, WasmTranslator}, + translator::{Instr, LabelRef, LabelRegistry, WasmTranslator}, BlockType, CompiledFuncEntity, TranslationError, @@ -223,6 +235,31 @@ impl FuncTranslator { &self.engine } + /// Translates the end of a Wasm `block` control frame. + fn translate_end_block(&mut self, frame: BlockControlFrame) -> Result<(), Error> { + todo!() + } + + /// Translates the end of a Wasm `loop` control frame. + fn translate_end_loop(&mut self, frame: LoopControlFrame) -> Result<(), Error> { + todo!() + } + + /// Translates the end of a Wasm `if` control frame. + fn translate_end_if(&mut self, frame: IfControlFrame) -> Result<(), Error> { + todo!() + } + + /// Translates the end of a Wasm `else` control frame. + fn translate_end_else(&mut self, frame: ElseControlFrame) -> Result<(), Error> { + todo!() + } + + /// Translates the end of an unreachable Wasm control frame. + fn translate_end_unreachable(&mut self, frame: UnreachableControlFrame) -> Result<(), Error> { + todo!() + } + /// Translates a commutative binary Wasm operator to Wasmi bytecode. fn translate_binary_commutative( &mut self, diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index ec27a172b4..ed3fc96c99 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -1,4 +1,4 @@ -use super::{FuncTranslator, LocalIdx}; +use super::{ControlFrame, FuncTranslator, LocalIdx}; use crate::{core::wasm, ir::Instruction, Error}; use wasmparser::VisitOperator; @@ -89,7 +89,13 @@ impl<'a> VisitOperator<'a> for FuncTranslator { } fn visit_end(&mut self) -> Self::Output { - todo!() + match self.stack.pop_control() { + ControlFrame::Block(frame) => self.translate_end_block(frame), + ControlFrame::Loop(frame) => self.translate_end_loop(frame), + ControlFrame::If(frame) => self.translate_end_if(frame), + ControlFrame::Else(frame) => self.translate_end_else(frame), + ControlFrame::Unreachable(frame) => self.translate_end_unreachable(frame), + } } fn visit_br(&mut self, relative_depth: u32) -> Self::Output { From ba45fa486615d2f3a97342b8dfc2a710f6b28259 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Fri, 6 Jun 2025 19:05:08 +0200 Subject: [PATCH 093/343] remove unused ControlFrame::is_branched_to method --- .../src/engine/translator/func2/stack/control.rs | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/stack/control.rs b/crates/wasmi/src/engine/translator/func2/stack/control.rs index 81eb7d74bd..04ba2ef220 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/control.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/control.rs @@ -267,21 +267,6 @@ impl From for ControlFrame { } impl ControlFrame { - /// Returns `true` if at least one branch targets the [`ControlFrame`]. - pub fn is_branched_to(&self) -> bool { - match self { - ControlFrame::Block(frame) => frame.is_branched_to(), - ControlFrame::Loop(frame) => frame.is_branched_to(), - ControlFrame::If(frame) => frame.is_branched_to(), - ControlFrame::Else(frame) => frame.is_branched_to(), - ControlFrame::Unreachable(frame) => { - panic!( - "invalid query for unreachable control frame: `ControlFrame::is_branched_to`" - ) - } - } - } - /// Makes the [`ControlFrame`] aware that there is a branch to it. pub fn branch_to(&mut self) { match self { From 26107e19e50bc69b550f7622afcbe7d7194fed25 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Sat, 7 Jun 2025 11:13:56 +0200 Subject: [PATCH 094/343] differentiate between func enclosing block in translation --- crates/wasmi/src/engine/translator/func2/mod.rs | 8 ++++++++ crates/wasmi/src/engine/translator/func2/stack/mod.rs | 5 +++++ 2 files changed, 13 insertions(+) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index faffcff891..9868d7671b 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -237,6 +237,14 @@ impl FuncTranslator { /// Translates the end of a Wasm `block` control frame. fn translate_end_block(&mut self, frame: BlockControlFrame) -> Result<(), Error> { + if self.stack.is_control_empty() { + return self.translate_end_func(frame); + } + todo!() + } + + /// Translates the end of the Wasm function enclosing Wasm `block`. + fn translate_end_func(&mut self, frame: BlockControlFrame) -> Result<(), Error> { todo!() } diff --git a/crates/wasmi/src/engine/translator/func2/stack/mod.rs b/crates/wasmi/src/engine/translator/func2/stack/mod.rs index 42099e171c..a77e8cc8e6 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/mod.rs @@ -99,6 +99,11 @@ impl Stack { self.operands.register_locals(amount, ty) } + /// Returns `true` if the control stack is empty. + pub fn is_control_empty(&self) -> bool { + self.controls.is_empty() + } + /// Returns the current height of the [`Stack`]. /// /// # Note From e8c438f5fd12a1a10f41a6e350a4c28d0943b436 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Sat, 7 Jun 2025 11:15:15 +0200 Subject: [PATCH 095/343] initial impl for translate_end_func --- crates/wasmi/src/engine/translator/func2/mod.rs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index 9868d7671b..ae463d6638 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -30,7 +30,7 @@ use self::{ use crate::{ core::{FuelCostsProvider, Typed, TypedVal}, engine::{ - translator::{Instr, LabelRef, LabelRegistry, WasmTranslator}, + translator::{utils::FuelInfo, Instr, LabelRef, LabelRegistry, WasmTranslator}, BlockType, CompiledFuncEntity, TranslationError, @@ -235,6 +235,11 @@ impl FuncTranslator { &self.engine } + /// Returns `true` if fuel metering is enabled. + fn is_fuel_metering_enabled(&self) -> bool { + self.engine.config().get_consume_fuel() + } + /// Translates the end of a Wasm `block` control frame. fn translate_end_block(&mut self, frame: BlockControlFrame) -> Result<(), Error> { if self.stack.is_control_empty() { @@ -245,6 +250,13 @@ impl FuncTranslator { /// Translates the end of the Wasm function enclosing Wasm `block`. fn translate_end_func(&mut self, frame: BlockControlFrame) -> Result<(), Error> { + let fuel_info = match (frame.consume_fuel_instr(), &self.fuel_costs) { + (Some(consume_fuel), Some(fuel_costs)) => { + FuelInfo::some(fuel_costs.clone(), consume_fuel) + } + (None, None) => FuelInfo::None, + _ => unreachable!(), + }; todo!() } From 28f25b4e523e4b296883d2283e697880964ac937 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Sat, 7 Jun 2025 15:11:29 +0200 Subject: [PATCH 096/343] implement translate_end_func method --- .../src/engine/translator/func2/instrs.rs | 1 - .../wasmi/src/engine/translator/func2/mod.rs | 111 +++++++++++++++++- 2 files changed, 106 insertions(+), 6 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/instrs.rs b/crates/wasmi/src/engine/translator/func2/instrs.rs index a0237d70ea..a59397f0e8 100644 --- a/crates/wasmi/src/engine/translator/func2/instrs.rs +++ b/crates/wasmi/src/engine/translator/func2/instrs.rs @@ -25,7 +25,6 @@ impl InstrEncoder { /// Pushes an [`Instruction`] to the [`InstrEncoder`]. /// /// Returns an [`Instr`] that refers to the pushed [`Instruction`]. - #[must_use] pub fn push_instr(&mut self, instruction: Instruction) -> Instr { let instr = self.next_instr(); self.instrs.push(instruction); diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index ae463d6638..55e70f8af7 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -28,14 +28,14 @@ use self::{ utils::Reset, }; use crate::{ - core::{FuelCostsProvider, Typed, TypedVal}, + core::{FuelCostsProvider, Typed, TypedVal, ValType}, engine::{ translator::{utils::FuelInfo, Instr, LabelRef, LabelRegistry, WasmTranslator}, BlockType, CompiledFuncEntity, TranslationError, }, - ir::{Const16, Instruction, Reg}, + ir::{BoundedRegSpan, Const16, Const32, Instruction, Reg, RegSpan}, module::{FuncIdx, ModuleHeader, WasmiValueType}, Engine, Error, @@ -250,14 +250,115 @@ impl FuncTranslator { /// Translates the end of the Wasm function enclosing Wasm `block`. fn translate_end_func(&mut self, frame: BlockControlFrame) -> Result<(), Error> { - let fuel_info = match (frame.consume_fuel_instr(), &self.fuel_costs) { - (Some(consume_fuel), Some(fuel_costs)) => { + let fuel_info = match (&self.fuel_costs, frame.consume_fuel_instr()) { + (Some(fuel_costs), Some(consume_fuel)) => { FuelInfo::some(fuel_costs.clone(), consume_fuel) } (None, None) => FuelInfo::None, _ => unreachable!(), }; - todo!() + let len_results = frame.ty().len_results(&self.engine); + if self.reachable && frame.is_branched_to() && len_results > 1 { + let height = frame.height(); + let len_results = usize::from(len_results); + for depth in 0..len_results { + let result = self + .layout + .temp_to_reg(OperandIdx::from(height + len_results - depth - 1))?; + match self.stack.operand_to_temp(depth) { + Some(Operand::Local(operand)) => { + let value = self.layout.local_to_reg(operand.local_index())?; + self.instrs.push_instr(Instruction::copy(result, value)); + } + Some(Operand::Immediate(operand)) => { + let val = operand.val(); + let instr = match operand.ty() { + ValType::I32 => Instruction::copy_imm32(result, i32::from(val)), + ValType::I64 => { + let val = i64::from(val); + match >::try_from(val) { + Ok(value) => Instruction::copy_i64imm32(result, value), + Err(_) => { + let value = self.layout.const_to_reg(val)?; + Instruction::copy(result, value) + } + } + } + ValType::F32 => Instruction::copy_imm32(result, f32::from(val)), + ValType::F64 => { + let val = f64::from(val); + match >::try_from(val) { + Ok(value) => Instruction::copy_f64imm32(result, value), + Err(_) => { + let value = self.layout.const_to_reg(val)?; + Instruction::copy(result, value) + } + } + } + ValType::V128 | ValType::FuncRef | ValType::ExternRef => { + let value = self.layout.const_to_reg(val)?; + Instruction::copy(result, value) + } + }; + self.instrs.push_instr(instr); + } + _ => {} + } + } + } + self.labels + .pin_label(frame.label(), self.instrs.next_instr()) + .unwrap_or_else(|err| panic!("failed to pin label: {err}")); + match len_results { + 0 => { + self.instrs.push_instr(Instruction::Return); + } + 1 => { + let instr = match self.stack.peek(0) { + Operand::Local(operand) => { + let value = self.layout.local_to_reg(operand.local_index())?; + Instruction::return_reg(value) + } + Operand::Temp(operand) => { + let value = self.layout.temp_to_reg(operand.operand_index())?; + Instruction::return_reg(value) + } + Operand::Immediate(operand) => { + let val = operand.val(); + match operand.ty() { + ValType::I32 => Instruction::return_imm32(i32::from(val)), + ValType::I64 => match >::try_from(i64::from(val)) { + Ok(value) => Instruction::return_i64imm32(value), + Err(_) => { + let value = self.layout.const_to_reg(val)?; + Instruction::return_reg(value) + } + }, + ValType::F32 => Instruction::return_imm32(f32::from(val)), + ValType::F64 => match >::try_from(f64::from(val)) { + Ok(value) => Instruction::return_f64imm32(value), + Err(_) => { + let value = self.layout.const_to_reg(val)?; + Instruction::return_reg(value) + } + }, + ValType::V128 | ValType::FuncRef | ValType::ExternRef => { + let value = self.layout.const_to_reg(val)?; + Instruction::return_reg(value) + } + } + } + }; + self.instrs.push_instr(instr); + } + n => { + let height = frame.height(); + let result = self.layout.temp_to_reg(OperandIdx::from(height - 1))?; + let values = BoundedRegSpan::new(RegSpan::new(result), len_results); + self.instrs.push_instr(Instruction::return_span(values)); + } + } + Ok(()) } /// Translates the end of a Wasm `loop` control frame. From bfb93edba14e73c49e796adcf21410a335262ae8 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Sat, 7 Jun 2025 15:21:34 +0200 Subject: [PATCH 097/343] unsilence unused_variables warning and fix occurrences --- .../src/engine/translator/func2/layout/mod.rs | 2 +- .../wasmi/src/engine/translator/func2/mod.rs | 14 +-- .../src/engine/translator/func2/simd/visit.rs | 76 ++++++------ .../engine/translator/func2/stack/control.rs | 16 ++- .../src/engine/translator/func2/stack/mod.rs | 6 +- .../engine/translator/func2/stack/operands.rs | 10 +- .../src/engine/translator/func2/visit.rs | 114 +++++++++--------- 7 files changed, 114 insertions(+), 124 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/layout/mod.rs b/crates/wasmi/src/engine/translator/func2/layout/mod.rs index d14a73ecd3..d000648eaf 100644 --- a/crates/wasmi/src/engine/translator/func2/layout/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/layout/mod.rs @@ -34,7 +34,7 @@ impl StackLayout { /// # Errors /// /// If too many local variables are being registered. - pub fn register_locals(&mut self, amount: u32, ty: ValType) -> Result<(), Error> { + pub fn register_locals(&mut self, amount: u32, _ty: ValType) -> Result<(), Error> { self.len_locals += amount as usize; Ok(()) } diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index 55e70f8af7..a40491b6ef 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -1,4 +1,4 @@ -#![expect(dead_code, unused_imports, unused_variables)] +#![expect(dead_code, unused_imports)] #[macro_use] mod utils; @@ -250,7 +250,7 @@ impl FuncTranslator { /// Translates the end of the Wasm function enclosing Wasm `block`. fn translate_end_func(&mut self, frame: BlockControlFrame) -> Result<(), Error> { - let fuel_info = match (&self.fuel_costs, frame.consume_fuel_instr()) { + let _fuel_info = match (&self.fuel_costs, frame.consume_fuel_instr()) { (Some(fuel_costs), Some(consume_fuel)) => { FuelInfo::some(fuel_costs.clone(), consume_fuel) } @@ -351,7 +351,7 @@ impl FuncTranslator { }; self.instrs.push_instr(instr); } - n => { + _ => { let height = frame.height(); let result = self.layout.temp_to_reg(OperandIdx::from(height - 1))?; let values = BoundedRegSpan::new(RegSpan::new(result), len_results); @@ -362,22 +362,22 @@ impl FuncTranslator { } /// Translates the end of a Wasm `loop` control frame. - fn translate_end_loop(&mut self, frame: LoopControlFrame) -> Result<(), Error> { + fn translate_end_loop(&mut self, _frame: LoopControlFrame) -> Result<(), Error> { todo!() } /// Translates the end of a Wasm `if` control frame. - fn translate_end_if(&mut self, frame: IfControlFrame) -> Result<(), Error> { + fn translate_end_if(&mut self, _frame: IfControlFrame) -> Result<(), Error> { todo!() } /// Translates the end of a Wasm `else` control frame. - fn translate_end_else(&mut self, frame: ElseControlFrame) -> Result<(), Error> { + fn translate_end_else(&mut self, _frame: ElseControlFrame) -> Result<(), Error> { todo!() } /// Translates the end of an unreachable Wasm control frame. - fn translate_end_unreachable(&mut self, frame: UnreachableControlFrame) -> Result<(), Error> { + fn translate_end_unreachable(&mut self, _frame: UnreachableControlFrame) -> Result<(), Error> { todo!() } diff --git a/crates/wasmi/src/engine/translator/func2/simd/visit.rs b/crates/wasmi/src/engine/translator/func2/simd/visit.rs index da8023629a..7845b92636 100644 --- a/crates/wasmi/src/engine/translator/func2/simd/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/simd/visit.rs @@ -2,155 +2,155 @@ use super::FuncTranslator; use wasmparser::VisitSimdOperator; impl VisitSimdOperator<'_> for FuncTranslator { - fn visit_v128_load(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + fn visit_v128_load(&mut self, _memarg: wasmparser::MemArg) -> Self::Output { todo!() } - fn visit_v128_load8x8_s(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + fn visit_v128_load8x8_s(&mut self, _memarg: wasmparser::MemArg) -> Self::Output { todo!() } - fn visit_v128_load8x8_u(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + fn visit_v128_load8x8_u(&mut self, _memarg: wasmparser::MemArg) -> Self::Output { todo!() } - fn visit_v128_load16x4_s(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + fn visit_v128_load16x4_s(&mut self, _memarg: wasmparser::MemArg) -> Self::Output { todo!() } - fn visit_v128_load16x4_u(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + fn visit_v128_load16x4_u(&mut self, _memarg: wasmparser::MemArg) -> Self::Output { todo!() } - fn visit_v128_load32x2_s(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + fn visit_v128_load32x2_s(&mut self, _memarg: wasmparser::MemArg) -> Self::Output { todo!() } - fn visit_v128_load32x2_u(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + fn visit_v128_load32x2_u(&mut self, _memarg: wasmparser::MemArg) -> Self::Output { todo!() } - fn visit_v128_load8_splat(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + fn visit_v128_load8_splat(&mut self, _memarg: wasmparser::MemArg) -> Self::Output { todo!() } - fn visit_v128_load16_splat(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + fn visit_v128_load16_splat(&mut self, _memarg: wasmparser::MemArg) -> Self::Output { todo!() } - fn visit_v128_load32_splat(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + fn visit_v128_load32_splat(&mut self, _memarg: wasmparser::MemArg) -> Self::Output { todo!() } - fn visit_v128_load64_splat(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + fn visit_v128_load64_splat(&mut self, _memarg: wasmparser::MemArg) -> Self::Output { todo!() } - fn visit_v128_load32_zero(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + fn visit_v128_load32_zero(&mut self, _memarg: wasmparser::MemArg) -> Self::Output { todo!() } - fn visit_v128_load64_zero(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + fn visit_v128_load64_zero(&mut self, _memarg: wasmparser::MemArg) -> Self::Output { todo!() } - fn visit_v128_store(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + fn visit_v128_store(&mut self, _memarg: wasmparser::MemArg) -> Self::Output { todo!() } - fn visit_v128_load8_lane(&mut self, memarg: wasmparser::MemArg, lane: u8) -> Self::Output { + fn visit_v128_load8_lane(&mut self, _memarg: wasmparser::MemArg, _lane: u8) -> Self::Output { todo!() } - fn visit_v128_load16_lane(&mut self, memarg: wasmparser::MemArg, lane: u8) -> Self::Output { + fn visit_v128_load16_lane(&mut self, _memarg: wasmparser::MemArg, _lane: u8) -> Self::Output { todo!() } - fn visit_v128_load32_lane(&mut self, memarg: wasmparser::MemArg, lane: u8) -> Self::Output { + fn visit_v128_load32_lane(&mut self, _memarg: wasmparser::MemArg, _lane: u8) -> Self::Output { todo!() } - fn visit_v128_load64_lane(&mut self, memarg: wasmparser::MemArg, lane: u8) -> Self::Output { + fn visit_v128_load64_lane(&mut self, _memarg: wasmparser::MemArg, _lane: u8) -> Self::Output { todo!() } - fn visit_v128_store8_lane(&mut self, memarg: wasmparser::MemArg, lane: u8) -> Self::Output { + fn visit_v128_store8_lane(&mut self, _memarg: wasmparser::MemArg, _lane: u8) -> Self::Output { todo!() } - fn visit_v128_store16_lane(&mut self, memarg: wasmparser::MemArg, lane: u8) -> Self::Output { + fn visit_v128_store16_lane(&mut self, _memarg: wasmparser::MemArg, _lane: u8) -> Self::Output { todo!() } - fn visit_v128_store32_lane(&mut self, memarg: wasmparser::MemArg, lane: u8) -> Self::Output { + fn visit_v128_store32_lane(&mut self, _memarg: wasmparser::MemArg, _lane: u8) -> Self::Output { todo!() } - fn visit_v128_store64_lane(&mut self, memarg: wasmparser::MemArg, lane: u8) -> Self::Output { + fn visit_v128_store64_lane(&mut self, _memarg: wasmparser::MemArg, _lane: u8) -> Self::Output { todo!() } - fn visit_v128_const(&mut self, value: wasmparser::V128) -> Self::Output { + fn visit_v128_const(&mut self, _value: wasmparser::V128) -> Self::Output { todo!() } - fn visit_i8x16_shuffle(&mut self, lanes: [u8; 16]) -> Self::Output { + fn visit_i8x16_shuffle(&mut self, _lanes: [u8; 16]) -> Self::Output { todo!() } - fn visit_i8x16_extract_lane_s(&mut self, lane: u8) -> Self::Output { + fn visit_i8x16_extract_lane_s(&mut self, _lane: u8) -> Self::Output { todo!() } - fn visit_i8x16_extract_lane_u(&mut self, lane: u8) -> Self::Output { + fn visit_i8x16_extract_lane_u(&mut self, _lane: u8) -> Self::Output { todo!() } - fn visit_i8x16_replace_lane(&mut self, lane: u8) -> Self::Output { + fn visit_i8x16_replace_lane(&mut self, _lane: u8) -> Self::Output { todo!() } - fn visit_i16x8_extract_lane_s(&mut self, lane: u8) -> Self::Output { + fn visit_i16x8_extract_lane_s(&mut self, _lane: u8) -> Self::Output { todo!() } - fn visit_i16x8_extract_lane_u(&mut self, lane: u8) -> Self::Output { + fn visit_i16x8_extract_lane_u(&mut self, _lane: u8) -> Self::Output { todo!() } - fn visit_i16x8_replace_lane(&mut self, lane: u8) -> Self::Output { + fn visit_i16x8_replace_lane(&mut self, _lane: u8) -> Self::Output { todo!() } - fn visit_i32x4_extract_lane(&mut self, lane: u8) -> Self::Output { + fn visit_i32x4_extract_lane(&mut self, _lane: u8) -> Self::Output { todo!() } - fn visit_i32x4_replace_lane(&mut self, lane: u8) -> Self::Output { + fn visit_i32x4_replace_lane(&mut self, _lane: u8) -> Self::Output { todo!() } - fn visit_i64x2_extract_lane(&mut self, lane: u8) -> Self::Output { + fn visit_i64x2_extract_lane(&mut self, _lane: u8) -> Self::Output { todo!() } - fn visit_i64x2_replace_lane(&mut self, lane: u8) -> Self::Output { + fn visit_i64x2_replace_lane(&mut self, _lane: u8) -> Self::Output { todo!() } - fn visit_f32x4_extract_lane(&mut self, lane: u8) -> Self::Output { + fn visit_f32x4_extract_lane(&mut self, _lane: u8) -> Self::Output { todo!() } - fn visit_f32x4_replace_lane(&mut self, lane: u8) -> Self::Output { + fn visit_f32x4_replace_lane(&mut self, _lane: u8) -> Self::Output { todo!() } - fn visit_f64x2_extract_lane(&mut self, lane: u8) -> Self::Output { + fn visit_f64x2_extract_lane(&mut self, _lane: u8) -> Self::Output { todo!() } - fn visit_f64x2_replace_lane(&mut self, lane: u8) -> Self::Output { + fn visit_f64x2_replace_lane(&mut self, _lane: u8) -> Self::Output { todo!() } diff --git a/crates/wasmi/src/engine/translator/func2/stack/control.rs b/crates/wasmi/src/engine/translator/func2/stack/control.rs index 04ba2ef220..91795f54d3 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/control.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/control.rs @@ -23,22 +23,21 @@ pub struct ElseOperands { /// The end indices of each `else` operands. ends: Vec, /// All operands of all allocated `else` control frames. - providers: Vec, + operands: Vec, } impl Reset for ElseOperands { fn reset(&mut self) { self.ends.clear(); - self.providers.clear(); + self.operands.clear(); } } impl ElseOperands { /// Pushes operands for a new Wasm `else` control frame. pub fn push(&mut self, operands: impl IntoIterator) { - self.providers.extend(operands); - let end = self.providers.len(); - let index = self.ends.len(); + self.operands.extend(operands); + let end = self.operands.len(); self.ends.push(end); } @@ -46,7 +45,7 @@ impl ElseOperands { pub fn pop(&mut self) -> Option> { let end = self.ends.pop()?; let start = self.ends.last().copied().unwrap_or(0); - Some(self.providers.drain(start..end)) + Some(self.operands.drain(start..end)) } } @@ -203,7 +202,6 @@ impl ControlStack { /// Acquires the target [`ControlFrame`] at the given relative `depth`. pub fn acquire_target(&mut self, depth: usize) -> AcquiredTarget { let is_root = self.is_root(depth); - let height = self.height(); let frame = self.get_mut(depth); if is_root { AcquiredTarget::Return(frame) @@ -274,7 +272,7 @@ impl ControlFrame { ControlFrame::Loop(frame) => frame.branch_to(), ControlFrame::If(frame) => frame.branch_to(), ControlFrame::Else(frame) => frame.branch_to(), - ControlFrame::Unreachable(frame) => { + ControlFrame::Unreachable(_) => { panic!("invalid query for unreachable control frame: `ControlFrame::branch_to`") } } @@ -289,7 +287,7 @@ impl ControlFrame { ControlFrame::Loop(frame) => frame.consume_fuel_instr(), ControlFrame::If(frame) => frame.consume_fuel_instr(), ControlFrame::Else(frame) => frame.consume_fuel_instr(), - ControlFrame::Unreachable(frame) => { + ControlFrame::Unreachable(_) => { panic!("invalid query for unreachable control frame: `ControlFrame::consume_fuel_instr`") } } diff --git a/crates/wasmi/src/engine/translator/func2/stack/mod.rs b/crates/wasmi/src/engine/translator/func2/stack/mod.rs index a77e8cc8e6..f95e6ba176 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/mod.rs @@ -307,11 +307,7 @@ impl Stack { /// # Errors /// /// If the stack height exceeds the maximum height. - pub fn push_unreachable( - &mut self, - ty: BlockType, - kind: UnreachableControlFrame, - ) -> Result<(), Error> { + pub fn push_unreachable(&mut self, kind: UnreachableControlFrame) -> Result<(), Error> { self.controls.push_unreachable(kind); Ok(()) } diff --git a/crates/wasmi/src/engine/translator/func2/stack/operands.rs b/crates/wasmi/src/engine/translator/func2/stack/operands.rs index 71e6be7176..c2f3ecd9d1 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/operands.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/operands.rs @@ -257,14 +257,10 @@ impl OperandStack { #[must_use] fn operand_to_temp_at(&mut self, index: OperandIdx) -> Option { let operand = self.get_at(index); - let ty = match operand { - StackOperand::Temp { ty, instr } => return None, + match operand { + StackOperand::Temp { .. } => return None, StackOperand::Immediate { val } => val.ty(), - StackOperand::Local { - local_index, - next_local, - prev_local, - } => self.locals.ty(local_index), + StackOperand::Local { local_index, .. } => self.locals.ty(local_index), }; self.unlink_local(operand); self.operands[usize::from(index)] = operand; diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index ed3fc96c99..5b4b953926 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -72,15 +72,15 @@ impl<'a> VisitOperator<'a> for FuncTranslator { todo!() } - fn visit_block(&mut self, blockty: wasmparser::BlockType) -> Self::Output { + fn visit_block(&mut self, _block_ty: wasmparser::BlockType) -> Self::Output { todo!() } - fn visit_loop(&mut self, blockty: wasmparser::BlockType) -> Self::Output { + fn visit_loop(&mut self, _block_ty: wasmparser::BlockType) -> Self::Output { todo!() } - fn visit_if(&mut self, blockty: wasmparser::BlockType) -> Self::Output { + fn visit_if(&mut self, _block_ty: wasmparser::BlockType) -> Self::Output { todo!() } @@ -98,15 +98,15 @@ impl<'a> VisitOperator<'a> for FuncTranslator { } } - fn visit_br(&mut self, relative_depth: u32) -> Self::Output { + fn visit_br(&mut self, _relative_depth: u32) -> Self::Output { todo!() } - fn visit_br_if(&mut self, relative_depth: u32) -> Self::Output { + fn visit_br_if(&mut self, _relative_depth: u32) -> Self::Output { todo!() } - fn visit_br_table(&mut self, targets: wasmparser::BrTable<'a>) -> Self::Output { + fn visit_br_table(&mut self, _targets: wasmparser::BrTable<'a>) -> Self::Output { todo!() } @@ -114,11 +114,11 @@ impl<'a> VisitOperator<'a> for FuncTranslator { todo!() } - fn visit_call(&mut self, function_index: u32) -> Self::Output { + fn visit_call(&mut self, _function_index: u32) -> Self::Output { todo!() } - fn visit_call_indirect(&mut self, type_index: u32, table_index: u32) -> Self::Output { + fn visit_call_indirect(&mut self, _type_index: u32, _table_index: u32) -> Self::Output { todo!() } @@ -136,119 +136,119 @@ impl<'a> VisitOperator<'a> for FuncTranslator { Ok(()) } - fn visit_local_set(&mut self, local_index: u32) -> Self::Output { + fn visit_local_set(&mut self, _local_index: u32) -> Self::Output { todo!() } - fn visit_local_tee(&mut self, local_index: u32) -> Self::Output { + fn visit_local_tee(&mut self, _local_index: u32) -> Self::Output { todo!() } - fn visit_global_get(&mut self, global_index: u32) -> Self::Output { + fn visit_global_get(&mut self, _global_index: u32) -> Self::Output { todo!() } - fn visit_global_set(&mut self, global_index: u32) -> Self::Output { + fn visit_global_set(&mut self, _global_index: u32) -> Self::Output { todo!() } - fn visit_i32_load(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + fn visit_i32_load(&mut self, _memarg: wasmparser::MemArg) -> Self::Output { todo!() } - fn visit_i64_load(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + fn visit_i64_load(&mut self, _memarg: wasmparser::MemArg) -> Self::Output { todo!() } - fn visit_f32_load(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + fn visit_f32_load(&mut self, _memarg: wasmparser::MemArg) -> Self::Output { todo!() } - fn visit_f64_load(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + fn visit_f64_load(&mut self, _memarg: wasmparser::MemArg) -> Self::Output { todo!() } - fn visit_i32_load8_s(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + fn visit_i32_load8_s(&mut self, _memarg: wasmparser::MemArg) -> Self::Output { todo!() } - fn visit_i32_load8_u(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + fn visit_i32_load8_u(&mut self, _memarg: wasmparser::MemArg) -> Self::Output { todo!() } - fn visit_i32_load16_s(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + fn visit_i32_load16_s(&mut self, _memarg: wasmparser::MemArg) -> Self::Output { todo!() } - fn visit_i32_load16_u(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + fn visit_i32_load16_u(&mut self, _memarg: wasmparser::MemArg) -> Self::Output { todo!() } - fn visit_i64_load8_s(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + fn visit_i64_load8_s(&mut self, _memarg: wasmparser::MemArg) -> Self::Output { todo!() } - fn visit_i64_load8_u(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + fn visit_i64_load8_u(&mut self, _memarg: wasmparser::MemArg) -> Self::Output { todo!() } - fn visit_i64_load16_s(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + fn visit_i64_load16_s(&mut self, _memarg: wasmparser::MemArg) -> Self::Output { todo!() } - fn visit_i64_load16_u(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + fn visit_i64_load16_u(&mut self, _memarg: wasmparser::MemArg) -> Self::Output { todo!() } - fn visit_i64_load32_s(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + fn visit_i64_load32_s(&mut self, _memarg: wasmparser::MemArg) -> Self::Output { todo!() } - fn visit_i64_load32_u(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + fn visit_i64_load32_u(&mut self, _memarg: wasmparser::MemArg) -> Self::Output { todo!() } - fn visit_i32_store(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + fn visit_i32_store(&mut self, _memarg: wasmparser::MemArg) -> Self::Output { todo!() } - fn visit_i64_store(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + fn visit_i64_store(&mut self, _memarg: wasmparser::MemArg) -> Self::Output { todo!() } - fn visit_f32_store(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + fn visit_f32_store(&mut self, _memarg: wasmparser::MemArg) -> Self::Output { todo!() } - fn visit_f64_store(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + fn visit_f64_store(&mut self, _memarg: wasmparser::MemArg) -> Self::Output { todo!() } - fn visit_i32_store8(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + fn visit_i32_store8(&mut self, _memarg: wasmparser::MemArg) -> Self::Output { todo!() } - fn visit_i32_store16(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + fn visit_i32_store16(&mut self, _memarg: wasmparser::MemArg) -> Self::Output { todo!() } - fn visit_i64_store8(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + fn visit_i64_store8(&mut self, _memarg: wasmparser::MemArg) -> Self::Output { todo!() } - fn visit_i64_store16(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + fn visit_i64_store16(&mut self, _memarg: wasmparser::MemArg) -> Self::Output { todo!() } - fn visit_i64_store32(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + fn visit_i64_store32(&mut self, _memarg: wasmparser::MemArg) -> Self::Output { todo!() } - fn visit_memory_size(&mut self, mem: u32) -> Self::Output { + fn visit_memory_size(&mut self, _mem: u32) -> Self::Output { todo!() } - fn visit_memory_grow(&mut self, mem: u32) -> Self::Output { + fn visit_memory_grow(&mut self, _mem: u32) -> Self::Output { todo!() } @@ -258,15 +258,15 @@ impl<'a> VisitOperator<'a> for FuncTranslator { Ok(()) } - fn visit_i64_const(&mut self, value: i64) -> Self::Output { + fn visit_i64_const(&mut self, _value: i64) -> Self::Output { todo!() } - fn visit_f32_const(&mut self, value: wasmparser::Ieee32) -> Self::Output { + fn visit_f32_const(&mut self, _value: wasmparser::Ieee32) -> Self::Output { todo!() } - fn visit_f64_const(&mut self, value: wasmparser::Ieee64) -> Self::Output { + fn visit_f64_const(&mut self, _value: wasmparser::Ieee64) -> Self::Output { todo!() } @@ -818,39 +818,39 @@ impl<'a> VisitOperator<'a> for FuncTranslator { todo!() } - fn visit_memory_init(&mut self, data_index: u32, mem: u32) -> Self::Output { + fn visit_memory_init(&mut self, _data_index: u32, _mem: u32) -> Self::Output { todo!() } - fn visit_data_drop(&mut self, data_index: u32) -> Self::Output { + fn visit_data_drop(&mut self, _data_index: u32) -> Self::Output { todo!() } - fn visit_memory_copy(&mut self, dst_mem: u32, src_mem: u32) -> Self::Output { + fn visit_memory_copy(&mut self, _dst_mem: u32, _src_mem: u32) -> Self::Output { todo!() } - fn visit_memory_fill(&mut self, mem: u32) -> Self::Output { + fn visit_memory_fill(&mut self, _mem: u32) -> Self::Output { todo!() } - fn visit_table_init(&mut self, elem_index: u32, table: u32) -> Self::Output { + fn visit_table_init(&mut self, _elem_index: u32, _table: u32) -> Self::Output { todo!() } - fn visit_elem_drop(&mut self, elem_index: u32) -> Self::Output { + fn visit_elem_drop(&mut self, _elem_index: u32) -> Self::Output { todo!() } - fn visit_table_copy(&mut self, dst_table: u32, src_table: u32) -> Self::Output { + fn visit_table_copy(&mut self, _dst_table: u32, _src_table: u32) -> Self::Output { todo!() } - fn visit_typed_select(&mut self, ty: wasmparser::ValType) -> Self::Output { + fn visit_typed_select(&mut self, _ty: wasmparser::ValType) -> Self::Output { todo!() } - fn visit_ref_null(&mut self, hty: wasmparser::HeapType) -> Self::Output { + fn visit_ref_null(&mut self, _hty: wasmparser::HeapType) -> Self::Output { todo!() } @@ -858,35 +858,35 @@ impl<'a> VisitOperator<'a> for FuncTranslator { todo!() } - fn visit_ref_func(&mut self, function_index: u32) -> Self::Output { + fn visit_ref_func(&mut self, _function_index: u32) -> Self::Output { todo!() } - fn visit_table_fill(&mut self, table: u32) -> Self::Output { + fn visit_table_fill(&mut self, _table: u32) -> Self::Output { todo!() } - fn visit_table_get(&mut self, table: u32) -> Self::Output { + fn visit_table_get(&mut self, _table: u32) -> Self::Output { todo!() } - fn visit_table_set(&mut self, table: u32) -> Self::Output { + fn visit_table_set(&mut self, _table: u32) -> Self::Output { todo!() } - fn visit_table_grow(&mut self, table: u32) -> Self::Output { + fn visit_table_grow(&mut self, _table: u32) -> Self::Output { todo!() } - fn visit_table_size(&mut self, table: u32) -> Self::Output { + fn visit_table_size(&mut self, _table: u32) -> Self::Output { todo!() } - fn visit_return_call(&mut self, function_index: u32) -> Self::Output { + fn visit_return_call(&mut self, _function_index: u32) -> Self::Output { todo!() } - fn visit_return_call_indirect(&mut self, type_index: u32, table_index: u32) -> Self::Output { + fn visit_return_call_indirect(&mut self, _type_index: u32, _table_index: u32) -> Self::Output { todo!() } From 98cf750b1eb5a446e252a88cd8e2a86130c3002d Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Sat, 7 Jun 2025 15:21:55 +0200 Subject: [PATCH 098/343] simplify translate_end_func when in unreachable state --- crates/wasmi/src/engine/translator/func2/mod.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index a40491b6ef..4d2d0b3807 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -250,6 +250,9 @@ impl FuncTranslator { /// Translates the end of the Wasm function enclosing Wasm `block`. fn translate_end_func(&mut self, frame: BlockControlFrame) -> Result<(), Error> { + if !self.reachable { + return Ok(()); + } let _fuel_info = match (&self.fuel_costs, frame.consume_fuel_instr()) { (Some(fuel_costs), Some(consume_fuel)) => { FuelInfo::some(fuel_costs.clone(), consume_fuel) @@ -258,7 +261,7 @@ impl FuncTranslator { _ => unreachable!(), }; let len_results = frame.ty().len_results(&self.engine); - if self.reachable && frame.is_branched_to() && len_results > 1 { + if frame.is_branched_to() && len_results > 1 { let height = frame.height(); let len_results = usize::from(len_results); for depth in 0..len_results { From e38e78b16374ebecc6287f4a07840bad2a642e40 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Sat, 7 Jun 2025 15:22:43 +0200 Subject: [PATCH 099/343] unsilence and fix occurrences of unused_imports --- crates/wasmi/src/engine/translator/func2/mod.rs | 6 +++--- crates/wasmi/src/engine/translator/func2/stack/mod.rs | 6 +----- crates/wasmi/src/engine/translator/func2/stack/operands.rs | 2 +- 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index 4d2d0b3807..e5b9ad4a67 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -1,4 +1,4 @@ -#![expect(dead_code, unused_imports)] +#![expect(dead_code)] #[macro_use] mod utils; @@ -11,7 +11,7 @@ mod visit; use self::{ instrs::InstrEncoder, - layout::{StackLayout, StackSpace}, + layout::StackLayout, stack::{ BlockControlFrame, ControlFrame, @@ -30,7 +30,7 @@ use self::{ use crate::{ core::{FuelCostsProvider, Typed, TypedVal, ValType}, engine::{ - translator::{utils::FuelInfo, Instr, LabelRef, LabelRegistry, WasmTranslator}, + translator::{utils::FuelInfo, Instr, LabelRegistry, WasmTranslator}, BlockType, CompiledFuncEntity, TranslationError, diff --git a/crates/wasmi/src/engine/translator/func2/stack/mod.rs b/crates/wasmi/src/engine/translator/func2/stack/mod.rs index f95e6ba176..56dcf48528 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/mod.rs @@ -23,18 +23,14 @@ use self::{ }; use super::Reset; use crate::{ - core::{TypedVal, UntypedVal, ValType}, + core::{TypedVal, ValType}, engine::{ translator::{Instr, LabelRef}, BlockType, - TranslationError, }, - ir::Reg, Engine, Error, }; -use alloc::vec::Vec; -use core::{array, mem, num::NonZero}; #[cfg(doc)] use crate::ir::Instruction; diff --git a/crates/wasmi/src/engine/translator/func2/stack/operands.rs b/crates/wasmi/src/engine/translator/func2/stack/operands.rs index c2f3ecd9d1..e8b1befb01 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/operands.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/operands.rs @@ -5,7 +5,7 @@ use crate::{ Error, }; use alloc::vec::Vec; -use core::{array, mem, num::NonZero, slice}; +use core::{num::NonZero, slice}; /// A [`StackOperand`] or [`Operand`] index on the [`OperandStack`]. #[derive(Debug, Copy, Clone, PartialEq, Eq)] From d68d2f4eb97e7108ada66ed54fa3bb0365d66a95 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Sun, 8 Jun 2025 22:17:11 +0200 Subject: [PATCH 100/343] add Stack::peek_control --- crates/wasmi/src/engine/translator/func2/stack/mod.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/crates/wasmi/src/engine/translator/func2/stack/mod.rs b/crates/wasmi/src/engine/translator/func2/stack/mod.rs index 56dcf48528..1dfd892d04 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/mod.rs @@ -319,6 +319,15 @@ impl Stack { .unwrap_or_else(|| panic!("tried to pop control from empty control stack")) } + /// Returns a shared reference to the [`ControlFrame`] at `depth`. + /// + /// # Panics + /// + /// If `depth` is out of bounds for `self`. + pub fn peek_control(&self, depth: usize) -> &ControlFrame { + self.controls.get(depth) + } + /// Pushes a local variable with index `local_idx` to the [`Stack`]. /// /// # Errors From eed2cfc7da2fce7331720b7717f357a620e1e088 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Sun, 8 Jun 2025 22:17:39 +0200 Subject: [PATCH 101/343] add InstrEncoder::{get_mut, bump_fuel_consumption} methods --- .../src/engine/translator/func2/instrs.rs | 34 ++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/func2/instrs.rs b/crates/wasmi/src/engine/translator/func2/instrs.rs index a59397f0e8..95e6abfe63 100644 --- a/crates/wasmi/src/engine/translator/func2/instrs.rs +++ b/crates/wasmi/src/engine/translator/func2/instrs.rs @@ -1,5 +1,10 @@ use super::{Instr, Reset}; -use crate::ir::Instruction; +use crate::{ + core::FuelCostsProvider, + engine::translator::{utils::FuelInfo, BumpFuelConsumption}, + ir::Instruction, + Error, +}; use alloc::vec::{self, Vec}; /// Creates and encodes the list of [`Instruction`]s for a function. @@ -47,6 +52,33 @@ impl InstrEncoder { &self.instrs[instr.into_usize()] } + /// Returns an exclusive reference to the [`Instruction`] associated to [`Instr`]. + /// + /// # Panics + /// + /// If `instr` is out of bounds for `self`. + pub fn get_mut(&mut self, instr: Instr) -> &mut Instruction { + &mut self.instrs[instr.into_usize()] + } + + /// Bumps consumed fuel for [`Instruction::ConsumeFuel`] of `instr` by `delta`. + /// + /// # Errors + /// + /// If consumed fuel is out of bounds after this operation. + pub fn bump_fuel_consumption(&mut self, fuel_info: &FuelInfo, f: F) -> Result<(), Error> + where + F: FnOnce(&FuelCostsProvider) -> u64, + { + let FuelInfo::Some { costs, instr } = fuel_info else { + // Fuel metering is disabled so we can bail out. + return Ok(()); + }; + let fuel_consumed = f(costs); + self.get_mut(*instr).bump_fuel_consumption(fuel_consumed)?; + Ok(()) + } + /// Returns an iterator yielding all [`Instruction`]s of the [`InstrEncoder`]. /// /// # Note From 474ee4cc95382a4843b005b96334976fb719633d Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Sun, 8 Jun 2025 22:18:16 +0200 Subject: [PATCH 102/343] refactor tranlate_end_func --- .../wasmi/src/engine/translator/func2/mod.rs | 26 +++++++------------ 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index e5b9ad4a67..48035d1693 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -253,13 +253,7 @@ impl FuncTranslator { if !self.reachable { return Ok(()); } - let _fuel_info = match (&self.fuel_costs, frame.consume_fuel_instr()) { - (Some(fuel_costs), Some(consume_fuel)) => { - FuelInfo::some(fuel_costs.clone(), consume_fuel) - } - (None, None) => FuelInfo::None, - _ => unreachable!(), - }; + // let _fuel_info = self.fuel_info(); let len_results = frame.ty().len_results(&self.engine); if frame.is_branched_to() && len_results > 1 { let height = frame.height(); @@ -309,13 +303,8 @@ impl FuncTranslator { } } } - self.labels - .pin_label(frame.label(), self.instrs.next_instr()) - .unwrap_or_else(|err| panic!("failed to pin label: {err}")); - match len_results { - 0 => { - self.instrs.push_instr(Instruction::Return); - } + let return_instr = match len_results { + 0 => self.instrs.push_instr(Instruction::Return), 1 => { let instr = match self.stack.peek(0) { Operand::Local(operand) => { @@ -352,15 +341,18 @@ impl FuncTranslator { } } }; - self.instrs.push_instr(instr); + self.instrs.push_instr(instr) } _ => { let height = frame.height(); let result = self.layout.temp_to_reg(OperandIdx::from(height - 1))?; let values = BoundedRegSpan::new(RegSpan::new(result), len_results); - self.instrs.push_instr(Instruction::return_span(values)); + self.instrs.push_instr(Instruction::return_span(values)) } - } + }; + self.labels + .pin_label(frame.label(), return_instr) + .unwrap_or_else(|err| panic!("failed to pin label: {err}")); Ok(()) } From 12e74f5165abc428f4cfe98cf4bce206617d2fe6 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 9 Jun 2025 12:53:30 +0200 Subject: [PATCH 103/343] add ControlFrame::len_branch_params --- .../engine/translator/func2/stack/control.rs | 42 +++++++++++++++++-- 1 file changed, 39 insertions(+), 3 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/stack/control.rs b/crates/wasmi/src/engine/translator/func2/stack/control.rs index 91795f54d3..04f25ee12b 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/control.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/control.rs @@ -1,7 +1,10 @@ use super::{Operand, Reset}; -use crate::engine::{ - translator::{Instr, LabelRef}, - BlockType, +use crate::{ + engine::{ + translator::{Instr, LabelRef}, + BlockType, + }, + Engine, }; use alloc::vec::{Drain, Vec}; @@ -278,6 +281,19 @@ impl ControlFrame { } } + /// Returns the number of operands required for branching to the [`ControlFrame`]. + pub fn len_branch_params(&self, engine: &Engine) -> u16 { + match self { + ControlFrame::Block(frame) => frame.len_branch_params(engine), + ControlFrame::Loop(frame) => frame.len_branch_params(engine), + ControlFrame::If(frame) => frame.len_branch_params(engine), + ControlFrame::Else(frame) => frame.len_branch_params(engine), + ControlFrame::Unreachable(_) => { + panic!("invalid query for unreachable control frame: `ControlFrame::len_branch_params`") + } + } + } + /// Returns a reference to the [`Instruction::ConsumeFuel`] of the [`ControlFrame`] if any. /// /// Returns `None` if fuel metering is disabled. @@ -319,6 +335,11 @@ impl BlockControlFrame { self.ty } + /// Returns the number of operands required for branching to the [`BlockControlFrame`]. + pub fn len_branch_params(&self, engine: &Engine) -> u16 { + self.ty.len_results(engine) + } + /// Returns the height of the [`BlockControlFrame`]. pub fn height(&self) -> usize { self.height @@ -372,6 +393,11 @@ impl LoopControlFrame { self.ty } + /// Returns the number of operands required for branching to the [`LoopControlFrame`]. + pub fn len_branch_params(&self, engine: &Engine) -> u16 { + self.ty.len_params(engine) + } + /// Returns the height of the [`LoopControlFrame`]. pub fn height(&self) -> usize { self.height @@ -427,6 +453,11 @@ impl IfControlFrame { self.ty } + /// Returns the number of operands required for branching to the [`IfControlFrame`]. + pub fn len_branch_params(&self, engine: &Engine) -> u16 { + self.ty.len_results(engine) + } + /// Returns the height of the [`IfControlFrame`]. pub fn height(&self) -> usize { self.height @@ -570,6 +601,11 @@ impl ElseControlFrame { self.ty } + /// Returns the number of operands required for branching to the [`ElseControlFrame`]. + pub fn len_branch_params(&self, engine: &Engine) -> u16 { + self.ty.len_results(engine) + } + /// Returns the height of the [`ElseControlFrame`]. pub fn height(&self) -> usize { self.height From 8ca512316c64d58447b7256c1f3e0d029fd122dd Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 9 Jun 2025 12:53:51 +0200 Subject: [PATCH 104/343] refactor and simplify translate_end --- .../wasmi/src/engine/translator/func2/mod.rs | 207 ++++++++++-------- 1 file changed, 112 insertions(+), 95 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index 48035d1693..e5d6e4d658 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -30,7 +30,7 @@ use self::{ use crate::{ core::{FuelCostsProvider, Typed, TypedVal, ValType}, engine::{ - translator::{utils::FuelInfo, Instr, LabelRegistry, WasmTranslator}, + translator::{Instr, LabelRegistry, WasmTranslator}, BlockType, CompiledFuncEntity, TranslationError, @@ -215,9 +215,13 @@ impl FuncTranslator { /// Returns the [`FuncType`] of the function that is currently translated. fn func_type(&self) -> FuncType { + self.func_type_with(FuncType::clone) + } + + /// Applies `f` to the [`FuncType`] of the function that is currently translated. + fn func_type_with(&self, f: impl FnOnce(&FuncType) -> R) -> R { let dedup_func_type = self.module.get_type_of_func(self.func); - self.engine() - .resolve_func_type(dedup_func_type, Clone::clone) + self.engine().resolve_func_type(dedup_func_type, f) } /// Consumes `self` and returns the underlying reusable [`FuncTranslatorAllocations`]. @@ -240,119 +244,132 @@ impl FuncTranslator { self.engine.config().get_consume_fuel() } - /// Translates the end of a Wasm `block` control frame. - fn translate_end_block(&mut self, frame: BlockControlFrame) -> Result<(), Error> { - if self.stack.is_control_empty() { - return self.translate_end_func(frame); + /// Convert all [`Operand`]s up to `depth` into [`Operand::Temp`]s by copying if necessary. + /// + /// # Note + /// + /// Does nothing if an [`Operand`] is already an [`Operand::Temp`]. + fn copy_operands_to_temp(&mut self, depth: usize) -> Result<(), Error> { + for n in 0..depth { + self.copy_operand_to_temp(n)?; } - todo!() + Ok(()) } - /// Translates the end of the Wasm function enclosing Wasm `block`. - fn translate_end_func(&mut self, frame: BlockControlFrame) -> Result<(), Error> { - if !self.reachable { - return Ok(()); - } - // let _fuel_info = self.fuel_info(); - let len_results = frame.ty().len_results(&self.engine); - if frame.is_branched_to() && len_results > 1 { - let height = frame.height(); - let len_results = usize::from(len_results); - for depth in 0..len_results { - let result = self - .layout - .temp_to_reg(OperandIdx::from(height + len_results - depth - 1))?; - match self.stack.operand_to_temp(depth) { - Some(Operand::Local(operand)) => { - let value = self.layout.local_to_reg(operand.local_index())?; - self.instrs.push_instr(Instruction::copy(result, value)); - } - Some(Operand::Immediate(operand)) => { - let val = operand.val(); - let instr = match operand.ty() { - ValType::I32 => Instruction::copy_imm32(result, i32::from(val)), - ValType::I64 => { - let val = i64::from(val); - match >::try_from(val) { - Ok(value) => Instruction::copy_i64imm32(result, value), - Err(_) => { - let value = self.layout.const_to_reg(val)?; - Instruction::copy(result, value) - } - } - } - ValType::F32 => Instruction::copy_imm32(result, f32::from(val)), - ValType::F64 => { - let val = f64::from(val); - match >::try_from(val) { - Ok(value) => Instruction::copy_f64imm32(result, value), - Err(_) => { - let value = self.layout.const_to_reg(val)?; - Instruction::copy(result, value) - } - } + /// Convert the [`Operand`] at `depth` into an [`Operand::Temp`] by copying if necessary. + /// + /// # Note + /// + /// Does nothing if the [`Operand`] is already an [`Operand::Temp`]. + fn copy_operand_to_temp(&mut self, depth: usize) -> Result<(), Error> { + let instr = match self.stack.operand_to_temp(depth) { + Some(Operand::Local(operand)) => { + let result = self.layout.temp_to_reg(operand.operand_index())?; + let value = self.layout.local_to_reg(operand.local_index())?; + Instruction::copy(result, value) + } + Some(Operand::Immediate(operand)) => { + let result = self.layout.temp_to_reg(operand.operand_index())?; + let val = operand.val(); + match operand.ty() { + ValType::I32 => Instruction::copy_imm32(result, i32::from(val)), + ValType::I64 => { + let val = i64::from(val); + match >::try_from(val) { + Ok(value) => Instruction::copy_i64imm32(result, value), + Err(_) => { + let value = self.layout.const_to_reg(val)?; + Instruction::copy(result, value) } - ValType::V128 | ValType::FuncRef | ValType::ExternRef => { + } + } + ValType::F32 => Instruction::copy_imm32(result, f32::from(val)), + ValType::F64 => { + let val = f64::from(val); + match >::try_from(val) { + Ok(value) => Instruction::copy_f64imm32(result, value), + Err(_) => { let value = self.layout.const_to_reg(val)?; Instruction::copy(result, value) } - }; - self.instrs.push_instr(instr); + } + } + ValType::V128 | ValType::FuncRef | ValType::ExternRef => { + let value = self.layout.const_to_reg(val)?; + Instruction::copy(result, value) } - _ => {} } } - } - let return_instr = match len_results { - 0 => self.instrs.push_instr(Instruction::Return), - 1 => { - let instr = match self.stack.peek(0) { - Operand::Local(operand) => { - let value = self.layout.local_to_reg(operand.local_index())?; - Instruction::return_reg(value) - } - Operand::Temp(operand) => { - let value = self.layout.temp_to_reg(operand.operand_index())?; - Instruction::return_reg(value) - } - Operand::Immediate(operand) => { - let val = operand.val(); - match operand.ty() { - ValType::I32 => Instruction::return_imm32(i32::from(val)), - ValType::I64 => match >::try_from(i64::from(val)) { - Ok(value) => Instruction::return_i64imm32(value), - Err(_) => { - let value = self.layout.const_to_reg(val)?; - Instruction::return_reg(value) - } - }, - ValType::F32 => Instruction::return_imm32(f32::from(val)), - ValType::F64 => match >::try_from(f64::from(val)) { - Ok(value) => Instruction::return_f64imm32(value), - Err(_) => { - let value = self.layout.const_to_reg(val)?; - Instruction::return_reg(value) - } - }, - ValType::V128 | ValType::FuncRef | ValType::ExternRef => { + _ => return Ok(()), + }; + self.instrs.push_instr(instr); + Ok(()) + } + + /// Translates a generic return instruction. + fn translate_return(&mut self) -> Result { + let len_results = self.func_type_with(FuncType::len_results); + let instr = match len_results { + 0 => Instruction::Return, + 1 => match self.stack.pop() { + Operand::Local(operand) => { + let value = self.layout.local_to_reg(operand.local_index())?; + Instruction::return_reg(value) + } + Operand::Temp(operand) => { + let value = self.layout.temp_to_reg(operand.operand_index())?; + Instruction::return_reg(value) + } + Operand::Immediate(operand) => { + let val = operand.val(); + match operand.ty() { + ValType::I32 => Instruction::return_imm32(i32::from(val)), + ValType::I64 => match >::try_from(i64::from(val)) { + Ok(value) => Instruction::return_i64imm32(value), + Err(_) => { let value = self.layout.const_to_reg(val)?; Instruction::return_reg(value) } + }, + ValType::F32 => Instruction::return_imm32(f32::from(val)), + ValType::F64 => match >::try_from(f64::from(val)) { + Ok(value) => Instruction::return_f64imm32(value), + Err(_) => { + let value = self.layout.const_to_reg(val)?; + Instruction::return_reg(value) + } + }, + ValType::V128 | ValType::FuncRef | ValType::ExternRef => { + let value = self.layout.const_to_reg(val)?; + Instruction::return_reg(value) } } - }; - self.instrs.push_instr(instr) - } + } + }, _ => { - let height = frame.height(); - let result = self.layout.temp_to_reg(OperandIdx::from(height - 1))?; + let depth = usize::from(len_results); + self.copy_operands_to_temp(depth)?; + let first_idx = self.stack.peek(depth).index(); + let result = self.layout.temp_to_reg(first_idx)?; let values = BoundedRegSpan::new(RegSpan::new(result), len_results); - self.instrs.push_instr(Instruction::return_span(values)) + Instruction::return_span(values) } }; + Ok(self.instrs.push_instr(instr)) + } + + /// Translates the end of a Wasm `block` control frame. + fn translate_end_block(&mut self, frame: BlockControlFrame) -> Result<(), Error> { + let len_values = frame.len_branch_params(&self.engine); + if self.reachable && frame.is_branched_to() && len_values > 1 { + self.copy_operands_to_temp(usize::from(len_values))?; + } self.labels - .pin_label(frame.label(), return_instr) + .pin_label(frame.label(), self.instrs.next_instr()) .unwrap_or_else(|err| panic!("failed to pin label: {err}")); + if self.stack.is_control_empty() { + self.translate_return()?; + } Ok(()) } From 6b908e453b184fe322de8518feafd4644466dc62 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 9 Jun 2025 12:55:50 +0200 Subject: [PATCH 105/343] format the code in a more readable way --- crates/wasmi/src/engine/translator/func2/mod.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index e5d6e4d658..b92bc3eeab 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -364,9 +364,12 @@ impl FuncTranslator { if self.reachable && frame.is_branched_to() && len_values > 1 { self.copy_operands_to_temp(usize::from(len_values))?; } - self.labels + if let Err(err) = self + .labels .pin_label(frame.label(), self.instrs.next_instr()) - .unwrap_or_else(|err| panic!("failed to pin label: {err}")); + { + panic!("failed to pin label: {err}") + } if self.stack.is_control_empty() { self.translate_return()?; } From 0feb63fd62b368d7aa01d85a7b00a3203cecc032 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 9 Jun 2025 14:10:54 +0200 Subject: [PATCH 106/343] no longer pop operands in translate_return --- crates/wasmi/src/engine/translator/func2/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index b92bc3eeab..c700d9e409 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -311,7 +311,7 @@ impl FuncTranslator { let len_results = self.func_type_with(FuncType::len_results); let instr = match len_results { 0 => Instruction::Return, - 1 => match self.stack.pop() { + 1 => match self.stack.peek(0) { Operand::Local(operand) => { let value = self.layout.local_to_reg(operand.local_index())?; Instruction::return_reg(value) From 594a8566fc359c947f49d320a8881a99490a7225 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 9 Jun 2025 14:11:24 +0200 Subject: [PATCH 107/343] trunc operands in translate_end_block --- crates/wasmi/src/engine/translator/func2/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index c700d9e409..5486d20a82 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -373,6 +373,7 @@ impl FuncTranslator { if self.stack.is_control_empty() { self.translate_return()?; } + self.stack.trunc(frame.height()); Ok(()) } From a0f45d5e4f792098f983e3e53d83349c0018d9b6 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 9 Jun 2025 14:11:44 +0200 Subject: [PATCH 108/343] fix Stack::trunc method --- crates/wasmi/src/engine/translator/func2/stack/mod.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/func2/stack/mod.rs b/crates/wasmi/src/engine/translator/func2/stack/mod.rs index 1dfd892d04..dbbba86233 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/mod.rs @@ -126,7 +126,10 @@ impl Stack { /// /// If `height` is greater than the current height of `self`. pub fn trunc(&mut self, height: usize) { - self.operands.trunc(height); + debug_assert!(height <= self.height()); + while self.height() > height { + self.pop(); + } } /// Returns `true` is fuel metering is enabled for the associated [`Engine`]. From d7ebd2851ea6a726eb18bb5b4ba6e027001f016a Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 9 Jun 2025 14:26:11 +0200 Subject: [PATCH 109/343] no longer manipulate stack in copy_operand[s]_to_temp methods --- crates/wasmi/src/engine/translator/func2/mod.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index 5486d20a82..b5aacd71bb 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -262,13 +262,13 @@ impl FuncTranslator { /// /// Does nothing if the [`Operand`] is already an [`Operand::Temp`]. fn copy_operand_to_temp(&mut self, depth: usize) -> Result<(), Error> { - let instr = match self.stack.operand_to_temp(depth) { - Some(Operand::Local(operand)) => { + let instr = match self.stack.peek(depth) { + Operand::Local(operand) => { let result = self.layout.temp_to_reg(operand.operand_index())?; let value = self.layout.local_to_reg(operand.local_index())?; Instruction::copy(result, value) } - Some(Operand::Immediate(operand)) => { + Operand::Immediate(operand) => { let result = self.layout.temp_to_reg(operand.operand_index())?; let val = operand.val(); match operand.ty() { @@ -300,7 +300,7 @@ impl FuncTranslator { } } } - _ => return Ok(()), + Operand::Temp(_) => return Ok(()), }; self.instrs.push_instr(instr); Ok(()) From 119310f3e2f388bdef98c82d4094feab9dc8a2d9 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 9 Jun 2025 14:26:52 +0200 Subject: [PATCH 110/343] move trivial match arm to top --- crates/wasmi/src/engine/translator/func2/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index b5aacd71bb..dbf4ef53fb 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -263,6 +263,7 @@ impl FuncTranslator { /// Does nothing if the [`Operand`] is already an [`Operand::Temp`]. fn copy_operand_to_temp(&mut self, depth: usize) -> Result<(), Error> { let instr = match self.stack.peek(depth) { + Operand::Temp(_) => return Ok(()), Operand::Local(operand) => { let result = self.layout.temp_to_reg(operand.operand_index())?; let value = self.layout.local_to_reg(operand.local_index())?; @@ -300,7 +301,6 @@ impl FuncTranslator { } } } - Operand::Temp(_) => return Ok(()), }; self.instrs.push_instr(instr); Ok(()) From 71e9457f76c7b80f14c7ec35af2777a8e4630a4d Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 9 Jun 2025 14:38:51 +0200 Subject: [PATCH 111/343] refactor FuncTranslator::is_fuel_metering_enabled --- crates/wasmi/src/engine/translator/func2/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index dbf4ef53fb..fff0672897 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -241,7 +241,7 @@ impl FuncTranslator { /// Returns `true` if fuel metering is enabled. fn is_fuel_metering_enabled(&self) -> bool { - self.engine.config().get_consume_fuel() + self.fuel_costs.is_some() } /// Convert all [`Operand`]s up to `depth` into [`Operand::Temp`]s by copying if necessary. From abd129a3f1fac3103c18dbc4412759ca049a7091 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 9 Jun 2025 14:46:24 +0200 Subject: [PATCH 112/343] return Operand in Stack::operator_to_temp --- .../src/engine/translator/func2/stack/mod.rs | 10 ++++---- .../engine/translator/func2/stack/operands.rs | 23 ++++++++----------- 2 files changed, 14 insertions(+), 19 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/stack/mod.rs b/crates/wasmi/src/engine/translator/func2/stack/mod.rs index dbbba86233..19ba7fe80a 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/mod.rs @@ -215,9 +215,7 @@ impl Stack { let len_params = usize::from(ty.len_params(&self.engine)); let block_height = self.height() - len_params; for depth in 0..block_height { - if let Some(operand) = self.operand_to_temp(depth) { - f(operand)?; - } + f(self.operand_to_temp(depth))?; } self.controls .push_loop(ty, block_height, label, consume_fuel); @@ -429,17 +427,17 @@ impl Stack { self.operands.preserve_locals(local_index) } - /// Converts and returns the [`StackOperand`] at `depth` into a [`Operand::Temp`]. + /// Converts and returns the [`Operand`] at `depth` into a [`Operand::Temp`]. /// /// # Note /// - /// Returns `None` if operand at `depth` is [`Operand::Temp`] already. + /// Returns the [`Operand`] at `depth` before being converted to an [`Operand::Temp`]. /// /// # Panics /// /// If `depth` is out of bounds for the [`Stack`] of operands. #[must_use] - pub fn operand_to_temp(&mut self, depth: usize) -> Option { + pub fn operand_to_temp(&mut self, depth: usize) -> Operand { self.operands.operand_to_temp(depth) } } diff --git a/crates/wasmi/src/engine/translator/func2/stack/operands.rs b/crates/wasmi/src/engine/translator/func2/stack/operands.rs index e8b1befb01..276fd4cb15 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/operands.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/operands.rs @@ -229,20 +229,20 @@ impl OperandStack { self.operands[usize::from(index)] } - /// Converts and returns the [`StackOperand`] at `depth` into a [`Operand::Temp`]. + /// Converts and returns the [`Operand`] at `depth` into a [`Operand::Temp`]. /// /// # Note /// - /// Returns `None` if operand at `depth` is [`Operand::Temp`] already. + /// Returns the [`Operand`] at `depth` before being converted to an [`Operand::Temp`]. /// /// # Panics /// - /// If `depth` is out of bounds for `self`. + /// If `depth` is out of bounds for the [`Stack`] of operands. #[must_use] - pub fn operand_to_temp(&mut self, depth: usize) -> Option { + pub fn operand_to_temp(&mut self, depth: usize) -> Operand { let index = self.depth_to_index(depth); - let operand = self.operand_to_temp_at(index)?; - Some(Operand::new(index, operand, &self.locals)) + let operand = self.operand_to_temp_at(index); + Operand::new(index, operand, &self.locals) } /// Converts and returns the [`StackOperand`] at `index` into a [`StackOperand::Temp`]. @@ -255,16 +255,16 @@ impl OperandStack { /// /// If `index` is out of bounds for `self`. #[must_use] - fn operand_to_temp_at(&mut self, index: OperandIdx) -> Option { + fn operand_to_temp_at(&mut self, index: OperandIdx) -> StackOperand { let operand = self.get_at(index); match operand { - StackOperand::Temp { .. } => return None, + StackOperand::Temp { ty, .. } => ty, StackOperand::Immediate { val } => val.ty(), StackOperand::Local { local_index, .. } => self.locals.ty(local_index), }; self.unlink_local(operand); self.operands[usize::from(index)] = operand; - Some(operand) + operand } /// Preserve all locals on the [`OperandStack`] that refer to `local_index`. @@ -360,10 +360,7 @@ impl Iterator for PreservedLocalsIter<'_> { fn next(&mut self) -> Option { let index = self.index?; - let operand = self - .operands - .operand_to_temp_at(index) - .expect("local operand in linked list"); + let operand = self.operands.operand_to_temp_at(index); self.index = match operand { StackOperand::Local { next_local, .. } => next_local, op => panic!("expected `StackOperand::Local` but found: {op:?}"), From 7600fc0c57fe83576d39c4143a04323f7f4f9446 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 9 Jun 2025 14:59:06 +0200 Subject: [PATCH 113/343] refactor new translator --- .../wasmi/src/engine/translator/func2/mod.rs | 32 ++++++++++++++----- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index fff0672897..f518966213 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -244,14 +244,31 @@ impl FuncTranslator { self.fuel_costs.is_some() } - /// Convert all [`Operand`]s up to `depth` into [`Operand::Temp`]s by copying if necessary. + /// Convert all branch params up to `depth` to [`Operand::Temp`]. /// /// # Note /// - /// Does nothing if an [`Operand`] is already an [`Operand::Temp`]. + /// - The top-most `depth` operands on the [`Stack`] will be [`Operand::Temp`] upon completion. + /// - Does nothing if an [`Operand`] is already an [`Operand::Temp`]. + fn copy_branch_params(&mut self, depth: usize) -> Result<(), Error> { + for n in 0..depth { + let operand = self.stack.operand_to_temp(n); + self.copy_operand_to_temp(operand)?; + } + Ok(()) + } + + /// Copy all [`Operand`]s up to `depth` into [`Operand::Temp`]s by copying if necessary. + /// + /// + /// # Note + /// + /// - This does _not_ manipulate the [`Stack`]. + /// - Does nothing if an [`Operand`] is already an [`Operand::Temp`]. fn copy_operands_to_temp(&mut self, depth: usize) -> Result<(), Error> { for n in 0..depth { - self.copy_operand_to_temp(n)?; + let operand = self.stack.peek(n); + self.copy_operand_to_temp(operand)?; } Ok(()) } @@ -261,8 +278,8 @@ impl FuncTranslator { /// # Note /// /// Does nothing if the [`Operand`] is already an [`Operand::Temp`]. - fn copy_operand_to_temp(&mut self, depth: usize) -> Result<(), Error> { - let instr = match self.stack.peek(depth) { + fn copy_operand_to_temp(&mut self, operand: Operand) -> Result<(), Error> { + let instr = match operand { Operand::Temp(_) => return Ok(()), Operand::Local(operand) => { let result = self.layout.temp_to_reg(operand.operand_index())?; @@ -361,8 +378,8 @@ impl FuncTranslator { /// Translates the end of a Wasm `block` control frame. fn translate_end_block(&mut self, frame: BlockControlFrame) -> Result<(), Error> { let len_values = frame.len_branch_params(&self.engine); - if self.reachable && frame.is_branched_to() && len_values > 1 { - self.copy_operands_to_temp(usize::from(len_values))?; + if self.reachable && frame.is_branched_to() { + self.copy_branch_params(usize::from(len_values))?; } if let Err(err) = self .labels @@ -373,7 +390,6 @@ impl FuncTranslator { if self.stack.is_control_empty() { self.translate_return()?; } - self.stack.trunc(frame.height()); Ok(()) } From e90f33aa1a5bd1d8aa7aa21bd7baaec8adaa5014 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 9 Jun 2025 15:02:03 +0200 Subject: [PATCH 114/343] move into_allocations method higher in file --- .../wasmi/src/engine/translator/func2/mod.rs | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index f518966213..d0f787cc43 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -213,6 +213,16 @@ impl FuncTranslator { Ok(()) } + /// Consumes `self` and returns the underlying reusable [`FuncTranslatorAllocations`]. + fn into_allocations(self) -> FuncTranslatorAllocations { + FuncTranslatorAllocations { + stack: self.stack.into_allocations(), + layout: self.layout, + labels: self.labels, + instrs: self.instrs, + } + } + /// Returns the [`FuncType`] of the function that is currently translated. fn func_type(&self) -> FuncType { self.func_type_with(FuncType::clone) @@ -224,16 +234,6 @@ impl FuncTranslator { self.engine().resolve_func_type(dedup_func_type, f) } - /// Consumes `self` and returns the underlying reusable [`FuncTranslatorAllocations`]. - fn into_allocations(self) -> FuncTranslatorAllocations { - FuncTranslatorAllocations { - stack: self.stack.into_allocations(), - layout: self.layout, - labels: self.labels, - instrs: self.instrs, - } - } - /// Returns the [`Engine`] for which the function is compiled. fn engine(&self) -> &Engine { &self.engine From ccd60e7463cd12277cfe5676f564e4c094239b7d Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 9 Jun 2025 17:28:16 +0200 Subject: [PATCH 115/343] fix intra doc link --- crates/wasmi/src/engine/translator/func2/stack/operands.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/func2/stack/operands.rs b/crates/wasmi/src/engine/translator/func2/stack/operands.rs index 276fd4cb15..50079c0446 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/operands.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/operands.rs @@ -237,7 +237,7 @@ impl OperandStack { /// /// # Panics /// - /// If `depth` is out of bounds for the [`Stack`] of operands. + /// If `depth` is out of bounds for the [`OperandStack`] of operands. #[must_use] pub fn operand_to_temp(&mut self, depth: usize) -> Operand { let index = self.depth_to_index(depth); From d5abbd6ca4c23b9c3903478f9305c9e9f4cfb911 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 9 Jun 2025 17:33:44 +0200 Subject: [PATCH 116/343] add experimental-translator crate feature to wasmi_wast crate --- crates/wast/Cargo.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/wast/Cargo.toml b/crates/wast/Cargo.toml index 52b280f389..9c189ad104 100644 --- a/crates/wast/Cargo.toml +++ b/crates/wast/Cargo.toml @@ -17,3 +17,6 @@ exclude.workspace = true wasmi = { workspace = true, features = ["std", "simd"] } wast = { workspace = true, features = ["wasm-module"] } anyhow = "1.0" + +[features] +experimental-translator = ["wasmi/experimental-translator"] From 7d6105674a7fd3c2c7c595e79dbe03e662b6e835 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 9 Jun 2025 17:37:18 +0200 Subject: [PATCH 117/343] properly update reachability in translate_end_block --- crates/wasmi/src/engine/translator/func2/mod.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index d0f787cc43..5dc9d38316 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -387,7 +387,8 @@ impl FuncTranslator { { panic!("failed to pin label: {err}") } - if self.stack.is_control_empty() { + self.reachable |= frame.is_branched_to(); + if self.reachable && self.stack.is_control_empty() { self.translate_return()?; } Ok(()) From 12e6630a62b8fe5cce2201919ebc851558068534 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 9 Jun 2025 17:40:46 +0200 Subject: [PATCH 118/343] implement translate_end_loop --- crates/wasmi/src/engine/translator/func2/mod.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index 5dc9d38316..02d709cbc4 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -396,7 +396,12 @@ impl FuncTranslator { /// Translates the end of a Wasm `loop` control frame. fn translate_end_loop(&mut self, _frame: LoopControlFrame) -> Result<(), Error> { - todo!() + debug_assert!( + !self.stack.is_control_empty(), + "control stack must not be empty since its first element is always a `block`" + ); + // Nothing needs to be done since Wasm `loop` control frames always only have a single exit. + Ok(()) } /// Translates the end of a Wasm `if` control frame. From 3b719ba0cfdda27490151534e65a62570ac2318b Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 11 Jun 2025 14:03:49 +0200 Subject: [PATCH 119/343] add ReusableAllocations trait --- crates/wasmi/src/engine/translator/func2/mod.rs | 2 +- crates/wasmi/src/engine/translator/func2/utils.rs | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index 02d709cbc4..75bd68c1a6 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -25,7 +25,7 @@ use self::{ StackAllocations, UnreachableControlFrame, }, - utils::Reset, + utils::{Reset, ReusableAllocations}, }; use crate::{ core::{FuelCostsProvider, Typed, TypedVal, ValType}, diff --git a/crates/wasmi/src/engine/translator/func2/utils.rs b/crates/wasmi/src/engine/translator/func2/utils.rs index 8c3b47040e..36e1644ef2 100644 --- a/crates/wasmi/src/engine/translator/func2/utils.rs +++ b/crates/wasmi/src/engine/translator/func2/utils.rs @@ -28,3 +28,12 @@ pub trait Reset: Sized { this } } + +/// Types that have reusable heap allocations. +pub trait ReusableAllocations { + /// The type of the reusable heap allocations. + type Allocations: Default + Reset; + + /// Returns the reusable heap allocations of `self`. + fn into_allocations(self) -> Self::Allocations; +} From e187591027a54773fad9e5321af880fc9484eb4e Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 11 Jun 2025 14:04:29 +0200 Subject: [PATCH 120/343] add InstrEncoderAllocations --- .../src/engine/translator/func2/instrs.rs | 32 ++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/func2/instrs.rs b/crates/wasmi/src/engine/translator/func2/instrs.rs index 95e6abfe63..ba0a9a9acd 100644 --- a/crates/wasmi/src/engine/translator/func2/instrs.rs +++ b/crates/wasmi/src/engine/translator/func2/instrs.rs @@ -1,4 +1,4 @@ -use super::{Instr, Reset}; +use super::{Instr, Reset, ReusableAllocations}; use crate::{ core::FuelCostsProvider, engine::translator::{utils::FuelInfo, BumpFuelConsumption}, @@ -14,6 +14,29 @@ pub struct InstrEncoder { instrs: Vec, } +impl ReusableAllocations for InstrEncoder { + type Allocations = InstrEncoderAllocations; + + fn into_allocations(self) -> Self::Allocations { + Self::Allocations { + instrs: self.instrs, + } + } +} + +/// The reusable heap allocations of the [`InstrEncoder`]. +#[derive(Debug, Default)] +pub struct InstrEncoderAllocations { + /// The list of constructed instructions and their parameters. + instrs: Vec, +} + +impl Reset for InstrEncoderAllocations { + fn reset(&mut self) { + self.instrs.clear(); + } +} + impl Reset for InstrEncoder { fn reset(&mut self) { self.instrs.clear(); @@ -21,6 +44,13 @@ impl Reset for InstrEncoder { } impl InstrEncoder { + /// Creates a new [`InstrEncoder`]. + pub fn new(engine: &Engine, alloc: InstrEncoderAllocations) -> Self { + Self { + instrs: alloc.instrs, + } + } + /// Returns the next [`Instr`]. #[must_use] pub fn next_instr(&self) -> Instr { From 06d01a23ed1e1f90232ea4b065e881534a9d77ad Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 11 Jun 2025 14:04:53 +0200 Subject: [PATCH 121/343] add fuel_costs to InstrEncoder --- crates/wasmi/src/engine/translator/func2/instrs.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crates/wasmi/src/engine/translator/func2/instrs.rs b/crates/wasmi/src/engine/translator/func2/instrs.rs index ba0a9a9acd..90bfca5108 100644 --- a/crates/wasmi/src/engine/translator/func2/instrs.rs +++ b/crates/wasmi/src/engine/translator/func2/instrs.rs @@ -3,6 +3,7 @@ use crate::{ core::FuelCostsProvider, engine::translator::{utils::FuelInfo, BumpFuelConsumption}, ir::Instruction, + Engine, Error, }; use alloc::vec::{self, Vec}; @@ -12,6 +13,8 @@ use alloc::vec::{self, Vec}; pub struct InstrEncoder { /// The list of constructed instructions and their parameters. instrs: Vec, + /// The fuel costs of instructions. + fuel_costs: FuelCostsProvider, } impl ReusableAllocations for InstrEncoder { @@ -48,6 +51,7 @@ impl InstrEncoder { pub fn new(engine: &Engine, alloc: InstrEncoderAllocations) -> Self { Self { instrs: alloc.instrs, + fuel_costs: engine.config().fuel_costs().clone(), } } From 3b49f253aaa2308300bd57b9b2eadbf8eb48f1fe Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 11 Jun 2025 14:05:59 +0200 Subject: [PATCH 122/343] make full use of the new ReusableAllocations trait --- .../wasmi/src/engine/translator/func2/mod.rs | 28 +++++++++-------- .../src/engine/translator/func2/stack/mod.rs | 30 +++++++++---------- 2 files changed, 30 insertions(+), 28 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index 75bd68c1a6..c63d8c5124 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -10,7 +10,7 @@ mod stack; mod visit; use self::{ - instrs::InstrEncoder, + instrs::{InstrEncoder, InstrEncoderAllocations}, layout::StackLayout, stack::{ BlockControlFrame, @@ -92,7 +92,7 @@ pub struct FuncTranslatorAllocations { /// Registers and pins labels and tracks their users. labels: LabelRegistry, /// Constructs and encodes function instructions. - instrs: InstrEncoder, + instrs: InstrEncoderAllocations, } impl Reset for FuncTranslatorAllocations { @@ -149,6 +149,19 @@ impl WasmTranslator<'_> for FuncTranslator { } } +impl ReusableAllocations for FuncTranslator { + type Allocations = FuncTranslatorAllocations; + + fn into_allocations(self) -> Self::Allocations { + Self::Allocations { + stack: self.stack.into_allocations(), + layout: self.layout, + labels: self.labels, + instrs: self.instrs.into_allocations(), + } + } +} + impl FuncTranslator { /// Creates a new [`FuncTranslator`]. pub fn new( @@ -174,6 +187,7 @@ impl FuncTranslator { instrs, } = alloc.into_reset(); let stack = Stack::new(&engine, stack); + let instrs = InstrEncoder::new(&engine, instrs); let mut translator = Self { func, engine, @@ -213,16 +227,6 @@ impl FuncTranslator { Ok(()) } - /// Consumes `self` and returns the underlying reusable [`FuncTranslatorAllocations`]. - fn into_allocations(self) -> FuncTranslatorAllocations { - FuncTranslatorAllocations { - stack: self.stack.into_allocations(), - layout: self.layout, - labels: self.labels, - instrs: self.instrs, - } - } - /// Returns the [`FuncType`] of the function that is currently translated. fn func_type(&self) -> FuncType { self.func_type_with(FuncType::clone) diff --git a/crates/wasmi/src/engine/translator/func2/stack/mod.rs b/crates/wasmi/src/engine/translator/func2/stack/mod.rs index 19ba7fe80a..de129212d6 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/mod.rs @@ -21,7 +21,7 @@ use self::{ locals::LocalsRegistry, operands::{OperandStack, StackOperand}, }; -use super::Reset; +use super::{Reset, ReusableAllocations}; use crate::{ core::{TypedVal, ValType}, engine::{ @@ -62,28 +62,26 @@ impl Reset for StackAllocations { } } -impl Stack { - /// Creates a new empty [`Stack`] from the given `engine`. - pub fn new(engine: &Engine, alloc: StackAllocations) -> Self { - Self { - engine: engine.clone(), - operands: alloc.operands, - controls: alloc.controls, - } - } +impl ReusableAllocations for Stack { + type Allocations = StackAllocations; - /// Returns the reusable [`StackAllocations`] of `self`. - pub fn into_allocations(self) -> StackAllocations { + fn into_allocations(self) -> StackAllocations { StackAllocations { operands: self.operands, controls: self.controls, } } +} - /// Resets the [`Stack`] for reuse. - pub fn reset(&mut self) { - self.operands.reset(); - self.controls.reset(); +impl Stack { + /// Creates a new empty [`Stack`] from the given `engine`. + pub fn new(engine: &Engine, alloc: StackAllocations) -> Self { + let StackAllocations { operands, controls } = alloc.into_reset(); + Self { + engine: engine.clone(), + operands, + controls, + } } /// Register `amount` local variables of common type `ty`. From 6f5b6b6b45a9a6277d98915187149d7e3777644e Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 11 Jun 2025 16:22:21 +0200 Subject: [PATCH 123/343] add InstrEncoder::push_consume_fuel_instr --- .../wasmi/src/engine/translator/func2/instrs.rs | 15 +++++++++++++++ crates/wasmi/src/engine/translator/func2/mod.rs | 3 ++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/func2/instrs.rs b/crates/wasmi/src/engine/translator/func2/instrs.rs index 90bfca5108..4d611c6077 100644 --- a/crates/wasmi/src/engine/translator/func2/instrs.rs +++ b/crates/wasmi/src/engine/translator/func2/instrs.rs @@ -61,6 +61,21 @@ impl InstrEncoder { Instr::from_usize(self.instrs.len()) } + /// Pushes an [`Instruction::ConsumeFuel`] instruction to `self`. + /// + /// # Note + /// + /// The pushes [`Instruction::ConsumeFuel`] is initialized with base fuel costs. + pub fn push_consume_fuel_instr(&mut self) -> Result { + let base_costs = self.fuel_costs.base(); + let Ok(base_costs) = u32::try_from(base_costs) else { + panic!("out of bounds base fuel costs: {base_costs}"); + }; + let instr = self.next_instr(); + self.instrs.push(Instruction::consume_fuel(base_costs)); + Ok(instr) + } + /// Pushes an [`Instruction`] to the [`InstrEncoder`]. /// /// Returns an [`Instr`] that refers to the pushed [`Instruction`]. diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index c63d8c5124..49b0965684 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -212,7 +212,8 @@ impl FuncTranslator { let consume_fuel = self .fuel_costs .as_ref() - .map(|_| self.instrs.push_instr(Instruction::consume_fuel(1))); + .map(|_| self.instrs.push_consume_fuel_instr()) + .transpose()?; self.stack .push_func_block(block_ty, end_label, consume_fuel)?; Ok(()) From 59231860992247269fe889586332fd8e3f0778e6 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 11 Jun 2025 16:41:49 +0200 Subject: [PATCH 124/343] add Stack::consume_fuel_instr getter --- crates/wasmi/src/engine/translator/func2/stack/mod.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/crates/wasmi/src/engine/translator/func2/stack/mod.rs b/crates/wasmi/src/engine/translator/func2/stack/mod.rs index de129212d6..4eaa930801 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/mod.rs @@ -438,4 +438,13 @@ impl Stack { pub fn operand_to_temp(&mut self, depth: usize) -> Operand { self.operands.operand_to_temp(depth) } + + /// Returns the current [`Instruction::ConsumeFuel`] if fuel metering is enabled. + /// + /// Returns `None` otherwise. + pub fn consume_fuel_instr(&self) -> Option { + std::println!("self.controls = {:?}", self.controls); + debug_assert!(!self.controls.is_empty()); + self.controls.get(0).consume_fuel_instr() + } } From 42c19f8b8ea2821a6ad938aed3a80eeae3850bb2 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 11 Jun 2025 16:42:12 +0200 Subject: [PATCH 125/343] apply rustfmt --- crates/wasmi/src/engine/translator/func2/instrs.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/instrs.rs b/crates/wasmi/src/engine/translator/func2/instrs.rs index 4d611c6077..b90986c22e 100644 --- a/crates/wasmi/src/engine/translator/func2/instrs.rs +++ b/crates/wasmi/src/engine/translator/func2/instrs.rs @@ -62,9 +62,9 @@ impl InstrEncoder { } /// Pushes an [`Instruction::ConsumeFuel`] instruction to `self`. - /// + /// /// # Note - /// + /// /// The pushes [`Instruction::ConsumeFuel`] is initialized with base fuel costs. pub fn push_consume_fuel_instr(&mut self) -> Result { let base_costs = self.fuel_costs.base(); From 42312933610091769fa6159abb9a1c193028cac7 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 11 Jun 2025 16:43:14 +0200 Subject: [PATCH 126/343] add fuel metering support to InstrEncoder APIs --- .../src/engine/translator/func2/instrs.rs | 29 ++++++----- .../wasmi/src/engine/translator/func2/mod.rs | 48 ++++++++++++++----- 2 files changed, 55 insertions(+), 22 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/instrs.rs b/crates/wasmi/src/engine/translator/func2/instrs.rs index b90986c22e..6983b17730 100644 --- a/crates/wasmi/src/engine/translator/func2/instrs.rs +++ b/crates/wasmi/src/engine/translator/func2/instrs.rs @@ -1,7 +1,7 @@ use super::{Instr, Reset, ReusableAllocations}; use crate::{ core::FuelCostsProvider, - engine::translator::{utils::FuelInfo, BumpFuelConsumption}, + engine::translator::BumpFuelConsumption, ir::Instruction, Engine, Error, @@ -79,10 +79,16 @@ impl InstrEncoder { /// Pushes an [`Instruction`] to the [`InstrEncoder`]. /// /// Returns an [`Instr`] that refers to the pushed [`Instruction`]. - pub fn push_instr(&mut self, instruction: Instruction) -> Instr { + pub fn push_instr( + &mut self, + instruction: Instruction, + consume_fuel: Option, + f: impl FnOnce(&FuelCostsProvider) -> u64, + ) -> Result { + self.bump_fuel_consumption(consume_fuel, f)?; let instr = self.next_instr(); self.instrs.push(instruction); - instr + Ok(instr) } /// Pushes an [`Instruction`] parameter to the [`InstrEncoder`]. @@ -115,16 +121,17 @@ impl InstrEncoder { /// # Errors /// /// If consumed fuel is out of bounds after this operation. - pub fn bump_fuel_consumption(&mut self, fuel_info: &FuelInfo, f: F) -> Result<(), Error> - where - F: FnOnce(&FuelCostsProvider) -> u64, - { - let FuelInfo::Some { costs, instr } = fuel_info else { - // Fuel metering is disabled so we can bail out. + pub fn bump_fuel_consumption( + &mut self, + consume_fuel: Option, + f: impl FnOnce(&FuelCostsProvider) -> u64, + ) -> Result<(), Error> { + let Some(consume_fuel) = consume_fuel else { return Ok(()); }; - let fuel_consumed = f(costs); - self.get_mut(*instr).bump_fuel_consumption(fuel_consumed)?; + let fuel_consumed = f(&self.fuel_costs); + self.get_mut(consume_fuel) + .bump_fuel_consumption(fuel_consumed)?; Ok(()) } diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index 49b0965684..18247359e5 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -255,10 +255,14 @@ impl FuncTranslator { /// /// - The top-most `depth` operands on the [`Stack`] will be [`Operand::Temp`] upon completion. /// - Does nothing if an [`Operand`] is already an [`Operand::Temp`]. - fn copy_branch_params(&mut self, depth: usize) -> Result<(), Error> { + fn copy_branch_params( + &mut self, + depth: usize, + consume_fuel: Option, + ) -> Result<(), Error> { for n in 0..depth { let operand = self.stack.operand_to_temp(n); - self.copy_operand_to_temp(operand)?; + self.copy_operand_to_temp(operand, consume_fuel)?; } Ok(()) } @@ -271,9 +275,10 @@ impl FuncTranslator { /// - This does _not_ manipulate the [`Stack`]. /// - Does nothing if an [`Operand`] is already an [`Operand::Temp`]. fn copy_operands_to_temp(&mut self, depth: usize) -> Result<(), Error> { + let consume_fuel = self.stack.consume_fuel_instr(); for n in 0..depth { let operand = self.stack.peek(n); - self.copy_operand_to_temp(operand)?; + self.copy_operand_to_temp(operand, consume_fuel)?; } Ok(()) } @@ -283,7 +288,11 @@ impl FuncTranslator { /// # Note /// /// Does nothing if the [`Operand`] is already an [`Operand::Temp`]. - fn copy_operand_to_temp(&mut self, operand: Operand) -> Result<(), Error> { + fn copy_operand_to_temp( + &mut self, + operand: Operand, + consume_fuel: Option, + ) -> Result<(), Error> { let instr = match operand { Operand::Temp(_) => return Ok(()), Operand::Local(operand) => { @@ -324,12 +333,13 @@ impl FuncTranslator { } } }; - self.instrs.push_instr(instr); + self.instrs + .push_instr(instr, consume_fuel, FuelCostsProvider::base)?; Ok(()) } /// Translates a generic return instruction. - fn translate_return(&mut self) -> Result { + fn translate_return(&mut self, consume_fuel: Option) -> Result { let len_results = self.func_type_with(FuncType::len_results); let instr = match len_results { 0 => Instruction::Return, @@ -377,14 +387,18 @@ impl FuncTranslator { Instruction::return_span(values) } }; - Ok(self.instrs.push_instr(instr)) + let instr = self + .instrs + .push_instr(instr, consume_fuel, FuelCostsProvider::base)?; + Ok(instr) } /// Translates the end of a Wasm `block` control frame. fn translate_end_block(&mut self, frame: BlockControlFrame) -> Result<(), Error> { let len_values = frame.len_branch_params(&self.engine); + let consume_fuel_instr = frame.consume_fuel_instr(); if self.reachable && frame.is_branched_to() { - self.copy_branch_params(usize::from(len_values))?; + self.copy_branch_params(usize::from(len_values), consume_fuel_instr)?; } if let Err(err) = self .labels @@ -394,7 +408,7 @@ impl FuncTranslator { } self.reachable |= frame.is_branched_to(); if self.reachable && self.stack.is_control_empty() { - self.translate_return()?; + self.translate_return(consume_fuel_instr)?; } Ok(()) } @@ -436,6 +450,7 @@ impl FuncTranslator { R: Into + Typed, { bail_unreachable!(self); + let consume_fuel = self.stack.consume_fuel_instr(); match self.stack.pop2() { (Operand::Immediate(lhs), Operand::Immediate(rhs)) => { let value = consteval(lhs.val().into(), rhs.val().into()); @@ -455,7 +470,11 @@ impl FuncTranslator { make_instr(result, lhs, rhs) } }; - assert_eq!(self.instrs.push_instr(instr), iidx); + assert_eq!( + self.instrs + .push_instr(instr, consume_fuel, FuelCostsProvider::base)?, + iidx + ); Ok(()) } (lhs, rhs) => { @@ -465,7 +484,14 @@ impl FuncTranslator { let result = self .layout .temp_to_reg(self.stack.push_temp(::TY, Some(iidx))?)?; - assert_eq!(self.instrs.push_instr(make_instr(result, lhs, rhs)), iidx); + assert_eq!( + self.instrs.push_instr( + make_instr(result, lhs, rhs), + consume_fuel, + FuelCostsProvider::base + )?, + iidx + ); Ok(()) } } From 5c990046aa2f8d2bcf5ebf3b5401af1e97b20bd3 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 11 Jun 2025 20:34:49 +0200 Subject: [PATCH 127/343] make InstrEncoder aware if fuel metering is enabled --- .../src/engine/translator/func2/instrs.rs | 33 ++++++++++++++----- .../wasmi/src/engine/translator/func2/mod.rs | 6 +--- 2 files changed, 26 insertions(+), 13 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/instrs.rs b/crates/wasmi/src/engine/translator/func2/instrs.rs index 6983b17730..99f5169976 100644 --- a/crates/wasmi/src/engine/translator/func2/instrs.rs +++ b/crates/wasmi/src/engine/translator/func2/instrs.rs @@ -14,7 +14,9 @@ pub struct InstrEncoder { /// The list of constructed instructions and their parameters. instrs: Vec, /// The fuel costs of instructions. - fuel_costs: FuelCostsProvider, + /// + /// This is `Some` if fuel metering is enabled, otherwise `None`. + fuel_costs: Option, } impl ReusableAllocations for InstrEncoder { @@ -49,9 +51,14 @@ impl Reset for InstrEncoder { impl InstrEncoder { /// Creates a new [`InstrEncoder`]. pub fn new(engine: &Engine, alloc: InstrEncoderAllocations) -> Self { + let config = engine.config(); + let fuel_costs = config + .get_consume_fuel() + .then(|| config.fuel_costs()) + .cloned(); Self { instrs: alloc.instrs, - fuel_costs: engine.config().fuel_costs().clone(), + fuel_costs, } } @@ -66,14 +73,17 @@ impl InstrEncoder { /// # Note /// /// The pushes [`Instruction::ConsumeFuel`] is initialized with base fuel costs. - pub fn push_consume_fuel_instr(&mut self) -> Result { - let base_costs = self.fuel_costs.base(); + pub fn push_consume_fuel_instr(&mut self) -> Result, Error> { + let Some(fuel_costs) = &self.fuel_costs else { + return Ok(None); + }; + let base_costs = fuel_costs.base(); let Ok(base_costs) = u32::try_from(base_costs) else { panic!("out of bounds base fuel costs: {base_costs}"); }; let instr = self.next_instr(); self.instrs.push(Instruction::consume_fuel(base_costs)); - Ok(instr) + Ok(Some(instr)) } /// Pushes an [`Instruction`] to the [`InstrEncoder`]. @@ -126,10 +136,17 @@ impl InstrEncoder { consume_fuel: Option, f: impl FnOnce(&FuelCostsProvider) -> u64, ) -> Result<(), Error> { - let Some(consume_fuel) = consume_fuel else { - return Ok(()); + let (fuel_costs, consume_fuel) = match (&self.fuel_costs, consume_fuel) { + (None, None) => return Ok(()), + (Some(fuel_costs), Some(consume_fuel)) => (fuel_costs, consume_fuel), + _ => { + panic!( + "fuel metering state mismatch: fuel_costs: {:?}, fuel_instr: {:?}", + self.fuel_costs, consume_fuel, + ); + } }; - let fuel_consumed = f(&self.fuel_costs); + let fuel_consumed = f(fuel_costs); self.get_mut(consume_fuel) .bump_fuel_consumption(fuel_consumed)?; Ok(()) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index 18247359e5..d29e8e84bd 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -209,11 +209,7 @@ impl FuncTranslator { let func_ty = self.module.get_type_of_func(self.func); let block_ty = BlockType::func_type(func_ty); let end_label = self.labels.new_label(); - let consume_fuel = self - .fuel_costs - .as_ref() - .map(|_| self.instrs.push_consume_fuel_instr()) - .transpose()?; + let consume_fuel = self.instrs.push_consume_fuel_instr()?; self.stack .push_func_block(block_ty, end_label, consume_fuel)?; Ok(()) From 32dbe4f53d2773176b738a0559f46dae8e120ebe Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 11 Jun 2025 20:35:05 +0200 Subject: [PATCH 128/343] add FuncTranslator::pin_label utility method --- crates/wasmi/src/engine/translator/func2/mod.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index d29e8e84bd..8709a7ba29 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -30,7 +30,7 @@ use self::{ use crate::{ core::{FuelCostsProvider, Typed, TypedVal, ValType}, engine::{ - translator::{Instr, LabelRegistry, WasmTranslator}, + translator::{Instr, LabelRef, LabelRegistry, WasmTranslator}, BlockType, CompiledFuncEntity, TranslationError, @@ -279,6 +279,13 @@ impl FuncTranslator { Ok(()) } + /// Pins the `label` to the next [`Instr`]. + fn pin_label(&mut self, label: LabelRef) { + self.labels + .pin_label(label, self.instrs.next_instr()) + .unwrap_or_else(|err| panic!("failed to pin label to next instruction: {err}")); + } + /// Convert the [`Operand`] at `depth` into an [`Operand::Temp`] by copying if necessary. /// /// # Note From 3809414b4765bda14882719e0a4389563c7eddee Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 11 Jun 2025 20:37:14 +0200 Subject: [PATCH 129/343] refactor Stack::push_loop It is not possible to push the copy instructions after pinning the label hich is required for the method. Therefore this needs to happen before the call to push_loop and push_loop itself has to (debug) assert that this has been done properly. --- .../src/engine/translator/func2/stack/mod.rs | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/stack/mod.rs b/crates/wasmi/src/engine/translator/func2/stack/mod.rs index 4eaa930801..0d681a841c 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/mod.rs @@ -189,14 +189,10 @@ impl Stack { /// Pushes a Wasm `loop` onto the [`Stack`]. /// - /// # Note - /// - /// Calls `f` for every non [`Operand::Temp`] operand on the [`Stack`] - /// that is also a parameter to the pushed Wasm `loop` control frame. - /// /// # Panics (debug) /// - /// If `consume_fuel` is `None` and fuel metering is enabled. + /// - If `consume_fuel` is `None` and fuel metering is enabled. + /// - If any of the Wasm `loop` operand parameters are _not_ [`Operand::Temp`]. /// /// # Errors /// @@ -206,15 +202,15 @@ impl Stack { ty: BlockType, label: LabelRef, consume_fuel: Option, - mut f: impl FnMut(Operand) -> Result<(), Error>, ) -> Result<(), Error> { debug_assert!(!self.controls.is_empty()); debug_assert!(self.is_fuel_metering_enabled() == consume_fuel.is_some()); let len_params = usize::from(ty.len_params(&self.engine)); let block_height = self.height() - len_params; - for depth in 0..block_height { - f(self.operand_to_temp(depth))?; - } + debug_assert!(self + .operands + .peek(len_params) + .all(|operand| operand.is_temp())); self.controls .push_loop(ty, block_height, label, consume_fuel); Ok(()) From 9b089d12baa02edac54804af812f2dbfe5c2b26f Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 11 Jun 2025 20:37:29 +0200 Subject: [PATCH 130/343] improve and update docs --- crates/wasmi/src/engine/translator/func2/stack/mod.rs | 3 ++- crates/wasmi/src/engine/translator/func2/stack/operands.rs | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/stack/mod.rs b/crates/wasmi/src/engine/translator/func2/stack/mod.rs index 0d681a841c..722c5ffa6d 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/mod.rs @@ -425,7 +425,8 @@ impl Stack { /// /// # Note /// - /// Returns the [`Operand`] at `depth` before being converted to an [`Operand::Temp`]. + /// - Returns the [`Operand`] at `depth` before being converted to an [`Operand::Temp`]. + /// - [`Operand::Temp`] will have their optional `instr` set to `None`. /// /// # Panics /// diff --git a/crates/wasmi/src/engine/translator/func2/stack/operands.rs b/crates/wasmi/src/engine/translator/func2/stack/operands.rs index 50079c0446..fb63ff17f5 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/operands.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/operands.rs @@ -233,7 +233,8 @@ impl OperandStack { /// /// # Note /// - /// Returns the [`Operand`] at `depth` before being converted to an [`Operand::Temp`]. + /// - Returns the [`Operand`] at `depth` before being converted to an [`Operand::Temp`]. + /// - [`Operand::Temp`] will have their optional `instr` set to `None`. /// /// # Panics /// @@ -249,7 +250,8 @@ impl OperandStack { /// /// # Note /// - /// Returns `None` if operand at `index` is [`StackOperand::Temp`] already. + /// - Returns the [`Operand`] at `index` before being converted to an [`Operand::Temp`]. + /// - [`Operand::Temp`] will have their optional `instr` set to `None`. /// /// # Panics /// From a3427440de2d69f1d2c69b0f706ec80a07dc3a22 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 11 Jun 2025 20:37:35 +0200 Subject: [PATCH 131/343] remove debug println --- crates/wasmi/src/engine/translator/func2/stack/mod.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/func2/stack/mod.rs b/crates/wasmi/src/engine/translator/func2/stack/mod.rs index 722c5ffa6d..629f31099b 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/mod.rs @@ -440,7 +440,6 @@ impl Stack { /// /// Returns `None` otherwise. pub fn consume_fuel_instr(&self) -> Option { - std::println!("self.controls = {:?}", self.controls); debug_assert!(!self.controls.is_empty()); self.controls.get(0).consume_fuel_instr() } From 77508a168a694c725a334cb0242bf347ac4f3c7b Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 11 Jun 2025 20:37:51 +0200 Subject: [PATCH 132/343] add StackOperand::ty getter method --- .../src/engine/translator/func2/stack/operands.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/crates/wasmi/src/engine/translator/func2/stack/operands.rs b/crates/wasmi/src/engine/translator/func2/stack/operands.rs index fb63ff17f5..e63af312e4 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/operands.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/operands.rs @@ -55,6 +55,17 @@ pub enum StackOperand { }, } +impl StackOperand { + /// Returns the [`ValType`] of the [`StackOperand`]. + pub fn ty(&self, locals: &LocalsRegistry) -> ValType { + match self { + StackOperand::Temp { ty, .. } => *ty, + StackOperand::Immediate { val } => val.ty(), + StackOperand::Local { local_index, .. } => locals.ty(*local_index), + } + } +} + /// The Wasm operand (or value) stack. #[derive(Debug, Default)] pub struct OperandStack { From 42a525386055ab036a789974ea8950e7843e46fc Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 11 Jun 2025 20:38:07 +0200 Subject: [PATCH 133/343] refactor and fix OperandStack::operand_to_temp_at method --- .../wasmi/src/engine/translator/func2/stack/operands.rs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/stack/operands.rs b/crates/wasmi/src/engine/translator/func2/stack/operands.rs index e63af312e4..fc05003eed 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/operands.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/operands.rs @@ -270,13 +270,9 @@ impl OperandStack { #[must_use] fn operand_to_temp_at(&mut self, index: OperandIdx) -> StackOperand { let operand = self.get_at(index); - match operand { - StackOperand::Temp { ty, .. } => ty, - StackOperand::Immediate { val } => val.ty(), - StackOperand::Local { local_index, .. } => self.locals.ty(local_index), - }; + let ty = operand.ty(&self.locals); self.unlink_local(operand); - self.operands[usize::from(index)] = operand; + self.operands[usize::from(index)] = StackOperand::Temp { ty, instr: None }; operand } From 5eeaedf890d40a7ef2b523eb7bc9bac5fb403e75 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 11 Jun 2025 20:38:34 +0200 Subject: [PATCH 134/343] implement FuncTranslator::visit_block trait method --- .../wasmi/src/engine/translator/func2/visit.rs | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index 5b4b953926..942c73603b 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -1,5 +1,5 @@ -use super::{ControlFrame, FuncTranslator, LocalIdx}; -use crate::{core::wasm, ir::Instruction, Error}; +use super::{ControlFrame, FuncTranslator, LocalIdx, UnreachableControlFrame}; +use crate::{core::wasm, engine::BlockType, ir::Instruction, Error}; use wasmparser::VisitOperator; macro_rules! impl_visit_operator { @@ -72,8 +72,16 @@ impl<'a> VisitOperator<'a> for FuncTranslator { todo!() } - fn visit_block(&mut self, _block_ty: wasmparser::BlockType) -> Self::Output { - todo!() + fn visit_block(&mut self, block_ty: wasmparser::BlockType) -> Self::Output { + if !self.reachable { + self.stack + .push_unreachable(UnreachableControlFrame::Block)?; + return Ok(()); + } + let block_ty = BlockType::new(block_ty, &self.module); + let end_label = self.labels.new_label(); + self.stack.push_block(block_ty, end_label)?; + Ok(()) } fn visit_loop(&mut self, _block_ty: wasmparser::BlockType) -> Self::Output { From dcfb8433f63071611a1b5fde6ae39f79f58c9722 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 11 Jun 2025 20:38:45 +0200 Subject: [PATCH 135/343] implement FuncTranslator::visit_loop trait method --- .../wasmi/src/engine/translator/func2/visit.rs | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index 942c73603b..376b7caa8a 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -84,8 +84,21 @@ impl<'a> VisitOperator<'a> for FuncTranslator { Ok(()) } - fn visit_loop(&mut self, _block_ty: wasmparser::BlockType) -> Self::Output { - todo!() + fn visit_loop(&mut self, block_ty: wasmparser::BlockType) -> Self::Output { + if !self.reachable { + self.stack.push_unreachable(UnreachableControlFrame::Loop)?; + return Ok(()); + } + let block_ty = BlockType::new(block_ty, &self.module); + let len_params = block_ty.len_params(&self.engine); + let continue_label = self.labels.new_label(); + let consume_fuel = self.stack.consume_fuel_instr(); + self.copy_branch_params(usize::from(len_params), consume_fuel)?; + self.pin_label(continue_label); + let consume_fuel = self.instrs.push_consume_fuel_instr()?; + self.stack + .push_loop(block_ty, continue_label, consume_fuel)?; + Ok(()) } fn visit_if(&mut self, _block_ty: wasmparser::BlockType) -> Self::Output { From d6f3f078f0bee1cf6b6a2dab0d78974297bacfca Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Thu, 12 Jun 2025 18:52:55 +0200 Subject: [PATCH 136/343] reorder exports --- .../wasmi/src/engine/translator/func2/stack/mod.rs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/stack/mod.rs b/crates/wasmi/src/engine/translator/func2/stack/mod.rs index 629f31099b..eb82b72b9a 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/mod.rs @@ -3,12 +3,19 @@ mod locals; mod operand; mod operands; +use self::{ + control::ControlStack, + locals::LocalsRegistry, + operands::{OperandStack, StackOperand}, +}; pub use self::{ control::{ BlockControlFrame, ControlFrame, ElseControlFrame, + ElseReachability, IfControlFrame, + IfReachability, LoopControlFrame, UnreachableControlFrame, }, @@ -16,11 +23,6 @@ pub use self::{ operand::Operand, operands::{OperandIdx, PreservedLocalsIter}, }; -use self::{ - control::{ControlStack, ElseReachability, IfReachability}, - locals::LocalsRegistry, - operands::{OperandStack, StackOperand}, -}; use super::{Reset, ReusableAllocations}; use crate::{ core::{TypedVal, ValType}, From dd04341a1ee59c99b17dae9649829ab41860d843 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Thu, 12 Jun 2025 18:53:40 +0200 Subject: [PATCH 137/343] push else operands only for IfReachability::Both Otherwise a Wasm `if` acts similar to a Wasm `block` since it is known upfront that operands are not required to be input twice. --- crates/wasmi/src/engine/translator/func2/stack/control.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/func2/stack/control.rs b/crates/wasmi/src/engine/translator/func2/stack/control.rs index 04f25ee12b..8f0ee26c1f 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/control.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/control.rs @@ -127,7 +127,9 @@ impl ControlStack { label, reachability, })); - self.else_operands.push(else_operands); + if matches!(reachability, IfReachability::Both { .. }) { + self.else_operands.push(else_operands); + } } /// Pushes a new Wasm `else` onto the [`ControlStack`]. From 95f5c35e5a84c87ef0613e903c00772d985b9fdc Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Thu, 12 Jun 2025 18:54:00 +0200 Subject: [PATCH 138/343] start implementation of FuncTranslator::visit_if --- .../src/engine/translator/func2/visit.rs | 35 ++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index 376b7caa8a..2b634f54e4 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -1,5 +1,13 @@ use super::{ControlFrame, FuncTranslator, LocalIdx, UnreachableControlFrame}; -use crate::{core::wasm, engine::BlockType, ir::Instruction, Error}; +use crate::{ + core::wasm, + engine::{ + translator::func2::{stack::IfReachability, Operand}, + BlockType, + }, + ir::Instruction, + Error, +}; use wasmparser::VisitOperator; macro_rules! impl_visit_operator { @@ -102,6 +110,31 @@ impl<'a> VisitOperator<'a> for FuncTranslator { } fn visit_if(&mut self, _block_ty: wasmparser::BlockType) -> Self::Output { + if !self.reachable { + self.stack.push_unreachable(UnreachableControlFrame::If)?; + return Ok(()); + } + let end_label = self.labels.new_label(); + let condition = self.stack.pop(); + let (reachability, consume_fuel_instr) = match condition { + Operand::Immediate(operand) => { + let condition = i32::from(operand.val()); + let reachability = match condition { + 0 => { + self.reachable = false; + IfReachability::OnlyElse + } + _ => IfReachability::OnlyThen, + }; + let consume_fuel_instr = self.stack.consume_fuel_instr(); + (reachability, consume_fuel_instr) + } + operand => { + let else_label = self.labels.new_label(); + let reachability = IfReachability::Both { else_label }; + todo!() + } + }; todo!() } From 107cbeeb49415a9f225a1f06c181d096ab1f1dec Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Fri, 13 Jun 2025 15:10:34 +0200 Subject: [PATCH 139/343] convert condition operand to Reg --- crates/wasmi/src/engine/translator/func2/visit.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index 2b634f54e4..02ad9cf26d 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -130,6 +130,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { (reachability, consume_fuel_instr) } operand => { + let condition = self.layout.operand_to_reg(operand)?; let else_label = self.labels.new_label(); let reachability = IfReachability::Both { else_label }; todo!() From 9d6f38319e3a2f5c979b242733a8703390675a21 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Sat, 14 Jun 2025 10:29:11 +0200 Subject: [PATCH 140/343] fix some internal doc links --- .../src/engine/translator/func2/stack/control.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/stack/control.rs b/crates/wasmi/src/engine/translator/func2/stack/control.rs index 8f0ee26c1f..b8f3a9b5a6 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/control.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/control.rs @@ -390,7 +390,7 @@ pub struct LoopControlFrame { } impl LoopControlFrame { - /// Returns the [`BlockType`] of the [`BlockControlFrame`]. + /// Returns the [`BlockType`] of the [`LoopControlFrame`]. pub fn ty(&self) -> BlockType { self.ty } @@ -431,13 +431,13 @@ impl LoopControlFrame { /// A Wasm `if` control frame including its `then` part. #[derive(Debug)] pub struct IfControlFrame { - /// The block type of the [`LoopControlFrame`]. + /// The block type of the [`IfControlFrame`]. ty: BlockType, - /// The value stack height upon entering the [`LoopControlFrame`]. + /// The value stack height upon entering the [`IfControlFrame`]. height: usize, - /// This is `true` if there is at least one branch to this [`LoopControlFrame`]. + /// This is `true` if there is at least one branch to this [`IfControlFrame`]. is_branched_to: bool, - /// The [`LoopControlFrame`]'s [`Instruction::ConsumeFuel`] if fuel metering is enabled. + /// The [`IfControlFrame`]'s [`Instruction::ConsumeFuel`] if fuel metering is enabled. /// /// # Note /// @@ -548,11 +548,11 @@ pub enum IfReachability { /// A Wasm `else` control frame part of Wasm `if`. #[derive(Debug)] pub struct ElseControlFrame { - /// The block type of the [`LoopControlFrame`]. + /// The block type of the [`ElseControlFrame`]. ty: BlockType, - /// The value stack height upon entering the [`LoopControlFrame`]. + /// The value stack height upon entering the [`ElseControlFrame`]. height: usize, - /// This is `true` if there is at least one branch to this [`LoopControlFrame`]. + /// This is `true` if there is at least one branch to this [`ElseControlFrame`]. is_branched_to: bool, /// The [`LoopControlFrame`]'s [`Instruction::ConsumeFuel`] if fuel metering is enabled. /// From 819c4f755abf468b2adb8011dd9ae218d45180b2 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Sat, 14 Jun 2025 10:33:49 +0200 Subject: [PATCH 141/343] make InstrEncoder::get_mut private --- crates/wasmi/src/engine/translator/func2/instrs.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/func2/instrs.rs b/crates/wasmi/src/engine/translator/func2/instrs.rs index 99f5169976..7009dc4401 100644 --- a/crates/wasmi/src/engine/translator/func2/instrs.rs +++ b/crates/wasmi/src/engine/translator/func2/instrs.rs @@ -122,7 +122,7 @@ impl InstrEncoder { /// # Panics /// /// If `instr` is out of bounds for `self`. - pub fn get_mut(&mut self, instr: Instr) -> &mut Instruction { + fn get_mut(&mut self, instr: Instr) -> &mut Instruction { &mut self.instrs[instr.into_usize()] } From a889514b27a268c0bb7abef22b62093861db7cab Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Sat, 14 Jun 2025 13:59:24 +0200 Subject: [PATCH 142/343] fix compile error --- crates/wasmi/src/engine/translator/func2/instrs.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/func2/instrs.rs b/crates/wasmi/src/engine/translator/func2/instrs.rs index 7009dc4401..78468e04f9 100644 --- a/crates/wasmi/src/engine/translator/func2/instrs.rs +++ b/crates/wasmi/src/engine/translator/func2/instrs.rs @@ -1,7 +1,7 @@ use super::{Instr, Reset, ReusableAllocations}; use crate::{ core::FuelCostsProvider, - engine::translator::BumpFuelConsumption, + engine::translator::utils::BumpFuelConsumption as _, ir::Instruction, Engine, Error, From 3f25bb17d0425dce3395b92798895ee259788eaf Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Sun, 15 Jun 2025 10:03:51 +0200 Subject: [PATCH 143/343] silence warnings temporarily for visit_if --- crates/wasmi/src/engine/translator/func2/visit.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index 02ad9cf26d..8117af3e38 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -109,6 +109,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { Ok(()) } + #[allow(unused_variables)] // TODO: remove fn visit_if(&mut self, _block_ty: wasmparser::BlockType) -> Self::Output { if !self.reachable { self.stack.push_unreachable(UnreachableControlFrame::If)?; From a10e1db34f59792ed7058567ff9120b1193d17d2 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Sun, 15 Jun 2025 10:04:39 +0200 Subject: [PATCH 144/343] remove Reset impl for InstrEncoder This impl is just needed for *Allocation types such as InstrEncoderAllocations. --- crates/wasmi/src/engine/translator/func2/instrs.rs | 6 ------ 1 file changed, 6 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/instrs.rs b/crates/wasmi/src/engine/translator/func2/instrs.rs index 78468e04f9..01fbb5ba22 100644 --- a/crates/wasmi/src/engine/translator/func2/instrs.rs +++ b/crates/wasmi/src/engine/translator/func2/instrs.rs @@ -42,12 +42,6 @@ impl Reset for InstrEncoderAllocations { } } -impl Reset for InstrEncoder { - fn reset(&mut self) { - self.instrs.clear(); - } -} - impl InstrEncoder { /// Creates a new [`InstrEncoder`]. pub fn new(engine: &Engine, alloc: InstrEncoderAllocations) -> Self { From 5ba6253dfdcec28318b55f5ef387c4fc3f0eb8da Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Sun, 15 Jun 2025 10:05:19 +0200 Subject: [PATCH 145/343] add last_instr field to InstrEncoder --- crates/wasmi/src/engine/translator/func2/instrs.rs | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/instrs.rs b/crates/wasmi/src/engine/translator/func2/instrs.rs index 01fbb5ba22..84b956b4a8 100644 --- a/crates/wasmi/src/engine/translator/func2/instrs.rs +++ b/crates/wasmi/src/engine/translator/func2/instrs.rs @@ -17,6 +17,8 @@ pub struct InstrEncoder { /// /// This is `Some` if fuel metering is enabled, otherwise `None`. fuel_costs: Option, + /// The last pushed non-parameter [`Instruction`]. + last_instr: Option, } impl ReusableAllocations for InstrEncoder { @@ -53,6 +55,7 @@ impl InstrEncoder { Self { instrs: alloc.instrs, fuel_costs, + last_instr: None, } } @@ -75,12 +78,11 @@ impl InstrEncoder { let Ok(base_costs) = u32::try_from(base_costs) else { panic!("out of bounds base fuel costs: {base_costs}"); }; - let instr = self.next_instr(); - self.instrs.push(Instruction::consume_fuel(base_costs)); + let instr = self.push_instr_impl(Instruction::consume_fuel(base_costs))?; Ok(Some(instr)) } - /// Pushes an [`Instruction`] to the [`InstrEncoder`]. + /// Pushes a non-parameter [`Instruction`] to the [`InstrEncoder`]. /// /// Returns an [`Instr`] that refers to the pushed [`Instruction`]. pub fn push_instr( @@ -90,8 +92,14 @@ impl InstrEncoder { f: impl FnOnce(&FuelCostsProvider) -> u64, ) -> Result { self.bump_fuel_consumption(consume_fuel, f)?; + self.push_instr_impl(instruction) + } + + /// Pushes a non-parameter [`Instruction`] to the [`InstrEncoder`]. + fn push_instr_impl(&mut self, instruction: Instruction) -> Result { let instr = self.next_instr(); self.instrs.push(instruction); + self.last_instr = Some(instr); Ok(instr) } From 97fe4cf118690a8f9e477933b9a88ecf52a9e033 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Sun, 15 Jun 2025 10:05:45 +0200 Subject: [PATCH 146/343] add (debug) asserts for is_instruction_parameter --- crates/wasmi/src/engine/translator/func2/instrs.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/func2/instrs.rs b/crates/wasmi/src/engine/translator/func2/instrs.rs index 84b956b4a8..461c130e7a 100644 --- a/crates/wasmi/src/engine/translator/func2/instrs.rs +++ b/crates/wasmi/src/engine/translator/func2/instrs.rs @@ -1,7 +1,7 @@ use super::{Instr, Reset, ReusableAllocations}; use crate::{ core::FuelCostsProvider, - engine::translator::utils::BumpFuelConsumption as _, + engine::translator::utils::{BumpFuelConsumption as _, IsInstructionParameter as _}, ir::Instruction, Engine, Error, @@ -97,6 +97,10 @@ impl InstrEncoder { /// Pushes a non-parameter [`Instruction`] to the [`InstrEncoder`]. fn push_instr_impl(&mut self, instruction: Instruction) -> Result { + debug_assert!( + !instruction.is_instruction_parameter(), + "parameter: {instruction:?}" + ); let instr = self.next_instr(); self.instrs.push(instruction); self.last_instr = Some(instr); @@ -107,6 +111,10 @@ impl InstrEncoder { /// /// The parameter is associated to the last pushed [`Instruction`]. pub fn push_param(&mut self, instruction: Instruction) { + debug_assert!( + instruction.is_instruction_parameter(), + "non-parameter: {instruction:?}" + ); self.instrs.push(instruction); } From 3702c30deb1702a94960f4348503e114dc1046ab Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Sun, 15 Jun 2025 10:07:13 +0200 Subject: [PATCH 147/343] add AllocConst impl for StackLayout This allows to use StackLayout as parameter to try_into_cmp_branch_instr ext-trait method. --- crates/wasmi/src/engine/translator/func2/layout/mod.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/func2/layout/mod.rs b/crates/wasmi/src/engine/translator/func2/layout/mod.rs index d000648eaf..5319b63c96 100644 --- a/crates/wasmi/src/engine/translator/func2/layout/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/layout/mod.rs @@ -4,7 +4,7 @@ use self::consts::{ConstRegistry, ConstRegistryIter}; use super::{LocalIdx, Operand, OperandIdx, Reset}; use crate::{ core::{UntypedVal, ValType}, - engine::TranslationError, + engine::{translator::comparator::AllocConst, TranslationError}, ir::Reg, Error, }; @@ -127,6 +127,12 @@ impl StackLayout { } } +impl AllocConst for StackLayout { + fn alloc_const>(&mut self, value: T) -> Result { + self.const_to_reg(value) + } +} + /// The [`StackSpace`] of a [`Reg`]. #[derive(Debug, Copy, Clone)] pub enum StackSpace { From d2f7b649a7d81a256e818ba80aabc679133df993 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Sun, 15 Jun 2025 10:07:33 +0200 Subject: [PATCH 148/343] add InstrEncoder::try_replace_instr method --- .../src/engine/translator/func2/instrs.rs | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/crates/wasmi/src/engine/translator/func2/instrs.rs b/crates/wasmi/src/engine/translator/func2/instrs.rs index 461c130e7a..dea3ce505b 100644 --- a/crates/wasmi/src/engine/translator/func2/instrs.rs +++ b/crates/wasmi/src/engine/translator/func2/instrs.rs @@ -107,6 +107,35 @@ impl InstrEncoder { Ok(instr) } + /// Replaces `instr` with `new_instr` in `self`. + /// + /// - Returns `Ok(true)` if replacement was successful. + /// - Returns `Ok(false)` if replacement was unsuccessful. + /// + /// # Panics (Debug) + /// + /// If `instr` or `new_instr` are [`Instruction`] parameters. + pub fn try_replace_instr( + &mut self, + instr: Instr, + new_instr: Instruction, + ) -> Result { + debug_assert!( + !new_instr.is_instruction_parameter(), + "parameter: {new_instr:?}" + ); + let Some(last_instr) = self.last_instr else { + return Ok(false); + }; + let replace = self.get_mut(instr); + debug_assert!(!replace.is_instruction_parameter(), "parameter: {instr:?}"); + if instr != last_instr { + return Ok(false); + } + *replace = new_instr; + Ok(true) + } + /// Pushes an [`Instruction`] parameter to the [`InstrEncoder`]. /// /// The parameter is associated to the last pushed [`Instruction`]. From da4c301d2f75274afc52817ba5a1b7c44d91f48b Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Sun, 15 Jun 2025 10:58:12 +0200 Subject: [PATCH 149/343] implement translate_br_if variants + fusion --- .../wasmi/src/engine/translator/func2/mod.rs | 155 +++++++++++++++++- .../src/engine/translator/func2/stack/mod.rs | 2 +- 2 files changed, 153 insertions(+), 4 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index 8709a7ba29..572979f47a 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -11,7 +11,7 @@ mod visit; use self::{ instrs::{InstrEncoder, InstrEncoderAllocations}, - layout::StackLayout, + layout::{StackLayout, StackSpace}, stack::{ BlockControlFrame, ControlFrame, @@ -23,6 +23,7 @@ use self::{ OperandIdx, Stack, StackAllocations, + TempOperand, UnreachableControlFrame, }, utils::{Reset, ReusableAllocations}, @@ -30,12 +31,29 @@ use self::{ use crate::{ core::{FuelCostsProvider, Typed, TypedVal, ValType}, engine::{ - translator::{Instr, LabelRef, LabelRegistry, WasmTranslator}, + translator::{ + comparator::{CompareResult as _, NegateCmpInstr as _, TryIntoCmpBranchInstr as _}, + Instr, + LabelRef, + LabelRegistry, + WasmTranslator, + }, BlockType, CompiledFuncEntity, TranslationError, }, - ir::{BoundedRegSpan, Const16, Const32, Instruction, Reg, RegSpan}, + ir::{ + BoundedRegSpan, + BranchOffset, + BranchOffset16, + Comparator, + ComparatorAndOffset, + Const16, + Const32, + Instruction, + Reg, + RegSpan, + }, module::{FuncIdx, ModuleHeader, WasmiValueType}, Engine, Error, @@ -341,6 +359,17 @@ impl FuncTranslator { Ok(()) } + /// Pushes the `instr` to the function with the associated `fuel_costs`. + fn push_instr( + &mut self, + instr: Instruction, + fuel_costs: impl FnOnce(&FuelCostsProvider) -> u64, + ) -> Result<(), Error> { + let consume_fuel = self.stack.consume_fuel_instr(); + self.instrs.push_instr(instr, consume_fuel, fuel_costs)?; + Ok(()) + } + /// Translates a generic return instruction. fn translate_return(&mut self, consume_fuel: Option) -> Result { let len_results = self.func_type_with(FuncType::len_results); @@ -441,6 +470,126 @@ impl FuncTranslator { todo!() } + /// Translates a `i32.eqz`+`br_if` or `if` conditional branch instruction. + fn translate_br_eqz(&mut self, condition: Operand, label: LabelRef) -> Result<(), Error> { + self.translate_br_if(condition, label, true) + } + + /// Translates a `br_if` conditional branch instruction. + fn translate_br_nez(&mut self, condition: Operand, label: LabelRef) -> Result<(), Error> { + self.translate_br_if(condition, label, false) + } + + /// Translates a generic `br_if` fused conditional branch instruction. + fn translate_br_if( + &mut self, + condition: Operand, + label: LabelRef, + branch_eqz: bool, + ) -> Result<(), Error> { + if self.try_fuse_branch_cmp(condition, label, branch_eqz)? { + return Ok(()); + } + let condition = match condition { + Operand::Local(condition) => self.layout.local_to_reg(condition.local_index())?, + Operand::Temp(condition) => self.layout.temp_to_reg(condition.operand_index())?, + Operand::Immediate(_condition) => { + todo!() // TODO: translate as `br` + } + }; + let instr = self.instrs.next_instr(); + let offset = self.labels.try_resolve_label(label, instr)?; + let instr = match BranchOffset16::try_from(offset) { + Ok(offset) => match branch_eqz { + true => Instruction::branch_i32_eq_imm16(condition, 0, offset), + false => Instruction::branch_i32_ne_imm16(condition, 0, offset), + }, + Err(_) => { + let zero = self.layout.const_to_reg(0_i32)?; + self.make_branch_cmp_fallback(Comparator::I32Eq, condition, zero, offset)? + } + }; + self.push_instr(instr, FuelCostsProvider::base) + } + + /// Create an [`Instruction::BranchCmpFallback`]. + fn make_branch_cmp_fallback( + &mut self, + cmp: Comparator, + lhs: Reg, + rhs: Reg, + offset: BranchOffset, + ) -> Result { + let params = self + .layout + .const_to_reg(ComparatorAndOffset::new(cmp, offset))?; + Ok(Instruction::branch_cmp_fallback(lhs, rhs, params)) + } + + /// Try to fuse a cmp+branch [`Instruction`] with optional negation. + fn try_fuse_branch_cmp( + &mut self, + condition: Operand, + label: LabelRef, + negate: bool, + ) -> Result { + let Operand::Temp(condition) = condition else { + return Ok(false); + }; + debug_assert_eq!(condition.ty(), ValType::I32); + let Some(origin) = condition.instr() else { + return Ok(false); + }; + let fused_instr = self.try_make_fused_branch_cmp_instr(origin, condition, label, negate)?; + let Some(fused_instr) = fused_instr else { + return Ok(false); + }; + self.instrs.try_replace_instr(origin, fused_instr) + } + + /// Try to return a fused cmp+branch [`Instruction`] from the given parameters. + /// + /// + /// # Note + /// + /// - The `instr` parameter refers to the to-be-fused cmp instruction. + /// - Returns `Ok(Some)` if cmp+branch fusion was successful. + /// - Returns `Ok(None)`, otherwise. + fn try_make_fused_branch_cmp_instr( + &mut self, + instr: Instr, + condition: TempOperand, + label: LabelRef, + negate: bool, + ) -> Result, Error> { + let cmp_instr = *self.instrs.get(instr); + let Some(result) = cmp_instr.compare_result() else { + // Note: cannot fuse non-cmp instructions or cmp-instructions without result. + return Ok(None); + }; + if matches!(self.layout.stack_space(result), StackSpace::Local) { + // Note: cannot fuse cmp instructions with observable semantics. + return Ok(None); + } + if result != self.layout.temp_to_reg(condition.operand_index())? { + // Note: cannot fuse cmp instruction with a result that differs + // from the condition operand. + return Ok(None); + } + let cmp_instr = match negate { + false => cmp_instr, + true => match cmp_instr.negate_cmp_instr() { + Some(negated) => negated, + None => { + // Note: cannot negate cmp instruction, thus not possible to fuse. + return Ok(None); + } + }, + }; + let offset = self.labels.try_resolve_label(label, instr)?; + cmp_instr.try_into_cmp_branch_instr(offset, &mut self.layout) + } + /// Translates a commutative binary Wasm operator to Wasmi bytecode. fn translate_binary_commutative( &mut self, diff --git a/crates/wasmi/src/engine/translator/func2/stack/mod.rs b/crates/wasmi/src/engine/translator/func2/stack/mod.rs index eb82b72b9a..f1062a526e 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/mod.rs @@ -20,7 +20,7 @@ pub use self::{ UnreachableControlFrame, }, locals::LocalIdx, - operand::Operand, + operand::{Operand, TempOperand}, operands::{OperandIdx, PreservedLocalsIter}, }; use super::{Reset, ReusableAllocations}; From afca6025257fd572630fe01c494b1c1cf8a60d33 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Sun, 15 Jun 2025 11:03:05 +0200 Subject: [PATCH 150/343] implement visit_if --- crates/wasmi/src/engine/translator/func2/visit.rs | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index 8117af3e38..70e122aeaa 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -109,8 +109,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { Ok(()) } - #[allow(unused_variables)] // TODO: remove - fn visit_if(&mut self, _block_ty: wasmparser::BlockType) -> Self::Output { + fn visit_if(&mut self, block_ty: wasmparser::BlockType) -> Self::Output { if !self.reachable { self.stack.push_unreachable(UnreachableControlFrame::If)?; return Ok(()); @@ -130,14 +129,18 @@ impl<'a> VisitOperator<'a> for FuncTranslator { let consume_fuel_instr = self.stack.consume_fuel_instr(); (reachability, consume_fuel_instr) } - operand => { - let condition = self.layout.operand_to_reg(operand)?; + _ => { let else_label = self.labels.new_label(); + self.translate_br_eqz(condition, else_label)?; let reachability = IfReachability::Both { else_label }; - todo!() + let consume_fuel_instr = self.instrs.push_consume_fuel_instr()?; + (reachability, consume_fuel_instr) } }; - todo!() + let block_ty = BlockType::new(block_ty, &self.module); + self.stack + .push_if(block_ty, end_label, reachability, consume_fuel_instr)?; + Ok(()) } fn visit_else(&mut self) -> Self::Output { From 8848d7f67aef6f45a033402f672e8d051e26c805 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Sun, 15 Jun 2025 11:07:13 +0200 Subject: [PATCH 151/343] fix bug in translate_br_if fallback --- crates/wasmi/src/engine/translator/func2/mod.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index 572979f47a..f5cc2f32a8 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -506,7 +506,11 @@ impl FuncTranslator { }, Err(_) => { let zero = self.layout.const_to_reg(0_i32)?; - self.make_branch_cmp_fallback(Comparator::I32Eq, condition, zero, offset)? + let comparator = match branch_eqz { + true => Comparator::I32Eq, + false => Comparator::I32Ne, + }; + self.make_branch_cmp_fallback(comparator, condition, zero, offset)? } }; self.push_instr(instr, FuelCostsProvider::base) From 222441b32e2f8302ed4872370030987c584bd55f Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 16 Jun 2025 11:26:56 +0200 Subject: [PATCH 152/343] reorder exports --- crates/wasmi/src/engine/translator/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/mod.rs b/crates/wasmi/src/engine/translator/mod.rs index 4657476fd3..7d02e20af8 100644 --- a/crates/wasmi/src/engine/translator/mod.rs +++ b/crates/wasmi/src/engine/translator/mod.rs @@ -25,8 +25,8 @@ pub use self::func2::{FuncTranslator, FuncTranslatorAllocations}; pub use self::{ driver::FuncTranslationDriver, error::TranslationError, - utils::Instr, labels::{LabelRef, LabelRegistry}, + utils::Instr, }; use super::code_map::CompiledFuncEntity; use crate::{ From 093ed5390995b593a5b50db398970c8578788c76 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 16 Jun 2025 11:34:11 +0200 Subject: [PATCH 153/343] fix imports in func2 translator --- crates/wasmi/src/engine/translator/func2/instrs.rs | 4 ++-- crates/wasmi/src/engine/translator/func2/mod.rs | 5 ++--- crates/wasmi/src/engine/translator/func2/stack/control.rs | 2 +- crates/wasmi/src/engine/translator/func2/stack/mod.rs | 2 +- crates/wasmi/src/engine/translator/func2/stack/operand.rs | 2 +- crates/wasmi/src/engine/translator/func2/stack/operands.rs | 2 +- crates/wasmi/src/engine/translator/mod.rs | 7 +------ 7 files changed, 9 insertions(+), 15 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/instrs.rs b/crates/wasmi/src/engine/translator/func2/instrs.rs index dea3ce505b..59c34601d6 100644 --- a/crates/wasmi/src/engine/translator/func2/instrs.rs +++ b/crates/wasmi/src/engine/translator/func2/instrs.rs @@ -1,7 +1,7 @@ -use super::{Instr, Reset, ReusableAllocations}; +use super::{Reset, ReusableAllocations}; use crate::{ core::FuelCostsProvider, - engine::translator::utils::{BumpFuelConsumption as _, IsInstructionParameter as _}, + engine::translator::utils::{BumpFuelConsumption as _, Instr, IsInstructionParameter as _}, ir::Instruction, Engine, Error, diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index f5cc2f32a8..7421eb0ed8 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -33,9 +33,8 @@ use crate::{ engine::{ translator::{ comparator::{CompareResult as _, NegateCmpInstr as _, TryIntoCmpBranchInstr as _}, - Instr, - LabelRef, - LabelRegistry, + labels::{LabelRef, LabelRegistry}, + utils::Instr, WasmTranslator, }, BlockType, diff --git a/crates/wasmi/src/engine/translator/func2/stack/control.rs b/crates/wasmi/src/engine/translator/func2/stack/control.rs index b8f3a9b5a6..f2325d817b 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/control.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/control.rs @@ -1,7 +1,7 @@ use super::{Operand, Reset}; use crate::{ engine::{ - translator::{Instr, LabelRef}, + translator::{labels::LabelRef, utils::Instr}, BlockType, }, Engine, diff --git a/crates/wasmi/src/engine/translator/func2/stack/mod.rs b/crates/wasmi/src/engine/translator/func2/stack/mod.rs index f1062a526e..ab942fcfa8 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/mod.rs @@ -27,7 +27,7 @@ use super::{Reset, ReusableAllocations}; use crate::{ core::{TypedVal, ValType}, engine::{ - translator::{Instr, LabelRef}, + translator::{labels::LabelRef, utils::Instr}, BlockType, }, Engine, diff --git a/crates/wasmi/src/engine/translator/func2/stack/operand.rs b/crates/wasmi/src/engine/translator/func2/stack/operand.rs index e147f1bde8..5f6c1a9e60 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/operand.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/operand.rs @@ -1,7 +1,7 @@ use super::{LocalIdx, LocalsRegistry, OperandIdx, StackOperand}; use crate::{ core::{TypedVal, ValType}, - engine::translator::Instr, + engine::translator::utils::Instr, }; #[cfg(doc)] diff --git a/crates/wasmi/src/engine/translator/func2/stack/operands.rs b/crates/wasmi/src/engine/translator/func2/stack/operands.rs index fc05003eed..3558aba5fb 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/operands.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/operands.rs @@ -1,7 +1,7 @@ use super::{LocalIdx, LocalsRegistry, Operand, Reset}; use crate::{ core::{TypedVal, ValType}, - engine::translator::Instr, + engine::translator::utils::Instr, Error, }; use alloc::vec::Vec; diff --git a/crates/wasmi/src/engine/translator/mod.rs b/crates/wasmi/src/engine/translator/mod.rs index 7d02e20af8..395f7b87f9 100644 --- a/crates/wasmi/src/engine/translator/mod.rs +++ b/crates/wasmi/src/engine/translator/mod.rs @@ -22,12 +22,7 @@ pub use self::func::{FuncTranslator, FuncTranslatorAllocations}; #[cfg(feature = "experimental-translator")] pub use self::func2::{FuncTranslator, FuncTranslatorAllocations}; -pub use self::{ - driver::FuncTranslationDriver, - error::TranslationError, - labels::{LabelRef, LabelRegistry}, - utils::Instr, -}; +pub use self::{driver::FuncTranslationDriver, error::TranslationError}; use super::code_map::CompiledFuncEntity; use crate::{ engine::EngineFunc, From 6e1bfd4ea9c761ff300650b31e60f0162e24614e Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 16 Jun 2025 11:36:33 +0200 Subject: [PATCH 154/343] cfg-guard func and func2 modules --- crates/wasmi/src/engine/translator/mod.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/wasmi/src/engine/translator/mod.rs b/crates/wasmi/src/engine/translator/mod.rs index 395f7b87f9..2ac2149cfb 100644 --- a/crates/wasmi/src/engine/translator/mod.rs +++ b/crates/wasmi/src/engine/translator/mod.rs @@ -3,7 +3,9 @@ mod comparator; mod driver; mod error; +#[cfg(not(feature = "experimental-translator"))] mod func; +#[cfg(feature = "experimental-translator")] mod func2; mod labels; mod relink_result; From 95a3eba0c467eb35a27f6f29a61e59fd1130a5d2 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Tue, 17 Jun 2025 10:33:18 +0200 Subject: [PATCH 155/343] silence warnings instead of conditionally compile func module --- crates/wasmi/src/engine/translator/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/mod.rs b/crates/wasmi/src/engine/translator/mod.rs index 2ac2149cfb..478e84f206 100644 --- a/crates/wasmi/src/engine/translator/mod.rs +++ b/crates/wasmi/src/engine/translator/mod.rs @@ -3,7 +3,7 @@ mod comparator; mod driver; mod error; -#[cfg(not(feature = "experimental-translator"))] +#[cfg_attr(feature = "experimental-translator", expect(dead_code))] mod func; #[cfg(feature = "experimental-translator")] mod func2; From 7d395dfb208027616b2a73985a933e64cece4747 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Tue, 17 Jun 2025 10:47:50 +0200 Subject: [PATCH 156/343] implement translate_br --- .../wasmi/src/engine/translator/func2/mod.rs | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index 7421eb0ed8..d3e1a914eb 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -469,6 +469,15 @@ impl FuncTranslator { todo!() } + /// Translates an unconditional Wasm `branch` instruction. + fn translate_br(&mut self, label: LabelRef) -> Result<(), Error> { + let instr = self.instrs.next_instr(); + let offset = self.labels.try_resolve_label(label, instr)?; + self.push_instr(Instruction::branch(offset), FuelCostsProvider::base)?; + self.reachable = false; + Ok(()) + } + /// Translates a `i32.eqz`+`br_if` or `if` conditional branch instruction. fn translate_br_eqz(&mut self, condition: Operand, label: LabelRef) -> Result<(), Error> { self.translate_br_if(condition, label, true) @@ -492,8 +501,16 @@ impl FuncTranslator { let condition = match condition { Operand::Local(condition) => self.layout.local_to_reg(condition.local_index())?, Operand::Temp(condition) => self.layout.temp_to_reg(condition.operand_index())?, - Operand::Immediate(_condition) => { - todo!() // TODO: translate as `br` + Operand::Immediate(condition) => { + let condition = i32::from(condition.val()); + let take_branch = match branch_eqz { + true => condition == 0, + false => condition != 0, + }; + match take_branch { + true => return self.translate_br(label), + false => return Ok(()), + } } }; let instr = self.instrs.next_instr(); From 41d4b854c20248024643729390755d8d44174587 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Tue, 17 Jun 2025 12:38:39 +0200 Subject: [PATCH 157/343] refactor Stack::push_else method It now takes the parent IfControlFrame which simplifies the caller site a lot and makes it harder to misuse the API. --- .../engine/translator/func2/stack/control.rs | 22 ++++++++++++++----- .../src/engine/translator/func2/stack/mod.rs | 17 ++++---------- 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/stack/control.rs b/crates/wasmi/src/engine/translator/func2/stack/control.rs index f2325d817b..4dbc53498a 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/control.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/control.rs @@ -137,17 +137,19 @@ impl ControlStack { /// Returns iterator yielding the memorized `else` operands. pub fn push_else( &mut self, - ty: BlockType, - height: usize, - label: LabelRef, + if_frame: IfControlFrame, consume_fuel: Option, - reachability: ElseReachability, is_end_of_then_reachable: bool, ) -> Drain { + let ty = if_frame.ty(); + let height = if_frame.height(); + let label = if_frame.label(); + let is_branched_to = if_frame.is_branched_to(); + let reachability = ElseReachability::from(if_frame.reachability); self.frames.push(ControlFrame::from(ElseControlFrame { ty, height, - is_branched_to: false, + is_branched_to, consume_fuel, label, reachability, @@ -597,6 +599,16 @@ pub enum ElseReachability { OnlyElse, } +impl From for ElseReachability { + fn from(reachability: IfReachability) -> Self { + match reachability { + IfReachability::Both { .. } => Self::Both, + IfReachability::OnlyThen => Self::OnlyThen, + IfReachability::OnlyElse => Self::OnlyElse, + } + } +} + impl ElseControlFrame { /// Returns the [`BlockType`] of the [`ElseControlFrame`]. pub fn ty(&self) -> BlockType { diff --git a/crates/wasmi/src/engine/translator/func2/stack/mod.rs b/crates/wasmi/src/engine/translator/func2/stack/mod.rs index ab942fcfa8..400eaca5f6 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/mod.rs @@ -262,23 +262,14 @@ impl Stack { /// If the stack height exceeds the maximum height. pub fn push_else( &mut self, - ty: BlockType, - label: LabelRef, - reachability: ElseReachability, + if_frame: IfControlFrame, is_end_of_then_reachable: bool, consume_fuel: Option, ) -> Result<(), Error> { debug_assert!(self.is_fuel_metering_enabled() == consume_fuel.is_some()); - let len_params = usize::from(ty.len_params(&self.engine)); - let block_height = self.height() - len_params; - let else_operands = self.controls.push_else( - ty, - block_height, - label, - consume_fuel, - reachability, - is_end_of_then_reachable, - ); + let else_operands = + self.controls + .push_else(if_frame, consume_fuel, is_end_of_then_reachable); for operand in else_operands { match operand { Operand::Local(op) => { From 33fe300e517d3a40b0c9c98f9a4e2a811416401f Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Tue, 17 Jun 2025 12:38:50 +0200 Subject: [PATCH 158/343] implement FuncTranslator::visit_else --- .../src/engine/translator/func2/visit.rs | 35 ++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index 70e122aeaa..d38fbd901c 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -144,7 +144,40 @@ impl<'a> VisitOperator<'a> for FuncTranslator { } fn visit_else(&mut self) -> Self::Output { - todo!() + let mut frame = match self.stack.pop_control() { + ControlFrame::If(frame) => frame, + ControlFrame::Unreachable(UnreachableControlFrame::If) => { + debug_assert!(!self.reachable); + self.stack.push_unreachable(UnreachableControlFrame::Else)?; + return Ok(()); + } + unexpected => panic!("expected `if` control frame but found: {unexpected:?}"), + }; + // After `then` block, before `else` block: + // - Copy `if` branch parameters. + // - Branch from end of `then` to end of `if`. + let is_end_of_then_reachable = self.reachable; + if let Some(else_label) = frame.else_label() { + debug_assert!(frame.is_then_reachable() && frame.is_else_reachable()); + if is_end_of_then_reachable { + let len_values = usize::from(frame.ty().len_results(&self.engine)); + let consume_fuel_instr = frame.consume_fuel_instr(); + self.copy_branch_params(len_values, consume_fuel_instr)?; + frame.branch_to(); + self.translate_br(else_label)?; + } + } + // Start of `else` block: + if let Some(else_label) = frame.else_label() { + self.labels + .pin_label(else_label, self.instrs.next_instr()) + .unwrap(); + } + let consume_fuel_instr = self.instrs.push_consume_fuel_instr()?; + self.reachable = frame.is_else_reachable(); + self.stack + .push_else(frame, is_end_of_then_reachable, consume_fuel_instr)?; + Ok(()) } fn visit_end(&mut self) -> Self::Output { From 5d4a5656ba2e382fbc3d0dd27b999a5250d3658c Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Tue, 17 Jun 2025 12:42:22 +0200 Subject: [PATCH 159/343] implement FuncTranslator::visit_nop --- crates/wasmi/src/engine/translator/func2/visit.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index d38fbd901c..898893c90c 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -77,7 +77,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { } fn visit_nop(&mut self) -> Self::Output { - todo!() + Ok(()) } fn visit_block(&mut self, block_ty: wasmparser::BlockType) -> Self::Output { From 8a4a60cf548deec03b54a29df68fac9e170d03fa Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Tue, 17 Jun 2025 12:44:37 +0200 Subject: [PATCH 160/343] implement FuncTranslator::visit_unreachable --- crates/wasmi/src/engine/translator/func2/visit.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index 898893c90c..304c909287 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -1,6 +1,6 @@ use super::{ControlFrame, FuncTranslator, LocalIdx, UnreachableControlFrame}; use crate::{ - core::wasm, + core::{wasm, FuelCostsProvider, TrapCode}, engine::{ translator::func2::{stack::IfReachability, Operand}, BlockType, @@ -73,7 +73,13 @@ impl<'a> VisitOperator<'a> for FuncTranslator { wasmparser::for_each_visit_operator!(impl_visit_operator); fn visit_unreachable(&mut self) -> Self::Output { - todo!() + bail_unreachable!(self); + self.push_instr( + Instruction::trap(TrapCode::UnreachableCodeReached), + FuelCostsProvider::base, + )?; + self.reachable = false; + Ok(()) } fn visit_nop(&mut self) -> Self::Output { From 1a6f1a0ee2f7c031be707b9de79a9ff69c190c5b Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Tue, 17 Jun 2025 12:50:21 +0200 Subject: [PATCH 161/343] implement FuncTranslator::end_unreachable Here we just do what the old Wasmi translator is doing: nothing. --- crates/wasmi/src/engine/translator/func2/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index d3e1a914eb..2644bf3608 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -466,7 +466,7 @@ impl FuncTranslator { /// Translates the end of an unreachable Wasm control frame. fn translate_end_unreachable(&mut self, _frame: UnreachableControlFrame) -> Result<(), Error> { - todo!() + Ok(()) } /// Translates an unconditional Wasm `branch` instruction. From b3f315895ce1281c8ef5fabcca62f735b2393992 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Tue, 17 Jun 2025 13:07:04 +0200 Subject: [PATCH 162/343] add StackHeight utility type This reduces size_of of ControlFrame from 48->36 bytes. --- .../engine/translator/func2/stack/control.rs | 43 +++++++++++++------ 1 file changed, 31 insertions(+), 12 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/stack/control.rs b/crates/wasmi/src/engine/translator/func2/stack/control.rs index 4dbc53498a..9be5ef80cb 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/control.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/control.rs @@ -11,6 +11,25 @@ use alloc::vec::{Drain, Vec}; #[cfg(doc)] use crate::ir::Instruction; +/// The height of the operand stack upon entering a [`ControlFrame`]. +#[derive(Debug, Copy, Clone)] +pub struct StackHeight(u16); + +impl From for usize { + fn from(height: StackHeight) -> Self { + usize::from(height.0) + } +} + +impl From for StackHeight { + fn from(height: usize) -> Self { + let Ok(height) = u16::try_from(height) else { + panic!("out of bounds stack height: {height}") + }; + Self(height) + } +} + /// The Wasm control stack. #[derive(Debug, Default)] pub struct ControlStack { @@ -85,7 +104,7 @@ impl ControlStack { ) { self.frames.push(ControlFrame::from(BlockControlFrame { ty, - height, + height: StackHeight::from(height), is_branched_to: false, consume_fuel, label, @@ -102,7 +121,7 @@ impl ControlStack { ) { self.frames.push(ControlFrame::from(LoopControlFrame { ty, - height, + height: StackHeight::from(height), is_branched_to: false, consume_fuel, label, @@ -121,7 +140,7 @@ impl ControlStack { ) { self.frames.push(ControlFrame::from(IfControlFrame { ty, - height, + height: StackHeight::from(height), is_branched_to: false, consume_fuel, label, @@ -148,7 +167,7 @@ impl ControlStack { let reachability = ElseReachability::from(if_frame.reachability); self.frames.push(ControlFrame::from(ElseControlFrame { ty, - height, + height: StackHeight::from(height), is_branched_to, consume_fuel, label, @@ -320,7 +339,7 @@ pub struct BlockControlFrame { /// The block type of the [`BlockControlFrame`]. ty: BlockType, /// The value stack height upon entering the [`BlockControlFrame`]. - height: usize, + height: StackHeight, /// This is `true` if there is at least one branch to this [`BlockControlFrame`]. is_branched_to: bool, /// The [`BlockControlFrame`]'s [`Instruction::ConsumeFuel`] if fuel metering is enabled. @@ -346,7 +365,7 @@ impl BlockControlFrame { /// Returns the height of the [`BlockControlFrame`]. pub fn height(&self) -> usize { - self.height + self.height.into() } /// Returns `true` if there are branches to this [`BlockControlFrame`]. @@ -378,7 +397,7 @@ pub struct LoopControlFrame { /// The block type of the [`LoopControlFrame`]. ty: BlockType, /// The value stack height upon entering the [`LoopControlFrame`]. - height: usize, + height: StackHeight, /// This is `true` if there is at least one branch to this [`LoopControlFrame`]. is_branched_to: bool, /// The [`LoopControlFrame`]'s [`Instruction::ConsumeFuel`] if fuel metering is enabled. @@ -404,7 +423,7 @@ impl LoopControlFrame { /// Returns the height of the [`LoopControlFrame`]. pub fn height(&self) -> usize { - self.height + self.height.into() } /// Returns `true` if there are branches to this [`LoopControlFrame`]. @@ -436,7 +455,7 @@ pub struct IfControlFrame { /// The block type of the [`IfControlFrame`]. ty: BlockType, /// The value stack height upon entering the [`IfControlFrame`]. - height: usize, + height: StackHeight, /// This is `true` if there is at least one branch to this [`IfControlFrame`]. is_branched_to: bool, /// The [`IfControlFrame`]'s [`Instruction::ConsumeFuel`] if fuel metering is enabled. @@ -464,7 +483,7 @@ impl IfControlFrame { /// Returns the height of the [`IfControlFrame`]. pub fn height(&self) -> usize { - self.height + self.height.into() } /// Returns `true` if there are branches to this [`IfControlFrame`]. @@ -553,7 +572,7 @@ pub struct ElseControlFrame { /// The block type of the [`ElseControlFrame`]. ty: BlockType, /// The value stack height upon entering the [`ElseControlFrame`]. - height: usize, + height: StackHeight, /// This is `true` if there is at least one branch to this [`ElseControlFrame`]. is_branched_to: bool, /// The [`LoopControlFrame`]'s [`Instruction::ConsumeFuel`] if fuel metering is enabled. @@ -622,7 +641,7 @@ impl ElseControlFrame { /// Returns the height of the [`ElseControlFrame`]. pub fn height(&self) -> usize { - self.height + self.height.into() } /// Returns `true` if there are branches to this [`ElseControlFrame`]. From 5f100633e6e2acf0007417beb261392ebde75cf8 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Fri, 20 Jun 2025 11:43:47 +0200 Subject: [PATCH 163/343] put debug_assert to various translate_end_* methods --- crates/wasmi/src/engine/translator/func2/mod.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index 2644bf3608..4f3a7a4034 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -446,26 +446,26 @@ impl FuncTranslator { /// Translates the end of a Wasm `loop` control frame. fn translate_end_loop(&mut self, _frame: LoopControlFrame) -> Result<(), Error> { - debug_assert!( - !self.stack.is_control_empty(), - "control stack must not be empty since its first element is always a `block`" - ); + debug_assert!(!self.stack.is_control_empty()); // Nothing needs to be done since Wasm `loop` control frames always only have a single exit. Ok(()) } /// Translates the end of a Wasm `if` control frame. fn translate_end_if(&mut self, _frame: IfControlFrame) -> Result<(), Error> { + debug_assert!(!self.stack.is_control_empty()); todo!() } /// Translates the end of a Wasm `else` control frame. fn translate_end_else(&mut self, _frame: ElseControlFrame) -> Result<(), Error> { + debug_assert!(!self.stack.is_control_empty()); todo!() } /// Translates the end of an unreachable Wasm control frame. fn translate_end_unreachable(&mut self, _frame: UnreachableControlFrame) -> Result<(), Error> { + debug_assert!(!self.stack.is_control_empty()); Ok(()) } From a5c4cc3584b99b8bc4f742025ddec30213deda62 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Fri, 20 Jun 2025 17:04:49 +0200 Subject: [PATCH 164/343] improve ControlStack::pop docs --- crates/wasmi/src/engine/translator/func2/stack/control.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/func2/stack/control.rs b/crates/wasmi/src/engine/translator/func2/stack/control.rs index 9be5ef80cb..033602131b 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/control.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/control.rs @@ -63,7 +63,7 @@ impl ElseOperands { self.ends.push(end); } - /// Pops the top-most Wasm `else` operands from `self` and returns it. + /// Pops the top-most Wasm `else` operands from `self` and returns them. pub fn pop(&mut self) -> Option> { let end = self.ends.pop()?; let start = self.ends.last().copied().unwrap_or(0); From 2a0a3eaff02ecd0b24151b75147528d3429489fd Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Fri, 20 Jun 2025 17:06:51 +0200 Subject: [PATCH 165/343] rename UnreachableControlFrame -> ControlFrameKind --- .../wasmi/src/engine/translator/func2/mod.rs | 4 ++-- .../engine/translator/func2/stack/control.rs | 20 +++++++++---------- .../src/engine/translator/func2/stack/mod.rs | 4 ++-- .../src/engine/translator/func2/visit.rs | 13 ++++++------ 4 files changed, 20 insertions(+), 21 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index 4f3a7a4034..9c345857e9 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -15,6 +15,7 @@ use self::{ stack::{ BlockControlFrame, ControlFrame, + ControlFrameKind, ElseControlFrame, IfControlFrame, LocalIdx, @@ -24,7 +25,6 @@ use self::{ Stack, StackAllocations, TempOperand, - UnreachableControlFrame, }, utils::{Reset, ReusableAllocations}, }; @@ -464,7 +464,7 @@ impl FuncTranslator { } /// Translates the end of an unreachable Wasm control frame. - fn translate_end_unreachable(&mut self, _frame: UnreachableControlFrame) -> Result<(), Error> { + fn translate_end_unreachable(&mut self, _frame: ControlFrameKind) -> Result<(), Error> { debug_assert!(!self.stack.is_control_empty()); Ok(()) } diff --git a/crates/wasmi/src/engine/translator/func2/stack/control.rs b/crates/wasmi/src/engine/translator/func2/stack/control.rs index 033602131b..67f2da650e 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/control.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/control.rs @@ -90,7 +90,7 @@ impl ControlStack { } /// Pushes a new unreachable Wasm control frame onto the [`ControlStack`]. - pub fn push_unreachable(&mut self, kind: UnreachableControlFrame) { + pub fn push_unreachable(&mut self, kind: ControlFrameKind) { self.frames.push(ControlFrame::from(kind)) } @@ -257,7 +257,7 @@ pub enum ControlFrame { /// A Wasm `else` control frame. Else(ElseControlFrame), /// A generic unreachable control frame. - Unreachable(UnreachableControlFrame), + Unreachable(ControlFrameKind), } impl From for ControlFrame { @@ -284,8 +284,8 @@ impl From for ControlFrame { } } -impl From for ControlFrame { - fn from(frame: UnreachableControlFrame) -> Self { +impl From for ControlFrame { + fn from(frame: ControlFrameKind) -> Self { Self::Unreachable(frame) } } @@ -696,15 +696,15 @@ impl ElseControlFrame { } } -/// A generic unreachable Wasm control frame. +/// The kind of a Wasm control frame. #[derive(Debug, Copy, Clone)] -pub enum UnreachableControlFrame { - /// An unreachable Wasm `block` control frame. +pub enum ControlFrameKind { + /// An Wasm `block` control frame. Block, - /// An unreachable Wasm `loop` control frame. + /// An Wasm `loop` control frame. Loop, - /// An unreachable Wasm `if` control frame. + /// An Wasm `if` control frame. If, - /// An unreachable Wasm `else` control frame. + /// An Wasm `else` control frame. Else, } diff --git a/crates/wasmi/src/engine/translator/func2/stack/mod.rs b/crates/wasmi/src/engine/translator/func2/stack/mod.rs index 400eaca5f6..0f011f035f 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/mod.rs @@ -12,12 +12,12 @@ pub use self::{ control::{ BlockControlFrame, ControlFrame, + ControlFrameKind, ElseControlFrame, ElseReachability, IfControlFrame, IfReachability, LoopControlFrame, - UnreachableControlFrame, }, locals::LocalIdx, operand::{Operand, TempOperand}, @@ -291,7 +291,7 @@ impl Stack { /// # Errors /// /// If the stack height exceeds the maximum height. - pub fn push_unreachable(&mut self, kind: UnreachableControlFrame) -> Result<(), Error> { + pub fn push_unreachable(&mut self, kind: ControlFrameKind) -> Result<(), Error> { self.controls.push_unreachable(kind); Ok(()) } diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index 304c909287..d671c01882 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -1,4 +1,4 @@ -use super::{ControlFrame, FuncTranslator, LocalIdx, UnreachableControlFrame}; +use super::{ControlFrame, ControlFrameKind, FuncTranslator, LocalIdx}; use crate::{ core::{wasm, FuelCostsProvider, TrapCode}, engine::{ @@ -88,8 +88,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { fn visit_block(&mut self, block_ty: wasmparser::BlockType) -> Self::Output { if !self.reachable { - self.stack - .push_unreachable(UnreachableControlFrame::Block)?; + self.stack.push_unreachable(ControlFrameKind::Block)?; return Ok(()); } let block_ty = BlockType::new(block_ty, &self.module); @@ -100,7 +99,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { fn visit_loop(&mut self, block_ty: wasmparser::BlockType) -> Self::Output { if !self.reachable { - self.stack.push_unreachable(UnreachableControlFrame::Loop)?; + self.stack.push_unreachable(ControlFrameKind::Loop)?; return Ok(()); } let block_ty = BlockType::new(block_ty, &self.module); @@ -117,7 +116,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { fn visit_if(&mut self, block_ty: wasmparser::BlockType) -> Self::Output { if !self.reachable { - self.stack.push_unreachable(UnreachableControlFrame::If)?; + self.stack.push_unreachable(ControlFrameKind::If)?; return Ok(()); } let end_label = self.labels.new_label(); @@ -152,9 +151,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { fn visit_else(&mut self) -> Self::Output { let mut frame = match self.stack.pop_control() { ControlFrame::If(frame) => frame, - ControlFrame::Unreachable(UnreachableControlFrame::If) => { + ControlFrame::Unreachable(ControlFrameKind::If) => { debug_assert!(!self.reachable); - self.stack.push_unreachable(UnreachableControlFrame::Else)?; + self.stack.push_unreachable(ControlFrameKind::Else)?; return Ok(()); } unexpected => panic!("expected `if` control frame but found: {unexpected:?}"), From deecddc58e33ac0106022c879cd3adcbb1f5c9bd Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Fri, 20 Jun 2025 17:07:39 +0200 Subject: [PATCH 166/343] pop else operands only when available --- .../engine/translator/func2/stack/control.rs | 16 ++++++++++---- .../src/engine/translator/func2/stack/mod.rs | 22 ++++++++++--------- 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/stack/control.rs b/crates/wasmi/src/engine/translator/func2/stack/control.rs index 67f2da650e..0d2617eea8 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/control.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/control.rs @@ -159,7 +159,7 @@ impl ControlStack { if_frame: IfControlFrame, consume_fuel: Option, is_end_of_then_reachable: bool, - ) -> Drain { + ) -> Option> { let ty = if_frame.ty(); let height = if_frame.height(); let label = if_frame.label(); @@ -174,9 +174,17 @@ impl ControlStack { reachability, is_end_of_then_reachable, })); - self.else_operands - .pop() - .unwrap_or_else(|| panic!("missing operands for `else` control frame")) + self.expect_else = false; + match reachability { + ElseReachability::OnlyThen | ElseReachability::OnlyElse => None, + ElseReachability::Both => { + let else_operands = self + .else_operands + .pop() + .unwrap_or_else(|| panic!("missing operands for `else` control frame")); + Some(else_operands) + } + } } /// Pops the top-most [`ControlFrame`] and returns it if any. diff --git a/crates/wasmi/src/engine/translator/func2/stack/mod.rs b/crates/wasmi/src/engine/translator/func2/stack/mod.rs index 0f011f035f..6ae232c629 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/mod.rs @@ -270,16 +270,18 @@ impl Stack { let else_operands = self.controls .push_else(if_frame, consume_fuel, is_end_of_then_reachable); - for operand in else_operands { - match operand { - Operand::Local(op) => { - self.operands.push_local(op.local_index())?; - } - Operand::Temp(op) => { - self.operands.push_temp(op.ty(), op.instr())?; - } - Operand::Immediate(op) => { - self.operands.push_immediate(op.val())?; + if let Some(else_operands) = else_operands { + for operand in else_operands { + match operand { + Operand::Local(op) => { + self.operands.push_local(op.local_index())?; + } + Operand::Temp(op) => { + self.operands.push_temp(op.ty(), op.instr())?; + } + Operand::Immediate(op) => { + self.operands.push_immediate(op.val())?; + } } } } From d5bab0c33fe012cdc39b01c6c8ec8a483b6c95a5 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Fri, 20 Jun 2025 17:07:52 +0200 Subject: [PATCH 167/343] drop else operands when possible --- .../engine/translator/func2/stack/control.rs | 38 ++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/func2/stack/control.rs b/crates/wasmi/src/engine/translator/func2/stack/control.rs index 0d2617eea8..2a1f64eed5 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/control.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/control.rs @@ -37,6 +37,8 @@ pub struct ControlStack { frames: Vec, /// Special operand stack to memorize operands for `else` control frames. else_operands: ElseOperands, + /// This is `true` if an `if` with else providers was just popped from the stack. + expect_else: bool, } /// Duplicated operands for Wasm `else` control frames. @@ -69,6 +71,13 @@ impl ElseOperands { let start = self.ends.last().copied().unwrap_or(0); Some(self.operands.drain(start..end)) } + + /// Drops the top-most Wasm `else` operands from `self`. + pub fn drop(&mut self) { + self.ends.pop().expect("tried to drop empty else operands"); + let start = self.ends.last().copied().unwrap_or(0); + self.operands.truncate(start); + } } impl Reset for ControlStack { @@ -91,6 +100,7 @@ impl ControlStack { /// Pushes a new unreachable Wasm control frame onto the [`ControlStack`]. pub fn push_unreachable(&mut self, kind: ControlFrameKind) { + self.try_drop_else_providers(); self.frames.push(ControlFrame::from(kind)) } @@ -102,6 +112,7 @@ impl ControlStack { label: LabelRef, consume_fuel: Option, ) { + self.try_drop_else_providers(); self.frames.push(ControlFrame::from(BlockControlFrame { ty, height: StackHeight::from(height), @@ -119,6 +130,7 @@ impl ControlStack { label: LabelRef, consume_fuel: Option, ) { + self.try_drop_else_providers(); self.frames.push(ControlFrame::from(LoopControlFrame { ty, height: StackHeight::from(height), @@ -138,6 +150,7 @@ impl ControlStack { reachability: IfReachability, else_operands: impl IntoIterator, ) { + self.try_drop_else_providers(); self.frames.push(ControlFrame::from(IfControlFrame { ty, height: StackHeight::from(height), @@ -189,7 +202,25 @@ impl ControlStack { /// Pops the top-most [`ControlFrame`] and returns it if any. pub fn pop(&mut self) -> Option { - self.frames.pop() + self.try_drop_else_providers(); + let frame = self.frames.pop()?; + self.expect_else = match &frame { + ControlFrame::If(frame) => { + matches!(frame.reachability, IfReachability::Both { .. }) + } + _ => false, + }; + Some(frame) + } + + /// Drops the top-most else operands if `expect_else` is `true`. + /// + /// Otherwise do nothing. + pub fn try_drop_else_providers(&mut self) { + if self.expect_else { + self.else_operands.drop(); + } + self.expect_else = false; } /// Returns a shared reference to the [`ControlFrame`] at `depth` if any. @@ -516,6 +547,11 @@ impl IfControlFrame { self.label } + /// Returns the [`IfReachability`] of the [`IfControlFrame`]. + pub fn reachability(&self) -> IfReachability { + self.reachability + } + /// Returns the label to the `else` branch of the [`IfControlFrame`]. pub fn else_label(&self) -> Option { match self.reachability { From bccd1d7e0386325d337c4d78a13e1b48a907dcd0 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Fri, 20 Jun 2025 17:32:08 +0200 Subject: [PATCH 168/343] pull let binding into if's then block --- crates/wasmi/src/engine/translator/func2/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index 9c345857e9..6fc9bde71c 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -426,9 +426,9 @@ impl FuncTranslator { /// Translates the end of a Wasm `block` control frame. fn translate_end_block(&mut self, frame: BlockControlFrame) -> Result<(), Error> { - let len_values = frame.len_branch_params(&self.engine); let consume_fuel_instr = frame.consume_fuel_instr(); if self.reachable && frame.is_branched_to() { + let len_values = frame.len_branch_params(&self.engine); self.copy_branch_params(usize::from(len_values), consume_fuel_instr)?; } if let Err(err) = self From 462726cedc2acf50dc9d190bb9bf222b3c2f0d35 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Fri, 20 Jun 2025 19:45:42 +0200 Subject: [PATCH 169/343] implement translate_end_if --- .../wasmi/src/engine/translator/func2/mod.rs | 46 ++++++++++++++++++- 1 file changed, 44 insertions(+), 2 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index 6fc9bde71c..762975daba 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -18,6 +18,7 @@ use self::{ ControlFrameKind, ElseControlFrame, IfControlFrame, + IfReachability, LocalIdx, LoopControlFrame, Operand, @@ -452,9 +453,50 @@ impl FuncTranslator { } /// Translates the end of a Wasm `if` control frame. - fn translate_end_if(&mut self, _frame: IfControlFrame) -> Result<(), Error> { + fn translate_end_if(&mut self, frame: IfControlFrame) -> Result<(), Error> { debug_assert!(!self.stack.is_control_empty()); - todo!() + let IfReachability::Both { else_label } = frame.reachability() else { + return self.translate_end_if_then_or_else_only(frame); + }; + let end_of_then_reachable = self.reachable; + let len_results = frame.ty().len_results(self.engine()); + let has_results = len_results >= 1; + if end_of_then_reachable && has_results { + let consume_fuel_instr = frame.consume_fuel_instr(); + self.copy_branch_params(usize::from(len_results), consume_fuel_instr)?; + let end_offset = self + .labels + .try_resolve_label(frame.label(), self.instrs.next_instr()) + .unwrap(); + self.instrs.push_instr( + Instruction::branch(end_offset), + consume_fuel_instr, + FuelCostsProvider::base, + )?; + } + self.labels + .try_pin_label(else_label, self.instrs.next_instr()); + self.reachable = true; + Ok(()) + } + + /// Translates the end of a Wasm `if` control frame where only one branch is known to be reachable. + fn translate_end_if_then_or_else_only(&mut self, frame: IfControlFrame) -> Result<(), Error> { + let end_is_reachable = match frame.reachability() { + IfReachability::OnlyThen => self.reachable, + IfReachability::OnlyElse => true, + IfReachability::Both { .. } => unreachable!(), + }; + if end_is_reachable && frame.is_branched_to() { + let len_values = frame.len_branch_params(&self.engine); + let consume_fuel_instr = frame.consume_fuel_instr(); + self.copy_branch_params(usize::from(len_values), consume_fuel_instr)?; + } + self.labels + .pin_label(frame.label(), self.instrs.next_instr()) + .unwrap(); + self.reachable = end_is_reachable || frame.is_branched_to(); + Ok(()) } /// Translates the end of a Wasm `else` control frame. From e52da651206030c88b52d01f082abc6a9ed1e416 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Fri, 20 Jun 2025 20:50:27 +0200 Subject: [PATCH 170/343] add and use ControlFrameBase trait --- .../wasmi/src/engine/translator/func2/mod.rs | 1 + .../engine/translator/func2/stack/control.rs | 216 ++++++++++-------- .../src/engine/translator/func2/stack/mod.rs | 1 + .../src/engine/translator/func2/visit.rs | 2 +- 4 files changed, 124 insertions(+), 96 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index 762975daba..76539acebe 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -15,6 +15,7 @@ use self::{ stack::{ BlockControlFrame, ControlFrame, + ControlFrameBase, ControlFrameKind, ElseControlFrame, IfControlFrame, diff --git a/crates/wasmi/src/engine/translator/func2/stack/control.rs b/crates/wasmi/src/engine/translator/func2/stack/control.rs index 2a1f64eed5..a04fcaeaeb 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/control.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/control.rs @@ -329,9 +329,54 @@ impl From for ControlFrame { } } -impl ControlFrame { - /// Makes the [`ControlFrame`] aware that there is a branch to it. - pub fn branch_to(&mut self) { +/// Trait implemented by control frame types that share a common API. +pub trait ControlFrameBase { + /// Returns the branch label of `self`. + fn label(&self) -> LabelRef; + + /// Returns `true` if there exists a branch to `self.` + fn is_branched_to(&self) -> bool; + + /// Makes `self` aware that there is a branch to it. + fn branch_to(&mut self); + + /// Returns the number of operands required for branching to `self`. + fn len_branch_params(&self, engine: &Engine) -> u16; + + /// Returns a reference to the [`Instruction::ConsumeFuel`] of `self`. + /// + /// Returns `None` if fuel metering is disabled. + fn consume_fuel_instr(&self) -> Option; +} + +impl ControlFrameBase for ControlFrame { + fn label(&self) -> LabelRef { + match self { + ControlFrame::Block(frame) => frame.label(), + ControlFrame::Loop(frame) => frame.label(), + ControlFrame::If(frame) => frame.label(), + ControlFrame::Else(frame) => frame.label(), + ControlFrame::Unreachable(_) => { + panic!("invalid query for unreachable control frame: `ControlFrame::label`") + } + } + } + + fn is_branched_to(&self) -> bool { + match self { + ControlFrame::Block(frame) => frame.is_branched_to(), + ControlFrame::Loop(frame) => frame.is_branched_to(), + ControlFrame::If(frame) => frame.is_branched_to(), + ControlFrame::Else(frame) => frame.is_branched_to(), + ControlFrame::Unreachable(_) => { + panic!( + "invalid query for unreachable control frame: `ControlFrame::is_branched_to`" + ) + } + } + } + + fn branch_to(&mut self) { match self { ControlFrame::Block(frame) => frame.branch_to(), ControlFrame::Loop(frame) => frame.branch_to(), @@ -343,8 +388,7 @@ impl ControlFrame { } } - /// Returns the number of operands required for branching to the [`ControlFrame`]. - pub fn len_branch_params(&self, engine: &Engine) -> u16 { + fn len_branch_params(&self, engine: &Engine) -> u16 { match self { ControlFrame::Block(frame) => frame.len_branch_params(engine), ControlFrame::Loop(frame) => frame.len_branch_params(engine), @@ -356,10 +400,7 @@ impl ControlFrame { } } - /// Returns a reference to the [`Instruction::ConsumeFuel`] of the [`ControlFrame`] if any. - /// - /// Returns `None` if fuel metering is disabled. - pub fn consume_fuel_instr(&self) -> Option { + fn consume_fuel_instr(&self) -> Option { match self { ControlFrame::Block(frame) => frame.consume_fuel_instr(), ControlFrame::Loop(frame) => frame.consume_fuel_instr(), @@ -397,36 +438,31 @@ impl BlockControlFrame { self.ty } - /// Returns the number of operands required for branching to the [`BlockControlFrame`]. - pub fn len_branch_params(&self, engine: &Engine) -> u16 { - self.ty.len_results(engine) - } - /// Returns the height of the [`BlockControlFrame`]. pub fn height(&self) -> usize { self.height.into() } +} + +impl ControlFrameBase for BlockControlFrame { + fn label(&self) -> LabelRef { + self.label + } - /// Returns `true` if there are branches to this [`BlockControlFrame`]. - pub fn is_branched_to(&self) -> bool { + fn is_branched_to(&self) -> bool { self.is_branched_to } - /// Makes the [`BlockControlFrame`] aware that there is a branch to it. - pub fn branch_to(&mut self) { + fn branch_to(&mut self) { self.is_branched_to = true; } - /// Returns a reference to the [`Instruction::ConsumeFuel`] of the [`BlockControlFrame`] if any. - /// - /// Returns `None` if fuel metering is disabled. - pub fn consume_fuel_instr(&self) -> Option { - self.consume_fuel + fn len_branch_params(&self, engine: &Engine) -> u16 { + self.ty.len_results(engine) } - /// Returns the branch label of the [`BlockControlFrame`]. - pub fn label(&self) -> LabelRef { - self.label + fn consume_fuel_instr(&self) -> Option { + self.consume_fuel } } @@ -455,36 +491,31 @@ impl LoopControlFrame { self.ty } - /// Returns the number of operands required for branching to the [`LoopControlFrame`]. - pub fn len_branch_params(&self, engine: &Engine) -> u16 { - self.ty.len_params(engine) - } - /// Returns the height of the [`LoopControlFrame`]. pub fn height(&self) -> usize { self.height.into() } +} - /// Returns `true` if there are branches to this [`LoopControlFrame`]. - pub fn is_branched_to(&self) -> bool { +impl ControlFrameBase for LoopControlFrame { + fn label(&self) -> LabelRef { + self.label + } + + fn is_branched_to(&self) -> bool { self.is_branched_to } - /// Makes the [`LoopControlFrame`] aware that there is a branch to it. - pub fn branch_to(&mut self) { + fn branch_to(&mut self) { self.is_branched_to = true; } - /// Returns a reference to the [`Instruction::ConsumeFuel`] of the [`LoopControlFrame`] if any. - /// - /// Returns `None` if fuel metering is disabled. - pub fn consume_fuel_instr(&self) -> Option { - self.consume_fuel + fn len_branch_params(&self, engine: &Engine) -> u16 { + self.ty.len_params(engine) } - /// Returns the branch label of the [`LoopControlFrame`]. - pub fn label(&self) -> LabelRef { - self.label + fn consume_fuel_instr(&self) -> Option { + self.consume_fuel } } @@ -515,38 +546,11 @@ impl IfControlFrame { self.ty } - /// Returns the number of operands required for branching to the [`IfControlFrame`]. - pub fn len_branch_params(&self, engine: &Engine) -> u16 { - self.ty.len_results(engine) - } - /// Returns the height of the [`IfControlFrame`]. pub fn height(&self) -> usize { self.height.into() } - /// Returns `true` if there are branches to this [`IfControlFrame`]. - pub fn is_branched_to(&self) -> bool { - self.is_branched_to - } - - /// Makes the [`IfControlFrame`] aware that there is a branch to it. - pub fn branch_to(&mut self) { - self.is_branched_to = true; - } - - /// Returns a reference to the [`Instruction::ConsumeFuel`] of the [`IfControlFrame`] if any. - /// - /// Returns `None` if fuel metering is disabled. - pub fn consume_fuel_instr(&self) -> Option { - self.consume_fuel - } - - /// Returns the branch label of the [`IfControlFrame`]. - pub fn label(&self) -> LabelRef { - self.label - } - /// Returns the [`IfReachability`] of the [`IfControlFrame`]. pub fn reachability(&self) -> IfReachability { self.reachability @@ -585,6 +589,28 @@ impl IfControlFrame { } } +impl ControlFrameBase for IfControlFrame { + fn label(&self) -> LabelRef { + self.label + } + + fn is_branched_to(&self) -> bool { + self.is_branched_to + } + + fn branch_to(&mut self) { + self.is_branched_to = true; + } + + fn len_branch_params(&self, engine: &Engine) -> u16 { + self.ty.len_results(engine) + } + + fn consume_fuel_instr(&self) -> Option { + self.consume_fuel + } +} + /// The reachability of the `if` control flow frame. #[derive(Debug, Copy, Clone)] pub enum IfReachability { @@ -678,36 +704,14 @@ impl ElseControlFrame { self.ty } - /// Returns the number of operands required for branching to the [`ElseControlFrame`]. - pub fn len_branch_params(&self, engine: &Engine) -> u16 { - self.ty.len_results(engine) - } - /// Returns the height of the [`ElseControlFrame`]. pub fn height(&self) -> usize { self.height.into() } - /// Returns `true` if there are branches to this [`ElseControlFrame`]. - pub fn is_branched_to(&self) -> bool { - self.is_branched_to - } - - /// Makes the [`ElseControlFrame`] aware that there is a branch to it. - pub fn branch_to(&mut self) { - self.is_branched_to = true; - } - - /// Returns a reference to the [`Instruction::ConsumeFuel`] of the [`ElseControlFrame`] if any. - /// - /// Returns `None` if fuel metering is disabled. - pub fn consume_fuel_instr(&self) -> Option { - self.consume_fuel - } - - /// Returns the branch label of the [`ElseControlFrame`]. - pub fn label(&self) -> LabelRef { - self.label + /// Returns the [`ElseReachability`] of the [`ElseReachability`]. + pub fn reachability(&self) -> ElseReachability { + self.reachability } /// Returns `true` if the `then` branch is reachable. @@ -740,6 +744,28 @@ impl ElseControlFrame { } } +impl ControlFrameBase for ElseControlFrame { + fn label(&self) -> LabelRef { + self.label + } + + fn is_branched_to(&self) -> bool { + self.is_branched_to + } + + fn branch_to(&mut self) { + self.is_branched_to = true; + } + + fn len_branch_params(&self, engine: &Engine) -> u16 { + self.ty.len_results(engine) + } + + fn consume_fuel_instr(&self) -> Option { + self.consume_fuel + } +} + /// The kind of a Wasm control frame. #[derive(Debug, Copy, Clone)] pub enum ControlFrameKind { diff --git a/crates/wasmi/src/engine/translator/func2/stack/mod.rs b/crates/wasmi/src/engine/translator/func2/stack/mod.rs index 6ae232c629..f23ad124fb 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/mod.rs @@ -12,6 +12,7 @@ pub use self::{ control::{ BlockControlFrame, ControlFrame, + ControlFrameBase, ControlFrameKind, ElseControlFrame, ElseReachability, diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index d671c01882..5f0f43dea4 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -2,7 +2,7 @@ use super::{ControlFrame, ControlFrameKind, FuncTranslator, LocalIdx}; use crate::{ core::{wasm, FuelCostsProvider, TrapCode}, engine::{ - translator::func2::{stack::IfReachability, Operand}, + translator::func2::{stack::IfReachability, ControlFrameBase, Operand}, BlockType, }, ir::Instruction, From 4227cf03b63cebe1937d38087600a7ac45213ec4 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Fri, 20 Jun 2025 20:52:09 +0200 Subject: [PATCH 171/343] use ControlFrameBase to generalize end_if translation --- .../wasmi/src/engine/translator/func2/mod.rs | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index 76539acebe..dd5805e85e 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -457,7 +457,8 @@ impl FuncTranslator { fn translate_end_if(&mut self, frame: IfControlFrame) -> Result<(), Error> { debug_assert!(!self.stack.is_control_empty()); let IfReachability::Both { else_label } = frame.reachability() else { - return self.translate_end_if_then_or_else_only(frame); + let reachability = frame.reachability().into(); + return self.translate_end_if_or_else_only(frame, reachability); }; let end_of_then_reachable = self.reachable; let len_results = frame.ty().len_results(self.engine()); @@ -481,12 +482,16 @@ impl FuncTranslator { Ok(()) } - /// Translates the end of a Wasm `if` control frame where only one branch is known to be reachable. - fn translate_end_if_then_or_else_only(&mut self, frame: IfControlFrame) -> Result<(), Error> { - let end_is_reachable = match frame.reachability() { - IfReachability::OnlyThen => self.reachable, - IfReachability::OnlyElse => true, - IfReachability::Both { .. } => unreachable!(), + /// Translates the end of a Wasm `else` control frame where only one branch is known to be reachable. + fn translate_end_if_or_else_only( + &mut self, + frame: impl ControlFrameBase, + reachability: ElseReachability, + ) -> Result<(), Error> { + let end_is_reachable = match reachability { + ElseReachability::OnlyThen => self.reachable, + ElseReachability::OnlyElse => true, + ElseReachability::Both => unreachable!(), }; if end_is_reachable && frame.is_branched_to() { let len_values = frame.len_branch_params(&self.engine); From b55c0f18dd4f27fe64da6fcedebe3133e8b26de0 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Fri, 20 Jun 2025 21:02:21 +0200 Subject: [PATCH 172/343] fix translate_end_if label pinning --- crates/wasmi/src/engine/translator/func2/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index dd5805e85e..40d5808af3 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -477,7 +477,7 @@ impl FuncTranslator { )?; } self.labels - .try_pin_label(else_label, self.instrs.next_instr()); + .try_pin_label(frame.label(), self.instrs.next_instr()); self.reachable = true; Ok(()) } From d5e6a87cc71b9809d1052b790a5842ce4287b579 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Fri, 20 Jun 2025 21:02:34 +0200 Subject: [PATCH 173/343] implement translate_end_else --- .../wasmi/src/engine/translator/func2/mod.rs | 35 +++++++++++++++---- 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index 40d5808af3..c24d748b7b 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -18,6 +18,7 @@ use self::{ ControlFrameBase, ControlFrameKind, ElseControlFrame, + ElseReachability, IfControlFrame, IfReachability, LocalIdx, @@ -482,6 +483,34 @@ impl FuncTranslator { Ok(()) } + /// Translates the end of a Wasm `else` control frame. + fn translate_end_else(&mut self, frame: ElseControlFrame) -> Result<(), Error> { + debug_assert!(!self.stack.is_control_empty()); + let reachability = frame.reachability(); + if matches!( + reachability, + ElseReachability::OnlyThen | ElseReachability::OnlyElse + ) { + return self.translate_end_if_or_else_only(frame, reachability); + } + let end_of_then_reachable = frame.is_end_of_then_reachable(); + let end_of_else_reachable = self.reachable; + let reachable = match (end_of_then_reachable, end_of_else_reachable) { + (false, false) => frame.is_branched_to(), + _ => true, + }; + if end_of_else_reachable { + let len_values = frame.len_branch_params(&self.engine); + let consume_fuel_instr: Option = frame.consume_fuel_instr(); + self.copy_branch_params(usize::from(len_values), consume_fuel_instr)?; + } + self.labels + .pin_label(frame.label(), self.instrs.next_instr()) + .unwrap(); + self.reachable = reachable; + Ok(()) + } + /// Translates the end of a Wasm `else` control frame where only one branch is known to be reachable. fn translate_end_if_or_else_only( &mut self, @@ -505,12 +534,6 @@ impl FuncTranslator { Ok(()) } - /// Translates the end of a Wasm `else` control frame. - fn translate_end_else(&mut self, _frame: ElseControlFrame) -> Result<(), Error> { - debug_assert!(!self.stack.is_control_empty()); - todo!() - } - /// Translates the end of an unreachable Wasm control frame. fn translate_end_unreachable(&mut self, _frame: ControlFrameKind) -> Result<(), Error> { debug_assert!(!self.stack.is_control_empty()); From 5afeb60b4efe1daf0ea83864bedb50c413cd2208 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Fri, 20 Jun 2025 21:12:28 +0200 Subject: [PATCH 174/343] fix pinning of the else_label --- crates/wasmi/src/engine/translator/func2/mod.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index c24d748b7b..b02e98c406 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -477,8 +477,9 @@ impl FuncTranslator { FuelCostsProvider::base, )?; } - self.labels - .try_pin_label(frame.label(), self.instrs.next_instr()); + let next_instr = self.instrs.next_instr(); + self.labels.try_pin_label(else_label, next_instr); + self.labels.pin_label(frame.label(), next_instr).unwrap(); self.reachable = true; Ok(()) } From 268967530778cc5daf9b083fd0ce03c2cc529ef1 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Fri, 20 Jun 2025 23:45:01 +0200 Subject: [PATCH 175/343] implement FuncTranslator::visit_return --- crates/wasmi/src/engine/translator/func2/visit.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index 5f0f43dea4..534c26adc2 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -7,6 +7,7 @@ use crate::{ }, ir::Instruction, Error, + FuncType, }; use wasmparser::VisitOperator; @@ -208,7 +209,15 @@ impl<'a> VisitOperator<'a> for FuncTranslator { } fn visit_return(&mut self) -> Self::Output { - todo!() + bail_unreachable!(self); + let consume_fuel_instr = self.stack.consume_fuel_instr(); + self.translate_return(consume_fuel_instr)?; + let len_results = self.func_type_with(FuncType::len_results); + for _ in 0..len_results { + self.stack.pop(); + } + self.reachable = false; + Ok(()) } fn visit_call(&mut self, _function_index: u32) -> Self::Output { From 1e7178ba0eee883fa37ede30cc91d23abf14126f Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Fri, 20 Jun 2025 23:47:36 +0200 Subject: [PATCH 176/343] rename translate_return -> encode_return --- crates/wasmi/src/engine/translator/func2/mod.rs | 6 +++--- crates/wasmi/src/engine/translator/func2/visit.rs | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index b02e98c406..2136937811 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -372,8 +372,8 @@ impl FuncTranslator { Ok(()) } - /// Translates a generic return instruction. - fn translate_return(&mut self, consume_fuel: Option) -> Result { + /// Encodes a generic return instruction. + fn encode_return(&mut self, consume_fuel: Option) -> Result { let len_results = self.func_type_with(FuncType::len_results); let instr = match len_results { 0 => Instruction::Return, @@ -442,7 +442,7 @@ impl FuncTranslator { } self.reachable |= frame.is_branched_to(); if self.reachable && self.stack.is_control_empty() { - self.translate_return(consume_fuel_instr)?; + self.encode_return(consume_fuel_instr)?; } Ok(()) } diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index 534c26adc2..0e4d5a12df 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -211,7 +211,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { fn visit_return(&mut self) -> Self::Output { bail_unreachable!(self); let consume_fuel_instr = self.stack.consume_fuel_instr(); - self.translate_return(consume_fuel_instr)?; + self.encode_return(consume_fuel_instr)?; let len_results = self.func_type_with(FuncType::len_results); for _ in 0..len_results { self.stack.pop(); From aa6456febcfe4ad6f92751062ccaa54410cc7dfa Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Sun, 22 Jun 2025 18:11:56 +0200 Subject: [PATCH 177/343] make ControlStack::get_mut return ControlFrameMut wrapper --- .../engine/translator/func2/stack/control.rs | 45 ++++++++++++++++--- 1 file changed, 38 insertions(+), 7 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/stack/control.rs b/crates/wasmi/src/engine/translator/func2/stack/control.rs index a04fcaeaeb..5a46bb07a3 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/control.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/control.rs @@ -234,13 +234,44 @@ impl ControlStack { } /// Returns an exclusive reference to the [`ControlFrame`] at `depth` if any. - pub fn get_mut(&mut self, depth: usize) -> &mut ControlFrame { + pub fn get_mut(&mut self, depth: usize) -> ControlFrameMut { let height = self.height(); - self.frames.iter_mut().rev().nth(depth).unwrap_or_else(|| { - panic!( + self.frames + .iter_mut() + .rev() + .nth(depth) + .map(ControlFrameMut) + .unwrap_or_else(|| { + panic!( "out of bounds control frame at depth (={depth}) for stack of height (={height})" ) - }) + }) + } +} + +/// An exclusive reference to a [`ControlFrame`]. +#[derive(Debug)] +pub struct ControlFrameMut<'a>(&'a mut ControlFrame); + +impl<'a> ControlFrameBase for ControlFrameMut<'a> { + fn label(&self) -> LabelRef { + self.0.label() + } + + fn is_branched_to(&self) -> bool { + self.0.is_branched_to() + } + + fn branch_to(&mut self) { + self.0.branch_to() + } + + fn len_branch_params(&self, engine: &Engine) -> u16 { + self.0.len_branch_params(engine) + } + + fn consume_fuel_instr(&self) -> Option { + self.0.consume_fuel_instr() } } @@ -248,14 +279,14 @@ impl ControlStack { #[derive(Debug)] pub enum AcquiredTarget<'stack> { /// The branch targets the function enclosing `block` and therefore is a `return`. - Return(&'stack mut ControlFrame), + Return(ControlFrameMut<'stack>), /// The branch targets a regular [`ControlFrame`]. - Branch(&'stack mut ControlFrame), + Branch(ControlFrameMut<'stack>), } impl<'stack> AcquiredTarget<'stack> { /// Returns an exclusive reference to the [`ControlFrame`] of the [`AcquiredTarget`]. - pub fn control_frame(&'stack mut self) -> &'stack mut ControlFrame { + pub fn control_frame(self) -> ControlFrameMut<'stack> { match self { Self::Return(frame) => frame, Self::Branch(frame) => frame, From c7a379b7ed1d00eff9d14de7d39b34ebc9c6e064 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Sun, 22 Jun 2025 18:12:07 +0200 Subject: [PATCH 178/343] add Stack::acquire_target method --- crates/wasmi/src/engine/translator/func2/stack/mod.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/crates/wasmi/src/engine/translator/func2/stack/mod.rs b/crates/wasmi/src/engine/translator/func2/stack/mod.rs index f23ad124fb..495f237432 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/mod.rs @@ -10,6 +10,7 @@ use self::{ }; pub use self::{ control::{ + AcquiredTarget, BlockControlFrame, ControlFrame, ControlFrameBase, @@ -319,6 +320,15 @@ impl Stack { self.controls.get(depth) } + /// Acquires the branch target at `depth`. + /// + /// # Panics + /// + /// If `depth` is out of bounds for `self`. + pub fn acquire_target(&mut self, depth: usize) -> AcquiredTarget { + self.controls.acquire_target(depth) + } + /// Pushes a local variable with index `local_idx` to the [`Stack`]. /// /// # Errors From 971f41df10be79796e6bc386f0bec477f43db699 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 23 Jun 2025 10:55:58 +0200 Subject: [PATCH 179/343] rename acquire_target -> peek_control_mut + docs --- crates/wasmi/src/engine/translator/func2/stack/mod.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/stack/mod.rs b/crates/wasmi/src/engine/translator/func2/stack/mod.rs index 495f237432..511d0c2d5e 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/mod.rs @@ -320,12 +320,18 @@ impl Stack { self.controls.get(depth) } - /// Acquires the branch target at `depth`. + /// Returns an exclusive reference to the [`ControlFrame`] at `depth`. + /// + /// # Note + /// + /// This returns an [`AcquiredTarget`] to differentiate between the function + /// body Wasm `block` and other control frames in order to know whether a branching + /// target returns or branches. /// /// # Panics /// /// If `depth` is out of bounds for `self`. - pub fn acquire_target(&mut self, depth: usize) -> AcquiredTarget { + pub fn peek_control_mut(&mut self, depth: usize) -> AcquiredTarget { self.controls.acquire_target(depth) } From 58b5f5a5e7801f64e96dfdd50c70c82cf3ff992c Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 23 Jun 2025 11:10:38 +0200 Subject: [PATCH 180/343] apply rustfmt --- crates/wasmi/src/engine/translator/func2/stack/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/stack/mod.rs b/crates/wasmi/src/engine/translator/func2/stack/mod.rs index 511d0c2d5e..8f133ff7c0 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/mod.rs @@ -321,9 +321,9 @@ impl Stack { } /// Returns an exclusive reference to the [`ControlFrame`] at `depth`. - /// + /// /// # Note - /// + /// /// This returns an [`AcquiredTarget`] to differentiate between the function /// body Wasm `block` and other control frames in order to know whether a branching /// target returns or branches. From 55a50ebf5a5095454c320a4f3207992b896da753 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 23 Jun 2025 11:10:51 +0200 Subject: [PATCH 181/343] rename some FuncTranslator helper methods --- .../wasmi/src/engine/translator/func2/mod.rs | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index 2136937811..2106b74569 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -541,8 +541,8 @@ impl FuncTranslator { Ok(()) } - /// Translates an unconditional Wasm `branch` instruction. - fn translate_br(&mut self, label: LabelRef) -> Result<(), Error> { + /// Encodes an unconditional Wasm `branch` instruction. + fn encode_br(&mut self, label: LabelRef) -> Result<(), Error> { let instr = self.instrs.next_instr(); let offset = self.labels.try_resolve_label(label, instr)?; self.push_instr(Instruction::branch(offset), FuelCostsProvider::base)?; @@ -550,18 +550,18 @@ impl FuncTranslator { Ok(()) } - /// Translates a `i32.eqz`+`br_if` or `if` conditional branch instruction. - fn translate_br_eqz(&mut self, condition: Operand, label: LabelRef) -> Result<(), Error> { - self.translate_br_if(condition, label, true) + /// Encodes a `i32.eqz`+`br_if` or `if` conditional branch instruction. + fn encode_br_eqz(&mut self, condition: Operand, label: LabelRef) -> Result<(), Error> { + self.encode_br_if(condition, label, true) } - /// Translates a `br_if` conditional branch instruction. - fn translate_br_nez(&mut self, condition: Operand, label: LabelRef) -> Result<(), Error> { - self.translate_br_if(condition, label, false) + /// Encodes a `br_if` conditional branch instruction. + fn encode_br_nez(&mut self, condition: Operand, label: LabelRef) -> Result<(), Error> { + self.encode_br_if(condition, label, false) } - /// Translates a generic `br_if` fused conditional branch instruction. - fn translate_br_if( + /// Encodes a generic `br_if` fused conditional branch instruction. + fn encode_br_if( &mut self, condition: Operand, label: LabelRef, @@ -580,7 +580,7 @@ impl FuncTranslator { false => condition != 0, }; match take_branch { - true => return self.translate_br(label), + true => return self.encode_br(label), false => return Ok(()), } } From 73bcb3c56fbb03786320c363f85f479a7f7b9d82 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 23 Jun 2025 11:11:04 +0200 Subject: [PATCH 182/343] fix after renamings --- crates/wasmi/src/engine/translator/func2/visit.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index 0e4d5a12df..c1770f42a7 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -137,7 +137,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { } _ => { let else_label = self.labels.new_label(); - self.translate_br_eqz(condition, else_label)?; + self.encode_br_eqz(condition, else_label)?; let reachability = IfReachability::Both { else_label }; let consume_fuel_instr = self.instrs.push_consume_fuel_instr()?; (reachability, consume_fuel_instr) @@ -170,7 +170,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { let consume_fuel_instr = frame.consume_fuel_instr(); self.copy_branch_params(len_values, consume_fuel_instr)?; frame.branch_to(); - self.translate_br(else_label)?; + self.encode_br(else_label)?; } } // Start of `else` block: From 756485512a3d1712efbdcc7a32c712fbbcfb69e1 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 23 Jun 2025 11:22:30 +0200 Subject: [PATCH 183/343] implement FuncTranslator::visit_br --- .../src/engine/translator/func2/visit.rs | 25 ++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index c1770f42a7..1008e555ce 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -2,7 +2,11 @@ use super::{ControlFrame, ControlFrameKind, FuncTranslator, LocalIdx}; use crate::{ core::{wasm, FuelCostsProvider, TrapCode}, engine::{ - translator::func2::{stack::IfReachability, ControlFrameBase, Operand}, + translator::func2::{ + stack::{AcquiredTarget, IfReachability}, + ControlFrameBase, + Operand, + }, BlockType, }, ir::Instruction, @@ -196,8 +200,23 @@ impl<'a> VisitOperator<'a> for FuncTranslator { } } - fn visit_br(&mut self, _relative_depth: u32) -> Self::Output { - todo!() + fn visit_br(&mut self, depth: u32) -> Self::Output { + bail_unreachable!(self); + let Ok(depth) = usize::try_from(depth) else { + panic!("out of bounds depth: {depth}") + }; + let consume_fuel_instr = self.stack.consume_fuel_instr(); + match self.stack.peek_control_mut(depth) { + AcquiredTarget::Return(_) => self.visit_return(), + AcquiredTarget::Branch(mut frame) => { + frame.branch_to(); + let label = frame.label(); + self.copy_branch_params(depth, consume_fuel_instr)?; + self.encode_br(label)?; + self.reachable = false; + Ok(()) + } + } } fn visit_br_if(&mut self, _relative_depth: u32) -> Self::Output { From 5732bbd5077b9b22727616ca9284eaac4b491100 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 23 Jun 2025 13:31:17 +0200 Subject: [PATCH 184/343] add ty and height methods to ControlFrameBase trait --- .../engine/translator/func2/stack/control.rs | 94 ++++++++++++------- 1 file changed, 60 insertions(+), 34 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/stack/control.rs b/crates/wasmi/src/engine/translator/func2/stack/control.rs index 5a46bb07a3..705a3fa989 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/control.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/control.rs @@ -254,6 +254,14 @@ impl ControlStack { pub struct ControlFrameMut<'a>(&'a mut ControlFrame); impl<'a> ControlFrameBase for ControlFrameMut<'a> { + fn ty(&self) -> BlockType { + self.0.ty() + } + + fn height(&self) -> usize { + self.0.height() + } + fn label(&self) -> LabelRef { self.0.label() } @@ -362,6 +370,12 @@ impl From for ControlFrame { /// Trait implemented by control frame types that share a common API. pub trait ControlFrameBase { + /// Returns the [`BlockType`] of the [`BlockControlFrame`]. + fn ty(&self) -> BlockType; + + /// Returns the height of the [`BlockControlFrame`]. + fn height(&self) -> usize; + /// Returns the branch label of `self`. fn label(&self) -> LabelRef; @@ -381,6 +395,30 @@ pub trait ControlFrameBase { } impl ControlFrameBase for ControlFrame { + fn ty(&self) -> BlockType { + match self { + ControlFrame::Block(frame) => frame.ty(), + ControlFrame::Loop(frame) => frame.ty(), + ControlFrame::If(frame) => frame.ty(), + ControlFrame::Else(frame) => frame.ty(), + ControlFrame::Unreachable(_) => { + panic!("invalid query for unreachable control frame: `ControlFrameBase::ty`") + } + } + } + + fn height(&self) -> usize { + match self { + ControlFrame::Block(frame) => frame.height(), + ControlFrame::Loop(frame) => frame.height(), + ControlFrame::If(frame) => frame.height(), + ControlFrame::Else(frame) => frame.height(), + ControlFrame::Unreachable(_) => { + panic!("invalid query for unreachable control frame: `ControlFrameBase::height`") + } + } + } + fn label(&self) -> LabelRef { match self { ControlFrame::Block(frame) => frame.label(), @@ -463,19 +501,15 @@ pub struct BlockControlFrame { label: LabelRef, } -impl BlockControlFrame { - /// Returns the [`BlockType`] of the [`BlockControlFrame`]. - pub fn ty(&self) -> BlockType { +impl ControlFrameBase for BlockControlFrame { + fn ty(&self) -> BlockType { self.ty } - /// Returns the height of the [`BlockControlFrame`]. - pub fn height(&self) -> usize { + fn height(&self) -> usize { self.height.into() } -} -impl ControlFrameBase for BlockControlFrame { fn label(&self) -> LabelRef { self.label } @@ -516,19 +550,15 @@ pub struct LoopControlFrame { label: LabelRef, } -impl LoopControlFrame { - /// Returns the [`BlockType`] of the [`LoopControlFrame`]. - pub fn ty(&self) -> BlockType { +impl ControlFrameBase for LoopControlFrame { + fn ty(&self) -> BlockType { self.ty } - /// Returns the height of the [`LoopControlFrame`]. - pub fn height(&self) -> usize { + fn height(&self) -> usize { self.height.into() } -} -impl ControlFrameBase for LoopControlFrame { fn label(&self) -> LabelRef { self.label } @@ -572,16 +602,6 @@ pub struct IfControlFrame { } impl IfControlFrame { - /// Returns the [`BlockType`] of the [`IfControlFrame`]. - pub fn ty(&self) -> BlockType { - self.ty - } - - /// Returns the height of the [`IfControlFrame`]. - pub fn height(&self) -> usize { - self.height.into() - } - /// Returns the [`IfReachability`] of the [`IfControlFrame`]. pub fn reachability(&self) -> IfReachability { self.reachability @@ -621,6 +641,14 @@ impl IfControlFrame { } impl ControlFrameBase for IfControlFrame { + fn ty(&self) -> BlockType { + self.ty + } + + fn height(&self) -> usize { + self.height.into() + } + fn label(&self) -> LabelRef { self.label } @@ -730,16 +758,6 @@ impl From for ElseReachability { } impl ElseControlFrame { - /// Returns the [`BlockType`] of the [`ElseControlFrame`]. - pub fn ty(&self) -> BlockType { - self.ty - } - - /// Returns the height of the [`ElseControlFrame`]. - pub fn height(&self) -> usize { - self.height.into() - } - /// Returns the [`ElseReachability`] of the [`ElseReachability`]. pub fn reachability(&self) -> ElseReachability { self.reachability @@ -776,6 +794,14 @@ impl ElseControlFrame { } impl ControlFrameBase for ElseControlFrame { + fn ty(&self) -> BlockType { + self.ty + } + + fn height(&self) -> usize { + self.height.into() + } + fn label(&self) -> LabelRef { self.label } From 8256ed9707fc02876658c1dc0d93c96658060fa1 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 23 Jun 2025 13:31:35 +0200 Subject: [PATCH 185/343] implement FuncTranslator::visit_br_if method --- .../wasmi/src/engine/translator/func2/mod.rs | 16 ++++++++ .../src/engine/translator/func2/visit.rs | 38 ++++++++++++++++++- 2 files changed, 52 insertions(+), 2 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index 2106b74569..3169fb8394 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -299,6 +299,22 @@ impl FuncTranslator { Ok(()) } + /// Returns `true` if the [`ControlFrame`] at `depth` requires copying for its branch parameters. + /// + /// # Note + /// + /// Some instructions can be encoded in a more efficient way if no branch parameter copies are required. + fn requires_branch_param_copies(&self, depth: usize) -> bool { + let frame = self.stack.peek_control(depth); + let len_branch_params = usize::from(frame.len_branch_params(&self.engine)); + let frame_height = frame.height(); + frame_height == (self.stack.height() - usize::from(len_branch_params)) + && (0..len_branch_params) + .into_iter() + .map(|depth| self.stack.peek(depth)) + .all(|o| o.is_temp()) + } + /// Pins the `label` to the next [`Instr`]. fn pin_label(&mut self, label: LabelRef) { self.labels diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index 1008e555ce..2d40050cfe 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -219,8 +219,42 @@ impl<'a> VisitOperator<'a> for FuncTranslator { } } - fn visit_br_if(&mut self, _relative_depth: u32) -> Self::Output { - todo!() + fn visit_br_if(&mut self, depth: u32) -> Self::Output { + bail_unreachable!(self); + let condition = self.stack.pop(); + if let Operand::Immediate(condition) = condition { + if i32::from(condition.val()) != 0 { + // Case (true): always takes the branch + self.visit_br(depth)?; + } + return Ok(()); + } + let Ok(depth) = usize::try_from(depth) else { + panic!("out of bounds depth: {depth}") + }; + let mut frame = self.stack.peek_control_mut(depth).control_frame(); + frame.branch_to(); + let len_branch_params = frame.len_branch_params(&self.engine); + let label = frame.label(); + if len_branch_params == 0 { + // Case: no branch values are required to be copied + self.encode_br_nez(condition, label)?; + return Ok(()); + } + if !self.requires_branch_param_copies(depth) { + // Case: no branch values are required to be copied + self.encode_br_nez(condition, label)?; + return Ok(()); + } + // Case: fallback to copy branch parameters conditionally + let skip_label = self.labels.new_label(); + self.encode_br_eqz(condition, label)?; + self.copy_operands_to_temp(depth)?; + self.encode_br(label)?; + self.labels + .pin_label(skip_label, self.instrs.next_instr()) + .unwrap(); + Ok(()) } fn visit_br_table(&mut self, _targets: wasmparser::BrTable<'a>) -> Self::Output { From 2c6c530371819a70042d48449865b2e871dbbe4a Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 23 Jun 2025 13:33:09 +0200 Subject: [PATCH 186/343] implement FuncTranslator::visit_drop method --- crates/wasmi/src/engine/translator/func2/visit.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index 2d40050cfe..8da6385b64 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -282,7 +282,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { } fn visit_drop(&mut self) -> Self::Output { - todo!() + bail_unreachable!(self); + _ = self.stack.pop(); + Ok(()) } fn visit_select(&mut self) -> Self::Output { From 9eabfe6dcbd2c37d156e0282d7452eb8d5e50314 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Tue, 24 Jun 2025 14:26:59 +0200 Subject: [PATCH 187/343] translate all commitative binary Wasm operators --- .../src/engine/translator/func2/visit.rs | 86 +++++++++++++++---- 1 file changed, 71 insertions(+), 15 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index 8da6385b64..c3d5d4abe9 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -432,15 +432,25 @@ impl<'a> VisitOperator<'a> for FuncTranslator { } fn visit_i32_eqz(&mut self) -> Self::Output { - todo!() + bail_unreachable!(self); + self.stack.push_immediate(0_i32)?; + self.visit_i32_eq() } fn visit_i32_eq(&mut self) -> Self::Output { - todo!() + self.translate_binary_commutative::( + Instruction::i32_eq, + Instruction::i32_eq_imm16, + wasm::i32_eq, + ) } fn visit_i32_ne(&mut self) -> Self::Output { - todo!() + self.translate_binary_commutative::( + Instruction::i32_ne, + Instruction::i32_ne_imm16, + wasm::i32_ne, + ) } fn visit_i32_lt_s(&mut self) -> Self::Output { @@ -476,15 +486,25 @@ impl<'a> VisitOperator<'a> for FuncTranslator { } fn visit_i64_eqz(&mut self) -> Self::Output { - todo!() + bail_unreachable!(self); + self.stack.push_immediate(0_i64)?; + self.visit_i64_eq() } fn visit_i64_eq(&mut self) -> Self::Output { - todo!() + self.translate_binary_commutative::( + Instruction::i64_eq, + Instruction::i64_eq_imm16, + wasm::i64_eq, + ) } fn visit_i64_ne(&mut self) -> Self::Output { - todo!() + self.translate_binary_commutative::( + Instruction::i64_ne, + Instruction::i64_ne_imm16, + wasm::i64_ne, + ) } fn visit_i64_lt_s(&mut self) -> Self::Output { @@ -592,7 +612,11 @@ impl<'a> VisitOperator<'a> for FuncTranslator { } fn visit_i32_mul(&mut self) -> Self::Output { - todo!() + self.translate_binary_commutative::( + Instruction::i32_mul, + Instruction::i32_mul_imm16, + wasm::i32_mul, + ) } fn visit_i32_div_s(&mut self) -> Self::Output { @@ -612,15 +636,27 @@ impl<'a> VisitOperator<'a> for FuncTranslator { } fn visit_i32_and(&mut self) -> Self::Output { - todo!() + self.translate_binary_commutative::( + Instruction::i32_and, + Instruction::i32_and_imm16, + wasm::i32_bitand, + ) } fn visit_i32_or(&mut self) -> Self::Output { - todo!() + self.translate_binary_commutative::( + Instruction::i32_or, + Instruction::i32_or_imm16, + wasm::i32_bitor, + ) } fn visit_i32_xor(&mut self) -> Self::Output { - todo!() + self.translate_binary_commutative::( + Instruction::i32_xor, + Instruction::i32_xor_imm16, + wasm::i32_bitxor, + ) } fn visit_i32_shl(&mut self) -> Self::Output { @@ -656,7 +692,11 @@ impl<'a> VisitOperator<'a> for FuncTranslator { } fn visit_i64_add(&mut self) -> Self::Output { - todo!() + self.translate_binary_commutative::( + Instruction::i64_add, + Instruction::i64_add_imm16, + wasm::i64_add, + ) } fn visit_i64_sub(&mut self) -> Self::Output { @@ -664,7 +704,11 @@ impl<'a> VisitOperator<'a> for FuncTranslator { } fn visit_i64_mul(&mut self) -> Self::Output { - todo!() + self.translate_binary_commutative::( + Instruction::i64_mul, + Instruction::i64_mul_imm16, + wasm::i64_mul, + ) } fn visit_i64_div_s(&mut self) -> Self::Output { @@ -684,15 +728,27 @@ impl<'a> VisitOperator<'a> for FuncTranslator { } fn visit_i64_and(&mut self) -> Self::Output { - todo!() + self.translate_binary_commutative::( + Instruction::i64_and, + Instruction::i64_and_imm16, + wasm::i64_bitand, + ) } fn visit_i64_or(&mut self) -> Self::Output { - todo!() + self.translate_binary_commutative::( + Instruction::i64_or, + Instruction::i64_or_imm16, + wasm::i64_bitor, + ) } fn visit_i64_xor(&mut self) -> Self::Output { - todo!() + self.translate_binary_commutative::( + Instruction::i64_xor, + Instruction::i64_xor_imm16, + wasm::i64_bitxor, + ) } fn visit_i64_shl(&mut self) -> Self::Output { From 32ea09c9fc856bf5a4d8309515c29b431b98095f Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Tue, 24 Jun 2025 14:34:02 +0200 Subject: [PATCH 188/343] apply clippy suggestions --- crates/wasmi/src/engine/translator/func2/mod.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index 3169fb8394..978ecd489b 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -308,9 +308,8 @@ impl FuncTranslator { let frame = self.stack.peek_control(depth); let len_branch_params = usize::from(frame.len_branch_params(&self.engine)); let frame_height = frame.height(); - frame_height == (self.stack.height() - usize::from(len_branch_params)) + frame_height == (self.stack.height() - len_branch_params) && (0..len_branch_params) - .into_iter() .map(|depth| self.stack.peek(depth)) .all(|o| o.is_temp()) } From d76ae0703ffaf86f2aaf5584c8d8010a60927bd0 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Tue, 24 Jun 2025 15:29:26 +0200 Subject: [PATCH 189/343] add translation for i{32,64} unary operators --- .../wasmi/src/engine/translator/func2/mod.rs | 34 +++++++++++++++++++ .../src/engine/translator/func2/visit.rs | 12 +++---- 2 files changed, 40 insertions(+), 6 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index 978ecd489b..edf0cc1822 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -697,6 +697,40 @@ impl FuncTranslator { cmp_instr.try_into_cmp_branch_instr(offset, &mut self.layout) } + /// Translates a unary Wasm instruction to Wasmi bytecode. + fn translate_unary( + &mut self, + make_instr: fn(result: Reg, input: Reg) -> Instruction, + consteval: fn(input: T) -> R, + ) -> Result<(), Error> + where + T: From, + R: Into + Typed, + { + bail_unreachable!(self); + let input = self.stack.pop(); + if let Operand::Immediate(input) = input { + self.stack + .push_immediate(consteval(input.val().into()).into())?; + return Ok(()); + } + let consume_fuel_instr = self.stack.consume_fuel_instr(); + let input = self.layout.operand_to_reg(input)?; + let iidx = self.instrs.next_instr(); + let result = self + .layout + .temp_to_reg(self.stack.push_temp(::TY, Some(iidx))?)?; + assert_eq!( + self.instrs.push_instr( + make_instr(result, input), + consume_fuel_instr, + FuelCostsProvider::base + )?, + iidx + ); + Ok(()) + } + /// Translates a commutative binary Wasm operator to Wasmi bytecode. fn translate_binary_commutative( &mut self, diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index c3d5d4abe9..939daa93bf 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -588,15 +588,15 @@ impl<'a> VisitOperator<'a> for FuncTranslator { } fn visit_i32_clz(&mut self) -> Self::Output { - todo!() + self.translate_unary::(Instruction::i32_clz, wasm::i32_clz) } fn visit_i32_ctz(&mut self) -> Self::Output { - todo!() + self.translate_unary::(Instruction::i32_ctz, wasm::i32_ctz) } fn visit_i32_popcnt(&mut self) -> Self::Output { - todo!() + self.translate_unary::(Instruction::i32_popcnt, wasm::i32_popcnt) } fn visit_i32_add(&mut self) -> Self::Output { @@ -680,15 +680,15 @@ impl<'a> VisitOperator<'a> for FuncTranslator { } fn visit_i64_clz(&mut self) -> Self::Output { - todo!() + self.translate_unary::(Instruction::i64_clz, wasm::i64_clz) } fn visit_i64_ctz(&mut self) -> Self::Output { - todo!() + self.translate_unary::(Instruction::i64_ctz, wasm::i64_ctz) } fn visit_i64_popcnt(&mut self) -> Self::Output { - todo!() + self.translate_unary::(Instruction::i64_popcnt, wasm::i64_popcnt) } fn visit_i64_add(&mut self) -> Self::Output { From fc64313b330b67864f36dbeab6de4e0728c177d3 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Tue, 24 Jun 2025 15:45:48 +0200 Subject: [PATCH 190/343] add FuncTranslator::push_instr_with_result utility --- .../wasmi/src/engine/translator/func2/mod.rs | 81 +++++++++---------- 1 file changed, 40 insertions(+), 41 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index edf0cc1822..3140e396a5 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -387,6 +387,25 @@ impl FuncTranslator { Ok(()) } + /// Pushes the `instr` to the function with the associated `fuel_costs`. + fn push_instr_with_result( + &mut self, + result_ty: ValType, + make_instr: impl FnOnce(Reg) -> Instruction, + fuel_costs: impl FnOnce(&FuelCostsProvider) -> u64, + ) -> Result<(), Error> { + let consume_fuel_instr = self.stack.consume_fuel_instr(); + let expected_iidx = self.instrs.next_instr(); + let result = self + .layout + .temp_to_reg(self.stack.push_temp(result_ty, Some(expected_iidx))?)?; + let actual_iidx = + self.instrs + .push_instr(make_instr(result), consume_fuel_instr, fuel_costs)?; + assert_eq!(expected_iidx, actual_iidx); + Ok(()) + } + /// Encodes a generic return instruction. fn encode_return(&mut self, consume_fuel: Option) -> Result { let len_results = self.func_type_with(FuncType::len_results); @@ -714,21 +733,12 @@ impl FuncTranslator { .push_immediate(consteval(input.val().into()).into())?; return Ok(()); } - let consume_fuel_instr = self.stack.consume_fuel_instr(); let input = self.layout.operand_to_reg(input)?; - let iidx = self.instrs.next_instr(); - let result = self - .layout - .temp_to_reg(self.stack.push_temp(::TY, Some(iidx))?)?; - assert_eq!( - self.instrs.push_instr( - make_instr(result, input), - consume_fuel_instr, - FuelCostsProvider::base - )?, - iidx - ); - Ok(()) + self.push_instr_with_result( + ::TY, + |result| make_instr(result, input), + FuelCostsProvider::base, + ) } /// Translates a commutative binary Wasm operator to Wasmi bytecode. @@ -743,7 +753,6 @@ impl FuncTranslator { R: Into + Typed, { bail_unreachable!(self); - let consume_fuel = self.stack.consume_fuel_instr(); match self.stack.pop2() { (Operand::Immediate(lhs), Operand::Immediate(rhs)) => { let value = consteval(lhs.val().into(), rhs.val().into()); @@ -752,40 +761,30 @@ impl FuncTranslator { } (val, Operand::Immediate(imm)) | (Operand::Immediate(imm), val) => { let lhs = self.layout.operand_to_reg(val)?; - let iidx = self.instrs.next_instr(); - let result = self - .layout - .temp_to_reg(self.stack.push_temp(::TY, Some(iidx))?)?; - let instr = match T::from(imm.val()).try_into() { - Ok(rhs) => make_instr_imm16(result, lhs, rhs), + let rhs16 = match T::from(imm.val()).try_into() { + Ok(rhs) => Ok(rhs), Err(_) => { let rhs = self.layout.const_to_reg(imm.val())?; - make_instr(result, lhs, rhs) + Err(rhs) } }; - assert_eq!( - self.instrs - .push_instr(instr, consume_fuel, FuelCostsProvider::base)?, - iidx - ); - Ok(()) + self.push_instr_with_result( + ::TY, + |result| match rhs16 { + Ok(rhs) => make_instr_imm16(result, lhs, rhs), + Err(rhs) => make_instr(result, lhs, rhs), + }, + FuelCostsProvider::base, + ) } (lhs, rhs) => { let lhs = self.layout.operand_to_reg(lhs)?; let rhs = self.layout.operand_to_reg(rhs)?; - let iidx = self.instrs.next_instr(); - let result = self - .layout - .temp_to_reg(self.stack.push_temp(::TY, Some(iidx))?)?; - assert_eq!( - self.instrs.push_instr( - make_instr(result, lhs, rhs), - consume_fuel, - FuelCostsProvider::base - )?, - iidx - ); - Ok(()) + self.push_instr_with_result( + ::TY, + |result| make_instr(result, lhs, rhs), + FuelCostsProvider::base, + ) } } } From 19867c9c965d8d0c8f2c90b7d20675f9b1669be8 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Tue, 24 Jun 2025 21:18:12 +0200 Subject: [PATCH 191/343] add translate_trap utility method --- crates/wasmi/src/engine/translator/func2/mod.rs | 9 +++++++++ crates/wasmi/src/engine/translator/func2/visit.rs | 7 +------ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index 3140e396a5..5a06fd67e5 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -788,4 +788,13 @@ impl FuncTranslator { } } } + + fn translate_trap(&mut self, trap: TrapCode) -> Result<(), Error> { + self.push_instr( + Instruction::trap(TrapCode::UnreachableCodeReached), + FuelCostsProvider::base, + )?; + self.reachable = false; + Ok(()) + } } diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index 939daa93bf..abf038a145 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -79,12 +79,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { fn visit_unreachable(&mut self) -> Self::Output { bail_unreachable!(self); - self.push_instr( - Instruction::trap(TrapCode::UnreachableCodeReached), - FuelCostsProvider::base, - )?; - self.reachable = false; - Ok(()) + self.translate_trap(TrapCode::UnreachableCodeReached) } fn visit_nop(&mut self) -> Self::Output { From c3e88a69711233a7018580d2c8446b636b667cf7 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Tue, 24 Jun 2025 21:18:20 +0200 Subject: [PATCH 192/343] add missing import --- crates/wasmi/src/engine/translator/func2/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index 5a06fd67e5..c8a1d69dec 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -32,7 +32,7 @@ use self::{ utils::{Reset, ReusableAllocations}, }; use crate::{ - core::{FuelCostsProvider, Typed, TypedVal, ValType}, + core::{FuelCostsProvider, Typed, TypedVal, ValType, TrapCode}, engine::{ translator::{ comparator::{CompareResult as _, NegateCmpInstr as _, TryIntoCmpBranchInstr as _}, From 7b56c9b173f7d41dd9bf62c508f2e16b7953e57f Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 25 Jun 2025 10:32:13 +0200 Subject: [PATCH 193/343] remove no longer import --- crates/wasmi/src/engine/translator/func2/visit.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index abf038a145..7de48ff0b1 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -1,6 +1,6 @@ use super::{ControlFrame, ControlFrameKind, FuncTranslator, LocalIdx}; use crate::{ - core::{wasm, FuelCostsProvider, TrapCode}, + core::{wasm, TrapCode}, engine::{ translator::func2::{ stack::{AcquiredTarget, IfReachability}, From 571a02cc7a48e176d2c91d0ea6f977dfdbdfce7c Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 25 Jun 2025 10:33:26 +0200 Subject: [PATCH 194/343] add Operand16 type and make_imm16 utility --- .../wasmi/src/engine/translator/func2/mod.rs | 39 ++++++++++++------- .../src/engine/translator/func2/utils.rs | 10 +++++ 2 files changed, 35 insertions(+), 14 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index c8a1d69dec..0f0b384afb 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -29,15 +29,15 @@ use self::{ StackAllocations, TempOperand, }, - utils::{Reset, ReusableAllocations}, + utils::{Operand16, Reset, ReusableAllocations}, }; use crate::{ - core::{FuelCostsProvider, Typed, TypedVal, ValType, TrapCode}, + core::{FuelCostsProvider, TrapCode, Typed, TypedVal, UntypedVal, ValType}, engine::{ translator::{ comparator::{CompareResult as _, NegateCmpInstr as _, TryIntoCmpBranchInstr as _}, labels::{LabelRef, LabelRegistry}, - utils::Instr, + utils::{Instr, WasmInteger}, WasmTranslator, }, BlockType, @@ -741,6 +741,20 @@ impl FuncTranslator { ) } + /// Creates a new 16-bit encoded [`Operand16`] from the given `value`. + pub fn make_imm16(&mut self, value: T) -> Result, Error> + where + T: Into + Copy + TryInto>, + { + match value.try_into() { + Ok(rhs) => Ok(Operand16::Immediate(rhs)), + Err(_) => { + let rhs = self.layout.const_to_reg(value)?; + Ok(Operand16::Reg(rhs)) + } + } + } + /// Translates a commutative binary Wasm operator to Wasmi bytecode. fn translate_binary_commutative( &mut self, @@ -749,30 +763,27 @@ impl FuncTranslator { consteval: fn(T, T) -> R, ) -> Result<(), Error> where - T: Copy + From + TryInto>, + T: WasmInteger + TryInto>, R: Into + Typed, { bail_unreachable!(self); match self.stack.pop2() { (Operand::Immediate(lhs), Operand::Immediate(rhs)) => { - let value = consteval(lhs.val().into(), rhs.val().into()); + let lhs = lhs.val().into(); + let rhs = rhs.val().into(); + let value = consteval(lhs, rhs); self.stack.push_immediate(value)?; Ok(()) } (val, Operand::Immediate(imm)) | (Operand::Immediate(imm), val) => { let lhs = self.layout.operand_to_reg(val)?; - let rhs16 = match T::from(imm.val()).try_into() { - Ok(rhs) => Ok(rhs), - Err(_) => { - let rhs = self.layout.const_to_reg(imm.val())?; - Err(rhs) - } - }; + let rhs = imm.val().into(); + let rhs16 = self.make_imm16(rhs)?; self.push_instr_with_result( ::TY, |result| match rhs16 { - Ok(rhs) => make_instr_imm16(result, lhs, rhs), - Err(rhs) => make_instr(result, lhs, rhs), + Operand16::Immediate(rhs) => make_instr_imm16(result, lhs, rhs), + Operand16::Reg(rhs) => make_instr(result, lhs, rhs), }, FuelCostsProvider::base, ) diff --git a/crates/wasmi/src/engine/translator/func2/utils.rs b/crates/wasmi/src/engine/translator/func2/utils.rs index 36e1644ef2..daef37812b 100644 --- a/crates/wasmi/src/engine/translator/func2/utils.rs +++ b/crates/wasmi/src/engine/translator/func2/utils.rs @@ -1,3 +1,5 @@ +use crate::ir::{Const16, Reg}; + /// Bail out early in case the current code is unreachable. /// /// # Note @@ -37,3 +39,11 @@ pub trait ReusableAllocations { /// Returns the reusable heap allocations of `self`. fn into_allocations(self) -> Self::Allocations; } + +/// A 16-bit encoded operand for a Wasmi instruction. +pub enum Operand16 { + /// A [`Reg`] operand. + Reg(Reg), + /// A 16-bit encoded immediate value operand. + Immediate(Const16), +} From 3f9a81b89a2ac60bf7ee0b7a50724ff862e4fdf6 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 25 Jun 2025 10:58:17 +0200 Subject: [PATCH 195/343] fix translate_trap not respecting its trap parameter --- crates/wasmi/src/engine/translator/func2/mod.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index 0f0b384afb..3b6064b7de 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -801,10 +801,7 @@ impl FuncTranslator { } fn translate_trap(&mut self, trap: TrapCode) -> Result<(), Error> { - self.push_instr( - Instruction::trap(TrapCode::UnreachableCodeReached), - FuelCostsProvider::base, - )?; + self.push_instr(Instruction::trap(trap), FuelCostsProvider::base)?; self.reachable = false; Ok(()) } From ae869856bd29993cf40d8b14f2a1375493b5939e Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 25 Jun 2025 10:58:40 +0200 Subject: [PATCH 196/343] implement i{32,64}_{div,rem}_{s,u} translations --- .../wasmi/src/engine/translator/func2/mod.rs | 69 +++++++++++++++++++ .../src/engine/translator/func2/visit.rs | 57 ++++++++++++--- 2 files changed, 118 insertions(+), 8 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index 3b6064b7de..e237818781 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -800,6 +800,75 @@ impl FuncTranslator { } } + /// Translates integer division and remainder Wasm operators to Wasmi bytecode. + fn translate_divrem( + &mut self, + make_instr: fn(result: Reg, lhs: Reg, rhs: Reg) -> Instruction, + make_instr_imm16_rhs: fn(result: Reg, lhs: Reg, rhs: Const16) -> Instruction, + make_instr_imm16_lhs: fn(result: Reg, lhs: Const16, rhs: Reg) -> Instruction, + consteval: fn(T, T) -> Result, + ) -> Result<(), Error> + where + T: WasmInteger, + NonZero: Copy + TryFrom + TryInto> + Into, + { + bail_unreachable!(self); + match self.stack.pop2() { + (Operand::Immediate(lhs), Operand::Immediate(rhs)) => { + let lhs = lhs.val().into(); + let rhs = rhs.val().into(); + match consteval(lhs, rhs) { + Ok(value) => { + self.stack.push_immediate(value)?; + } + Err(trap) => { + self.translate_trap(trap)?; + } + } + Ok(()) + } + (lhs, Operand::Immediate(rhs)) => { + let lhs = self.layout.operand_to_reg(lhs)?; + let rhs = T::from(rhs.val()); + let Some(non_zero_rhs) = NonZero::try_from(rhs).ok() else { + // Optimization: division by zero always traps + return self.translate_trap(TrapCode::IntegerDivisionByZero); + }; + let rhs16 = self.make_imm16(non_zero_rhs)?; + self.push_instr_with_result( + ::TY, + |result| match rhs16 { + Operand16::Immediate(rhs) => make_instr_imm16_rhs(result, lhs, rhs), + Operand16::Reg(rhs) => make_instr(result, lhs, rhs), + }, + FuelCostsProvider::base, + ) + } + (Operand::Immediate(lhs), rhs) => { + let lhs = T::from(lhs.val()); + let lhs16 = self.make_imm16(lhs)?; + let rhs = self.layout.operand_to_reg(rhs)?; + self.push_instr_with_result( + ::TY, + |result| match lhs16 { + Operand16::Immediate(lhs) => make_instr_imm16_lhs(result, lhs, rhs), + Operand16::Reg(lhs) => make_instr(result, lhs, rhs), + }, + FuelCostsProvider::base, + ) + } + (lhs, rhs) => { + let lhs = self.layout.operand_to_reg(lhs)?; + let rhs = self.layout.operand_to_reg(rhs)?; + self.push_instr_with_result( + ::TY, + |result| make_instr(result, lhs, rhs), + FuelCostsProvider::base, + ) + } + } + } + fn translate_trap(&mut self, trap: TrapCode) -> Result<(), Error> { self.push_instr(Instruction::trap(trap), FuelCostsProvider::base)?; self.reachable = false; diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index 7de48ff0b1..8c24328e42 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -13,6 +13,7 @@ use crate::{ Error, FuncType, }; +use core::num::NonZero; use wasmparser::VisitOperator; macro_rules! impl_visit_operator { @@ -615,19 +616,39 @@ impl<'a> VisitOperator<'a> for FuncTranslator { } fn visit_i32_div_s(&mut self) -> Self::Output { - todo!() + self.translate_divrem::>( + Instruction::i32_div_s, + Instruction::i32_div_s_imm16_rhs, + Instruction::i32_div_s_imm16_lhs, + wasm::i32_div_s, + ) } fn visit_i32_div_u(&mut self) -> Self::Output { - todo!() + self.translate_divrem::>( + Instruction::i32_div_u, + Instruction::i32_div_u_imm16_rhs, + Instruction::i32_div_u_imm16_lhs, + wasm::i32_div_u, + ) } fn visit_i32_rem_s(&mut self) -> Self::Output { - todo!() + self.translate_divrem::>( + Instruction::i32_rem_s, + Instruction::i32_rem_s_imm16_rhs, + Instruction::i32_rem_s_imm16_lhs, + wasm::i32_rem_s, + ) } fn visit_i32_rem_u(&mut self) -> Self::Output { - todo!() + self.translate_divrem::>( + Instruction::i32_rem_u, + Instruction::i32_rem_u_imm16_rhs, + Instruction::i32_rem_u_imm16_lhs, + wasm::i32_rem_u, + ) } fn visit_i32_and(&mut self) -> Self::Output { @@ -707,19 +728,39 @@ impl<'a> VisitOperator<'a> for FuncTranslator { } fn visit_i64_div_s(&mut self) -> Self::Output { - todo!() + self.translate_divrem::>( + Instruction::i64_div_s, + Instruction::i64_div_s_imm16_rhs, + Instruction::i64_div_s_imm16_lhs, + wasm::i64_div_s, + ) } fn visit_i64_div_u(&mut self) -> Self::Output { - todo!() + self.translate_divrem::>( + Instruction::i64_div_u, + Instruction::i64_div_u_imm16_rhs, + Instruction::i64_div_u_imm16_lhs, + wasm::i64_div_u, + ) } fn visit_i64_rem_s(&mut self) -> Self::Output { - todo!() + self.translate_divrem::>( + Instruction::i64_rem_s, + Instruction::i64_rem_s_imm16_rhs, + Instruction::i64_rem_s_imm16_lhs, + wasm::i64_rem_s, + ) } fn visit_i64_rem_u(&mut self) -> Self::Output { - todo!() + self.translate_divrem::>( + Instruction::i64_rem_u, + Instruction::i64_rem_u_imm16_rhs, + Instruction::i64_rem_u_imm16_lhs, + wasm::i64_rem_u, + ) } fn visit_i64_and(&mut self) -> Self::Output { From 34f38035d8cbcdcb3505e41c4494791ec77a91db Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 25 Jun 2025 11:06:20 +0200 Subject: [PATCH 197/343] simplify translate_divrem generic params --- crates/wasmi/src/engine/translator/func2/mod.rs | 11 +++++++---- .../wasmi/src/engine/translator/func2/visit.rs | 17 ++++++++--------- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index e237818781..4a459ce9a3 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -801,16 +801,19 @@ impl FuncTranslator { } /// Translates integer division and remainder Wasm operators to Wasmi bytecode. - fn translate_divrem( + fn translate_divrem( &mut self, make_instr: fn(result: Reg, lhs: Reg, rhs: Reg) -> Instruction, - make_instr_imm16_rhs: fn(result: Reg, lhs: Reg, rhs: Const16) -> Instruction, + make_instr_imm16_rhs: fn( + result: Reg, + lhs: Reg, + rhs: Const16<::NonZero>, + ) -> Instruction, make_instr_imm16_lhs: fn(result: Reg, lhs: Const16, rhs: Reg) -> Instruction, consteval: fn(T, T) -> Result, ) -> Result<(), Error> where T: WasmInteger, - NonZero: Copy + TryFrom + TryInto> + Into, { bail_unreachable!(self); match self.stack.pop2() { @@ -830,7 +833,7 @@ impl FuncTranslator { (lhs, Operand::Immediate(rhs)) => { let lhs = self.layout.operand_to_reg(lhs)?; let rhs = T::from(rhs.val()); - let Some(non_zero_rhs) = NonZero::try_from(rhs).ok() else { + let Some(non_zero_rhs) = ::NonZero::try_from(rhs).ok() else { // Optimization: division by zero always traps return self.translate_trap(TrapCode::IntegerDivisionByZero); }; diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index 8c24328e42..a60907110b 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -13,7 +13,6 @@ use crate::{ Error, FuncType, }; -use core::num::NonZero; use wasmparser::VisitOperator; macro_rules! impl_visit_operator { @@ -616,7 +615,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { } fn visit_i32_div_s(&mut self) -> Self::Output { - self.translate_divrem::>( + self.translate_divrem::( Instruction::i32_div_s, Instruction::i32_div_s_imm16_rhs, Instruction::i32_div_s_imm16_lhs, @@ -625,7 +624,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { } fn visit_i32_div_u(&mut self) -> Self::Output { - self.translate_divrem::>( + self.translate_divrem::( Instruction::i32_div_u, Instruction::i32_div_u_imm16_rhs, Instruction::i32_div_u_imm16_lhs, @@ -634,7 +633,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { } fn visit_i32_rem_s(&mut self) -> Self::Output { - self.translate_divrem::>( + self.translate_divrem::( Instruction::i32_rem_s, Instruction::i32_rem_s_imm16_rhs, Instruction::i32_rem_s_imm16_lhs, @@ -643,7 +642,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { } fn visit_i32_rem_u(&mut self) -> Self::Output { - self.translate_divrem::>( + self.translate_divrem::( Instruction::i32_rem_u, Instruction::i32_rem_u_imm16_rhs, Instruction::i32_rem_u_imm16_lhs, @@ -728,7 +727,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { } fn visit_i64_div_s(&mut self) -> Self::Output { - self.translate_divrem::>( + self.translate_divrem::( Instruction::i64_div_s, Instruction::i64_div_s_imm16_rhs, Instruction::i64_div_s_imm16_lhs, @@ -737,7 +736,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { } fn visit_i64_div_u(&mut self) -> Self::Output { - self.translate_divrem::>( + self.translate_divrem::( Instruction::i64_div_u, Instruction::i64_div_u_imm16_rhs, Instruction::i64_div_u_imm16_lhs, @@ -746,7 +745,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { } fn visit_i64_rem_s(&mut self) -> Self::Output { - self.translate_divrem::>( + self.translate_divrem::( Instruction::i64_rem_s, Instruction::i64_rem_s_imm16_rhs, Instruction::i64_rem_s_imm16_lhs, @@ -755,7 +754,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { } fn visit_i64_rem_u(&mut self) -> Self::Output { - self.translate_divrem::>( + self.translate_divrem::( Instruction::i64_rem_u, Instruction::i64_rem_u_imm16_rhs, Instruction::i64_rem_u_imm16_lhs, From b885d36f6b8fdc9bf3ffbdc38b47d7648ee2ac09 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 25 Jun 2025 11:18:03 +0200 Subject: [PATCH 198/343] add missing docs --- crates/wasmi/src/engine/translator/func2/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index 4a459ce9a3..38e0218a2a 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -872,6 +872,7 @@ impl FuncTranslator { } } + /// Translates a generic trap instruction. fn translate_trap(&mut self, trap: TrapCode) -> Result<(), Error> { self.push_instr(Instruction::trap(trap), FuelCostsProvider::base)?; self.reachable = false; From 7cd54f5b6ad97a5534d6da06b2a682a98ed47358 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 30 Jun 2025 14:55:07 +0200 Subject: [PATCH 199/343] add tranlate_binary_consteval* helpers --- .../wasmi/src/engine/translator/func2/mod.rs | 59 ++++++++++++++----- .../src/engine/translator/func2/stack/mod.rs | 2 +- 2 files changed, 44 insertions(+), 17 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index 38e0218a2a..a69ef896fc 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -21,6 +21,7 @@ use self::{ ElseReachability, IfControlFrame, IfReachability, + ImmediateOperand, LocalIdx, LoopControlFrame, Operand, @@ -755,6 +756,46 @@ impl FuncTranslator { } } + /// Evaluates `consteval(lhs, rhs)` and pushed either its result or tranlates a `trap`. + fn translate_binary_consteval_fallible( + &mut self, + lhs: ImmediateOperand, + rhs: ImmediateOperand, + consteval: impl FnOnce(T, T) -> Result, + ) -> Result<(), Error> + where + T: From, + R: Into, + { + let lhs: T = lhs.val().into(); + let rhs: T = rhs.val().into(); + match consteval(lhs, rhs) { + Ok(value) => { + self.stack.push_immediate(value)?; + } + Err(trap) => { + self.translate_trap(trap)?; + } + } + Ok(()) + } + + /// Evaluates `consteval(lhs, rhs)` and pushed either its result or tranlates a `trap`. + fn translate_binary_consteval( + &mut self, + lhs: ImmediateOperand, + rhs: ImmediateOperand, + consteval: fn(T, T) -> R, + ) -> Result<(), Error> + where + T: From, + R: Into, + { + self.translate_binary_consteval_fallible::(lhs, rhs, |lhs, rhs| { + Ok(consteval(lhs, rhs)) + }) + } + /// Translates a commutative binary Wasm operator to Wasmi bytecode. fn translate_binary_commutative( &mut self, @@ -769,11 +810,7 @@ impl FuncTranslator { bail_unreachable!(self); match self.stack.pop2() { (Operand::Immediate(lhs), Operand::Immediate(rhs)) => { - let lhs = lhs.val().into(); - let rhs = rhs.val().into(); - let value = consteval(lhs, rhs); - self.stack.push_immediate(value)?; - Ok(()) + self.translate_binary_consteval::(lhs, rhs, consteval) } (val, Operand::Immediate(imm)) | (Operand::Immediate(imm), val) => { let lhs = self.layout.operand_to_reg(val)?; @@ -818,17 +855,7 @@ impl FuncTranslator { bail_unreachable!(self); match self.stack.pop2() { (Operand::Immediate(lhs), Operand::Immediate(rhs)) => { - let lhs = lhs.val().into(); - let rhs = rhs.val().into(); - match consteval(lhs, rhs) { - Ok(value) => { - self.stack.push_immediate(value)?; - } - Err(trap) => { - self.translate_trap(trap)?; - } - } - Ok(()) + self.translate_binary_consteval_fallible::(lhs, rhs, consteval) } (lhs, Operand::Immediate(rhs)) => { let lhs = self.layout.operand_to_reg(lhs)?; diff --git a/crates/wasmi/src/engine/translator/func2/stack/mod.rs b/crates/wasmi/src/engine/translator/func2/stack/mod.rs index 8f133ff7c0..448a8ad36e 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/mod.rs @@ -22,7 +22,7 @@ pub use self::{ LoopControlFrame, }, locals::LocalIdx, - operand::{Operand, TempOperand}, + operand::{ImmediateOperand, Operand, TempOperand}, operands::{OperandIdx, PreservedLocalsIter}, }; use super::{Reset, ReusableAllocations}; From d1653684feb3ccc109b3a521ade3fe3175cdd135 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 30 Jun 2025 15:19:05 +0200 Subject: [PATCH 200/343] implement visit_{i64,f32,f64}_const trait methods --- .../src/engine/translator/func2/visit.rs | 22 +++++++++++++------ 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index a60907110b..fd6489c056 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -1,6 +1,6 @@ use super::{ControlFrame, ControlFrameKind, FuncTranslator, LocalIdx}; use crate::{ - core::{wasm, TrapCode}, + core::{wasm, TrapCode, F32, F64}, engine::{ translator::func2::{ stack::{AcquiredTarget, IfReachability}, @@ -414,16 +414,24 @@ impl<'a> VisitOperator<'a> for FuncTranslator { Ok(()) } - fn visit_i64_const(&mut self, _value: i64) -> Self::Output { - todo!() + fn visit_i64_const(&mut self, value: i64) -> Self::Output { + bail_unreachable!(self); + self.stack.push_immediate(value)?; + Ok(()) } - fn visit_f32_const(&mut self, _value: wasmparser::Ieee32) -> Self::Output { - todo!() + fn visit_f32_const(&mut self, value: wasmparser::Ieee32) -> Self::Output { + bail_unreachable!(self); + let value = F32::from_bits(value.bits()); + self.stack.push_immediate(value)?; + Ok(()) } - fn visit_f64_const(&mut self, _value: wasmparser::Ieee64) -> Self::Output { - todo!() + fn visit_f64_const(&mut self, value: wasmparser::Ieee64) -> Self::Output { + bail_unreachable!(self); + let value = F64::from_bits(value.bits()); + self.stack.push_immediate(value)?; + Ok(()) } fn visit_i32_eqz(&mut self) -> Self::Output { From 5341a07bb1bab771cb6948a5a1fb995db394d3f0 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 30 Jun 2025 15:19:20 +0200 Subject: [PATCH 201/343] improve hygiene of bail_unreachable macro --- crates/wasmi/src/engine/translator/func2/utils.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/func2/utils.rs b/crates/wasmi/src/engine/translator/func2/utils.rs index daef37812b..87c2f75ebf 100644 --- a/crates/wasmi/src/engine/translator/func2/utils.rs +++ b/crates/wasmi/src/engine/translator/func2/utils.rs @@ -12,7 +12,7 @@ use crate::ir::{Const16, Reg}; macro_rules! bail_unreachable { ($this:ident) => {{ if !$this.reachable { - return Ok(()); + return ::core::result::Result::Ok(()); } }}; } From d8178656a9bf434fdc52aa86f2b7388488279eb8 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 30 Jun 2025 15:46:09 +0200 Subject: [PATCH 202/343] add swap_ops utility macro --- crates/wasmi/src/engine/translator/func2/utils.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/crates/wasmi/src/engine/translator/func2/utils.rs b/crates/wasmi/src/engine/translator/func2/utils.rs index 87c2f75ebf..7078e6bd9d 100644 --- a/crates/wasmi/src/engine/translator/func2/utils.rs +++ b/crates/wasmi/src/engine/translator/func2/utils.rs @@ -17,6 +17,17 @@ macro_rules! bail_unreachable { }}; } +/// Used to swap operands of binary [`Instruction`] constructor. +/// +/// [`Instruction`]: crate::ir::Instruction +macro_rules! swap_ops { + ($make_instr:path) => { + |result: $crate::ir::Reg, lhs, rhs| -> $crate::ir::Instruction { + $make_instr(result, rhs, lhs) + } + }; +} + /// Implemented by types that can be reset for reuse. pub trait Reset: Sized { /// Resets `self` for reuse. From 9a25bb84e239f8619e5871b1202c4ef08c5d0a44 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 30 Jun 2025 15:46:42 +0200 Subject: [PATCH 203/343] impl i{32,64} comparison translation --- .../wasmi/src/engine/translator/func2/mod.rs | 55 +++++++++ .../src/engine/translator/func2/visit.rs | 112 +++++++++++++++--- 2 files changed, 151 insertions(+), 16 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index a69ef896fc..3e9e42a225 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -899,6 +899,61 @@ impl FuncTranslator { } } + /// Translates binary non-commutative Wasm operators to Wasmi bytecode. + fn translate_binary( + &mut self, + make_instr: fn(result: Reg, lhs: Reg, rhs: Reg) -> Instruction, + make_instr_imm16_rhs: fn(result: Reg, lhs: Reg, rhs: Const16) -> Instruction, + make_instr_imm16_lhs: fn(result: Reg, lhs: Const16, rhs: Reg) -> Instruction, + consteval: fn(T, T) -> R, + ) -> Result<(), Error> + where + T: WasmInteger, + R: Into, + { + bail_unreachable!(self); + match self.stack.pop2() { + (Operand::Immediate(lhs), Operand::Immediate(rhs)) => { + self.translate_binary_consteval::(lhs, rhs, consteval) + } + (lhs, Operand::Immediate(rhs)) => { + let lhs = self.layout.operand_to_reg(lhs)?; + let rhs = T::from(rhs.val()); + let rhs16 = self.make_imm16(rhs)?; + self.push_instr_with_result( + ::TY, + |result| match rhs16 { + Operand16::Immediate(rhs) => make_instr_imm16_rhs(result, lhs, rhs), + Operand16::Reg(rhs) => make_instr(result, lhs, rhs), + }, + FuelCostsProvider::base, + ) + } + (Operand::Immediate(lhs), rhs) => { + let lhs = T::from(lhs.val()); + let lhs16 = self.make_imm16(lhs)?; + let rhs = self.layout.operand_to_reg(rhs)?; + self.push_instr_with_result( + ::TY, + |result| match lhs16 { + Operand16::Immediate(lhs) => make_instr_imm16_lhs(result, lhs, rhs), + Operand16::Reg(lhs) => make_instr(result, lhs, rhs), + }, + FuelCostsProvider::base, + ) + } + (lhs, rhs) => { + let lhs = self.layout.operand_to_reg(lhs)?; + let rhs = self.layout.operand_to_reg(rhs)?; + self.push_instr_with_result( + ::TY, + |result| make_instr(result, lhs, rhs), + FuelCostsProvider::base, + ) + } + } + } + /// Translates a generic trap instruction. fn translate_trap(&mut self, trap: TrapCode) -> Result<(), Error> { self.push_instr(Instruction::trap(trap), FuelCostsProvider::base)?; diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index fd6489c056..1a47cac81f 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -457,35 +457,75 @@ impl<'a> VisitOperator<'a> for FuncTranslator { } fn visit_i32_lt_s(&mut self) -> Self::Output { - todo!() + self.translate_binary::( + Instruction::i32_lt_s, + Instruction::i32_lt_s_imm16_rhs, + Instruction::i32_lt_s_imm16_lhs, + wasm::i32_lt_s, + ) } fn visit_i32_lt_u(&mut self) -> Self::Output { - todo!() + self.translate_binary::( + Instruction::i32_lt_u, + Instruction::i32_lt_u_imm16_rhs, + Instruction::i32_lt_u_imm16_lhs, + wasm::i32_lt_u, + ) } fn visit_i32_gt_s(&mut self) -> Self::Output { - todo!() + self.translate_binary::( + swap_ops!(Instruction::i32_lt_s), + swap_ops!(Instruction::i32_lt_s_imm16_lhs), + swap_ops!(Instruction::i32_lt_s_imm16_rhs), + wasm::i32_gt_s, + ) } fn visit_i32_gt_u(&mut self) -> Self::Output { - todo!() + self.translate_binary::( + swap_ops!(Instruction::i32_lt_u), + swap_ops!(Instruction::i32_lt_u_imm16_lhs), + swap_ops!(Instruction::i32_lt_u_imm16_rhs), + wasm::i32_gt_u, + ) } fn visit_i32_le_s(&mut self) -> Self::Output { - todo!() + self.translate_binary::( + Instruction::i32_le_s, + Instruction::i32_le_s_imm16_rhs, + Instruction::i32_le_s_imm16_lhs, + wasm::i32_le_s, + ) } fn visit_i32_le_u(&mut self) -> Self::Output { - todo!() + self.translate_binary::( + Instruction::i32_le_u, + Instruction::i32_le_u_imm16_rhs, + Instruction::i32_le_u_imm16_lhs, + wasm::i32_le_u, + ) } fn visit_i32_ge_s(&mut self) -> Self::Output { - todo!() + self.translate_binary::( + swap_ops!(Instruction::i32_le_s), + swap_ops!(Instruction::i32_le_s_imm16_lhs), + swap_ops!(Instruction::i32_le_s_imm16_rhs), + wasm::i32_ge_s, + ) } fn visit_i32_ge_u(&mut self) -> Self::Output { - todo!() + self.translate_binary::( + swap_ops!(Instruction::i32_le_u), + swap_ops!(Instruction::i32_le_u_imm16_lhs), + swap_ops!(Instruction::i32_le_u_imm16_rhs), + wasm::i32_ge_u, + ) } fn visit_i64_eqz(&mut self) -> Self::Output { @@ -511,35 +551,75 @@ impl<'a> VisitOperator<'a> for FuncTranslator { } fn visit_i64_lt_s(&mut self) -> Self::Output { - todo!() + self.translate_binary::( + Instruction::i64_lt_s, + Instruction::i64_lt_s_imm16_rhs, + Instruction::i64_lt_s_imm16_lhs, + wasm::i64_lt_s, + ) } fn visit_i64_lt_u(&mut self) -> Self::Output { - todo!() + self.translate_binary::( + Instruction::i64_lt_u, + Instruction::i64_lt_u_imm16_rhs, + Instruction::i64_lt_u_imm16_lhs, + wasm::i64_lt_u, + ) } fn visit_i64_gt_s(&mut self) -> Self::Output { - todo!() + self.translate_binary::( + swap_ops!(Instruction::i64_lt_s), + swap_ops!(Instruction::i64_lt_s_imm16_lhs), + swap_ops!(Instruction::i64_lt_s_imm16_rhs), + wasm::i64_gt_s, + ) } fn visit_i64_gt_u(&mut self) -> Self::Output { - todo!() + self.translate_binary::( + swap_ops!(Instruction::i64_lt_u), + swap_ops!(Instruction::i64_lt_u_imm16_lhs), + swap_ops!(Instruction::i64_lt_u_imm16_rhs), + wasm::i64_gt_u, + ) } fn visit_i64_le_s(&mut self) -> Self::Output { - todo!() + self.translate_binary::( + Instruction::i64_le_s, + Instruction::i64_le_s_imm16_rhs, + Instruction::i64_le_s_imm16_lhs, + wasm::i64_le_s, + ) } fn visit_i64_le_u(&mut self) -> Self::Output { - todo!() + self.translate_binary::( + Instruction::i64_le_u, + Instruction::i64_le_u_imm16_rhs, + Instruction::i64_le_u_imm16_lhs, + wasm::i64_le_u, + ) } fn visit_i64_ge_s(&mut self) -> Self::Output { - todo!() + self.translate_binary::( + swap_ops!(Instruction::i64_le_s), + swap_ops!(Instruction::i64_le_s_imm16_lhs), + swap_ops!(Instruction::i64_le_s_imm16_rhs), + wasm::i64_ge_s, + ) } fn visit_i64_ge_u(&mut self) -> Self::Output { - todo!() + self.translate_binary::( + swap_ops!(Instruction::i64_le_u), + swap_ops!(Instruction::i64_le_u_imm16_lhs), + swap_ops!(Instruction::i64_le_u_imm16_rhs), + wasm::i64_ge_u, + ) } fn visit_f32_eq(&mut self) -> Self::Output { From 139de47b0606a1d6617e54c63a5bf79c8c3bad0e Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 30 Jun 2025 18:12:17 +0200 Subject: [PATCH 204/343] double-brace swap_ops macro expansion --- crates/wasmi/src/engine/translator/func2/utils.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/utils.rs b/crates/wasmi/src/engine/translator/func2/utils.rs index 7078e6bd9d..ccff936792 100644 --- a/crates/wasmi/src/engine/translator/func2/utils.rs +++ b/crates/wasmi/src/engine/translator/func2/utils.rs @@ -21,11 +21,11 @@ macro_rules! bail_unreachable { /// /// [`Instruction`]: crate::ir::Instruction macro_rules! swap_ops { - ($make_instr:path) => { + ($make_instr:path) => {{ |result: $crate::ir::Reg, lhs, rhs| -> $crate::ir::Instruction { $make_instr(result, rhs, lhs) } - }; + }}; } /// Implemented by types that can be reset for reuse. From f7531f6c7f6c9a126ed0204160d3183c085b8274 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Tue, 1 Jul 2025 11:08:40 +0200 Subject: [PATCH 205/343] refactor WasmInteger trait --- crates/wasmi/src/engine/translator/func2/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index 3e9e42a225..cfa3def3f2 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -860,7 +860,7 @@ impl FuncTranslator { (lhs, Operand::Immediate(rhs)) => { let lhs = self.layout.operand_to_reg(lhs)?; let rhs = T::from(rhs.val()); - let Some(non_zero_rhs) = ::NonZero::try_from(rhs).ok() else { + let Some(non_zero_rhs) = ::non_zero(rhs) else { // Optimization: division by zero always traps return self.translate_trap(TrapCode::IntegerDivisionByZero); }; From 1f3bd374333de63ead0e9aba6a1ada8a661d5ba4 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 2 Jul 2025 11:25:18 +0200 Subject: [PATCH 206/343] implement shift and rotate translation --- .../wasmi/src/engine/translator/func2/mod.rs | 65 +++++++++++++++++ .../src/engine/translator/func2/stack/mod.rs | 16 +++++ .../src/engine/translator/func2/visit.rs | 70 ++++++++++++++++--- 3 files changed, 141 insertions(+), 10 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index cfa3def3f2..f69c56cc59 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -54,6 +54,7 @@ use crate::{ Const16, Const32, Instruction, + IntoShiftAmount, Reg, RegSpan, }, @@ -954,6 +955,70 @@ impl FuncTranslator { } } + /// Translates Wasm shift and rotate operators to Wasmi bytecode. + fn translate_shift( + &mut self, + make_instr: fn(result: Reg, lhs: Reg, rhs: Reg) -> Instruction, + make_instr_imm16_rhs: fn( + result: Reg, + lhs: Reg, + rhs: ::Output, + ) -> Instruction, + make_instr_imm16_lhs: fn(result: Reg, lhs: Const16, rhs: Reg) -> Instruction, + consteval: fn(T, T) -> T, + ) -> Result<(), Error> + where + T: WasmInteger + IntoShiftAmount>, + Const16: From, + { + bail_unreachable!(self); + match self.stack.pop2() { + (Operand::Immediate(lhs), Operand::Immediate(rhs)) => { + self.translate_binary_consteval::(lhs, rhs, consteval) + } + (lhs, Operand::Immediate(rhs)) => { + let Some(rhs) = T::into_shift_amount(rhs.val().into()) else { + // Optimization: Shifting or rotating by zero bits is a no-op. + self.stack.push_operand(lhs)?; + return Ok(()); + }; + let lhs = self.layout.operand_to_reg(lhs)?; + self.push_instr_with_result( + ::TY, + |result| make_instr_imm16_rhs(result, lhs, rhs), + FuelCostsProvider::base, + ) + } + (Operand::Immediate(lhs), rhs) => { + let lhs = T::from(lhs.val()); + if lhs.is_zero() { + // Optimization: Shifting or rotating a zero value is a no-op. + self.stack.push_immediate(lhs)?; + return Ok(()); + } + let lhs16 = self.make_imm16(T::from(lhs))?; + let rhs = self.layout.operand_to_reg(rhs)?; + self.push_instr_with_result( + ::TY, + |result| match lhs16 { + Operand16::Immediate(lhs) => make_instr_imm16_lhs(result, lhs, rhs), + Operand16::Reg(lhs) => make_instr(result, lhs, rhs), + }, + FuelCostsProvider::base, + ) + } + (lhs, rhs) => { + let lhs = self.layout.operand_to_reg(lhs)?; + let rhs = self.layout.operand_to_reg(rhs)?; + self.push_instr_with_result( + ::TY, + |result| make_instr(result, lhs, rhs), + FuelCostsProvider::base, + ) + } + } + } + /// Translates a generic trap instruction. fn translate_trap(&mut self, trap: TrapCode) -> Result<(), Error> { self.push_instr(Instruction::trap(trap), FuelCostsProvider::base)?; diff --git a/crates/wasmi/src/engine/translator/func2/stack/mod.rs b/crates/wasmi/src/engine/translator/func2/stack/mod.rs index 448a8ad36e..a1c1f8d7ad 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/mod.rs @@ -335,6 +335,22 @@ impl Stack { self.controls.acquire_target(depth) } + /// Pushes the [`Operand`] back to the [`Stack`]. + /// + /// Returns the new [`OperandIdx`]. + /// + /// # Errors + /// + /// - If too many operands have been pushed onto the [`Stack`]. + /// - If the local with `local_idx` does not exist. + pub fn push_operand(&mut self, operand: Operand) -> Result { + match operand { + Operand::Local(operand) => self.push_local(operand.local_index()), + Operand::Temp(operand) => self.push_temp(operand.ty(), operand.instr()), + Operand::Immediate(operand) => self.push_immediate(operand.val()), + } + } + /// Pushes a local variable with index `local_idx` to the [`Stack`]. /// /// # Errors diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index 1a47cac81f..5d78b19bb8 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -763,23 +763,48 @@ impl<'a> VisitOperator<'a> for FuncTranslator { } fn visit_i32_shl(&mut self) -> Self::Output { - todo!() + self.translate_shift::( + Instruction::i32_shl, + Instruction::i32_shl_by, + Instruction::i32_shl_imm16, + wasm::i32_shl, + ) } fn visit_i32_shr_s(&mut self) -> Self::Output { - todo!() + self.translate_shift::( + Instruction::i32_shr_s, + Instruction::i32_shr_s_by, + Instruction::i32_shr_s_imm16, + wasm::i32_shr_s, + ) } fn visit_i32_shr_u(&mut self) -> Self::Output { - todo!() + self.translate_shift::( + Instruction::i32_shr_u, + Instruction::i32_shr_u_by, + Instruction::i32_shr_u_imm16, + wasm::i32_shr_u, + ) } fn visit_i32_rotl(&mut self) -> Self::Output { - todo!() + self.translate_shift::( + Instruction::i32_rotl, + Instruction::i32_rotl_by, + Instruction::i32_rotl_imm16, + wasm::i32_rotl, + ) } fn visit_i32_rotr(&mut self) -> Self::Output { - todo!() + self.translate_shift::( + Instruction::i32_rotr, + Instruction::i32_rotr_by, + Instruction::i32_rotr_imm16, + wasm::i32_rotr, + ) } fn visit_i64_clz(&mut self) -> Self::Output { @@ -875,23 +900,48 @@ impl<'a> VisitOperator<'a> for FuncTranslator { } fn visit_i64_shl(&mut self) -> Self::Output { - todo!() + self.translate_shift::( + Instruction::i64_shl, + Instruction::i64_shl_by, + Instruction::i64_shl_imm16, + wasm::i64_shl, + ) } fn visit_i64_shr_s(&mut self) -> Self::Output { - todo!() + self.translate_shift::( + Instruction::i64_shr_s, + Instruction::i64_shr_s_by, + Instruction::i64_shr_s_imm16, + wasm::i64_shr_s, + ) } fn visit_i64_shr_u(&mut self) -> Self::Output { - todo!() + self.translate_shift::( + Instruction::i64_shr_u, + Instruction::i64_shr_u_by, + Instruction::i64_shr_u_imm16, + wasm::i64_shr_u, + ) } fn visit_i64_rotl(&mut self) -> Self::Output { - todo!() + self.translate_shift::( + Instruction::i64_rotl, + Instruction::i64_rotl_by, + Instruction::i64_rotl_imm16, + wasm::i64_rotl, + ) } fn visit_i64_rotr(&mut self) -> Self::Output { - todo!() + self.translate_shift::( + Instruction::i64_rotr, + Instruction::i64_rotr_by, + Instruction::i64_rotr_imm16, + wasm::i64_rotr, + ) } fn visit_f32_abs(&mut self) -> Self::Output { From c116b0f610ded11473cae5f8404eb3736823e63c Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 2 Jul 2025 11:28:00 +0200 Subject: [PATCH 207/343] remove unnecessary .into() call --- crates/wasmi/src/engine/translator/func2/mod.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index f69c56cc59..7c90c6c8e2 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -731,8 +731,7 @@ impl FuncTranslator { bail_unreachable!(self); let input = self.stack.pop(); if let Operand::Immediate(input) = input { - self.stack - .push_immediate(consteval(input.val().into()).into())?; + self.stack.push_immediate(consteval(input.val().into()))?; return Ok(()); } let input = self.layout.operand_to_reg(input)?; From 12e1f7830fcca91dec1b825823ba0c5593ef2f18 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 2 Jul 2025 11:37:32 +0200 Subject: [PATCH 208/343] apply clippy suggestion --- crates/wasmi/src/engine/translator/func2/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index 7c90c6c8e2..3128d20b03 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -995,7 +995,7 @@ impl FuncTranslator { self.stack.push_immediate(lhs)?; return Ok(()); } - let lhs16 = self.make_imm16(T::from(lhs))?; + let lhs16 = self.make_imm16(lhs)?; let rhs = self.layout.operand_to_reg(rhs)?; self.push_instr_with_result( ::TY, From 54f6cbd99b5f4977f0f24d95090824553796b900 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 2 Jul 2025 12:43:02 +0200 Subject: [PATCH 209/343] impl unary f{32,64} translation --- .../src/engine/translator/func2/visit.rs | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index 5d78b19bb8..2442c5200f 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -945,31 +945,31 @@ impl<'a> VisitOperator<'a> for FuncTranslator { } fn visit_f32_abs(&mut self) -> Self::Output { - todo!() + self.translate_unary::(Instruction::f32_abs, wasm::f32_abs) } fn visit_f32_neg(&mut self) -> Self::Output { - todo!() + self.translate_unary::(Instruction::f32_neg, wasm::f32_neg) } fn visit_f32_ceil(&mut self) -> Self::Output { - todo!() + self.translate_unary::(Instruction::f32_ceil, wasm::f32_ceil) } fn visit_f32_floor(&mut self) -> Self::Output { - todo!() + self.translate_unary::(Instruction::f32_floor, wasm::f32_floor) } fn visit_f32_trunc(&mut self) -> Self::Output { - todo!() + self.translate_unary::(Instruction::f32_trunc, wasm::f32_trunc) } fn visit_f32_nearest(&mut self) -> Self::Output { - todo!() + self.translate_unary::(Instruction::f32_nearest, wasm::f32_nearest) } fn visit_f32_sqrt(&mut self) -> Self::Output { - todo!() + self.translate_unary::(Instruction::f32_sqrt, wasm::f32_sqrt) } fn visit_f32_add(&mut self) -> Self::Output { @@ -1001,31 +1001,31 @@ impl<'a> VisitOperator<'a> for FuncTranslator { } fn visit_f64_abs(&mut self) -> Self::Output { - todo!() + self.translate_unary::(Instruction::f64_abs, wasm::f64_abs) } fn visit_f64_neg(&mut self) -> Self::Output { - todo!() + self.translate_unary::(Instruction::f64_neg, wasm::f64_neg) } fn visit_f64_ceil(&mut self) -> Self::Output { - todo!() + self.translate_unary::(Instruction::f64_ceil, wasm::f64_ceil) } fn visit_f64_floor(&mut self) -> Self::Output { - todo!() + self.translate_unary::(Instruction::f64_floor, wasm::f64_floor) } fn visit_f64_trunc(&mut self) -> Self::Output { - todo!() + self.translate_unary::(Instruction::f64_trunc, wasm::f64_trunc) } fn visit_f64_nearest(&mut self) -> Self::Output { - todo!() + self.translate_unary::(Instruction::f64_nearest, wasm::f64_nearest) } fn visit_f64_sqrt(&mut self) -> Self::Output { - todo!() + self.translate_unary::(Instruction::f64_sqrt, wasm::f64_sqrt) } fn visit_f64_add(&mut self) -> Self::Output { From a641c49605a5d9a98b81c28317271cc7e9236267 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 2 Jul 2025 12:46:20 +0200 Subject: [PATCH 210/343] remove unnecessary generic params --- crates/wasmi/src/engine/translator/func2/visit.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index 2442c5200f..07a84eaa04 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -1001,31 +1001,31 @@ impl<'a> VisitOperator<'a> for FuncTranslator { } fn visit_f64_abs(&mut self) -> Self::Output { - self.translate_unary::(Instruction::f64_abs, wasm::f64_abs) + self.translate_unary(Instruction::f64_abs, wasm::f64_abs) } fn visit_f64_neg(&mut self) -> Self::Output { - self.translate_unary::(Instruction::f64_neg, wasm::f64_neg) + self.translate_unary(Instruction::f64_neg, wasm::f64_neg) } fn visit_f64_ceil(&mut self) -> Self::Output { - self.translate_unary::(Instruction::f64_ceil, wasm::f64_ceil) + self.translate_unary(Instruction::f64_ceil, wasm::f64_ceil) } fn visit_f64_floor(&mut self) -> Self::Output { - self.translate_unary::(Instruction::f64_floor, wasm::f64_floor) + self.translate_unary(Instruction::f64_floor, wasm::f64_floor) } fn visit_f64_trunc(&mut self) -> Self::Output { - self.translate_unary::(Instruction::f64_trunc, wasm::f64_trunc) + self.translate_unary(Instruction::f64_trunc, wasm::f64_trunc) } fn visit_f64_nearest(&mut self) -> Self::Output { - self.translate_unary::(Instruction::f64_nearest, wasm::f64_nearest) + self.translate_unary(Instruction::f64_nearest, wasm::f64_nearest) } fn visit_f64_sqrt(&mut self) -> Self::Output { - self.translate_unary::(Instruction::f64_sqrt, wasm::f64_sqrt) + self.translate_unary(Instruction::f64_sqrt, wasm::f64_sqrt) } fn visit_f64_add(&mut self) -> Self::Output { From 8d871bd1d4b2680f5e73a41861d7745ce71494e8 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 2 Jul 2025 12:46:45 +0200 Subject: [PATCH 211/343] remove unnecessary generic params (2) --- crates/wasmi/src/engine/translator/func2/visit.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index 07a84eaa04..27671fd7a7 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -945,31 +945,31 @@ impl<'a> VisitOperator<'a> for FuncTranslator { } fn visit_f32_abs(&mut self) -> Self::Output { - self.translate_unary::(Instruction::f32_abs, wasm::f32_abs) + self.translate_unary(Instruction::f32_abs, wasm::f32_abs) } fn visit_f32_neg(&mut self) -> Self::Output { - self.translate_unary::(Instruction::f32_neg, wasm::f32_neg) + self.translate_unary(Instruction::f32_neg, wasm::f32_neg) } fn visit_f32_ceil(&mut self) -> Self::Output { - self.translate_unary::(Instruction::f32_ceil, wasm::f32_ceil) + self.translate_unary(Instruction::f32_ceil, wasm::f32_ceil) } fn visit_f32_floor(&mut self) -> Self::Output { - self.translate_unary::(Instruction::f32_floor, wasm::f32_floor) + self.translate_unary(Instruction::f32_floor, wasm::f32_floor) } fn visit_f32_trunc(&mut self) -> Self::Output { - self.translate_unary::(Instruction::f32_trunc, wasm::f32_trunc) + self.translate_unary(Instruction::f32_trunc, wasm::f32_trunc) } fn visit_f32_nearest(&mut self) -> Self::Output { - self.translate_unary::(Instruction::f32_nearest, wasm::f32_nearest) + self.translate_unary(Instruction::f32_nearest, wasm::f32_nearest) } fn visit_f32_sqrt(&mut self) -> Self::Output { - self.translate_unary::(Instruction::f32_sqrt, wasm::f32_sqrt) + self.translate_unary(Instruction::f32_sqrt, wasm::f32_sqrt) } fn visit_f32_add(&mut self) -> Self::Output { From 7f39a2c80378bab42e6ffa3b15250e3e93c43247 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 2 Jul 2025 18:07:07 +0200 Subject: [PATCH 212/343] impl translation for fallible f2i conversions --- .../wasmi/src/engine/translator/func2/mod.rs | 32 +++++++++++++++ .../src/engine/translator/func2/visit.rs | 40 +++++++++++++++---- 2 files changed, 64 insertions(+), 8 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index 3128d20b03..d218778178 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -742,6 +742,38 @@ impl FuncTranslator { ) } + /// Translates a unary Wasm instruction to Wasmi bytecode. + fn translate_unary_fallible( + &mut self, + make_instr: fn(result: Reg, input: Reg) -> Instruction, + consteval: fn(input: T) -> Result, + ) -> Result<(), Error> + where + T: From, + R: Into + Typed, + { + bail_unreachable!(self); + let input = self.stack.pop(); + if let Operand::Immediate(input) = input { + let input = T::from(input.val()); + match consteval(input) { + Ok(result) => { + self.stack.push_immediate(result)?; + } + Err(trap) => { + self.translate_trap(trap)?; + } + } + return Ok(()); + } + let input = self.layout.operand_to_reg(input)?; + self.push_instr_with_result( + ::TY, + |result| make_instr(result, input), + FuelCostsProvider::base, + ) + } + /// Creates a new 16-bit encoded [`Operand16`] from the given `value`. pub fn make_imm16(&mut self, value: T) -> Result, Error> where diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index 27671fd7a7..c8205fcf14 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -1061,19 +1061,31 @@ impl<'a> VisitOperator<'a> for FuncTranslator { } fn visit_i32_trunc_f32_s(&mut self) -> Self::Output { - todo!() + self.translate_unary_fallible::( + Instruction::i32_trunc_f32_s, + wasm::i32_trunc_f32_s, + ) } fn visit_i32_trunc_f32_u(&mut self) -> Self::Output { - todo!() + self.translate_unary_fallible::( + Instruction::i32_trunc_f32_u, + wasm::i32_trunc_f32_u, + ) } fn visit_i32_trunc_f64_s(&mut self) -> Self::Output { - todo!() + self.translate_unary_fallible::( + Instruction::i32_trunc_f64_s, + wasm::i32_trunc_f64_s, + ) } fn visit_i32_trunc_f64_u(&mut self) -> Self::Output { - todo!() + self.translate_unary_fallible::( + Instruction::i32_trunc_f64_u, + wasm::i32_trunc_f64_u, + ) } fn visit_i64_extend_i32_s(&mut self) -> Self::Output { @@ -1085,19 +1097,31 @@ impl<'a> VisitOperator<'a> for FuncTranslator { } fn visit_i64_trunc_f32_s(&mut self) -> Self::Output { - todo!() + self.translate_unary_fallible::( + Instruction::i64_trunc_f32_s, + wasm::i64_trunc_f32_s, + ) } fn visit_i64_trunc_f32_u(&mut self) -> Self::Output { - todo!() + self.translate_unary_fallible::( + Instruction::i64_trunc_f32_u, + wasm::i64_trunc_f32_u, + ) } fn visit_i64_trunc_f64_s(&mut self) -> Self::Output { - todo!() + self.translate_unary_fallible::( + Instruction::i64_trunc_f64_s, + wasm::i64_trunc_f64_s, + ) } fn visit_i64_trunc_f64_u(&mut self) -> Self::Output { - todo!() + self.translate_unary_fallible::( + Instruction::i64_trunc_f64_u, + wasm::i64_trunc_f64_u, + ) } fn visit_f32_convert_i32_s(&mut self) -> Self::Output { From a37e32d9b9f0f2d444dc197d32b00e6aaf919b62 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 2 Jul 2025 18:07:40 +0200 Subject: [PATCH 213/343] impl more conversion translations --- crates/wasmi/src/engine/translator/func2/visit.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index c8205fcf14..c553e191cb 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -1057,7 +1057,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { } fn visit_i32_wrap_i64(&mut self) -> Self::Output { - todo!() + self.translate_unary::(Instruction::i32_wrap_i64, wasm::i32_wrap_i64) } fn visit_i32_trunc_f32_s(&mut self) -> Self::Output { @@ -1089,7 +1089,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { } fn visit_i64_extend_i32_s(&mut self) -> Self::Output { - todo!() + self.translate_unary::(Instruction::i64_extend32_s, wasm::i64_extend_i32_s) } fn visit_i64_extend_i32_u(&mut self) -> Self::Output { From bb34ea9e774e2a38afa8fe26e80a8ed966102f6d Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 2 Jul 2025 18:08:45 +0200 Subject: [PATCH 214/343] initial i64.extend_i32_u translation We can go 2 ways from here: either encode a copy for local inputs; OR allow to overwrite local operand types, ignoring the type of the local. The first solution is inefficient, the second hacky. Maybe there is a third option? --- .../src/engine/translator/func2/visit.rs | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index c553e191cb..0e9277fafe 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -1,6 +1,6 @@ use super::{ControlFrame, ControlFrameKind, FuncTranslator, LocalIdx}; use crate::{ - core::{wasm, TrapCode, F32, F64}, + core::{wasm, TrapCode, F32, F64, ValType}, engine::{ translator::func2::{ stack::{AcquiredTarget, IfReachability}, @@ -1093,7 +1093,23 @@ impl<'a> VisitOperator<'a> for FuncTranslator { } fn visit_i64_extend_i32_u(&mut self) -> Self::Output { - todo!() + bail_unreachable!(self); + // Note: this Wasm operation is a no-op, we only have to change the types on the stack. + match self.stack.pop() { + Operand::Local(input) => { + debug_assert!(matches!(input.ty(), ValType::I32)); + todo!() // do we need a copy or should we allow to manipulate a local's type? + } + Operand::Temp(input) => { + debug_assert!(matches!(input.ty(), ValType::I32)); + self.stack.push_temp(ValType::I64, None)?; + } + Operand::Immediate(input) => { + let input = u32::from(input.val()); + self.stack.push_immediate(u64::from(input))?; + } + } + Ok(()) } fn visit_i64_trunc_f32_s(&mut self) -> Self::Output { From 63f8675f3fa55ddb618b31fc287b00f6b2e2b5d9 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 2 Jul 2025 18:10:04 +0200 Subject: [PATCH 215/343] apply rustfmt --- crates/wasmi/src/engine/translator/func2/visit.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index 0e9277fafe..b96dd68918 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -1,6 +1,6 @@ use super::{ControlFrame, ControlFrameKind, FuncTranslator, LocalIdx}; use crate::{ - core::{wasm, TrapCode, F32, F64, ValType}, + core::{wasm, TrapCode, ValType, F32, F64}, engine::{ translator::func2::{ stack::{AcquiredTarget, IfReachability}, From 006566f06173302f6f450291593f9d1f0ededba3 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 2 Jul 2025 18:10:25 +0200 Subject: [PATCH 216/343] impl i2f conversion instruction translation --- .../wasmi/src/engine/translator/func2/visit.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index b96dd68918..87b5918fc2 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -1141,19 +1141,19 @@ impl<'a> VisitOperator<'a> for FuncTranslator { } fn visit_f32_convert_i32_s(&mut self) -> Self::Output { - todo!() + self.translate_unary::(Instruction::f32_convert_i32_s, wasm::f32_convert_i32_s) } fn visit_f32_convert_i32_u(&mut self) -> Self::Output { - todo!() + self.translate_unary::(Instruction::f32_convert_i32_u, wasm::f32_convert_i32_u) } fn visit_f32_convert_i64_s(&mut self) -> Self::Output { - todo!() + self.translate_unary::(Instruction::f32_convert_i64_s, wasm::f32_convert_i64_s) } fn visit_f32_convert_i64_u(&mut self) -> Self::Output { - todo!() + self.translate_unary::(Instruction::f32_convert_i64_u, wasm::f32_convert_i64_u) } fn visit_f32_demote_f64(&mut self) -> Self::Output { @@ -1161,19 +1161,19 @@ impl<'a> VisitOperator<'a> for FuncTranslator { } fn visit_f64_convert_i32_s(&mut self) -> Self::Output { - todo!() + self.translate_unary::(Instruction::f64_convert_i32_s, wasm::f64_convert_i32_s) } fn visit_f64_convert_i32_u(&mut self) -> Self::Output { - todo!() + self.translate_unary::(Instruction::f64_convert_i32_u, wasm::f64_convert_i32_u) } fn visit_f64_convert_i64_s(&mut self) -> Self::Output { - todo!() + self.translate_unary::(Instruction::f64_convert_i64_s, wasm::f64_convert_i64_s) } fn visit_f64_convert_i64_u(&mut self) -> Self::Output { - todo!() + self.translate_unary::(Instruction::f64_convert_i64_u, wasm::f64_convert_i64_u) } fn visit_f64_promote_f32(&mut self) -> Self::Output { From 336ebe3dd7bd70e65c8d938616264c28fe843395 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 2 Jul 2025 18:11:03 +0200 Subject: [PATCH 217/343] impl promote and demote instruction translation --- crates/wasmi/src/engine/translator/func2/visit.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index 87b5918fc2..0ce3f7d43e 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -1157,7 +1157,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { } fn visit_f32_demote_f64(&mut self) -> Self::Output { - todo!() + self.translate_unary::(Instruction::f32_demote_f64, wasm::f32_demote_f64) } fn visit_f64_convert_i32_s(&mut self) -> Self::Output { @@ -1177,7 +1177,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { } fn visit_f64_promote_f32(&mut self) -> Self::Output { - todo!() + self.translate_unary::(Instruction::f64_promote_f32, wasm::f64_promote_f32) } fn visit_i32_reinterpret_f32(&mut self) -> Self::Output { From ea6c50907993de345cebec97d499b9c19f49a1ae Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Thu, 3 Jul 2025 14:45:43 +0200 Subject: [PATCH 218/343] add translate_reinterpret helper method --- .../wasmi/src/engine/translator/func2/mod.rs | 28 +++++++++++++++++++ .../src/engine/translator/func2/visit.rs | 20 ++----------- 2 files changed, 30 insertions(+), 18 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index d218778178..c23be1a08e 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -774,6 +774,34 @@ impl FuncTranslator { ) } + /// Translate a generic Wasm reinterpret-like operation. + /// + /// # Note + /// + /// This Wasm operation is a no-op. Ideally we only have to change the types on the stack. + fn translate_reinterpret(&mut self, consteval: fn(T) -> R) -> Result<(), Error> + where + T: From, + R: Into, + { + bail_unreachable!(self); + match self.stack.pop() { + Operand::Local(input) => { + debug_assert!(matches!(input.ty(), ValType::I32)); + todo!() // do we need a copy or should we allow to manipulate a local's type? + } + Operand::Temp(input) => { + debug_assert!(matches!(input.ty(), ValType::I32)); + self.stack.push_temp(ValType::I64, None)?; + } + Operand::Immediate(input) => { + let input: T = input.val().into(); + self.stack.push_immediate(consteval(input))?; + } + } + Ok(()) + } + /// Creates a new 16-bit encoded [`Operand16`] from the given `value`. pub fn make_imm16(&mut self, value: T) -> Result, Error> where diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index 0ce3f7d43e..b935d3f90b 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -1,6 +1,6 @@ use super::{ControlFrame, ControlFrameKind, FuncTranslator, LocalIdx}; use crate::{ - core::{wasm, TrapCode, ValType, F32, F64}, + core::{wasm, TrapCode, F32, F64}, engine::{ translator::func2::{ stack::{AcquiredTarget, IfReachability}, @@ -1093,23 +1093,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { } fn visit_i64_extend_i32_u(&mut self) -> Self::Output { - bail_unreachable!(self); - // Note: this Wasm operation is a no-op, we only have to change the types on the stack. - match self.stack.pop() { - Operand::Local(input) => { - debug_assert!(matches!(input.ty(), ValType::I32)); - todo!() // do we need a copy or should we allow to manipulate a local's type? - } - Operand::Temp(input) => { - debug_assert!(matches!(input.ty(), ValType::I32)); - self.stack.push_temp(ValType::I64, None)?; - } - Operand::Immediate(input) => { - let input = u32::from(input.val()); - self.stack.push_immediate(u64::from(input))?; - } - } - Ok(()) + self.translate_reinterpret(wasm::i64_extend_i32_u) } fn visit_i64_trunc_f32_s(&mut self) -> Self::Output { From 6956d468db02d5d319371472fdcc2d4684a5e45c Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Thu, 3 Jul 2025 14:45:59 +0200 Subject: [PATCH 219/343] implement Wasm reinterpret operators translation --- crates/wasmi/src/engine/translator/func2/visit.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index b935d3f90b..b15744280a 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -1165,19 +1165,19 @@ impl<'a> VisitOperator<'a> for FuncTranslator { } fn visit_i32_reinterpret_f32(&mut self) -> Self::Output { - todo!() + self.translate_reinterpret(wasm::i32_reinterpret_f32) } fn visit_i64_reinterpret_f64(&mut self) -> Self::Output { - todo!() + self.translate_reinterpret(wasm::i64_reinterpret_f64) } fn visit_f32_reinterpret_i32(&mut self) -> Self::Output { - todo!() + self.translate_reinterpret(wasm::f32_reinterpret_i32) } fn visit_f64_reinterpret_i64(&mut self) -> Self::Output { - todo!() + self.translate_reinterpret(wasm::f64_reinterpret_i64) } fn visit_i32_extend8_s(&mut self) -> Self::Output { From 3f1e6c1ab57408824052fa0f71411028ce7fc221 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Thu, 3 Jul 2025 14:55:22 +0200 Subject: [PATCH 220/343] remove superflous generic parameters --- .../src/engine/translator/func2/visit.rs | 62 ++++++------------- 1 file changed, 19 insertions(+), 43 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index b15744280a..a6921683b5 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -1057,35 +1057,23 @@ impl<'a> VisitOperator<'a> for FuncTranslator { } fn visit_i32_wrap_i64(&mut self) -> Self::Output { - self.translate_unary::(Instruction::i32_wrap_i64, wasm::i32_wrap_i64) + self.translate_unary(Instruction::i32_wrap_i64, wasm::i32_wrap_i64) } fn visit_i32_trunc_f32_s(&mut self) -> Self::Output { - self.translate_unary_fallible::( - Instruction::i32_trunc_f32_s, - wasm::i32_trunc_f32_s, - ) + self.translate_unary_fallible(Instruction::i32_trunc_f32_s, wasm::i32_trunc_f32_s) } fn visit_i32_trunc_f32_u(&mut self) -> Self::Output { - self.translate_unary_fallible::( - Instruction::i32_trunc_f32_u, - wasm::i32_trunc_f32_u, - ) + self.translate_unary_fallible(Instruction::i32_trunc_f32_u, wasm::i32_trunc_f32_u) } fn visit_i32_trunc_f64_s(&mut self) -> Self::Output { - self.translate_unary_fallible::( - Instruction::i32_trunc_f64_s, - wasm::i32_trunc_f64_s, - ) + self.translate_unary_fallible(Instruction::i32_trunc_f64_s, wasm::i32_trunc_f64_s) } fn visit_i32_trunc_f64_u(&mut self) -> Self::Output { - self.translate_unary_fallible::( - Instruction::i32_trunc_f64_u, - wasm::i32_trunc_f64_u, - ) + self.translate_unary_fallible(Instruction::i32_trunc_f64_u, wasm::i32_trunc_f64_u) } fn visit_i64_extend_i32_s(&mut self) -> Self::Output { @@ -1097,71 +1085,59 @@ impl<'a> VisitOperator<'a> for FuncTranslator { } fn visit_i64_trunc_f32_s(&mut self) -> Self::Output { - self.translate_unary_fallible::( - Instruction::i64_trunc_f32_s, - wasm::i64_trunc_f32_s, - ) + self.translate_unary_fallible(Instruction::i64_trunc_f32_s, wasm::i64_trunc_f32_s) } fn visit_i64_trunc_f32_u(&mut self) -> Self::Output { - self.translate_unary_fallible::( - Instruction::i64_trunc_f32_u, - wasm::i64_trunc_f32_u, - ) + self.translate_unary_fallible(Instruction::i64_trunc_f32_u, wasm::i64_trunc_f32_u) } fn visit_i64_trunc_f64_s(&mut self) -> Self::Output { - self.translate_unary_fallible::( - Instruction::i64_trunc_f64_s, - wasm::i64_trunc_f64_s, - ) + self.translate_unary_fallible(Instruction::i64_trunc_f64_s, wasm::i64_trunc_f64_s) } fn visit_i64_trunc_f64_u(&mut self) -> Self::Output { - self.translate_unary_fallible::( - Instruction::i64_trunc_f64_u, - wasm::i64_trunc_f64_u, - ) + self.translate_unary_fallible(Instruction::i64_trunc_f64_u, wasm::i64_trunc_f64_u) } fn visit_f32_convert_i32_s(&mut self) -> Self::Output { - self.translate_unary::(Instruction::f32_convert_i32_s, wasm::f32_convert_i32_s) + self.translate_unary(Instruction::f32_convert_i32_s, wasm::f32_convert_i32_s) } fn visit_f32_convert_i32_u(&mut self) -> Self::Output { - self.translate_unary::(Instruction::f32_convert_i32_u, wasm::f32_convert_i32_u) + self.translate_unary(Instruction::f32_convert_i32_u, wasm::f32_convert_i32_u) } fn visit_f32_convert_i64_s(&mut self) -> Self::Output { - self.translate_unary::(Instruction::f32_convert_i64_s, wasm::f32_convert_i64_s) + self.translate_unary(Instruction::f32_convert_i64_s, wasm::f32_convert_i64_s) } fn visit_f32_convert_i64_u(&mut self) -> Self::Output { - self.translate_unary::(Instruction::f32_convert_i64_u, wasm::f32_convert_i64_u) + self.translate_unary(Instruction::f32_convert_i64_u, wasm::f32_convert_i64_u) } fn visit_f32_demote_f64(&mut self) -> Self::Output { - self.translate_unary::(Instruction::f32_demote_f64, wasm::f32_demote_f64) + self.translate_unary(Instruction::f32_demote_f64, wasm::f32_demote_f64) } fn visit_f64_convert_i32_s(&mut self) -> Self::Output { - self.translate_unary::(Instruction::f64_convert_i32_s, wasm::f64_convert_i32_s) + self.translate_unary(Instruction::f64_convert_i32_s, wasm::f64_convert_i32_s) } fn visit_f64_convert_i32_u(&mut self) -> Self::Output { - self.translate_unary::(Instruction::f64_convert_i32_u, wasm::f64_convert_i32_u) + self.translate_unary(Instruction::f64_convert_i32_u, wasm::f64_convert_i32_u) } fn visit_f64_convert_i64_s(&mut self) -> Self::Output { - self.translate_unary::(Instruction::f64_convert_i64_s, wasm::f64_convert_i64_s) + self.translate_unary(Instruction::f64_convert_i64_s, wasm::f64_convert_i64_s) } fn visit_f64_convert_i64_u(&mut self) -> Self::Output { - self.translate_unary::(Instruction::f64_convert_i64_u, wasm::f64_convert_i64_u) + self.translate_unary(Instruction::f64_convert_i64_u, wasm::f64_convert_i64_u) } fn visit_f64_promote_f32(&mut self) -> Self::Output { - self.translate_unary::(Instruction::f64_promote_f32, wasm::f64_promote_f32) + self.translate_unary(Instruction::f64_promote_f32, wasm::f64_promote_f32) } fn visit_i32_reinterpret_f32(&mut self) -> Self::Output { From 8b720ba371ade3add4456d1bcba685aa47dc3364 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Thu, 3 Jul 2025 15:01:26 +0200 Subject: [PATCH 221/343] implement i{32,64}_extend{8,16,32}_s translation --- crates/wasmi/src/engine/translator/func2/visit.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index a6921683b5..a08891cb2f 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -1157,23 +1157,23 @@ impl<'a> VisitOperator<'a> for FuncTranslator { } fn visit_i32_extend8_s(&mut self) -> Self::Output { - todo!() + self.translate_unary(Instruction::i32_extend8_s, wasm::i32_extend8_s) } fn visit_i32_extend16_s(&mut self) -> Self::Output { - todo!() + self.translate_unary(Instruction::i32_extend16_s, wasm::i32_extend16_s) } fn visit_i64_extend8_s(&mut self) -> Self::Output { - todo!() + self.translate_unary(Instruction::i64_extend8_s, wasm::i64_extend8_s) } fn visit_i64_extend16_s(&mut self) -> Self::Output { - todo!() + self.translate_unary(Instruction::i64_extend16_s, wasm::i64_extend16_s) } fn visit_i64_extend32_s(&mut self) -> Self::Output { - todo!() + self.translate_unary(Instruction::i64_extend32_s, wasm::i64_extend32_s) } fn visit_i32_trunc_sat_f32_s(&mut self) -> Self::Output { From c295777c0bf13aad50e92673278fbe4da0978978 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Thu, 3 Jul 2025 15:02:49 +0200 Subject: [PATCH 222/343] impl trunc-sat instruction translation --- .../wasmi/src/engine/translator/func2/visit.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index a08891cb2f..7aa79244bf 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -1177,35 +1177,35 @@ impl<'a> VisitOperator<'a> for FuncTranslator { } fn visit_i32_trunc_sat_f32_s(&mut self) -> Self::Output { - todo!() + self.translate_unary(Instruction::i32_trunc_sat_f32_s, wasm::i32_trunc_sat_f32_s) } fn visit_i32_trunc_sat_f32_u(&mut self) -> Self::Output { - todo!() + self.translate_unary(Instruction::i32_trunc_sat_f32_u, wasm::i32_trunc_sat_f32_u) } fn visit_i32_trunc_sat_f64_s(&mut self) -> Self::Output { - todo!() + self.translate_unary(Instruction::i32_trunc_sat_f64_s, wasm::i32_trunc_sat_f64_s) } fn visit_i32_trunc_sat_f64_u(&mut self) -> Self::Output { - todo!() + self.translate_unary(Instruction::i32_trunc_sat_f64_u, wasm::i32_trunc_sat_f64_u) } fn visit_i64_trunc_sat_f32_s(&mut self) -> Self::Output { - todo!() + self.translate_unary(Instruction::i64_trunc_sat_f32_s, wasm::i64_trunc_sat_f32_s) } fn visit_i64_trunc_sat_f32_u(&mut self) -> Self::Output { - todo!() + self.translate_unary(Instruction::i64_trunc_sat_f32_u, wasm::i64_trunc_sat_f32_u) } fn visit_i64_trunc_sat_f64_s(&mut self) -> Self::Output { - todo!() + self.translate_unary(Instruction::i64_trunc_sat_f64_s, wasm::i64_trunc_sat_f64_s) } fn visit_i64_trunc_sat_f64_u(&mut self) -> Self::Output { - todo!() + self.translate_unary(Instruction::i64_trunc_sat_f64_u, wasm::i64_trunc_sat_f64_u) } fn visit_memory_init(&mut self, _data_index: u32, _mem: u32) -> Self::Output { From fc6589c08ae8bd052d9a0dedcf8e5f21f35a120e Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Fri, 4 Jul 2025 14:42:36 +0200 Subject: [PATCH 223/343] add Typed supertrait to WasmFloat trait --- crates/wasmi/src/engine/translator/utils.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/utils.rs b/crates/wasmi/src/engine/translator/utils.rs index dfc33c4e74..74e46e16c3 100644 --- a/crates/wasmi/src/engine/translator/utils.rs +++ b/crates/wasmi/src/engine/translator/utils.rs @@ -85,7 +85,7 @@ impl_wasm_integer!(i32, u32, i64, u64); /// # Note /// /// This trait provides some utility methods useful for translation. -pub trait WasmFloat: Copy + Into + From { +pub trait WasmFloat: Typed + Copy + Into + From { /// Returns `true` if `self` is any kind of NaN value. fn is_nan(self) -> bool; From 18883bf0378052712f629f3046aee461cb5ffdf1 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Fri, 4 Jul 2025 14:45:53 +0200 Subject: [PATCH 224/343] implement f{32,64} comparison translation --- .../wasmi/src/engine/translator/func2/mod.rs | 26 ++++++++++++++++++- .../src/engine/translator/func2/visit.rs | 24 ++++++++--------- 2 files changed, 37 insertions(+), 13 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index c23be1a08e..878a17ab7e 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -38,7 +38,7 @@ use crate::{ translator::{ comparator::{CompareResult as _, NegateCmpInstr as _, TryIntoCmpBranchInstr as _}, labels::{LabelRef, LabelRegistry}, - utils::{Instr, WasmInteger}, + utils::{Instr, WasmFloat, WasmInteger}, WasmTranslator, }, BlockType, @@ -1078,6 +1078,30 @@ impl FuncTranslator { } } + /// Translate a binary float Wasm operation. + fn translate_fbinary( + &mut self, + make_instr: fn(result: Reg, lhs: Reg, rhs: Reg) -> Instruction, + consteval: fn(T, T) -> R, + ) -> Result<(), Error> + where + T: WasmFloat, + R: Into, + { + bail_unreachable!(self); + let (lhs, rhs) = self.stack.pop2(); + if let (Operand::Immediate(lhs), Operand::Immediate(rhs)) = (lhs, rhs) { + return self.translate_binary_consteval::(lhs, rhs, consteval); + } + let lhs = self.layout.operand_to_reg(lhs)?; + let rhs = self.layout.operand_to_reg(rhs)?; + self.push_instr_with_result( + ::TY, + |result| make_instr(result, lhs, rhs), + FuelCostsProvider::base, + ) + } + /// Translates a generic trap instruction. fn translate_trap(&mut self, trap: TrapCode) -> Result<(), Error> { self.push_instr(Instruction::trap(trap), FuelCostsProvider::base)?; diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index 7aa79244bf..f3abc96da5 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -623,51 +623,51 @@ impl<'a> VisitOperator<'a> for FuncTranslator { } fn visit_f32_eq(&mut self) -> Self::Output { - todo!() + self.translate_fbinary(Instruction::f32_eq, wasm::f32_eq) } fn visit_f32_ne(&mut self) -> Self::Output { - todo!() + self.translate_fbinary(Instruction::f32_ne, wasm::f32_ne) } fn visit_f32_lt(&mut self) -> Self::Output { - todo!() + self.translate_fbinary(Instruction::f32_lt, wasm::f32_lt) } fn visit_f32_gt(&mut self) -> Self::Output { - todo!() + self.translate_fbinary(swap_ops!(Instruction::f32_lt), wasm::f32_gt) } fn visit_f32_le(&mut self) -> Self::Output { - todo!() + self.translate_fbinary(Instruction::f32_le, wasm::f32_le) } fn visit_f32_ge(&mut self) -> Self::Output { - todo!() + self.translate_fbinary(swap_ops!(Instruction::f32_le), wasm::f32_ge) } fn visit_f64_eq(&mut self) -> Self::Output { - todo!() + self.translate_fbinary(Instruction::f64_eq, wasm::f64_eq) } fn visit_f64_ne(&mut self) -> Self::Output { - todo!() + self.translate_fbinary(Instruction::f64_ne, wasm::f64_ne) } fn visit_f64_lt(&mut self) -> Self::Output { - todo!() + self.translate_fbinary(Instruction::f64_lt, wasm::f64_lt) } fn visit_f64_gt(&mut self) -> Self::Output { - todo!() + self.translate_fbinary(swap_ops!(Instruction::f64_lt), wasm::f64_gt) } fn visit_f64_le(&mut self) -> Self::Output { - todo!() + self.translate_fbinary(Instruction::f64_le, wasm::f64_le) } fn visit_f64_ge(&mut self) -> Self::Output { - todo!() + self.translate_fbinary(swap_ops!(Instruction::f64_le), wasm::f64_ge) } fn visit_i32_clz(&mut self) -> Self::Output { From f6fb521862b7f056c718e947cb89e95b28272d40 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Fri, 4 Jul 2025 14:48:32 +0200 Subject: [PATCH 225/343] apply clippy suggestions for more explicit lifetimes --- crates/wasmi/src/engine/translator/func2/instrs.rs | 2 +- crates/wasmi/src/engine/translator/func2/layout/consts.rs | 2 +- crates/wasmi/src/engine/translator/func2/layout/mod.rs | 2 +- crates/wasmi/src/engine/translator/func2/stack/control.rs | 8 ++++---- crates/wasmi/src/engine/translator/func2/stack/mod.rs | 4 ++-- .../wasmi/src/engine/translator/func2/stack/operands.rs | 4 ++-- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/instrs.rs b/crates/wasmi/src/engine/translator/func2/instrs.rs index 59c34601d6..788015baca 100644 --- a/crates/wasmi/src/engine/translator/func2/instrs.rs +++ b/crates/wasmi/src/engine/translator/func2/instrs.rs @@ -196,7 +196,7 @@ impl InstrEncoder { /// # Note /// /// The [`InstrEncoder`] will be empty after this operation. - pub fn drain(&mut self) -> InstrEncoderIter { + pub fn drain(&mut self) -> InstrEncoderIter<'_> { InstrEncoderIter { iter: self.instrs.drain(..), } diff --git a/crates/wasmi/src/engine/translator/func2/layout/consts.rs b/crates/wasmi/src/engine/translator/func2/layout/consts.rs index d25862b09e..1a0c28b595 100644 --- a/crates/wasmi/src/engine/translator/func2/layout/consts.rs +++ b/crates/wasmi/src/engine/translator/func2/layout/consts.rs @@ -98,7 +98,7 @@ impl ConstRegistry { /// # Note /// /// The function local constant values are yielded in their allocation order. - pub fn iter(&self) -> ConstRegistryIter { + pub fn iter(&self) -> ConstRegistryIter<'_> { ConstRegistryIter::new(self) } } diff --git a/crates/wasmi/src/engine/translator/func2/layout/mod.rs b/crates/wasmi/src/engine/translator/func2/layout/mod.rs index 5319b63c96..6385f20bf7 100644 --- a/crates/wasmi/src/engine/translator/func2/layout/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/layout/mod.rs @@ -122,7 +122,7 @@ impl StackLayout { /// # Note /// /// The function local constant are yielded in reverse order of allocation. - pub fn consts(&self) -> ConstRegistryIter { + pub fn consts(&self) -> ConstRegistryIter<'_> { self.consts.iter() } } diff --git a/crates/wasmi/src/engine/translator/func2/stack/control.rs b/crates/wasmi/src/engine/translator/func2/stack/control.rs index 705a3fa989..fcd777db6f 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/control.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/control.rs @@ -66,7 +66,7 @@ impl ElseOperands { } /// Pops the top-most Wasm `else` operands from `self` and returns them. - pub fn pop(&mut self) -> Option> { + pub fn pop(&mut self) -> Option> { let end = self.ends.pop()?; let start = self.ends.last().copied().unwrap_or(0); Some(self.operands.drain(start..end)) @@ -172,7 +172,7 @@ impl ControlStack { if_frame: IfControlFrame, consume_fuel: Option, is_end_of_then_reachable: bool, - ) -> Option> { + ) -> Option> { let ty = if_frame.ty(); let height = if_frame.height(); let label = if_frame.label(); @@ -234,7 +234,7 @@ impl ControlStack { } /// Returns an exclusive reference to the [`ControlFrame`] at `depth` if any. - pub fn get_mut(&mut self, depth: usize) -> ControlFrameMut { + pub fn get_mut(&mut self, depth: usize) -> ControlFrameMut<'_> { let height = self.height(); self.frames .iter_mut() @@ -304,7 +304,7 @@ impl<'stack> AcquiredTarget<'stack> { impl ControlStack { /// Acquires the target [`ControlFrame`] at the given relative `depth`. - pub fn acquire_target(&mut self, depth: usize) -> AcquiredTarget { + pub fn acquire_target(&mut self, depth: usize) -> AcquiredTarget<'_> { let is_root = self.is_root(depth); let frame = self.get_mut(depth); if is_root { diff --git a/crates/wasmi/src/engine/translator/func2/stack/mod.rs b/crates/wasmi/src/engine/translator/func2/stack/mod.rs index a1c1f8d7ad..b980f83ff4 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/mod.rs @@ -331,7 +331,7 @@ impl Stack { /// # Panics /// /// If `depth` is out of bounds for `self`. - pub fn peek_control_mut(&mut self, depth: usize) -> AcquiredTarget { + pub fn peek_control_mut(&mut self, depth: usize) -> AcquiredTarget<'_> { self.controls.acquire_target(depth) } @@ -445,7 +445,7 @@ impl Stack { /// /// If the local at `local_index` is out of bounds. #[must_use] - pub fn preserve_locals(&mut self, local_index: LocalIdx) -> PreservedLocalsIter { + pub fn preserve_locals(&mut self, local_index: LocalIdx) -> PreservedLocalsIter<'_> { self.operands.preserve_locals(local_index) } diff --git a/crates/wasmi/src/engine/translator/func2/stack/operands.rs b/crates/wasmi/src/engine/translator/func2/stack/operands.rs index 3558aba5fb..510f8417a8 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/operands.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/operands.rs @@ -196,7 +196,7 @@ impl OperandStack { /// # Panics /// /// If `depth` is out of bounds for `self`. - pub fn peek(&self, depth: usize) -> PeekedOperands { + pub fn peek(&self, depth: usize) -> PeekedOperands<'_> { let index = self.depth_to_index(depth); let operands = &self.operands[usize::from(index)..]; PeekedOperands { @@ -289,7 +289,7 @@ impl OperandStack { /// /// If the local at `local_index` is out of bounds. #[must_use] - pub fn preserve_locals(&mut self, local_index: LocalIdx) -> PreservedLocalsIter { + pub fn preserve_locals(&mut self, local_index: LocalIdx) -> PreservedLocalsIter<'_> { let ty = self.locals.ty(local_index); let index = self.locals.replace_first_operand(local_index, None); PreservedLocalsIter { From b163e7adebb2c65edcd08e39a89f4e1dd2eecc66 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Fri, 4 Jul 2025 14:58:47 +0200 Subject: [PATCH 226/343] implement most of binary float instruction translation --- .../src/engine/translator/func2/visit.rs | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index f3abc96da5..ac2af77cf7 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -973,27 +973,27 @@ impl<'a> VisitOperator<'a> for FuncTranslator { } fn visit_f32_add(&mut self) -> Self::Output { - todo!() + self.translate_fbinary(Instruction::f32_add, wasm::f32_add) } fn visit_f32_sub(&mut self) -> Self::Output { - todo!() + self.translate_fbinary(Instruction::f32_sub, wasm::f32_sub) } fn visit_f32_mul(&mut self) -> Self::Output { - todo!() + self.translate_fbinary(Instruction::f32_mul, wasm::f32_mul) } fn visit_f32_div(&mut self) -> Self::Output { - todo!() + self.translate_fbinary(Instruction::f32_div, wasm::f32_div) } fn visit_f32_min(&mut self) -> Self::Output { - todo!() + self.translate_fbinary(Instruction::f32_min, wasm::f32_min) } fn visit_f32_max(&mut self) -> Self::Output { - todo!() + self.translate_fbinary(Instruction::f32_max, wasm::f32_max) } fn visit_f32_copysign(&mut self) -> Self::Output { @@ -1029,27 +1029,27 @@ impl<'a> VisitOperator<'a> for FuncTranslator { } fn visit_f64_add(&mut self) -> Self::Output { - todo!() + self.translate_fbinary(Instruction::f64_add, wasm::f64_add) } fn visit_f64_sub(&mut self) -> Self::Output { - todo!() + self.translate_fbinary(Instruction::f64_sub, wasm::f64_sub) } fn visit_f64_mul(&mut self) -> Self::Output { - todo!() + self.translate_fbinary(Instruction::f64_mul, wasm::f64_mul) } fn visit_f64_div(&mut self) -> Self::Output { - todo!() + self.translate_fbinary(Instruction::f64_div, wasm::f64_div) } fn visit_f64_min(&mut self) -> Self::Output { - todo!() + self.translate_fbinary(Instruction::f64_min, wasm::f64_min) } fn visit_f64_max(&mut self) -> Self::Output { - todo!() + self.translate_fbinary(Instruction::f64_max, wasm::f64_max) } fn visit_f64_copysign(&mut self) -> Self::Output { From 3e33600c6f63cb0783dac40e506421964b88717a Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Fri, 4 Jul 2025 15:43:56 +0200 Subject: [PATCH 227/343] renaming match bindings --- crates/wasmi/src/engine/translator/func2/stack/operand.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/stack/operand.rs b/crates/wasmi/src/engine/translator/func2/stack/operand.rs index 5f6c1a9e60..c4c965d15f 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/operand.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/operand.rs @@ -83,9 +83,9 @@ impl Operand { /// Returns the type of the [`Operand`]. pub fn ty(&self) -> ValType { match self { - Self::Local(local_operand) => local_operand.ty(), - Self::Temp(temp_operand) => temp_operand.ty(), - Self::Immediate(immediate_operand) => immediate_operand.ty(), + Self::Local(operand) => operand.ty(), + Self::Temp(operand) => operand.ty(), + Self::Immediate(operand) => operand.ty(), } } } From b35d018da728740c04fb0fdbba9bc994e70161fc Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Fri, 4 Jul 2025 15:48:19 +0200 Subject: [PATCH 228/343] add From impls for Operand --- .../engine/translator/func2/stack/operand.rs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/crates/wasmi/src/engine/translator/func2/stack/operand.rs b/crates/wasmi/src/engine/translator/func2/stack/operand.rs index c4c965d15f..8531d318ae 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/operand.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/operand.rs @@ -101,6 +101,12 @@ pub struct LocalOperand { ty: ValType, } +impl From for Operand { + fn from(operand: LocalOperand) -> Self { + Self::Local(operand) + } +} + impl LocalOperand { /// Returns the operand index of the [`LocalOperand`]. pub fn operand_index(&self) -> OperandIdx { @@ -129,6 +135,12 @@ pub struct TempOperand { instr: Option, } +impl From for Operand { + fn from(operand: TempOperand) -> Self { + Self::Temp(operand) + } +} + impl TempOperand { /// Returns the operand index of the [`TempOperand`]. pub fn operand_index(&self) -> OperandIdx { @@ -155,6 +167,12 @@ pub struct ImmediateOperand { val: TypedVal, } +impl From for Operand { + fn from(operand: ImmediateOperand) -> Self { + Self::Immediate(operand) + } +} + impl ImmediateOperand { /// Returns the operand index of the [`ImmediateOperand`]. pub fn operand_index(&self) -> OperandIdx { From 39f59815e135b1cd88422263a9b67607038d4e1f Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Sat, 5 Jul 2025 11:01:09 +0200 Subject: [PATCH 229/343] add make_copy_imm_instr utility method --- .../wasmi/src/engine/translator/func2/mod.rs | 61 ++++++++++--------- 1 file changed, 33 insertions(+), 28 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index 878a17ab7e..b4da6ff820 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -342,40 +342,45 @@ impl FuncTranslator { } Operand::Immediate(operand) => { let result = self.layout.temp_to_reg(operand.operand_index())?; - let val = operand.val(); - match operand.ty() { - ValType::I32 => Instruction::copy_imm32(result, i32::from(val)), - ValType::I64 => { - let val = i64::from(val); - match >::try_from(val) { - Ok(value) => Instruction::copy_i64imm32(result, value), - Err(_) => { - let value = self.layout.const_to_reg(val)?; - Instruction::copy(result, value) - } - } - } - ValType::F32 => Instruction::copy_imm32(result, f32::from(val)), - ValType::F64 => { - let val = f64::from(val); - match >::try_from(val) { - Ok(value) => Instruction::copy_f64imm32(result, value), - Err(_) => { - let value = self.layout.const_to_reg(val)?; - Instruction::copy(result, value) - } - } + self.make_copy_imm_instr(result, operand.val())? + } + }; + self.instrs + .push_instr(instr, consume_fuel, FuelCostsProvider::base)?; + Ok(()) + } + + /// Returns the copy instruction to copy the given immediate `value`. + fn make_copy_imm_instr(&mut self, result: Reg, value: TypedVal) -> Result { + let instr = match value.ty() { + ValType::I32 => Instruction::copy_imm32(result, i32::from(value)), + ValType::I64 => { + let value = i64::from(value); + match >::try_from(value) { + Ok(value) => Instruction::copy_i64imm32(result, value), + Err(_) => { + let value = self.layout.const_to_reg(value)?; + Instruction::copy(result, value) } - ValType::V128 | ValType::FuncRef | ValType::ExternRef => { - let value = self.layout.const_to_reg(val)?; + } + } + ValType::F32 => Instruction::copy_imm32(result, f32::from(value)), + ValType::F64 => { + let value = f64::from(value); + match >::try_from(value) { + Ok(value) => Instruction::copy_f64imm32(result, value), + Err(_) => { + let value = self.layout.const_to_reg(value)?; Instruction::copy(result, value) } } } + ValType::V128 | ValType::FuncRef | ValType::ExternRef => { + let value = self.layout.const_to_reg(value)?; + Instruction::copy(result, value) + } }; - self.instrs - .push_instr(instr, consume_fuel, FuelCostsProvider::base)?; - Ok(()) + Ok(instr) } /// Pushes the `instr` to the function with the associated `fuel_costs`. From 7c4516f8668b1f4d4121612150eb3170b0ca9101 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Sat, 5 Jul 2025 11:01:25 +0200 Subject: [PATCH 230/343] add local.{set,tee} translation --- .../src/engine/translator/func2/instrs.rs | 41 +++++++++- .../wasmi/src/engine/translator/func2/mod.rs | 82 +++++++++++++++++++ .../engine/translator/func2/stack/locals.rs | 2 +- .../src/engine/translator/func2/visit.rs | 8 +- 4 files changed, 126 insertions(+), 7 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/instrs.rs b/crates/wasmi/src/engine/translator/func2/instrs.rs index 788015baca..a890a9fe9e 100644 --- a/crates/wasmi/src/engine/translator/func2/instrs.rs +++ b/crates/wasmi/src/engine/translator/func2/instrs.rs @@ -1,8 +1,13 @@ use super::{Reset, ReusableAllocations}; use crate::{ core::FuelCostsProvider, - engine::translator::utils::{BumpFuelConsumption as _, Instr, IsInstructionParameter as _}, - ir::Instruction, + engine::translator::{ + func2::{StackLayout, StackSpace}, + relink_result::RelinkResult, + utils::{BumpFuelConsumption as _, Instr, IsInstructionParameter as _}, + }, + ir::{Instruction, Reg}, + module::ModuleHeader, Engine, Error, }; @@ -136,6 +141,38 @@ impl InstrEncoder { Ok(true) } + /// Tries to replace the result of the last instruction with `new_result` if possible. + /// + /// # Note + /// + /// - `old_result`: just required for additional safety to check if the last instruction + /// really is the source of the `local.set` or `local.tee`. + /// - `new_result`: the new result which shall replace the `old_result`. + pub fn try_replace_result( + &mut self, + new_result: Reg, + old_result: Reg, + layout: &StackLayout, + module: &ModuleHeader, + ) -> Result { + if !matches!(layout.stack_space(new_result), StackSpace::Local) { + // Case: cannot replace result if `new_result` isn't a local. + return Ok(false); + } + let Some(last_instr) = self.last_instr else { + // Case: cannot replace result without last instruction. + return Ok(false); + }; + if !self + .get_mut(last_instr) + .relink_result(module, new_result, old_result)? + { + // Case: it was impossible to relink the result of `last_instr. + return Ok(false); + } + Ok(true) + } + /// Pushes an [`Instruction`] parameter to the [`InstrEncoder`]. /// /// The parameter is associated to the last pushed [`Instruction`]. diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index b4da6ff820..21460017bb 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -582,6 +582,88 @@ impl FuncTranslator { Ok(()) } + /// Translate the Wasm `local.set` and `local.tee` operations. + /// + /// # Note + /// + /// This applies op-code fusion that replaces the result of the previous instruction + /// instead of encoding a copy instruction for the `local.set` or `local.tee` if possible. + fn translate_local_set(&mut self, local_index: u32, push_result: bool) -> Result<(), Error> { + bail_unreachable!(self); + let input = self.stack.pop(); + if let Operand::Local(input) = input { + if u32::from(input.local_index()) == local_index { + // Case: `(local.set $n (local.get $n))` is a no-op so we can ignore it. + // + // Note: This does not require any preservation since it won't change + // the value of `local $n`. + return Ok(()); + } + } + let local_idx = LocalIdx::from(local_index); + let consume_fuel_instr = self.stack.consume_fuel_instr(); + for preserved in self.stack.preserve_locals(local_idx) { + let result = self.layout.temp_to_reg(preserved)?; + let value = self.layout.local_to_reg(local_idx)?; + self.instrs.push_instr( + Instruction::copy(result, value), + consume_fuel_instr, + FuelCostsProvider::base, + )?; + } + if push_result { + match input { + Operand::Immediate(input) => { + self.stack.push_immediate(input.val())?; + } + _ => { + self.stack.push_local(local_idx)?; + } + } + } + if self.try_replace_result(local_idx, input)? { + // Case: it was possible to replace the result of the previous + // instructions so no copy instruction is required. + return Ok(()); + } + // At this point we need to encode a copy instruction. + let result = self.layout.local_to_reg(local_idx)?; + let instr = match input { + Operand::Immediate(operand) => self.make_copy_imm_instr(result, operand.val())?, + operand => { + let input = self.layout.operand_to_reg(operand)?; + Instruction::copy(result, input) + } + }; + self.instrs + .push_instr(instr, consume_fuel_instr, FuelCostsProvider::base)?; + Ok(()) + } + + /// Tries to replace the result of the previous instruction with `new_result` if possible. + /// + /// Returns `Ok(true)` if replacement was successful and `Ok(false)` otherwise. + fn try_replace_result( + &mut self, + new_result: LocalIdx, + old_result: Operand, + ) -> Result { + let result = self.layout.local_to_reg(new_result)?; + let old_result = match old_result { + Operand::Immediate(_) => { + // Case: cannot replace immediate value result. + return Ok(false); + } + Operand::Local(_) => { + // Case: cannot replace local with another local due to observable behavior. + return Ok(false); + } + Operand::Temp(operand) => self.layout.temp_to_reg(operand.operand_index())?, + }; + self.instrs + .try_replace_result(result, old_result, &self.layout, &self.module) + } + /// Encodes an unconditional Wasm `branch` instruction. fn encode_br(&mut self, label: LabelRef) -> Result<(), Error> { let instr = self.instrs.next_instr(); diff --git a/crates/wasmi/src/engine/translator/func2/stack/locals.rs b/crates/wasmi/src/engine/translator/func2/stack/locals.rs index 97c4b126a7..588a0263a9 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/locals.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/locals.rs @@ -4,7 +4,7 @@ use alloc::vec::Vec; use core::{cmp, iter}; /// A local variable index. -#[derive(Debug, Copy, Clone)] +#[derive(Debug, Copy, Clone, PartialEq, Eq)] pub struct LocalIdx(u32); impl From for LocalIdx { diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index ac2af77cf7..78782e5259 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -292,12 +292,12 @@ impl<'a> VisitOperator<'a> for FuncTranslator { Ok(()) } - fn visit_local_set(&mut self, _local_index: u32) -> Self::Output { - todo!() + fn visit_local_set(&mut self, local_index: u32) -> Self::Output { + self.translate_local_set(local_index, false) } - fn visit_local_tee(&mut self, _local_index: u32) -> Self::Output { - todo!() + fn visit_local_tee(&mut self, local_index: u32) -> Self::Output { + self.translate_local_set(local_index, true) } fn visit_global_get(&mut self, _global_index: u32) -> Self::Output { From bf1edba171ec5c7178bb773dd966040349f46123 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Sat, 5 Jul 2025 12:38:38 +0200 Subject: [PATCH 231/343] dedup code with new push_binary_instr_with_result utility --- .../wasmi/src/engine/translator/func2/mod.rs | 47 ++++++++----------- 1 file changed, 19 insertions(+), 28 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index 21460017bb..35eb9b767e 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -413,6 +413,21 @@ impl FuncTranslator { Ok(()) } + /// Pushes a binary instruction with a result and associated fuel costs. + fn push_binary_instr_with_result( + &mut self, + lhs: Operand, + rhs: Operand, + make_instr: impl FnOnce(Reg, Reg, Reg) -> Instruction, + fuel_costs: impl FnOnce(&FuelCostsProvider) -> u64, + ) -> Result<(), Error> { + debug_assert_eq!(lhs.ty(), rhs.ty()); + let ty = lhs.ty(); + let lhs = self.layout.operand_to_reg(lhs)?; + let rhs = self.layout.operand_to_reg(rhs)?; + self.push_instr_with_result(ty, |result| make_instr(result, lhs, rhs), fuel_costs) + } + /// Encodes a generic return instruction. fn encode_return(&mut self, consume_fuel: Option) -> Result { let len_results = self.func_type_with(FuncType::len_results); @@ -973,13 +988,7 @@ impl FuncTranslator { ) } (lhs, rhs) => { - let lhs = self.layout.operand_to_reg(lhs)?; - let rhs = self.layout.operand_to_reg(rhs)?; - self.push_instr_with_result( - ::TY, - |result| make_instr(result, lhs, rhs), - FuelCostsProvider::base, - ) + self.push_binary_instr_with_result(lhs, rhs, make_instr, FuelCostsProvider::base) } } } @@ -1035,13 +1044,7 @@ impl FuncTranslator { ) } (lhs, rhs) => { - let lhs = self.layout.operand_to_reg(lhs)?; - let rhs = self.layout.operand_to_reg(rhs)?; - self.push_instr_with_result( - ::TY, - |result| make_instr(result, lhs, rhs), - FuelCostsProvider::base, - ) + self.push_binary_instr_with_result(lhs, rhs, make_instr, FuelCostsProvider::base) } } } @@ -1154,13 +1157,7 @@ impl FuncTranslator { ) } (lhs, rhs) => { - let lhs = self.layout.operand_to_reg(lhs)?; - let rhs = self.layout.operand_to_reg(rhs)?; - self.push_instr_with_result( - ::TY, - |result| make_instr(result, lhs, rhs), - FuelCostsProvider::base, - ) + self.push_binary_instr_with_result(lhs, rhs, make_instr, FuelCostsProvider::base) } } } @@ -1180,13 +1177,7 @@ impl FuncTranslator { if let (Operand::Immediate(lhs), Operand::Immediate(rhs)) = (lhs, rhs) { return self.translate_binary_consteval::(lhs, rhs, consteval); } - let lhs = self.layout.operand_to_reg(lhs)?; - let rhs = self.layout.operand_to_reg(rhs)?; - self.push_instr_with_result( - ::TY, - |result| make_instr(result, lhs, rhs), - FuelCostsProvider::base, - ) + self.push_binary_instr_with_result(lhs, rhs, make_instr, FuelCostsProvider::base) } /// Translates a generic trap instruction. From b8c1f57259ed62fb88de081f6791a6fbdebf8cae Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Sat, 5 Jul 2025 12:38:53 +0200 Subject: [PATCH 232/343] update doc (clippy) --- crates/wasmi/src/engine/translator/func2/instrs.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/instrs.rs b/crates/wasmi/src/engine/translator/func2/instrs.rs index a890a9fe9e..a2b87391df 100644 --- a/crates/wasmi/src/engine/translator/func2/instrs.rs +++ b/crates/wasmi/src/engine/translator/func2/instrs.rs @@ -145,9 +145,11 @@ impl InstrEncoder { /// /// # Note /// - /// - `old_result`: just required for additional safety to check if the last instruction - /// really is the source of the `local.set` or `local.tee`. - /// - `new_result`: the new result which shall replace the `old_result`. + /// - `old_result`: + /// just required for additional safety to check if the last instruction + /// really is the source of the `local.set` or `local.tee`. + /// - `new_result`: + /// the new result which shall replace the `old_result`. pub fn try_replace_result( &mut self, new_result: Reg, From 9f79c5e6028f529cab10242603dba97bbff4dbc6 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Sat, 5 Jul 2025 12:39:12 +0200 Subject: [PATCH 233/343] add WasmInteger::wrapping_neg trait method --- crates/wasmi/src/engine/translator/utils.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/crates/wasmi/src/engine/translator/utils.rs b/crates/wasmi/src/engine/translator/utils.rs index 74e46e16c3..32b141ed00 100644 --- a/crates/wasmi/src/engine/translator/utils.rs +++ b/crates/wasmi/src/engine/translator/utils.rs @@ -59,6 +59,9 @@ pub trait WasmInteger: /// Returns `true` if `self` is equal to zero (0). fn is_zero(self) -> bool; + + /// Returns the wrapped negated `self`. + fn wrapping_neg(self) -> Self; } macro_rules! impl_wasm_integer { @@ -74,6 +77,10 @@ macro_rules! impl_wasm_integer { fn is_zero(self) -> bool { self == 0 } + + fn wrapping_neg(self) -> Self { + Self::wrapping_neg(self) + } } )* }; From de31929ea328a89656842ccf580883f3818dc7b0 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Sat, 5 Jul 2025 12:39:39 +0200 Subject: [PATCH 234/343] add i{32,64}.sub operators translation --- .../wasmi/src/engine/translator/func2/mod.rs | 51 ++++++++++++++++++- .../src/engine/translator/func2/visit.rs | 14 ++++- 2 files changed, 62 insertions(+), 3 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index 35eb9b767e..7c77f03ec0 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -1093,14 +1093,63 @@ impl FuncTranslator { ) } (lhs, rhs) => { + self.push_binary_instr_with_result(lhs, rhs, make_instr, FuelCostsProvider::base) + } + } + } + + /// Translates Wasm `i{32,64}.sub` operators to Wasmi bytecode. + fn translate_isub( + &mut self, + make_sub_rr: fn(result: Reg, lhs: Reg, rhs: Reg) -> Instruction, + make_add_ri: fn(result: Reg, lhs: Reg, rhs: Const16) -> Instruction, + make_sub_ir: fn(result: Reg, lhs: Const16, rhs: Reg) -> Instruction, + consteval: fn(T, T) -> R, + ) -> Result<(), Error> + where + T: WasmInteger, + R: Into, + { + bail_unreachable!(self); + match self.stack.pop2() { + (Operand::Immediate(lhs), Operand::Immediate(rhs)) => { + self.translate_binary_consteval::(lhs, rhs, consteval) + } + (lhs, Operand::Immediate(rhs)) => { let lhs = self.layout.operand_to_reg(lhs)?; + let rhs = T::from(rhs.val()); + let rhs16 = match rhs.wrapping_neg().try_into() { + Ok(rhs) => Operand16::Immediate(rhs), + Err(_) => { + let rhs = self.layout.const_to_reg(rhs)?; + Operand16::Reg(rhs) + } + }; + self.push_instr_with_result( + ::TY, + |result| match rhs16 { + Operand16::Immediate(rhs) => make_add_ri(result, lhs, rhs), + Operand16::Reg(rhs) => make_sub_rr(result, lhs, rhs), + }, + FuelCostsProvider::base, + ) + } + (Operand::Immediate(lhs), rhs) => { + let lhs = T::from(lhs.val()); + let lhs16 = self.make_imm16(lhs)?; let rhs = self.layout.operand_to_reg(rhs)?; self.push_instr_with_result( ::TY, - |result| make_instr(result, lhs, rhs), + |result| match lhs16 { + Operand16::Immediate(lhs) => make_sub_ir(result, lhs, rhs), + Operand16::Reg(lhs) => make_sub_rr(result, lhs, rhs), + }, FuelCostsProvider::base, ) } + (lhs, rhs) => { + self.push_binary_instr_with_result(lhs, rhs, make_sub_rr, FuelCostsProvider::base) + } } } diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index 78782e5259..9c9b858040 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -691,7 +691,12 @@ impl<'a> VisitOperator<'a> for FuncTranslator { } fn visit_i32_sub(&mut self) -> Self::Output { - todo!() + self.translate_isub( + Instruction::i32_sub, + Instruction::i32_add_imm16, + Instruction::i32_sub_imm16_lhs, + wasm::i32_sub, + ) } fn visit_i32_mul(&mut self) -> Self::Output { @@ -828,7 +833,12 @@ impl<'a> VisitOperator<'a> for FuncTranslator { } fn visit_i64_sub(&mut self) -> Self::Output { - todo!() + self.translate_isub( + Instruction::i64_sub, + Instruction::i64_add_imm16, + Instruction::i64_sub_imm16_lhs, + wasm::i64_sub, + ) } fn visit_i64_mul(&mut self) -> Self::Output { From c9baeebf417bfe6d664dbd6bbcdd25041909acb7 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Sat, 5 Jul 2025 13:04:40 +0200 Subject: [PATCH 235/343] fix i{32,64}.{and,or,xor} translation --- .../src/engine/translator/func2/visit.rs | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index 9c9b858040..0707bbedb4 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -745,24 +745,24 @@ impl<'a> VisitOperator<'a> for FuncTranslator { fn visit_i32_and(&mut self) -> Self::Output { self.translate_binary_commutative::( - Instruction::i32_and, - Instruction::i32_and_imm16, + Instruction::i32_bitand, + Instruction::i32_bitand_imm16, wasm::i32_bitand, ) } fn visit_i32_or(&mut self) -> Self::Output { self.translate_binary_commutative::( - Instruction::i32_or, - Instruction::i32_or_imm16, + Instruction::i32_bitor, + Instruction::i32_bitor_imm16, wasm::i32_bitor, ) } fn visit_i32_xor(&mut self) -> Self::Output { self.translate_binary_commutative::( - Instruction::i32_xor, - Instruction::i32_xor_imm16, + Instruction::i32_bitxor, + Instruction::i32_bitxor_imm16, wasm::i32_bitxor, ) } @@ -887,24 +887,24 @@ impl<'a> VisitOperator<'a> for FuncTranslator { fn visit_i64_and(&mut self) -> Self::Output { self.translate_binary_commutative::( - Instruction::i64_and, - Instruction::i64_and_imm16, + Instruction::i64_bitand, + Instruction::i64_bitand_imm16, wasm::i64_bitand, ) } fn visit_i64_or(&mut self) -> Self::Output { self.translate_binary_commutative::( - Instruction::i64_or, - Instruction::i64_or_imm16, + Instruction::i64_bitor, + Instruction::i64_bitor_imm16, wasm::i64_bitor, ) } fn visit_i64_xor(&mut self) -> Self::Output { self.translate_binary_commutative::( - Instruction::i64_xor, - Instruction::i64_xor_imm16, + Instruction::i64_bitxor, + Instruction::i64_bitxor_imm16, wasm::i64_bitxor, ) } From e0dfb6454ac6aef15c7139680257219524d6e73d Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Sat, 5 Jul 2025 13:05:04 +0200 Subject: [PATCH 236/343] fix OperandStack::peek yielding too many operands --- .../engine/translator/func2/stack/operands.rs | 28 ++++++++++++++----- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/stack/operands.rs b/crates/wasmi/src/engine/translator/func2/stack/operands.rs index 510f8417a8..153625ee96 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/operands.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/operands.rs @@ -191,16 +191,19 @@ impl OperandStack { Ok(idx) } - /// Returns an iterator that yields all [`Operand`]s up to `depth`. - /// + /// Returns an iterator that yields the last `n` [`Operand`]s. + /// /// # Panics /// - /// If `depth` is out of bounds for `self`. - pub fn peek(&self, depth: usize) -> PeekedOperands<'_> { - let index = self.depth_to_index(depth); - let operands = &self.operands[usize::from(index)..]; + /// If `n` is out of bounds for `self`. + pub fn peek(&self, n: usize) -> PeekedOperands<'_> { + let len_operands = self.operands.len(); + let first_index = len_operands - n; + let Some(operands) = self.operands.get(first_index..) else { + return PeekedOperands::empty(&self.locals) + }; PeekedOperands { - index: usize::from(index), + index: first_index, operands: operands.iter(), locals: &self.locals, } @@ -389,6 +392,17 @@ pub struct PeekedOperands<'stack> { locals: &'stack LocalsRegistry, } +impl<'stack> PeekedOperands<'stack> { + /// Creates a [`PeekedOperands`] iterator that yields no operands. + pub fn empty(locals: &'stack LocalsRegistry) -> Self { + Self { + index: 0, + operands: [].iter(), + locals, + } + } +} + impl Iterator for PeekedOperands<'_> { type Item = Operand; From d562c7f6371e543d1e9360bb4cb9aa6d360bd272 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Sat, 5 Jul 2025 13:24:18 +0200 Subject: [PATCH 237/343] apply rustfmt --- crates/wasmi/src/engine/translator/func2/stack/operands.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/stack/operands.rs b/crates/wasmi/src/engine/translator/func2/stack/operands.rs index 153625ee96..f3a2b4a64f 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/operands.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/operands.rs @@ -192,7 +192,7 @@ impl OperandStack { } /// Returns an iterator that yields the last `n` [`Operand`]s. - /// + /// /// # Panics /// /// If `n` is out of bounds for `self`. @@ -200,7 +200,7 @@ impl OperandStack { let len_operands = self.operands.len(); let first_index = len_operands - n; let Some(operands) = self.operands.get(first_index..) else { - return PeekedOperands::empty(&self.locals) + return PeekedOperands::empty(&self.locals); }; PeekedOperands { index: first_index, From b8aef2d65c722721ef27de82eb4b0431f99da8b3 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Sat, 5 Jul 2025 13:25:21 +0200 Subject: [PATCH 238/343] cfg-guard WasmInteger::wrapping_neg --- crates/wasmi/src/engine/translator/utils.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/wasmi/src/engine/translator/utils.rs b/crates/wasmi/src/engine/translator/utils.rs index 32b141ed00..2fb670b94c 100644 --- a/crates/wasmi/src/engine/translator/utils.rs +++ b/crates/wasmi/src/engine/translator/utils.rs @@ -61,6 +61,7 @@ pub trait WasmInteger: fn is_zero(self) -> bool; /// Returns the wrapped negated `self`. + #[cfg(feature = "experimental-translator")] fn wrapping_neg(self) -> Self; } @@ -78,6 +79,7 @@ macro_rules! impl_wasm_integer { self == 0 } + #[cfg(feature = "experimental-translator")] fn wrapping_neg(self) -> Self { Self::wrapping_neg(self) } From e99c7e4c0cfbb7ba0d58e8c01c150908cc155e7e Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Sat, 5 Jul 2025 13:25:35 +0200 Subject: [PATCH 239/343] add TODO comment to remove cfg-guard --- crates/wasmi/src/engine/translator/utils.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/wasmi/src/engine/translator/utils.rs b/crates/wasmi/src/engine/translator/utils.rs index 2fb670b94c..4aed2967ce 100644 --- a/crates/wasmi/src/engine/translator/utils.rs +++ b/crates/wasmi/src/engine/translator/utils.rs @@ -61,7 +61,7 @@ pub trait WasmInteger: fn is_zero(self) -> bool; /// Returns the wrapped negated `self`. - #[cfg(feature = "experimental-translator")] + #[cfg(feature = "experimental-translator")] // TODO: remove fn wrapping_neg(self) -> Self; } @@ -79,7 +79,7 @@ macro_rules! impl_wasm_integer { self == 0 } - #[cfg(feature = "experimental-translator")] + #[cfg(feature = "experimental-translator")] // TODO: remove fn wrapping_neg(self) -> Self { Self::wrapping_neg(self) } From d0e4c36fc887c703b02a24f3ce724bd9fa75f28c Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Sat, 5 Jul 2025 15:40:43 +0200 Subject: [PATCH 240/343] add f{32,64}.copysign operators translation --- .../wasmi/src/engine/translator/func2/mod.rs | 41 +++++++++++++++++++ .../engine/translator/func2/stack/operand.rs | 8 ++-- .../src/engine/translator/func2/visit.rs | 12 +++++- 3 files changed, 55 insertions(+), 6 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index 7c77f03ec0..484f5a12de 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -57,6 +57,7 @@ use crate::{ IntoShiftAmount, Reg, RegSpan, + Sign, }, module::{FuncIdx, ModuleHeader, WasmiValueType}, Engine, @@ -1229,6 +1230,46 @@ impl FuncTranslator { self.push_binary_instr_with_result(lhs, rhs, make_instr, FuelCostsProvider::base) } + /// Translate Wasmi `{f32,f64}.copysign` instructions. + /// + /// # Note + /// + /// - This applies some optimization that are valid for copysign instructions. + /// - Applies constant evaluation if both operands are constant values. + fn translate_fcopysign( + &mut self, + make_instr: fn(result: Reg, lhs: Reg, rhs: Reg) -> Instruction, + make_instr_imm: fn(result: Reg, lhs: Reg, rhs: Sign) -> Instruction, + consteval: fn(T, T) -> T, + ) -> Result<(), Error> + where + T: WasmFloat, + { + bail_unreachable!(self); + match self.stack.pop2() { + (Operand::Immediate(lhs), Operand::Immediate(rhs)) => { + self.translate_binary_consteval::(lhs, rhs, consteval) + } + (lhs, Operand::Immediate(rhs)) => { + let lhs = self.layout.operand_to_reg(lhs)?; + let sign = T::from(rhs.val()).sign(); + self.push_instr_with_result( + ::TY, + |result| make_instr_imm(result, lhs, sign), + FuelCostsProvider::base, + ) + } + (lhs, rhs) => { + if lhs == rhs { + // Optimization: `copysign x x` is always just `x` + self.stack.push_operand(lhs)?; + return Ok(()); + } + self.push_binary_instr_with_result(lhs, rhs, make_instr, FuelCostsProvider::base) + } + } + } + /// Translates a generic trap instruction. fn translate_trap(&mut self, trap: TrapCode) -> Result<(), Error> { self.push_instr(Instruction::trap(trap), FuelCostsProvider::base)?; diff --git a/crates/wasmi/src/engine/translator/func2/stack/operand.rs b/crates/wasmi/src/engine/translator/func2/stack/operand.rs index 8531d318ae..2f67985678 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/operand.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/operand.rs @@ -8,7 +8,7 @@ use crate::{ use super::Stack; /// An operand on the [`Stack`]. -#[derive(Debug, Copy, Clone)] +#[derive(Debug, Copy, Clone, PartialEq, Eq)] pub enum Operand { /// A local variable operand. Local(LocalOperand), @@ -91,7 +91,7 @@ impl Operand { } /// A local variable on the [`Stack`]. -#[derive(Debug, Copy, Clone)] +#[derive(Debug, Copy, Clone, PartialEq, Eq)] pub struct LocalOperand { /// The index of the operand. operand_index: OperandIdx, @@ -125,7 +125,7 @@ impl LocalOperand { } /// A temporary on the [`Stack`]. -#[derive(Debug, Copy, Clone)] +#[derive(Debug, Copy, Clone, PartialEq, Eq)] pub struct TempOperand { /// The index of the operand. operand_index: OperandIdx, @@ -159,7 +159,7 @@ impl TempOperand { } /// An immediate value on the [`Stack`]. -#[derive(Debug, Copy, Clone)] +#[derive(Debug, Copy, Clone, PartialEq, Eq)] pub struct ImmediateOperand { /// The index of the operand. operand_index: OperandIdx, diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index 0707bbedb4..5f2a3100b4 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -1007,7 +1007,11 @@ impl<'a> VisitOperator<'a> for FuncTranslator { } fn visit_f32_copysign(&mut self) -> Self::Output { - todo!() + self.translate_fcopysign::( + Instruction::f32_copysign, + Instruction::f32_copysign_imm, + wasm::f32_copysign, + ) } fn visit_f64_abs(&mut self) -> Self::Output { @@ -1063,7 +1067,11 @@ impl<'a> VisitOperator<'a> for FuncTranslator { } fn visit_f64_copysign(&mut self) -> Self::Output { - todo!() + self.translate_fcopysign::( + Instruction::f64_copysign, + Instruction::f64_copysign_imm, + wasm::f64_copysign, + ) } fn visit_i32_wrap_i64(&mut self) -> Self::Output { From 48d5f610c6b5ffd94e3929b256d50b0635650bf3 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 7 Jul 2025 15:22:01 +0200 Subject: [PATCH 241/343] replace Operand::[partial_]eq with Operand::is_same --- .../wasmi/src/engine/translator/func2/mod.rs | 2 +- .../engine/translator/func2/stack/operand.rs | 18 ++++++++++++++---- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index 484f5a12de..0a14fa9387 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -1260,7 +1260,7 @@ impl FuncTranslator { ) } (lhs, rhs) => { - if lhs == rhs { + if lhs.is_same(&rhs) { // Optimization: `copysign x x` is always just `x` self.stack.push_operand(lhs)?; return Ok(()); diff --git a/crates/wasmi/src/engine/translator/func2/stack/operand.rs b/crates/wasmi/src/engine/translator/func2/stack/operand.rs index 2f67985678..7e396f5968 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/operand.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/operand.rs @@ -8,7 +8,7 @@ use crate::{ use super::Stack; /// An operand on the [`Stack`]. -#[derive(Debug, Copy, Clone, PartialEq, Eq)] +#[derive(Debug, Copy, Clone)] pub enum Operand { /// A local variable operand. Local(LocalOperand), @@ -28,6 +28,16 @@ impl Operand { } } + /// Returns `true` if `self` and `other` evaluate to the same value. + pub fn is_same(&self, other: &Self) -> bool { + match (self, other) { + (Operand::Local(lhs), Operand::Local(rhs)) => lhs.local_index() == rhs.local_index(), + (Operand::Temp(lhs), Operand::Temp(rhs)) => lhs.operand_index() == rhs.operand_index(), + (Operand::Immediate(lhs), Operand::Immediate(rhs)) => lhs.val() == rhs.val(), + _ => false, + } + } + /// Creates a local [`Operand`]. pub(super) fn local( operand_index: OperandIdx, @@ -91,7 +101,7 @@ impl Operand { } /// A local variable on the [`Stack`]. -#[derive(Debug, Copy, Clone, PartialEq, Eq)] +#[derive(Debug, Copy, Clone)] pub struct LocalOperand { /// The index of the operand. operand_index: OperandIdx, @@ -125,7 +135,7 @@ impl LocalOperand { } /// A temporary on the [`Stack`]. -#[derive(Debug, Copy, Clone, PartialEq, Eq)] +#[derive(Debug, Copy, Clone)] pub struct TempOperand { /// The index of the operand. operand_index: OperandIdx, @@ -159,7 +169,7 @@ impl TempOperand { } /// An immediate value on the [`Stack`]. -#[derive(Debug, Copy, Clone, PartialEq, Eq)] +#[derive(Debug, Copy, Clone)] pub struct ImmediateOperand { /// The index of the operand. operand_index: OperandIdx, From baf9b2f1fdd9605de3477cac1a1ba9fa8e9894c9 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 7 Jul 2025 15:24:50 +0200 Subject: [PATCH 242/343] add translation for Wasm select instructions --- .../src/engine/translator/func2/instrs.rs | 53 +++++++++++++- .../wasmi/src/engine/translator/func2/mod.rs | 73 +++++++++++++++++++ .../src/engine/translator/func2/visit.rs | 8 +- 3 files changed, 129 insertions(+), 5 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/instrs.rs b/crates/wasmi/src/engine/translator/func2/instrs.rs index a2b87391df..c927ba1755 100644 --- a/crates/wasmi/src/engine/translator/func2/instrs.rs +++ b/crates/wasmi/src/engine/translator/func2/instrs.rs @@ -1,8 +1,9 @@ use super::{Reset, ReusableAllocations}; use crate::{ - core::FuelCostsProvider, + core::{FuelCostsProvider, ValType}, engine::translator::{ - func2::{StackLayout, StackSpace}, + comparator::{CmpSelectFusion, CompareResult as _, TryIntoCmpSelectInstr as _}, + func2::{Stack, StackLayout, StackSpace}, relink_result::RelinkResult, utils::{BumpFuelConsumption as _, Instr, IsInstructionParameter as _}, }, @@ -175,6 +176,54 @@ impl InstrEncoder { Ok(true) } + /// Tries to fuse a compare instruction with a Wasm `select` instruction. + /// + /// # Returns + /// + /// - Returns `Some` if fusion was successful. + /// - Returns `None` if fusion could not be applied. + pub fn try_fuse_select( + &mut self, + ty: ValType, + select_condition: Reg, + layout: &StackLayout, + stack: &mut Stack, + ) -> Result, Error> { + let Some(last_instr) = self.last_instr else { + // If there is no last instruction there is no comparison instruction to negate. + return Ok(None); + }; + let last_instruction = self.get(last_instr); + let Some(last_result) = last_instruction.compare_result() else { + // All negatable instructions have a single result register. + return Ok(None); + }; + if matches!(layout.stack_space(last_result), StackSpace::Local) { + // The instruction stores its result into a local variable which + // is an observable side effect which we are not allowed to mutate. + return Ok(None); + } + if last_result != select_condition { + // The result of the last instruction and the select's `condition` + // are not equal thus indicating that we cannot fuse the instructions. + return Ok(None); + } + let CmpSelectFusion::Applied { + fused, + swap_operands, + } = last_instruction.try_into_cmp_select_instr(|| { + let select_result = stack.push_temp(ty, Some(last_instr))?; + let select_result = layout.temp_to_reg(select_result)?; + Ok(select_result) + })? + else { + return Ok(None); + }; + let last_instr = self.get_mut(last_instr); + *last_instr = fused; + Ok(Some(swap_operands)) + } + /// Pushes an [`Instruction`] parameter to the [`InstrEncoder`]. /// /// The parameter is associated to the last pushed [`Instruction`]. diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index 0a14fa9387..137ae9fe8d 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -64,6 +64,7 @@ use crate::{ Error, FuncType, }; +use core::mem; use wasmparser::WasmFeatures; /// Type concerned with translating from Wasm bytecode to Wasmi bytecode. @@ -1276,4 +1277,76 @@ impl FuncTranslator { self.reachable = false; Ok(()) } + + /// Translates a Wasm `select` or `select ` instruction. + /// + /// # Note + /// + /// - This applies constant propagation in case `condition` is a constant value. + /// - If both `lhs` and `rhs` are equal registers or constant values `lhs` is forwarded. + /// - Fuses compare instructions with the associated select instructions if possible. + fn translate_select(&mut self, type_hint: Option) -> Result<(), Error> { + bail_unreachable!(self); + let (true_val, false_val, condition) = self.stack.pop3(); + if let Some(type_hint) = type_hint { + debug_assert_eq!(true_val.ty(), type_hint); + debug_assert_eq!(false_val.ty(), type_hint); + } + let ty = true_val.ty(); + if true_val.is_same(&false_val) { + // Optimization: both `lhs` and `rhs` either are the same register or constant values and + // thus `select` will always yield this same value irrespective of the condition. + self.stack.push_operand(true_val)?; + return Ok(()); + } + if let Operand::Immediate(condition) = condition { + // Optimization: since condition is a constant value we can const-fold the `select` + // instruction and simply push the selected value back to the provider stack. + let condition = i32::from(condition.val()) != 0; + let selected = match condition { + true => true_val, + false => false_val, + }; + if let Operand::Temp(selected) = selected { + // Case: the selected operand is a temporary which needs to be copied + // if it was the `false_val` since it changed its index. This is + // not the case for the `true_val` since `true_val` is the first + // value popped from the stack. + if !condition { + let selected = self.layout.temp_to_reg(selected.operand_index())?; + self.push_instr_with_result( + ty, + |result| Instruction::copy(result, selected), + FuelCostsProvider::base, + )?; + } + } + self.stack.push_operand(selected)?; + return Ok(()); + } + let condition = self.layout.operand_to_reg(condition)?; + let mut true_val = self.layout.operand_to_reg(true_val)?; + let mut false_val = self.layout.operand_to_reg(false_val)?; + match self + .instrs + .try_fuse_select(ty, condition, &self.layout, &mut self.stack)? + { + Some(swap_operands) => { + if swap_operands { + mem::swap(&mut true_val, &mut false_val); + } + } + None => { + self.push_instr_with_result( + ty, + |result| Instruction::select_i32_eq_imm16(result, condition, 0_i16), + FuelCostsProvider::base, + )?; + mem::swap(&mut true_val, &mut false_val); + } + }; + self.instrs + .push_param(Instruction::register2_ext(true_val, false_val)); + Ok(()) + } } diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index 5f2a3100b4..7d7669b31c 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -10,6 +10,7 @@ use crate::{ BlockType, }, ir::Instruction, + module::WasmiValueType, Error, FuncType, }; @@ -283,7 +284,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { } fn visit_select(&mut self) -> Self::Output { - todo!() + self.translate_select(None) } fn visit_local_get(&mut self, local_index: u32) -> Self::Output { @@ -1254,8 +1255,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { todo!() } - fn visit_typed_select(&mut self, _ty: wasmparser::ValType) -> Self::Output { - todo!() + fn visit_typed_select(&mut self, ty: wasmparser::ValType) -> Self::Output { + let type_hint = WasmiValueType::from(ty).into_inner(); + self.translate_select(Some(type_hint)) } fn visit_ref_null(&mut self, _hty: wasmparser::HeapType) -> Self::Output { From 0a3e30cf8898301bc9e29aaaacfb7a6efd0dc03f Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Tue, 8 Jul 2025 10:30:04 +0200 Subject: [PATCH 243/343] add InstrEncoder::last_instr getter --- crates/wasmi/src/engine/translator/func2/instrs.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/crates/wasmi/src/engine/translator/func2/instrs.rs b/crates/wasmi/src/engine/translator/func2/instrs.rs index c927ba1755..c8f2ff155c 100644 --- a/crates/wasmi/src/engine/translator/func2/instrs.rs +++ b/crates/wasmi/src/engine/translator/func2/instrs.rs @@ -289,6 +289,11 @@ impl InstrEncoder { iter: self.instrs.drain(..), } } + + /// Returns the last instruction of the [`InstrEncoder`] if any. + pub fn last_instr(&self) -> Option { + self.last_instr + } } /// Iterator yielding all [`Instruction`]s of the [`InstrEncoder`]. From 52d60c295c274731c434bcc59c0de6e8b74c2b6b Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Tue, 8 Jul 2025 10:30:37 +0200 Subject: [PATCH 244/343] add fuse_eqz and fuse_nez methods --- .../wasmi/src/engine/translator/func2/mod.rs | 105 +++++++++++++++++- 1 file changed, 104 insertions(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index 137ae9fe8d..1fb219d665 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -36,7 +36,12 @@ use crate::{ core::{FuelCostsProvider, TrapCode, Typed, TypedVal, UntypedVal, ValType}, engine::{ translator::{ - comparator::{CompareResult as _, NegateCmpInstr as _, TryIntoCmpBranchInstr as _}, + comparator::{ + CompareResult as _, + LogicalizeCmpInstr as _, + NegateCmpInstr as _, + TryIntoCmpBranchInstr as _, + }, labels::{LabelRef, LabelRegistry}, utils::{Instr, WasmFloat, WasmInteger}, WasmTranslator, @@ -1349,4 +1354,102 @@ impl FuncTranslator { .push_param(Instruction::register2_ext(true_val, false_val)); Ok(()) } + + /// Tries to fuse a Wasm `i32.eqz` (or `i32.eq` with 0 `rhs` value) instruction. + /// + /// Returns + /// + /// - `Ok(true)` if the intruction fusion was successful. + /// - `Ok(false)` if instruction fusion could not be applied. + /// - `Err(_)` if an error occurred. + pub fn fuse_eqz(&mut self, lhs: Operand, rhs: T) -> Result { + if !rhs.is_zero() { + // Case: cannot fuse with non-zero `rhs` + return Ok(false); + } + let lhs_reg = match lhs { + Operand::Immediate(_) => { + // Case: const-eval opt should take place instead since both operands are const + return Ok(false); + } + operand => self.layout.operand_to_reg(operand)?, + }; + let Some(last_instr) = self.instrs.last_instr() else { + // Case: cannot fuse without registered last instruction + return Ok(false); + }; + let last_instruction = *self.instrs.get(last_instr); + let Some(result) = last_instruction.compare_result() else { + // Case: cannot fuse non-cmp instructions + return Ok(false); + }; + if matches!(self.layout.stack_space(result), StackSpace::Local) { + // Case: cannot fuse cmp instructions with local result + // Note: local results have observable side effects which must not change + return Ok(false); + } + if result != lhs_reg { + // Case: the `cmp` instruction does not feed into the `eqz` and cannot be fused + return Ok(false); + } + let Some(negated) = last_instruction.negate_cmp_instr() else { + // Case: the `cmp` instruction cannot be negated + return Ok(false); + }; + if !self.instrs.try_replace_instr(last_instr, negated)? { + // Case: could not replace the `cmp` instruction with the fused one + return Ok(false); + } + self.stack.push_operand(lhs)?; + Ok(true) + } + + /// Tries to fuse a Wasm `i32.ne` instruction with 0 `rhs` value. + /// + /// Returns + /// + /// - `Ok(true)` if the intruction fusion was successful. + /// - `Ok(false)` if instruction fusion could not be applied. + /// - `Err(_)` if an error occurred. + pub fn fuse_nez(&mut self, lhs: Operand, rhs: T) -> Result { + if !rhs.is_zero() { + // Case: cannot fuse with non-zero `rhs` + return Ok(false); + } + let lhs_reg = match lhs { + Operand::Immediate(_) => { + // Case: const-eval opt should take place instead since both operands are const + return Ok(false); + } + operand => self.layout.operand_to_reg(operand)?, + }; + let Some(last_instr) = self.instrs.last_instr() else { + // Case: cannot fuse without registered last instruction + return Ok(false); + }; + let last_instruction = *self.instrs.get(last_instr); + let Some(result) = last_instruction.compare_result() else { + // Case: cannot fuse non-cmp instructions + return Ok(false); + }; + if matches!(self.layout.stack_space(result), StackSpace::Local) { + // Case: cannot fuse cmp instructions with local result + // Note: local results have observable side effects which must not change + return Ok(false); + } + if result != lhs_reg { + // Case: the `cmp` instruction does not feed into the `nez` and cannot be fused + return Ok(false); + } + let Some(logicalized) = last_instruction.logicalize_cmp_instr() else { + // Case: the `cmp` instruction cannot be logicalized + return Ok(false); + }; + if !self.instrs.try_replace_instr(last_instr, logicalized)? { + // Case: could not replace the `cmp` instruction with the fused one + return Ok(false); + } + self.stack.push_operand(lhs)?; + Ok(true) + } } From f1d13608b24c20518f97bada8d1953f69a1c77da Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Tue, 8 Jul 2025 10:31:17 +0200 Subject: [PATCH 245/343] support custom opts in translation of commutative binary ops --- .../wasmi/src/engine/translator/func2/mod.rs | 21 +++++++++++++------ .../src/engine/translator/func2/visit.rs | 14 +++++++++++++ 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index 1fb219d665..2360021df2 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -965,12 +965,18 @@ impl FuncTranslator { }) } + /// Convenience method to tell that there is no custom optimization. + fn no_opt_ri(&mut self, _lhs: Operand, _rhs: T) -> Result { + Ok(false) + } + /// Translates a commutative binary Wasm operator to Wasmi bytecode. fn translate_binary_commutative( &mut self, - make_instr: fn(result: Reg, lhs: Reg, rhs: Reg) -> Instruction, - make_instr_imm16: fn(result: Reg, lhs: Reg, rhs: Const16) -> Instruction, + make_rr: fn(result: Reg, lhs: Reg, rhs: Reg) -> Instruction, + make_ri: fn(result: Reg, lhs: Reg, rhs: Const16) -> Instruction, consteval: fn(T, T) -> R, + opt_ri: fn(this: &mut Self, lhs: Operand, rhs: T) -> Result, ) -> Result<(), Error> where T: WasmInteger + TryInto>, @@ -982,20 +988,23 @@ impl FuncTranslator { self.translate_binary_consteval::(lhs, rhs, consteval) } (val, Operand::Immediate(imm)) | (Operand::Immediate(imm), val) => { - let lhs = self.layout.operand_to_reg(val)?; let rhs = imm.val().into(); + if opt_ri(self, val, rhs)? { + return Ok(()); + } + let lhs = self.layout.operand_to_reg(val)?; let rhs16 = self.make_imm16(rhs)?; self.push_instr_with_result( ::TY, |result| match rhs16 { - Operand16::Immediate(rhs) => make_instr_imm16(result, lhs, rhs), - Operand16::Reg(rhs) => make_instr(result, lhs, rhs), + Operand16::Immediate(rhs) => make_ri(result, lhs, rhs), + Operand16::Reg(rhs) => make_rr(result, lhs, rhs), }, FuelCostsProvider::base, ) } (lhs, rhs) => { - self.push_binary_instr_with_result(lhs, rhs, make_instr, FuelCostsProvider::base) + self.push_binary_instr_with_result(lhs, rhs, make_rr, FuelCostsProvider::base) } } } diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index 7d7669b31c..8837b08d12 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -446,6 +446,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { Instruction::i32_eq, Instruction::i32_eq_imm16, wasm::i32_eq, + FuncTranslator::fuse_eqz, ) } @@ -454,6 +455,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { Instruction::i32_ne, Instruction::i32_ne_imm16, wasm::i32_ne, + FuncTranslator::fuse_nez, ) } @@ -540,6 +542,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { Instruction::i64_eq, Instruction::i64_eq_imm16, wasm::i64_eq, + FuncTranslator::fuse_eqz, ) } @@ -548,6 +551,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { Instruction::i64_ne, Instruction::i64_ne_imm16, wasm::i64_ne, + FuncTranslator::fuse_nez, ) } @@ -688,6 +692,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { Instruction::i32_add, Instruction::i32_add_imm16, wasm::i32_add, + FuncTranslator::no_opt_ri, ) } @@ -705,6 +710,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { Instruction::i32_mul, Instruction::i32_mul_imm16, wasm::i32_mul, + FuncTranslator::no_opt_ri, ) } @@ -749,6 +755,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { Instruction::i32_bitand, Instruction::i32_bitand_imm16, wasm::i32_bitand, + FuncTranslator::no_opt_ri, ) } @@ -757,6 +764,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { Instruction::i32_bitor, Instruction::i32_bitor_imm16, wasm::i32_bitor, + FuncTranslator::no_opt_ri, ) } @@ -765,6 +773,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { Instruction::i32_bitxor, Instruction::i32_bitxor_imm16, wasm::i32_bitxor, + FuncTranslator::no_opt_ri, ) } @@ -830,6 +839,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { Instruction::i64_add, Instruction::i64_add_imm16, wasm::i64_add, + FuncTranslator::no_opt_ri, ) } @@ -847,6 +857,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { Instruction::i64_mul, Instruction::i64_mul_imm16, wasm::i64_mul, + FuncTranslator::no_opt_ri, ) } @@ -891,6 +902,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { Instruction::i64_bitand, Instruction::i64_bitand_imm16, wasm::i64_bitand, + FuncTranslator::no_opt_ri, ) } @@ -899,6 +911,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { Instruction::i64_bitor, Instruction::i64_bitor_imm16, wasm::i64_bitor, + FuncTranslator::no_opt_ri, ) } @@ -907,6 +920,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { Instruction::i64_bitxor, Instruction::i64_bitxor_imm16, wasm::i64_bitxor, + FuncTranslator::no_opt_ri, ) } From 5bc4a8cf3002594ba343666f7aa27166b2ba84d4 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 9 Jul 2025 12:22:19 +0200 Subject: [PATCH 246/343] add Stack::pop_n method --- .../wasmi/src/engine/translator/func2/stack/mod.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/crates/wasmi/src/engine/translator/func2/stack/mod.rs b/crates/wasmi/src/engine/translator/func2/stack/mod.rs index b980f83ff4..f5ef55c7aa 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/mod.rs @@ -35,6 +35,7 @@ use crate::{ Engine, Error, }; +use alloc::vec::Vec; #[cfg(doc)] use crate::ir::Instruction; @@ -432,6 +433,18 @@ impl Stack { (o1, o2, o3) } + /// Pops `len` operands from the stack and store them into `buffer`. + /// + /// Operands stored into the buffer are placed in order. + pub fn pop_n(&mut self, len: usize, buffer: &mut Vec) { + buffer.clear(); + for _ in 0..len { + let operand = self.pop(); + buffer.push(operand); + } + buffer.reverse(); + } + /// Preserve all locals on the [`Stack`] that refer to `local_index`. /// /// This is done by converting those locals to [`Operand::Temp`] and yielding them. From 85d2454e8b9ea3ff3d95aa7548c4659d6c3f5222 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 9 Jul 2025 12:23:54 +0200 Subject: [PATCH 247/343] refactor copy_operands_to_temp - it now takes a consume_fuel_instr parameter - it now returns the OperandIdx of the first operand --- crates/wasmi/src/engine/translator/func2/mod.rs | 16 +++++++++++----- .../wasmi/src/engine/translator/func2/visit.rs | 3 ++- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index 2360021df2..e374ba6cc5 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -292,20 +292,26 @@ impl FuncTranslator { Ok(()) } - /// Copy all [`Operand`]s up to `depth` into [`Operand::Temp`]s by copying if necessary. + /// Copy the top-most `len` [`Operand`]s into [`Operand::Temp`]s by copying if necessary. /// + /// Returns the [`OperandIdx`] of the first [`Operand`]. /// /// # Note /// /// - This does _not_ manipulate the [`Stack`]. /// - Does nothing if an [`Operand`] is already an [`Operand::Temp`]. - fn copy_operands_to_temp(&mut self, depth: usize) -> Result<(), Error> { - let consume_fuel = self.stack.consume_fuel_instr(); - for n in 0..depth { + fn copy_operands_to_temp( + &mut self, + len: usize, + consume_fuel: Option, + ) -> Result, Error> { + let mut idx = None; + for n in 0..len { let operand = self.stack.peek(n); self.copy_operand_to_temp(operand, consume_fuel)?; + idx = Some(operand.index()); } - Ok(()) + Ok(idx) } /// Returns `true` if the [`ControlFrame`] at `depth` requires copying for its branch parameters. diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index 8837b08d12..f00bed40cf 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -243,9 +243,10 @@ impl<'a> VisitOperator<'a> for FuncTranslator { return Ok(()); } // Case: fallback to copy branch parameters conditionally + let consume_fuel_instr = self.stack.consume_fuel_instr(); let skip_label = self.labels.new_label(); self.encode_br_eqz(condition, label)?; - self.copy_operands_to_temp(depth)?; + self.copy_operands_to_temp(depth, consume_fuel_instr)?; self.encode_br(label)?; self.labels .pin_label(skip_label, self.instrs.next_instr()) From c9fd965566a73fe39a942c1da498e044d3ffab98 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 9 Jul 2025 12:24:41 +0200 Subject: [PATCH 248/343] make push_instr return the pushed Instr --- crates/wasmi/src/engine/translator/func2/mod.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index e374ba6cc5..7f04eccecc 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -401,10 +401,10 @@ impl FuncTranslator { &mut self, instr: Instruction, fuel_costs: impl FnOnce(&FuelCostsProvider) -> u64, - ) -> Result<(), Error> { + ) -> Result { let consume_fuel = self.stack.consume_fuel_instr(); - self.instrs.push_instr(instr, consume_fuel, fuel_costs)?; - Ok(()) + let instr = self.instrs.push_instr(instr, consume_fuel, fuel_costs)?; + Ok(instr) } /// Pushes the `instr` to the function with the associated `fuel_costs`. @@ -752,7 +752,8 @@ impl FuncTranslator { self.make_branch_cmp_fallback(comparator, condition, zero, offset)? } }; - self.push_instr(instr, FuelCostsProvider::base) + self.push_instr(instr, FuelCostsProvider::base)?; + Ok(()) } /// Create an [`Instruction::BranchCmpFallback`]. From 7976666d0762390dc99f6dbfe3637da95792bfcf Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 9 Jul 2025 12:25:14 +0200 Subject: [PATCH 249/343] extend encode_return with ReturnReg{2,3} support --- crates/wasmi/src/engine/translator/func2/mod.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index 7f04eccecc..dc98f858f0 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -481,6 +481,19 @@ impl FuncTranslator { } } }, + 2 => { + let [v0, v1] = [self.stack.peek(1), self.stack.peek(0)]; + let v0 = self.layout.operand_to_reg(v0)?; + let v1 = self.layout.operand_to_reg(v1)?; + Instruction::return_reg2_ext(v0, v1) + } + 3 => { + let [v0, v1, v2] = [self.stack.peek(2), self.stack.peek(1), self.stack.peek(0)]; + let v0 = self.layout.operand_to_reg(v0)?; + let v1 = self.layout.operand_to_reg(v1)?; + let v2 = self.layout.operand_to_reg(v2)?; + Instruction::return_reg3_ext(v0, v1, v2) + } _ => { let depth = usize::from(len_results); self.copy_operands_to_temp(depth)?; From a60f6d4b998dc7e4cf07fa57690bc30a27ccd167 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 9 Jul 2025 12:25:27 +0200 Subject: [PATCH 250/343] make use of copy_operands_to_temp return value --- crates/wasmi/src/engine/translator/func2/mod.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index dc98f858f0..15a2c4ec85 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -495,9 +495,10 @@ impl FuncTranslator { Instruction::return_reg3_ext(v0, v1, v2) } _ => { - let depth = usize::from(len_results); - self.copy_operands_to_temp(depth)?; - let first_idx = self.stack.peek(depth).index(); + let len_copies = usize::from(len_results); + let Some(first_idx) = self.copy_operands_to_temp(len_copies, consume_fuel)? else { + unreachable!("`first_idx` must be `Some` since `len_copies` is >0") + }; let result = self.layout.temp_to_reg(first_idx)?; let values = BoundedRegSpan::new(RegSpan::new(result), len_results); Instruction::return_span(values) From 86d86fc78fba083299b1cfbdcaa403bd6ece9b26 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 9 Jul 2025 12:25:51 +0200 Subject: [PATCH 251/343] add FuncTranslator::{resolve_func_type[_with]} methods --- crates/wasmi/src/engine/translator/func2/mod.rs | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index 15a2c4ec85..d1b42f0f00 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -260,7 +260,17 @@ impl FuncTranslator { /// Applies `f` to the [`FuncType`] of the function that is currently translated. fn func_type_with(&self, f: impl FnOnce(&FuncType) -> R) -> R { - let dedup_func_type = self.module.get_type_of_func(self.func); + self.resolve_func_type_with(self.func, f) + } + + /// Returns the [`FuncType`] of the function at `func_index`. + fn resolve_func_type(&self, func_index: FuncIdx) -> FuncType { + self.resolve_func_type_with(func_index, FuncType::clone) + } + + /// Applies `f` to the [`FuncType`] of the function at `func_index`. + fn resolve_func_type_with(&self, func_index: FuncIdx, f: impl FnOnce(&FuncType) -> R) -> R { + let dedup_func_type = self.module.get_type_of_func(func_index); self.engine().resolve_func_type(dedup_func_type, f) } From eec3540d57074bfdd58e44d03bf8ff2768e47961 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 9 Jul 2025 12:26:52 +0200 Subject: [PATCH 252/343] add temporary buffer for Operands to FuncTranslator --- crates/wasmi/src/engine/translator/func2/mod.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index d1b42f0f00..445c14ab71 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -69,6 +69,7 @@ use crate::{ Error, FuncType, }; +use alloc::vec::Vec; use core::mem; use wasmparser::WasmFeatures; @@ -109,6 +110,8 @@ pub struct FuncTranslator { labels: LabelRegistry, /// Constructs and encodes function instructions. instrs: InstrEncoder, + /// Temporary buffer for operands. + operands: Vec, } /// Heap allocated data structured used by the [`FuncTranslator`]. @@ -122,6 +125,8 @@ pub struct FuncTranslatorAllocations { labels: LabelRegistry, /// Constructs and encodes function instructions. instrs: InstrEncoderAllocations, + /// Temporary buffer for operands. + operands: Vec, } impl Reset for FuncTranslatorAllocations { @@ -130,6 +135,7 @@ impl Reset for FuncTranslatorAllocations { self.layout.reset(); self.labels.reset(); self.instrs.reset(); + self.operands.clear(); } } @@ -187,6 +193,7 @@ impl ReusableAllocations for FuncTranslator { layout: self.layout, labels: self.labels, instrs: self.instrs.into_allocations(), + operands: self.operands, } } } @@ -214,6 +221,7 @@ impl FuncTranslator { layout, labels, instrs, + operands, } = alloc.into_reset(); let stack = Stack::new(&engine, stack); let instrs = InstrEncoder::new(&engine, instrs); @@ -227,6 +235,7 @@ impl FuncTranslator { layout, labels, instrs, + operands, }; translator.init_func_body_block()?; translator.init_func_params()?; From af80cc24f881da67392dafe5c9c61a668201da89 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 9 Jul 2025 12:27:16 +0200 Subject: [PATCH 253/343] initial support for call instruction translation --- .../wasmi/src/engine/translator/func2/mod.rs | 81 +++++++++++++++++++ .../src/engine/translator/func2/visit.rs | 36 ++++++++- 2 files changed, 113 insertions(+), 4 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index 445c14ab71..7d959a0754 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -283,6 +283,87 @@ impl FuncTranslator { self.engine().resolve_func_type(dedup_func_type, f) } + /// Returns the [`RegSpan`] of a call instruction before manipulating the operand stack. + fn call_regspan(&self, len_params: usize) -> Result { + let height = self.stack.height(); + let Some(start) = height.checked_sub(len_params) else { + panic!("operand stack underflow while evaluating call `RegSpan`"); + }; + let start = self.layout.temp_to_reg(OperandIdx::from(start))?; + Ok(RegSpan::new(start)) + } + + /// Encode the top-most `len` operands on the stack as register list. + /// + /// # Note + /// + /// This is used for the following n-ary instructions: + /// + /// - [`Instruction::ReturnMany`] + /// - [`Instruction::CopyMany`] + /// - [`Instruction::CallInternal`] + /// - [`Instruction::CallImported`] + /// - [`Instruction::CallIndirect`] + /// - [`Instruction::ReturnCallInternal`] + /// - [`Instruction::ReturnCallImported`] + /// - [`Instruction::ReturnCallIndirect`] + pub fn encode_register_list(&mut self, len: usize) -> Result<(), Error> { + self.stack.pop_n(len, &mut self.operands); + let mut remaining = &self.operands[..]; + let mut operand_to_reg = + |operand: &Operand| -> Result { self.layout.operand_to_reg(*operand) }; + let instr = loop { + match remaining { + [] => return Ok(()), + [v0] => { + let v0 = operand_to_reg(v0)?; + break Instruction::register(v0); + } + [v0, v1] => { + let v0 = operand_to_reg(v0)?; + let v1 = operand_to_reg(v1)?; + break Instruction::register2_ext(v0, v1); + } + [v0, v1, v2] => { + let v0 = operand_to_reg(v0)?; + let v1 = operand_to_reg(v1)?; + let v2 = operand_to_reg(v2)?; + break Instruction::register3_ext(v0, v1, v2); + } + [v0, v1, v2, rest @ ..] => { + let v0 = operand_to_reg(v0)?; + let v1 = operand_to_reg(v1)?; + let v2 = operand_to_reg(v2)?; + let instr = Instruction::register_list_ext(v0, v1, v2); + self.instrs.push_param(instr); + remaining = rest; + } + }; + }; + self.instrs.push_param(instr); + Ok(()) + } + + /// Push `results` as [`TempOperand`] onto the [`Stack`] tagged to `instr`. + /// + /// Returns the [`RegSpan`] identifying the pushed operands if any. + fn push_results( + &mut self, + instr: Instr, + results: &[ValType], + ) -> Result, Error> { + let (first, rest) = match results.split_first() { + Some((first, rest)) => (first, rest), + None => return Ok(None), + }; + let first = self.stack.push_temp(*first, Some(instr))?; + for result in rest { + self.stack.push_temp(*result, Some(instr))?; + } + let start = self.layout.temp_to_reg(first)?; + Ok(Some(RegSpan::new(start))) + } + /// Returns the [`Engine`] for which the function is compiled. fn engine(&self) -> &Engine { &self.engine diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index f00bed40cf..38ccde6f63 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -1,6 +1,6 @@ use super::{ControlFrame, ControlFrameKind, FuncTranslator, LocalIdx}; use crate::{ - core::{wasm, TrapCode, F32, F64}, + core::{wasm, FuelCostsProvider, TrapCode, F32, F64}, engine::{ translator::func2::{ stack::{AcquiredTarget, IfReachability}, @@ -10,7 +10,7 @@ use crate::{ BlockType, }, ir::Instruction, - module::WasmiValueType, + module::{FuncIdx, WasmiValueType}, Error, FuncType, }; @@ -270,8 +270,36 @@ impl<'a> VisitOperator<'a> for FuncTranslator { Ok(()) } - fn visit_call(&mut self, _function_index: u32) -> Self::Output { - todo!() + fn visit_call(&mut self, function_index: u32) -> Self::Output { + bail_unreachable!(self); + let func_idx = FuncIdx::from(function_index); + let func_type = self.resolve_func_type(func_idx); + let len_params = usize::from(func_type.len_params()); + let results = self.call_regspan(len_params)?; + let instr = match self.module.get_engine_func(func_idx) { + Some(engine_func) => { + // Case: We are calling an internal function and can optimize + // this case by using the special instruction for it. + match len_params { + 0 => Instruction::call_internal_0(results, engine_func), + _ => Instruction::call_internal(results, engine_func), + } + } + None => { + // Case: We are calling an imported function and must use the + // general calling operator for it. + match len_params { + 0 => Instruction::call_imported_0(results, function_index), + _ => Instruction::call_imported(results, function_index), + } + } + }; + let call_instr = self.push_instr(instr, FuelCostsProvider::call)?; + self.encode_register_list(len_params)?; + if let Some(span) = self.push_results(call_instr, func_type.results())? { + debug_assert_eq!(span, results); + } + Ok(()) } fn visit_call_indirect(&mut self, _type_index: u32, _table_index: u32) -> Self::Output { From 1cfd63c576607eba6c08c1de513a1aa28234bb46 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 9 Jul 2025 12:31:49 +0200 Subject: [PATCH 254/343] add Stack::peek{2,3} methods --- .../wasmi/src/engine/translator/func2/mod.rs | 4 ++-- .../src/engine/translator/func2/stack/mod.rs | 23 +++++++++++++++++++ 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index 7d959a0754..7592f61307 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -582,13 +582,13 @@ impl FuncTranslator { } }, 2 => { - let [v0, v1] = [self.stack.peek(1), self.stack.peek(0)]; + let (v0, v1) = self.stack.peek2(); let v0 = self.layout.operand_to_reg(v0)?; let v1 = self.layout.operand_to_reg(v1)?; Instruction::return_reg2_ext(v0, v1) } 3 => { - let [v0, v1, v2] = [self.stack.peek(2), self.stack.peek(1), self.stack.peek(0)]; + let (v0, v1, v2) = self.stack.peek3(); let v0 = self.layout.operand_to_reg(v0)?; let v1 = self.layout.operand_to_reg(v1)?; let v2 = self.layout.operand_to_reg(v2)?; diff --git a/crates/wasmi/src/engine/translator/func2/stack/mod.rs b/crates/wasmi/src/engine/translator/func2/stack/mod.rs index f5ef55c7aa..d2f943ce5b 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/mod.rs @@ -393,6 +393,29 @@ impl Stack { self.operands.get(depth) } + /// Peeks the 2 top-most [`Operand`]s. + /// + /// # Panics + /// + /// If there aren't at least 2 [`Operand`]s on the [`Stack`]. + pub fn peek2(&self) -> (Operand, Operand) { + let v0 = self.peek(1); + let v1 = self.peek(0); + (v0, v1) + } + + /// Peeks the 3 top-most [`Operand`]s. + /// + /// # Panics + /// + /// If there aren't at least 2 [`Operand`]s on the [`Stack`]. + pub fn peek3(&self) -> (Operand, Operand, Operand) { + let v0 = self.peek(2); + let v1 = self.peek(1); + let v2 = self.peek(0); + (v0, v1, v2) + } + /// Pops the top-most [`Operand`] from the [`Stack`]. /// /// # Panics From 58a94540b8f54ac8a7d5377fd79885fa8339b4b9 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 9 Jul 2025 12:38:13 +0200 Subject: [PATCH 255/343] fix translate_reinterpret types --- crates/wasmi/src/engine/translator/func2/mod.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index 7592f61307..e6ab17321a 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -1011,18 +1011,18 @@ impl FuncTranslator { /// This Wasm operation is a no-op. Ideally we only have to change the types on the stack. fn translate_reinterpret(&mut self, consteval: fn(T) -> R) -> Result<(), Error> where - T: From, - R: Into, + T: From + Typed, + R: Into + Typed, { bail_unreachable!(self); match self.stack.pop() { Operand::Local(input) => { - debug_assert!(matches!(input.ty(), ValType::I32)); + debug_assert_eq!(input.ty(), ::TY); todo!() // do we need a copy or should we allow to manipulate a local's type? } Operand::Temp(input) => { - debug_assert!(matches!(input.ty(), ValType::I32)); - self.stack.push_temp(ValType::I64, None)?; + debug_assert_eq!(input.ty(), ::TY); + self.stack.push_temp(::TY, None)?; } Operand::Immediate(input) => { let input: T = input.val().into(); From 6265846b800e8ff02cae64518c3e6d45ae7ddecb Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 9 Jul 2025 12:55:08 +0200 Subject: [PATCH 256/343] fix return type of translate_binary --- crates/wasmi/src/engine/translator/func2/mod.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index e6ab17321a..0c746942c7 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -1196,7 +1196,7 @@ impl FuncTranslator { ) -> Result<(), Error> where T: WasmInteger, - R: Into, + R: Into + Typed, { bail_unreachable!(self); match self.stack.pop2() { @@ -1208,7 +1208,7 @@ impl FuncTranslator { let rhs = T::from(rhs.val()); let rhs16 = self.make_imm16(rhs)?; self.push_instr_with_result( - ::TY, + ::TY, |result| match rhs16 { Operand16::Immediate(rhs) => make_instr_imm16_rhs(result, lhs, rhs), Operand16::Reg(rhs) => make_instr(result, lhs, rhs), @@ -1221,7 +1221,7 @@ impl FuncTranslator { let lhs16 = self.make_imm16(lhs)?; let rhs = self.layout.operand_to_reg(rhs)?; self.push_instr_with_result( - ::TY, + ::TY, |result| match lhs16 { Operand16::Immediate(lhs) => make_instr_imm16_lhs(result, lhs, rhs), Operand16::Reg(lhs) => make_instr(result, lhs, rhs), From bca23ca55c877337b7cab0860210752118bfad1b Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 9 Jul 2025 16:51:50 +0200 Subject: [PATCH 257/343] account for bit-{and,or,xor} in try_fuse_branch_cmp debug_assert --- crates/wasmi/src/engine/translator/func2/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index 0c746942c7..7c7b2122e3 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -894,7 +894,7 @@ impl FuncTranslator { let Operand::Temp(condition) = condition else { return Ok(false); }; - debug_assert_eq!(condition.ty(), ValType::I32); + debug_assert!(matches!(condition.ty(), ValType::I32 | ValType::I64)); let Some(origin) = condition.instr() else { return Ok(false); }; From 68386939d16f9f5762af226d47d6eb1f51786efc Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 9 Jul 2025 17:30:59 +0200 Subject: [PATCH 258/343] avoid copies if the stack already forms a RegSpan --- .../wasmi/src/engine/translator/func2/mod.rs | 48 ++++++++++++++++--- 1 file changed, 42 insertions(+), 6 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index 7c7b2122e3..e7fb15c6ef 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -596,12 +596,22 @@ impl FuncTranslator { } _ => { let len_copies = usize::from(len_results); - let Some(first_idx) = self.copy_operands_to_temp(len_copies, consume_fuel)? else { - unreachable!("`first_idx` must be `Some` since `len_copies` is >0") - }; - let result = self.layout.temp_to_reg(first_idx)?; - let values = BoundedRegSpan::new(RegSpan::new(result), len_results); - Instruction::return_span(values) + match self.try_form_regspan(len_copies)? { + Some(span) => { + let values = BoundedRegSpan::new(span, len_results); + Instruction::return_span(values) + } + None => { + let Some(first_idx) = + self.copy_operands_to_temp(len_copies, consume_fuel)? + else { + unreachable!("`first_idx` must be `Some` since `len_copies` is >0") + }; + let result = self.layout.temp_to_reg(first_idx)?; + let values = BoundedRegSpan::new(RegSpan::new(result), len_results); + Instruction::return_span(values) + } + } } }; let instr = self @@ -610,6 +620,32 @@ impl FuncTranslator { Ok(instr) } + /// Tries to form a [`RegSpan`] from the top-most `len` operands on the [`Stack`]. + /// + /// Returns `None` if forming a [`RegSpan`] was not possible. + fn try_form_regspan(&self, len: usize) -> Result, Error> { + if len == 0 { + return Ok(None); + } + let mut start = match self.stack.peek(0) { + Operand::Immediate(_) => return Ok(None), + Operand::Local(operand) => self.layout.local_to_reg(operand.local_index())?, + Operand::Temp(operand) => self.layout.temp_to_reg(operand.operand_index())?, + }; + for depth in 1..len { + let cur = match self.stack.peek(depth) { + Operand::Immediate(_) => return Ok(None), + Operand::Local(operand) => self.layout.local_to_reg(operand.local_index())?, + Operand::Temp(operand) => self.layout.temp_to_reg(operand.operand_index())?, + }; + if start != cur.next() { + return Ok(None); + } + start = cur; + } + Ok(Some(RegSpan::new(start))) + } + /// Translates the end of a Wasm `block` control frame. fn translate_end_block(&mut self, frame: BlockControlFrame) -> Result<(), Error> { let consume_fuel_instr = frame.consume_fuel_instr(); From 73d8e61a7d06ab03789b1a8f60ea215ff513f739 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 9 Jul 2025 17:37:22 +0200 Subject: [PATCH 259/343] implement return_call translation --- .../src/engine/translator/func2/visit.rs | 29 +++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index 38ccde6f63..24602fdd17 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -1335,8 +1335,33 @@ impl<'a> VisitOperator<'a> for FuncTranslator { todo!() } - fn visit_return_call(&mut self, _function_index: u32) -> Self::Output { - todo!() + fn visit_return_call(&mut self, function_index: u32) -> Self::Output { + bail_unreachable!(self); + let func_idx = FuncIdx::from(function_index); + let func_type = self.resolve_func_type(func_idx); + let len_params = usize::from(func_type.len_params()); + let instr = match self.module.get_engine_func(func_idx) { + Some(engine_func) => { + // Case: We are calling an internal function and can optimize + // this case by using the special instruction for it. + match len_params { + 0 => Instruction::return_call_internal_0(engine_func), + _ => Instruction::return_call_internal(engine_func), + } + } + None => { + // Case: We are calling an imported function and must use the + // general calling operator for it. + match len_params { + 0 => Instruction::return_call_imported_0(function_index), + _ => Instruction::return_call_imported(function_index), + } + } + }; + self.push_instr(instr, FuelCostsProvider::call)?; + self.encode_register_list(len_params)?; + self.reachable = false; + Ok(()) } fn visit_return_call_indirect(&mut self, _type_index: u32, _table_index: u32) -> Self::Output { From 864d0ec429d8a6e98c71ab7187ca40e9b294313d Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Thu, 10 Jul 2025 14:16:10 +0200 Subject: [PATCH 260/343] add IndexType::ty conversion method --- crates/core/src/index_ty.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/crates/core/src/index_ty.rs b/crates/core/src/index_ty.rs index c6072147eb..d8b42d22e6 100644 --- a/crates/core/src/index_ty.rs +++ b/crates/core/src/index_ty.rs @@ -1,3 +1,5 @@ +use crate::ValType; + /// The index type used for addressing memories and tables. #[derive(Debug, Copy, Clone, PartialEq, Eq)] pub enum IndexType { @@ -8,6 +10,14 @@ pub enum IndexType { } impl IndexType { + /// Returns the [`ValType`] associated to `self`. + pub fn ty(&self) -> ValType { + match self { + IndexType::I32 => ValType::I32, + IndexType::I64 => ValType::I64, + } + } + /// Returns `true` if `self` is [`IndexType::I64`]. pub fn is_64(&self) -> bool { matches!(self, Self::I64) From 8101070bd06fbfba6f4e010bc4b8e71e1681ee88 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Thu, 10 Jul 2025 14:16:38 +0200 Subject: [PATCH 261/343] add FuncTranslator::make_index16 utility method --- .../wasmi/src/engine/translator/func2/mod.rs | 37 ++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index e7fb15c6ef..06e53f567d 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -33,7 +33,7 @@ use self::{ utils::{Operand16, Reset, ReusableAllocations}, }; use crate::{ - core::{FuelCostsProvider, TrapCode, Typed, TypedVal, UntypedVal, ValType}, + core::{FuelCostsProvider, IndexType, TrapCode, Typed, TypedVal, UntypedVal, ValType}, engine::{ translator::{ comparator::{ @@ -1082,6 +1082,41 @@ impl FuncTranslator { } } + /// Converts the `provider` to a 16-bit index-type constant value. + /// + /// # Note + /// + /// - Turns immediates that cannot be 16-bit encoded into function local constants. + /// - The behavior is different whether `memory64` is enabled or disabled. + pub(super) fn make_index16( + &mut self, + operand: Operand, + index_type: IndexType, + ) -> Result, Error> { + let value = match operand { + Operand::Immediate(value) => value.val(), + operand => { + debug_assert_eq!(operand.ty(), index_type.ty()); + let reg = self.layout.operand_to_reg(operand)?; + return Ok(Operand16::Reg(reg)); + } + }; + match index_type { + IndexType::I64 => { + if let Ok(value) = Const16::try_from(u64::from(value)) { + return Ok(Operand16::Immediate(value)); + } + } + IndexType::I32 => { + if let Ok(value) = Const16::try_from(u32::from(value)) { + return Ok(Operand16::Immediate(>::cast(value))); + } + } + } + let reg = self.layout.const_to_reg(value)?; + Ok(Operand16::Reg(reg)) + } + /// Evaluates `consteval(lhs, rhs)` and pushed either its result or tranlates a `trap`. fn translate_binary_consteval_fallible( &mut self, From 05ea309a43c0dfba5a59944483090c6191f8f6d2 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Thu, 10 Jul 2025 14:16:56 +0200 Subject: [PATCH 262/343] implement return_call[_indirect] translation --- .../wasmi/src/engine/translator/func2/mod.rs | 27 ++++++++- .../src/engine/translator/func2/visit.rs | 58 +++++++++++++++++-- 2 files changed, 80 insertions(+), 5 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index 06e53f567d..47bf38aef2 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -64,7 +64,7 @@ use crate::{ RegSpan, Sign, }, - module::{FuncIdx, ModuleHeader, WasmiValueType}, + module::{FuncIdx, FuncTypeIdx, ModuleHeader, TableIdx, WasmiValueType}, Engine, Error, FuncType, @@ -283,6 +283,14 @@ impl FuncTranslator { self.engine().resolve_func_type(dedup_func_type, f) } + /// Resolves the [`FuncType`] at the given Wasm module `type_index`. + fn resolve_type(&self, type_index: u32) -> FuncType { + let func_type_idx = FuncTypeIdx::from(type_index); + let dedup_func_type = self.module.get_func_type(func_type_idx); + self.engine() + .resolve_func_type(dedup_func_type, Clone::clone) + } + /// Returns the [`RegSpan`] of a call instruction before manipulating the operand stack. fn call_regspan(&self, len_params: usize) -> Result { let height = self.stack.height(); @@ -1556,6 +1564,23 @@ impl FuncTranslator { Ok(()) } + /// Create either [`Instruction::CallIndirectParams`] or [`Instruction::CallIndirectParamsImm16`] depending on the inputs. + fn call_indirect_params( + &mut self, + index: Operand, + table_index: u32, + ) -> Result { + let table_type = *self.module.get_type_of_table(TableIdx::from(table_index)); + let index = self.make_index16(index, table_type.index_ty())?; + let instr = match index { + Operand16::Reg(index) => Instruction::call_indirect_params(index, table_index), + Operand16::Immediate(index) => { + Instruction::call_indirect_params_imm16(index, table_index) + } + }; + Ok(instr) + } + /// Tries to fuse a Wasm `i32.eqz` (or `i32.eq` with 0 `rhs` value) instruction. /// /// Returns diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index 24602fdd17..1e0ec25cab 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -302,8 +302,34 @@ impl<'a> VisitOperator<'a> for FuncTranslator { Ok(()) } - fn visit_call_indirect(&mut self, _type_index: u32, _table_index: u32) -> Self::Output { - todo!() + fn visit_call_indirect(&mut self, type_index: u32, table_index: u32) -> Self::Output { + let func_type = self.resolve_type(type_index); + let index = self.stack.pop(); + let indirect_params = self.call_indirect_params(index, table_index)?; + let len_params = usize::from(func_type.len_params()); + let results = self.call_regspan(len_params)?; + let instr = match (len_params, indirect_params) { + (0, Instruction::CallIndirectParams { .. }) => { + Instruction::call_indirect_0(results, type_index) + } + (0, Instruction::CallIndirectParamsImm16 { .. }) => { + Instruction::call_indirect_0_imm16(results, type_index) + } + (_, Instruction::CallIndirectParams { .. }) => { + Instruction::call_indirect(results, type_index) + } + (_, Instruction::CallIndirectParamsImm16 { .. }) => { + Instruction::call_indirect_imm16(results, type_index) + } + _ => unreachable!(), + }; + let call_instr = self.push_instr(instr, FuelCostsProvider::call)?; + self.instrs.push_param(indirect_params); + self.encode_register_list(len_params)?; + if let Some(span) = self.push_results(call_instr, func_type.results())? { + debug_assert_eq!(span, results); + } + Ok(()) } fn visit_drop(&mut self) -> Self::Output { @@ -1364,8 +1390,32 @@ impl<'a> VisitOperator<'a> for FuncTranslator { Ok(()) } - fn visit_return_call_indirect(&mut self, _type_index: u32, _table_index: u32) -> Self::Output { - todo!() + fn visit_return_call_indirect(&mut self, type_index: u32, table_index: u32) -> Self::Output { + let func_type = self.resolve_type(type_index); + let index = self.stack.pop(); + let indirect_params = self.call_indirect_params(index, table_index)?; + let len_params = usize::from(func_type.len_params()); + // let results = self.call_regspan(len_params)?; + let instr = match (len_params, indirect_params) { + (0, Instruction::CallIndirectParams { .. }) => { + Instruction::return_call_indirect_0(type_index) + } + (0, Instruction::CallIndirectParamsImm16 { .. }) => { + Instruction::return_call_indirect_0_imm16(type_index) + } + (_, Instruction::CallIndirectParams { .. }) => { + Instruction::return_call_indirect(type_index) + } + (_, Instruction::CallIndirectParamsImm16 { .. }) => { + Instruction::return_call_indirect_imm16(type_index) + } + _ => unreachable!(), + }; + self.push_instr(instr, FuelCostsProvider::call)?; + self.instrs.push_param(indirect_params); + self.encode_register_list(len_params)?; + self.reachable = false; + Ok(()) } fn visit_i64_add128(&mut self) -> Self::Output { From 857ea8d62910c8082dfae6bcd153ad21c422a40c Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Thu, 10 Jul 2025 19:29:41 +0200 Subject: [PATCH 263/343] add global.get translation --- .../src/engine/translator/func2/visit.rs | 33 +++++++++++++++++-- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index 1e0ec25cab..2e9656b559 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -1,6 +1,6 @@ use super::{ControlFrame, ControlFrameKind, FuncTranslator, LocalIdx}; use crate::{ - core::{wasm, FuelCostsProvider, TrapCode, F32, F64}, + core::{wasm, FuelCostsProvider, Mutability, TrapCode, TypedVal, F32, F64}, engine::{ translator::func2::{ stack::{AcquiredTarget, IfReachability}, @@ -9,7 +9,9 @@ use crate::{ }, BlockType, }, + ir, ir::Instruction, + module, module::{FuncIdx, WasmiValueType}, Error, FuncType, @@ -356,8 +358,33 @@ impl<'a> VisitOperator<'a> for FuncTranslator { self.translate_local_set(local_index, true) } - fn visit_global_get(&mut self, _global_index: u32) -> Self::Output { - todo!() + fn visit_global_get(&mut self, global_index: u32) -> Self::Output { + bail_unreachable!(self); + let global_idx = module::GlobalIdx::from(global_index); + let (global_type, init_value) = self.module.get_global(global_idx); + let content = global_type.content(); + if let (Mutability::Const, Some(init_expr)) = (global_type.mutability(), init_value) { + if let Some(value) = init_expr.eval_const() { + // Case: access to immutable internally defined global variables + // can be replaced with their constant initialization value. + self.stack.push_immediate(TypedVal::new(content, value))?; + return Ok(()); + } + if let Some(func_index) = init_expr.funcref() { + // Case: forward to `ref.func x` translation. + self.visit_ref_func(func_index.into_u32())?; + return Ok(()); + } + } + // Case: The `global.get` instruction accesses a mutable or imported + // global variable and thus cannot be optimized away. + let global_idx = ir::index::Global::from(global_index); + self.push_instr_with_result( + content, + |result| Instruction::global_get(result, global_idx), + FuelCostsProvider::instance, + )?; + Ok(()) } fn visit_global_set(&mut self, _global_index: u32) -> Self::Output { From 832e19f3c95662a7f89a207aec0f9f90dec378b9 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Thu, 10 Jul 2025 19:37:23 +0200 Subject: [PATCH 264/343] add global.set translation --- .../src/engine/translator/func2/visit.rs | 56 +++++++++++++++++-- 1 file changed, 52 insertions(+), 4 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index 2e9656b559..d75ed412c4 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -1,6 +1,6 @@ use super::{ControlFrame, ControlFrameKind, FuncTranslator, LocalIdx}; use crate::{ - core::{wasm, FuelCostsProvider, Mutability, TrapCode, TypedVal, F32, F64}, + core::{wasm, FuelCostsProvider, Mutability, TrapCode, TypedVal, ValType, F32, F64}, engine::{ translator::func2::{ stack::{AcquiredTarget, IfReachability}, @@ -10,7 +10,7 @@ use crate::{ BlockType, }, ir, - ir::Instruction, + ir::{Const16, Instruction}, module, module::{FuncIdx, WasmiValueType}, Error, @@ -387,8 +387,56 @@ impl<'a> VisitOperator<'a> for FuncTranslator { Ok(()) } - fn visit_global_set(&mut self, _global_index: u32) -> Self::Output { - todo!() + fn visit_global_set(&mut self, global_index: u32) -> Self::Output { + bail_unreachable!(self); + let global = ir::index::Global::from(global_index); + let input = match self.stack.pop() { + Operand::Immediate(input) => input.val(), + input => { + // Case: `global.set` with simple register input. + let input = self.layout.operand_to_reg(input)?; + self.push_instr( + Instruction::global_set(input, global), + FuelCostsProvider::instance, + )?; + return Ok(()); + } + }; + // Note: at this point we handle the different immediate `global.set` instructions. + let (global_type, _init_value) = self + .module + .get_global(module::GlobalIdx::from(global_index)); + debug_assert_eq!(global_type.content(), input.ty()); + match global_type.content() { + ValType::I32 => { + if let Ok(value) = Const16::try_from(i32::from(input)) { + // Case: `global.set` with 16-bit encoded `i32` value. + self.push_instr( + Instruction::global_set_i32imm16(value, global), + FuelCostsProvider::instance, + )?; + return Ok(()); + } + } + ValType::I64 => { + if let Ok(value) = Const16::try_from(i64::from(input)) { + // Case: `global.set` with 16-bit encoded `i64` value. + self.push_instr( + Instruction::global_set_i64imm16(value, global), + FuelCostsProvider::instance, + )?; + return Ok(()); + } + } + _ => {} + }; + // Note: at this point we have to allocate a function local constant. + let cref = self.layout.const_to_reg(input)?; + self.push_instr( + Instruction::global_set(cref, global), + FuelCostsProvider::instance, + )?; + Ok(()) } fn visit_i32_load(&mut self, _memarg: wasmparser::MemArg) -> Self::Output { From 7a4761bea284c2fbad2bd162c8dcefff3468ed01 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Thu, 10 Jul 2025 19:45:30 +0200 Subject: [PATCH 265/343] fix copy_branch_params call in visit_br --- crates/wasmi/src/engine/translator/func2/visit.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index d75ed412c4..4601b9aeae 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -209,7 +209,8 @@ impl<'a> VisitOperator<'a> for FuncTranslator { AcquiredTarget::Branch(mut frame) => { frame.branch_to(); let label = frame.label(); - self.copy_branch_params(depth, consume_fuel_instr)?; + let len_params = usize::from(frame.len_branch_params(&self.engine)); + self.copy_branch_params(len_params, consume_fuel_instr)?; self.encode_br(label)?; self.reachable = false; Ok(()) From 9973c9a866c09495e364ea02c91f9e1e3a33491f Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Thu, 10 Jul 2025 20:33:22 +0200 Subject: [PATCH 266/343] update branch offsets upon finishing translation --- .../src/engine/translator/func2/instrs.rs | 24 +++++++++++++++++-- .../wasmi/src/engine/translator/func2/mod.rs | 14 +++++++++++ 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/instrs.rs b/crates/wasmi/src/engine/translator/func2/instrs.rs index c8f2ff155c..463a44c595 100644 --- a/crates/wasmi/src/engine/translator/func2/instrs.rs +++ b/crates/wasmi/src/engine/translator/func2/instrs.rs @@ -2,12 +2,17 @@ use super::{Reset, ReusableAllocations}; use crate::{ core::{FuelCostsProvider, ValType}, engine::translator::{ - comparator::{CmpSelectFusion, CompareResult as _, TryIntoCmpSelectInstr as _}, + comparator::{ + CmpSelectFusion, + CompareResult as _, + TryIntoCmpSelectInstr as _, + UpdateBranchOffset as _, + }, func2::{Stack, StackLayout, StackSpace}, relink_result::RelinkResult, utils::{BumpFuelConsumption as _, Instr, IsInstructionParameter as _}, }, - ir::{Instruction, Reg}, + ir::{BranchOffset, Instruction, Reg}, module::ModuleHeader, Engine, Error, @@ -253,6 +258,21 @@ impl InstrEncoder { &mut self.instrs[instr.into_usize()] } + /// Updates the branch offset of `instr` to `offset`. + /// + /// # Errors + /// + /// If the branch offset could not be updated for `instr`. + pub fn update_branch_offset( + &mut self, + instr: Instr, + offset: BranchOffset, + layout: &mut StackLayout, + ) -> Result<(), Error> { + self.get_mut(instr).update_branch_offset(layout, offset)?; + Ok(()) + } + /// Bumps consumed fuel for [`Instruction::ConsumeFuel`] of `instr` by `delta`. /// /// # Errors diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index 47bf38aef2..ebe1e73fbe 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -175,6 +175,7 @@ impl WasmTranslator<'_> for FuncTranslator { let Ok(max_height) = u16::try_from(self.stack.max_height()) else { return Err(Error::from(TranslationError::AllocatedTooManyRegisters)); }; + self.update_branch_offsets()?; finalize(CompiledFuncEntity::new( max_height, self.instrs.drain(), @@ -262,6 +263,19 @@ impl FuncTranslator { Ok(()) } + /// Updates the branch offsets of all branch instructions inplace. + /// + /// # Panics + /// + /// If this is used before all branching labels have been pinned. + fn update_branch_offsets(&mut self) -> Result<(), Error> { + for (user, offset) in self.labels.resolved_users() { + self.instrs + .update_branch_offset(user, offset?, &mut self.layout)?; + } + Ok(()) + } + /// Returns the [`FuncType`] of the function that is currently translated. fn func_type(&self) -> FuncType { self.func_type_with(FuncType::clone) From 7c08da76c4ddf5d5e6074d931516da67e7868a68 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Thu, 10 Jul 2025 22:21:26 +0200 Subject: [PATCH 267/343] add result_ty to push_binary_instr_with_result method This fixes some incorrect types on the translation stack. --- .../wasmi/src/engine/translator/func2/mod.rs | 74 +++++++++++++------ 1 file changed, 53 insertions(+), 21 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index ebe1e73fbe..83a567e3d8 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -551,16 +551,16 @@ impl FuncTranslator { /// Pushes a binary instruction with a result and associated fuel costs. fn push_binary_instr_with_result( &mut self, + result_ty: ValType, lhs: Operand, rhs: Operand, make_instr: impl FnOnce(Reg, Reg, Reg) -> Instruction, fuel_costs: impl FnOnce(&FuelCostsProvider) -> u64, ) -> Result<(), Error> { debug_assert_eq!(lhs.ty(), rhs.ty()); - let ty = lhs.ty(); let lhs = self.layout.operand_to_reg(lhs)?; let rhs = self.layout.operand_to_reg(rhs)?; - self.push_instr_with_result(ty, |result| make_instr(result, lhs, rhs), fuel_costs) + self.push_instr_with_result(result_ty, |result| make_instr(result, lhs, rhs), fuel_costs) } /// Encodes a generic return instruction. @@ -1217,9 +1217,13 @@ impl FuncTranslator { FuelCostsProvider::base, ) } - (lhs, rhs) => { - self.push_binary_instr_with_result(lhs, rhs, make_rr, FuelCostsProvider::base) - } + (lhs, rhs) => self.push_binary_instr_with_result( + ::TY, + lhs, + rhs, + make_rr, + FuelCostsProvider::base, + ), } } @@ -1273,9 +1277,13 @@ impl FuncTranslator { FuelCostsProvider::base, ) } - (lhs, rhs) => { - self.push_binary_instr_with_result(lhs, rhs, make_instr, FuelCostsProvider::base) - } + (lhs, rhs) => self.push_binary_instr_with_result( + ::TY, + lhs, + rhs, + make_instr, + FuelCostsProvider::base, + ), } } @@ -1322,9 +1330,13 @@ impl FuncTranslator { FuelCostsProvider::base, ) } - (lhs, rhs) => { - self.push_binary_instr_with_result(lhs, rhs, make_instr, FuelCostsProvider::base) - } + (lhs, rhs) => self.push_binary_instr_with_result( + ::TY, + lhs, + rhs, + make_instr, + FuelCostsProvider::base, + ), } } @@ -1338,7 +1350,7 @@ impl FuncTranslator { ) -> Result<(), Error> where T: WasmInteger, - R: Into, + R: Into + Typed, { bail_unreachable!(self); match self.stack.pop2() { @@ -1377,9 +1389,13 @@ impl FuncTranslator { FuelCostsProvider::base, ) } - (lhs, rhs) => { - self.push_binary_instr_with_result(lhs, rhs, make_sub_rr, FuelCostsProvider::base) - } + (lhs, rhs) => self.push_binary_instr_with_result( + ::TY, + lhs, + rhs, + make_sub_rr, + FuelCostsProvider::base, + ), } } @@ -1435,9 +1451,13 @@ impl FuncTranslator { FuelCostsProvider::base, ) } - (lhs, rhs) => { - self.push_binary_instr_with_result(lhs, rhs, make_instr, FuelCostsProvider::base) - } + (lhs, rhs) => self.push_binary_instr_with_result( + ::TY, + lhs, + rhs, + make_instr, + FuelCostsProvider::base, + ), } } @@ -1449,14 +1469,20 @@ impl FuncTranslator { ) -> Result<(), Error> where T: WasmFloat, - R: Into, + R: Into + Typed, { bail_unreachable!(self); let (lhs, rhs) = self.stack.pop2(); if let (Operand::Immediate(lhs), Operand::Immediate(rhs)) = (lhs, rhs) { return self.translate_binary_consteval::(lhs, rhs, consteval); } - self.push_binary_instr_with_result(lhs, rhs, make_instr, FuelCostsProvider::base) + self.push_binary_instr_with_result( + ::TY, + lhs, + rhs, + make_instr, + FuelCostsProvider::base, + ) } /// Translate Wasmi `{f32,f64}.copysign` instructions. @@ -1494,7 +1520,13 @@ impl FuncTranslator { self.stack.push_operand(lhs)?; return Ok(()); } - self.push_binary_instr_with_result(lhs, rhs, make_instr, FuelCostsProvider::base) + self.push_binary_instr_with_result( + ::TY, + lhs, + rhs, + make_instr, + FuelCostsProvider::base, + ) } } } From 19be57de3d76899c731eb8cd6f35ccf746a4bad7 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Thu, 10 Jul 2025 22:53:09 +0200 Subject: [PATCH 268/343] remove commented out line of code --- crates/wasmi/src/engine/translator/func2/visit.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index 4601b9aeae..2f4b00c939 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -1471,7 +1471,6 @@ impl<'a> VisitOperator<'a> for FuncTranslator { let index = self.stack.pop(); let indirect_params = self.call_indirect_params(index, table_index)?; let len_params = usize::from(func_type.len_params()); - // let results = self.call_regspan(len_params)?; let instr = match (len_params, indirect_params) { (0, Instruction::CallIndirectParams { .. }) => { Instruction::return_call_indirect_0(type_index) From a29a32919bbdc52710d54e8732a277617280b036 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Fri, 11 Jul 2025 10:37:09 +0200 Subject: [PATCH 269/343] fix return value of requires_branch_param_copies --- crates/wasmi/src/engine/translator/func2/mod.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index 83a567e3d8..449d0074bd 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -445,10 +445,11 @@ impl FuncTranslator { let frame = self.stack.peek_control(depth); let len_branch_params = usize::from(frame.len_branch_params(&self.engine)); let frame_height = frame.height(); - frame_height == (self.stack.height() - len_branch_params) - && (0..len_branch_params) - .map(|depth| self.stack.peek(depth)) - .all(|o| o.is_temp()) + let height_matches = frame_height == (self.stack.height() - len_branch_params); + let only_temps = (0..len_branch_params) + .map(|depth| self.stack.peek(depth)) + .all(|o| o.is_temp()); + !height_matches || !only_temps } /// Pins the `label` to the next [`Instr`]. From 9288f63897475f0e8aa8516bc7b263db663bfc85 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Fri, 11 Jul 2025 10:43:44 +0200 Subject: [PATCH 270/343] add missing bail_unreachable to visit[_return]_call_indirect --- crates/wasmi/src/engine/translator/func2/visit.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index 2f4b00c939..cd596f053f 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -306,6 +306,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { } fn visit_call_indirect(&mut self, type_index: u32, table_index: u32) -> Self::Output { + bail_unreachable!(self); let func_type = self.resolve_type(type_index); let index = self.stack.pop(); let indirect_params = self.call_indirect_params(index, table_index)?; @@ -1467,6 +1468,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { } fn visit_return_call_indirect(&mut self, type_index: u32, table_index: u32) -> Self::Output { + bail_unreachable!(self); let func_type = self.resolve_type(type_index); let index = self.stack.pop(); let indirect_params = self.call_indirect_params(index, table_index)?; From bf310da281545b4dd36bb38cbe99199cdd6a042f Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Fri, 11 Jul 2025 11:06:20 +0200 Subject: [PATCH 271/343] fix calculation of frame_size in FuncTranslator::finish --- crates/wasmi/src/engine/translator/func2/mod.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index 449d0074bd..d164b9d928 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -175,9 +175,15 @@ impl WasmTranslator<'_> for FuncTranslator { let Ok(max_height) = u16::try_from(self.stack.max_height()) else { return Err(Error::from(TranslationError::AllocatedTooManyRegisters)); }; + let Some(frame_size) = self + .func_type_with(FuncType::len_params) + .checked_add(max_height) + else { + return Err(Error::from(TranslationError::AllocatedTooManyRegisters)); + }; self.update_branch_offsets()?; finalize(CompiledFuncEntity::new( - max_height, + frame_size, self.instrs.drain(), self.layout.consts(), )); From 650c9730d6f9d3edac241c007faac40d32aecff9 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Fri, 11 Jul 2025 11:33:40 +0200 Subject: [PATCH 272/343] fix branch from end of `then` to end of `if` --- crates/wasmi/src/engine/translator/func2/visit.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index cd596f053f..08ec6ecf35 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -172,11 +172,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { let consume_fuel_instr = frame.consume_fuel_instr(); self.copy_branch_params(len_values, consume_fuel_instr)?; frame.branch_to(); - self.encode_br(else_label)?; + self.encode_br(frame.label())?; } - } - // Start of `else` block: - if let Some(else_label) = frame.else_label() { + // Start of `else` block: self.labels .pin_label(else_label, self.instrs.next_instr()) .unwrap(); From c368303dfcf91c23b096a9da31cafcb4ff7caeba Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Sat, 12 Jul 2025 12:49:38 +0200 Subject: [PATCH 273/343] move encode_register_list to InstrEncoder --- .../src/engine/translator/func2/instrs.rs | 56 ++++++++++++++++++- .../wasmi/src/engine/translator/func2/mod.rs | 51 ----------------- .../src/engine/translator/func2/visit.rs | 16 ++++-- 3 files changed, 67 insertions(+), 56 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/instrs.rs b/crates/wasmi/src/engine/translator/func2/instrs.rs index 463a44c595..b2b36cf4d2 100644 --- a/crates/wasmi/src/engine/translator/func2/instrs.rs +++ b/crates/wasmi/src/engine/translator/func2/instrs.rs @@ -8,7 +8,7 @@ use crate::{ TryIntoCmpSelectInstr as _, UpdateBranchOffset as _, }, - func2::{Stack, StackLayout, StackSpace}, + func2::{Operand, Stack, StackLayout, StackSpace}, relink_result::RelinkResult, utils::{BumpFuelConsumption as _, Instr, IsInstructionParameter as _}, }, @@ -299,6 +299,60 @@ impl InstrEncoder { Ok(()) } + /// Encode the top-most `len` operands on the stack as register list. + /// + /// # Note + /// + /// This is used for the following n-ary instructions: + /// + /// - [`Instruction::ReturnMany`] + /// - [`Instruction::CopyMany`] + /// - [`Instruction::CallInternal`] + /// - [`Instruction::CallImported`] + /// - [`Instruction::CallIndirect`] + /// - [`Instruction::ReturnCallInternal`] + /// - [`Instruction::ReturnCallImported`] + /// - [`Instruction::ReturnCallIndirect`] + pub fn encode_register_list( + &mut self, + operands: &[Operand], + layout: &mut StackLayout, + ) -> Result<(), Error> { + let mut remaining = &operands[..]; + let mut operand_to_reg = + |operand: &Operand| -> Result { layout.operand_to_reg(*operand) }; + let instr = loop { + match remaining { + [] => return Ok(()), + [v0] => { + let v0 = operand_to_reg(v0)?; + break Instruction::register(v0); + } + [v0, v1] => { + let v0 = operand_to_reg(v0)?; + let v1 = operand_to_reg(v1)?; + break Instruction::register2_ext(v0, v1); + } + [v0, v1, v2] => { + let v0 = operand_to_reg(v0)?; + let v1 = operand_to_reg(v1)?; + let v2 = operand_to_reg(v2)?; + break Instruction::register3_ext(v0, v1, v2); + } + [v0, v1, v2, rest @ ..] => { + let v0 = operand_to_reg(v0)?; + let v1 = operand_to_reg(v1)?; + let v2 = operand_to_reg(v2)?; + let instr = Instruction::register_list_ext(v0, v1, v2); + self.push_param(instr); + remaining = rest; + } + }; + }; + self.push_param(instr); + Ok(()) + } + /// Returns an iterator yielding all [`Instruction`]s of the [`InstrEncoder`]. /// /// # Note diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index d164b9d928..747c772476 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -321,57 +321,6 @@ impl FuncTranslator { Ok(RegSpan::new(start)) } - /// Encode the top-most `len` operands on the stack as register list. - /// - /// # Note - /// - /// This is used for the following n-ary instructions: - /// - /// - [`Instruction::ReturnMany`] - /// - [`Instruction::CopyMany`] - /// - [`Instruction::CallInternal`] - /// - [`Instruction::CallImported`] - /// - [`Instruction::CallIndirect`] - /// - [`Instruction::ReturnCallInternal`] - /// - [`Instruction::ReturnCallImported`] - /// - [`Instruction::ReturnCallIndirect`] - pub fn encode_register_list(&mut self, len: usize) -> Result<(), Error> { - self.stack.pop_n(len, &mut self.operands); - let mut remaining = &self.operands[..]; - let mut operand_to_reg = - |operand: &Operand| -> Result { self.layout.operand_to_reg(*operand) }; - let instr = loop { - match remaining { - [] => return Ok(()), - [v0] => { - let v0 = operand_to_reg(v0)?; - break Instruction::register(v0); - } - [v0, v1] => { - let v0 = operand_to_reg(v0)?; - let v1 = operand_to_reg(v1)?; - break Instruction::register2_ext(v0, v1); - } - [v0, v1, v2] => { - let v0 = operand_to_reg(v0)?; - let v1 = operand_to_reg(v1)?; - let v2 = operand_to_reg(v2)?; - break Instruction::register3_ext(v0, v1, v2); - } - [v0, v1, v2, rest @ ..] => { - let v0 = operand_to_reg(v0)?; - let v1 = operand_to_reg(v1)?; - let v2 = operand_to_reg(v2)?; - let instr = Instruction::register_list_ext(v0, v1, v2); - self.instrs.push_param(instr); - remaining = rest; - } - }; - }; - self.instrs.push_param(instr); - Ok(()) - } - /// Push `results` as [`TempOperand`] onto the [`Stack`] tagged to `instr`. /// /// Returns the [`RegSpan`] identifying the pushed operands if any. diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index 08ec6ecf35..56e1e86578 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -296,7 +296,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { } }; let call_instr = self.push_instr(instr, FuelCostsProvider::call)?; - self.encode_register_list(len_params)?; + self.stack.pop_n(len_params, &mut self.operands); + self.instrs + .encode_register_list(&self.operands, &mut self.layout)?; if let Some(span) = self.push_results(call_instr, func_type.results())? { debug_assert_eq!(span, results); } @@ -327,7 +329,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { }; let call_instr = self.push_instr(instr, FuelCostsProvider::call)?; self.instrs.push_param(indirect_params); - self.encode_register_list(len_params)?; + self.stack.pop_n(len_params, &mut self.operands); + self.instrs + .encode_register_list(&self.operands, &mut self.layout)?; if let Some(span) = self.push_results(call_instr, func_type.results())? { debug_assert_eq!(span, results); } @@ -1460,7 +1464,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { } }; self.push_instr(instr, FuelCostsProvider::call)?; - self.encode_register_list(len_params)?; + self.stack.pop_n(len_params, &mut self.operands); + self.instrs + .encode_register_list(&self.operands, &mut self.layout)?; self.reachable = false; Ok(()) } @@ -1488,7 +1494,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { }; self.push_instr(instr, FuelCostsProvider::call)?; self.instrs.push_param(indirect_params); - self.encode_register_list(len_params)?; + self.stack.pop_n(len_params, &mut self.operands); + self.instrs + .encode_register_list(&self.operands, &mut self.layout)?; self.reachable = false; Ok(()) } From 7d86d9a70fefc2bbb87e81e15ccbd08b14f2d02f Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Sat, 12 Jul 2025 17:06:34 +0200 Subject: [PATCH 274/343] add Stack::peek_n method --- crates/wasmi/src/engine/translator/func2/stack/mod.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/crates/wasmi/src/engine/translator/func2/stack/mod.rs b/crates/wasmi/src/engine/translator/func2/stack/mod.rs index d2f943ce5b..c897cde3f3 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/mod.rs @@ -416,6 +416,14 @@ impl Stack { (v0, v1, v2) } + /// Peeks the top-most `len` operands from the stack and store them into `buffer`. + /// + /// Operands stored into the buffer are placed in order. + pub fn peek_n(&mut self, len: usize, buffer: &mut Vec) { + buffer.clear(); + buffer.extend((0..len).rev().map(|depth| self.peek(depth))); + } + /// Pops the top-most [`Operand`] from the [`Stack`]. /// /// # Panics From 19538accbd13c6b06ba46e9813d4d844d6ce87a9 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Sat, 12 Jul 2025 17:06:47 +0200 Subject: [PATCH 275/343] add new copy_branch_params implementation (v2) --- .../wasmi/src/engine/translator/func2/mod.rs | 139 ++++++++++++++++++ 1 file changed, 139 insertions(+) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index 747c772476..6c781e3949 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -391,6 +391,145 @@ impl FuncTranslator { Ok(idx) } + /// Convert all branch params up to `depth` to [`Operand::Temp`]. + /// + /// # Note + /// + /// - The top-most `depth` operands on the [`Stack`] will be [`Operand::Temp`] upon completion. + /// - Does nothing if an [`Operand`] is already an [`Operand::Temp`]. + fn copy_branch_params_v2( + &mut self, + target: &impl ControlFrameBase, + consume_fuel_instr: Option, + ) -> Result<(), Error> { + let len_branch_params = target.len_branch_params(&self.engine); + let Some(branch_results) = self.frame_results(target)? else { + return Ok(()); + }; + self.encode_copies(branch_results, len_branch_params, consume_fuel_instr)?; + Ok(()) + } + + /// Encodes a copy instruction for the top-most `len_values` on the stack to `results`. + /// + /// # Note + /// + /// - This does _not_ pop values from the stack or manipulate the stack otherwise. + /// - This might allocate new function local constant values if necessary. + /// - This does _not_ encode a copy if the copy is a no-op. + fn encode_copies( + &mut self, + results: RegSpan, + len_values: u16, + consume_fuel_instr: Option, + ) -> Result<(), Error> { + match len_values { + 0 => Ok(()), + 1 => { + let result = results.head(); + let copy_instr = match self.stack.peek(0) { + Operand::Immediate(operand) => match operand.ty() { + ValType::I32 => { + let value = i32::from(operand.val()); + Instruction::copy_imm32(result, value) + } + ValType::I64 => { + let value = i64::from(operand.val()); + match Const32::try_from(value) { + Ok(value) => Instruction::copy_i64imm32(result, value), + Err(_) => { + let value = self.layout.const_to_reg(value)?; + Instruction::copy(result, value) + } + } + } + ValType::F32 => { + let value = f32::from(operand.val()); + Instruction::copy_imm32(result, value) + } + ValType::F64 => { + let value = f64::from(operand.val()); + match Const32::try_from(value) { + Ok(value) => Instruction::copy_f64imm32(result, value), + Err(_) => { + let value = self.layout.const_to_reg(value)?; + Instruction::copy(result, value) + } + } + } + ValType::V128 | ValType::FuncRef | ValType::ExternRef => { + let value = self.layout.const_to_reg(operand.val())?; + Instruction::copy(result, value) + } + }, + operand => { + let operand = self.layout.operand_to_reg(operand)?; + Instruction::copy(result, operand) + } + }; + self.instrs + .push_instr(copy_instr, consume_fuel_instr, FuelCostsProvider::base)?; + Ok(()) + } + 2 => { + let (fst, snd) = self.stack.peek2(); + let fst = self.layout.operand_to_reg(fst)?; + let snd = self.layout.operand_to_reg(snd)?; + self.instrs.push_instr( + Instruction::copy2_ext(results, fst, snd), + consume_fuel_instr, + FuelCostsProvider::base, + )?; + Ok(()) + } + _ => { + self.instrs + .bump_fuel_consumption(consume_fuel_instr, |costs| { + costs.fuel_for_copying_values(u64::from(len_values)) + })?; + if let Some(values) = self.try_form_regspan(usize::from(len_values))? { + // Case: can encode the copies as a more efficient `copy_span` + if results == values { + // Case: results and values are equal and therefore the copy is a no-op + return Ok(()); + } + debug_assert!(results.head() < values.head()); + self.instrs.push_instr( + Instruction::copy_span(results, values, len_values), + consume_fuel_instr, + FuelCostsProvider::base, + )?; + return Ok(()); + } + self.stack + .peek_n(usize::from(len_values), &mut self.operands); + let [fst, snd, rest @ ..] = &self.operands[..] else { + unreachable!("asserted that operands.len() >= 3") + }; + let fst = self.layout.operand_to_reg(*fst)?; + let snd = self.layout.operand_to_reg(*snd)?; + self.instrs.push_instr( + Instruction::copy_many_ext(results, fst, snd), + consume_fuel_instr, + FuelCostsProvider::base, + )?; + self.instrs.encode_register_list(rest, &mut self.layout)?; + Ok(()) + } + } + } + + /// Returns the results [`RegSpan`] of the `frame` if any. + fn frame_results(&self, frame: &impl ControlFrameBase) -> Result, Error> { + if frame.len_branch_params(&self.engine) == 0 { + return Ok(None); + } + let height = frame.height(); + let start = self.layout.temp_to_reg(OperandIdx::from(height))?; + let span = RegSpan::new(start); + Ok(Some(span)) + } + /// Returns `true` if the [`ControlFrame`] at `depth` requires copying for its branch parameters. /// /// # Note From b429a6ec51239754a141260c21a087eb0a577608 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 14 Jul 2025 11:25:51 +0200 Subject: [PATCH 276/343] add BlockType::func_type_with This requires us to put back information about a BlockType's exact ValTypes. --- crates/wasmi/src/engine/block_type.rs | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/crates/wasmi/src/engine/block_type.rs b/crates/wasmi/src/engine/block_type.rs index 285a661449..8f9803e597 100644 --- a/crates/wasmi/src/engine/block_type.rs +++ b/crates/wasmi/src/engine/block_type.rs @@ -1,3 +1,4 @@ +use core::slice; use crate::{ core::ValType, engine::DedupFuncType, @@ -18,7 +19,7 @@ pub enum BlockTypeInner { /// A block type with no parameters and no results. Empty, /// A block type with no parameters and exactly one result. - Returns, + Returns(ValType), /// A general block type with parameters and results. FuncType(DedupFuncType), } @@ -54,8 +55,8 @@ impl BlockType { } /// Creates a [`BlockType`] with no parameters and a single result type. - fn returns(_return_type: ValType) -> Self { - Self::from_inner(BlockTypeInner::Returns) + fn returns(return_type: ValType) -> Self { + Self::from_inner(BlockTypeInner::Returns(return_type)) } /// Creates a [`BlockType`] with parameters and results. @@ -66,7 +67,7 @@ impl BlockType { /// Returns the number of parameters of the [`BlockType`]. pub fn len_params(&self, engine: &Engine) -> u16 { match &self.inner { - BlockTypeInner::Empty | BlockTypeInner::Returns => 0, + BlockTypeInner::Empty | BlockTypeInner::Returns(_) => 0, BlockTypeInner::FuncType(func_type) => { engine.resolve_func_type(func_type, FuncType::len_params) } @@ -77,10 +78,19 @@ impl BlockType { pub fn len_results(&self, engine: &Engine) -> u16 { match &self.inner { BlockTypeInner::Empty => 0, - BlockTypeInner::Returns => 1, + BlockTypeInner::Returns(_) => 1, BlockTypeInner::FuncType(func_type) => { engine.resolve_func_type(func_type, FuncType::len_results) } } } + + /// Applies `f` to `self`'s [`FuncType`] and returns the result. + pub fn func_type_with(&self, engine: &Engine, f: impl for<'a> FnOnce(&FuncType) -> R) -> R { + match &self.inner { + BlockTypeInner::Empty => f(&FuncType::new([], [])), + BlockTypeInner::Returns(return_type) => f(&FuncType::new([], [*return_type])), + BlockTypeInner::FuncType(func_type) => engine.resolve_func_type(func_type, f), + } + } } From 691921384776cf6904618d595ab82475a692209a Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 14 Jul 2025 11:26:04 +0200 Subject: [PATCH 277/343] add FuncTranslator::push_frame_results utility method --- .../wasmi/src/engine/translator/func2/mod.rs | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index 6c781e3949..60997b8c11 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -410,6 +410,28 @@ impl FuncTranslator { Ok(()) } + /// Pushes the temporary results of the control `frame` onto the [`Stack`]. + /// + /// # Note + /// + /// - Before pushing the results, the [`Stack`] is truncated to the `frame`'s height. + /// - Not all control frames have temporary results, e.g. Wasm `loop`s, Wasm `if`s with + /// a compile-time known branch or Wasm `block`s that are never branched to, do not + /// require to call this function. + fn push_frame_results(&mut self, frame: &impl ControlFrameBase) -> Result<(), Error> { + let height = frame.height(); + self.stack.trunc(height); + frame + .ty() + .func_type_with(&self.engine, |func_ty| -> Result<(), Error> { + for result in func_ty.results() { + self.stack.push_temp(*result, None)?; + } + Ok(()) + })?; + Ok(()) + } + /// Encodes a copy instruction for the top-most `len_values` on the stack to `results`. /// /// # Note From ab841770c52b8850144ab985d151927051c6f4b1 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 14 Jul 2025 11:26:10 +0200 Subject: [PATCH 278/343] apply rustfmt --- crates/wasmi/src/engine/translator/func2/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index 60997b8c11..7296ee9643 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -433,9 +433,9 @@ impl FuncTranslator { } /// Encodes a copy instruction for the top-most `len_values` on the stack to `results`. - /// + /// /// # Note - /// + /// /// - This does _not_ pop values from the stack or manipulate the stack otherwise. /// - This might allocate new function local constant values if necessary. /// - This does _not_ encode a copy if the copy is a no-op. From 9bc1d1004baafb1b7a5a8b48a0ccd70b511dd182 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 14 Jul 2025 11:27:26 +0200 Subject: [PATCH 279/343] remove unused import --- crates/wasmi/src/engine/block_type.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/wasmi/src/engine/block_type.rs b/crates/wasmi/src/engine/block_type.rs index 8f9803e597..2d3b084e13 100644 --- a/crates/wasmi/src/engine/block_type.rs +++ b/crates/wasmi/src/engine/block_type.rs @@ -1,4 +1,3 @@ -use core::slice; use crate::{ core::ValType, engine::DedupFuncType, From 13f2e79cc6e9bd37434da0eb454d3bcab27f314a Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 14 Jul 2025 11:27:39 +0200 Subject: [PATCH 280/343] silence warnings for non-(experimental-translator) builds --- crates/wasmi/src/engine/block_type.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/wasmi/src/engine/block_type.rs b/crates/wasmi/src/engine/block_type.rs index 2d3b084e13..bf8f6402b4 100644 --- a/crates/wasmi/src/engine/block_type.rs +++ b/crates/wasmi/src/engine/block_type.rs @@ -1,3 +1,5 @@ +#![cfg_attr(not(feature = "experimental-translator"), allow(dead_code))] + use crate::{ core::ValType, engine::DedupFuncType, From 8def06379206242500f03d882d777a4d89a9de86 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 14 Jul 2025 11:27:55 +0200 Subject: [PATCH 281/343] add TODO annotation for later removal --- crates/wasmi/src/engine/block_type.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/block_type.rs b/crates/wasmi/src/engine/block_type.rs index bf8f6402b4..284bcd1b6b 100644 --- a/crates/wasmi/src/engine/block_type.rs +++ b/crates/wasmi/src/engine/block_type.rs @@ -1,4 +1,4 @@ -#![cfg_attr(not(feature = "experimental-translator"), allow(dead_code))] +#![cfg_attr(not(feature = "experimental-translator"), allow(dead_code))] // TODO: remove use crate::{ core::ValType, From b8bbd44c24f6429150828b0ef22a1c628fbe2d31 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 14 Jul 2025 11:45:38 +0200 Subject: [PATCH 282/343] add frame_results_impl API to fix borrow issues --- crates/wasmi/src/engine/translator/func2/mod.rs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index 7296ee9643..5a4946de8e 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -543,11 +543,20 @@ impl FuncTranslator { /// Returns the results [`RegSpan`] of the `frame` if any. fn frame_results(&self, frame: &impl ControlFrameBase) -> Result, Error> { - if frame.len_branch_params(&self.engine) == 0 { + Self::frame_results_impl(frame, &self.engine, &self.layout) + } + + /// Returns the results [`RegSpan`] of the `frame` if any. + fn frame_results_impl( + frame: &impl ControlFrameBase, + engine: &Engine, + layout: &StackLayout, + ) -> Result, Error> { + if frame.len_branch_params(engine) == 0 { return Ok(None); } let height = frame.height(); - let start = self.layout.temp_to_reg(OperandIdx::from(height))?; + let start = layout.temp_to_reg(OperandIdx::from(height))?; let span = RegSpan::new(start); Ok(Some(span)) } From d84cfa27c0cc557a181a43191eb3990d7db6d59a Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 14 Jul 2025 12:17:45 +0200 Subject: [PATCH 283/343] make conditionals a bit more readable --- crates/wasmi/src/engine/translator/func2/mod.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index 5a4946de8e..7b0e33617f 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -574,7 +574,8 @@ impl FuncTranslator { let only_temps = (0..len_branch_params) .map(|depth| self.stack.peek(depth)) .all(|o| o.is_temp()); - !height_matches || !only_temps + let can_avoid_copies = height_matches && only_temps; + !can_avoid_copies } /// Pins the `label` to the next [`Instr`]. From a923877b7433ea637cfbaa6762ba1eae20295015 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 14 Jul 2025 12:36:47 +0200 Subject: [PATCH 284/343] use new copy_branch_params_v2 API --- .../wasmi/src/engine/translator/func2/mod.rs | 34 ++++++++++++------- .../src/engine/translator/func2/visit.rs | 18 ++++++---- 2 files changed, 33 insertions(+), 19 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index 7b0e33617f..8942bd9d06 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -485,8 +485,12 @@ impl FuncTranslator { } }, operand => { - let operand = self.layout.operand_to_reg(operand)?; - Instruction::copy(result, operand) + let value = self.layout.operand_to_reg(operand)?; + if result == value { + // Case: no-op copy + return Ok(()); + } + Instruction::copy(result, value) } }; self.instrs @@ -799,8 +803,8 @@ impl FuncTranslator { fn translate_end_block(&mut self, frame: BlockControlFrame) -> Result<(), Error> { let consume_fuel_instr = frame.consume_fuel_instr(); if self.reachable && frame.is_branched_to() { - let len_values = frame.len_branch_params(&self.engine); - self.copy_branch_params(usize::from(len_values), consume_fuel_instr)?; + self.copy_branch_params_v2(&frame, consume_fuel_instr)?; + self.push_frame_results(&frame)?; } if let Err(err) = self .labels @@ -835,6 +839,7 @@ impl FuncTranslator { if end_of_then_reachable && has_results { let consume_fuel_instr = frame.consume_fuel_instr(); self.copy_branch_params(usize::from(len_results), consume_fuel_instr)?; + self.push_frame_results(&frame)?; let end_offset = self .labels .try_resolve_label(frame.label(), self.instrs.next_instr()) @@ -869,9 +874,9 @@ impl FuncTranslator { _ => true, }; if end_of_else_reachable { - let len_values = frame.len_branch_params(&self.engine); let consume_fuel_instr: Option = frame.consume_fuel_instr(); - self.copy_branch_params(usize::from(len_values), consume_fuel_instr)?; + self.copy_branch_params_v2(&frame, consume_fuel_instr)?; + self.push_frame_results(&frame)?; } self.labels .pin_label(frame.label(), self.instrs.next_instr()) @@ -892,9 +897,9 @@ impl FuncTranslator { ElseReachability::Both => unreachable!(), }; if end_is_reachable && frame.is_branched_to() { - let len_values = frame.len_branch_params(&self.engine); let consume_fuel_instr = frame.consume_fuel_instr(); - self.copy_branch_params(usize::from(len_values), consume_fuel_instr)?; + self.copy_branch_params_v2(&frame, consume_fuel_instr)?; + self.push_frame_results(&frame)?; } self.labels .pin_label(frame.label(), self.instrs.next_instr()) @@ -992,12 +997,11 @@ impl FuncTranslator { } /// Encodes an unconditional Wasm `branch` instruction. - fn encode_br(&mut self, label: LabelRef) -> Result<(), Error> { + fn encode_br(&mut self, label: LabelRef) -> Result { let instr = self.instrs.next_instr(); let offset = self.labels.try_resolve_label(label, instr)?; - self.push_instr(Instruction::branch(offset), FuelCostsProvider::base)?; - self.reachable = false; - Ok(()) + let br_instr = self.push_instr(Instruction::branch(offset), FuelCostsProvider::base)?; + Ok(br_instr) } /// Encodes a `i32.eqz`+`br_if` or `if` conditional branch instruction. @@ -1030,7 +1034,11 @@ impl FuncTranslator { false => condition != 0, }; match take_branch { - true => return self.encode_br(label), + true => { + self.encode_br(label)?; + self.reachable = false; + return Ok(()); + } false => return Ok(()), } } diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index 56e1e86578..867241e805 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -168,9 +168,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { if let Some(else_label) = frame.else_label() { debug_assert!(frame.is_then_reachable() && frame.is_else_reachable()); if is_end_of_then_reachable { - let len_values = usize::from(frame.ty().len_results(&self.engine)); let consume_fuel_instr = frame.consume_fuel_instr(); - self.copy_branch_params(len_values, consume_fuel_instr)?; + self.copy_branch_params_v2(&frame, consume_fuel_instr)?; + self.stack.trunc(frame.height()); frame.branch_to(); self.encode_br(frame.label())?; } @@ -207,8 +207,11 @@ impl<'a> VisitOperator<'a> for FuncTranslator { AcquiredTarget::Branch(mut frame) => { frame.branch_to(); let label = frame.label(); - let len_params = usize::from(frame.len_branch_params(&self.engine)); - self.copy_branch_params(len_params, consume_fuel_instr)?; + let len_params = frame.len_branch_params(&self.engine); + let branch_results = Self::frame_results_impl(&frame, &self.engine, &self.layout)?; + if let Some(branch_results) = branch_results { + self.encode_copies(branch_results, len_params, consume_fuel_instr)?; + } self.encode_br(label)?; self.reachable = false; Ok(()) @@ -232,6 +235,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { let mut frame = self.stack.peek_control_mut(depth).control_frame(); frame.branch_to(); let len_branch_params = frame.len_branch_params(&self.engine); + let branch_results = Self::frame_results_impl(&frame, &self.engine, &self.layout)?; let label = frame.label(); if len_branch_params == 0 { // Case: no branch values are required to be copied @@ -246,8 +250,10 @@ impl<'a> VisitOperator<'a> for FuncTranslator { // Case: fallback to copy branch parameters conditionally let consume_fuel_instr = self.stack.consume_fuel_instr(); let skip_label = self.labels.new_label(); - self.encode_br_eqz(condition, label)?; - self.copy_operands_to_temp(depth, consume_fuel_instr)?; + self.encode_br_eqz(condition, skip_label)?; + if let Some(branch_results) = branch_results { + self.encode_copies(branch_results, len_branch_params, consume_fuel_instr)?; + } self.encode_br(label)?; self.labels .pin_label(skip_label, self.instrs.next_instr()) From 3103a2afa5343666ab94543cd0317df7f4e3cbc6 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 14 Jul 2025 19:23:30 +0200 Subject: [PATCH 285/343] fix bugs with push_frame_results usage --- .../wasmi/src/engine/translator/func2/mod.rs | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index 8942bd9d06..e8d86242d5 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -802,8 +802,10 @@ impl FuncTranslator { /// Translates the end of a Wasm `block` control frame. fn translate_end_block(&mut self, frame: BlockControlFrame) -> Result<(), Error> { let consume_fuel_instr = frame.consume_fuel_instr(); - if self.reachable && frame.is_branched_to() { - self.copy_branch_params_v2(&frame, consume_fuel_instr)?; + if frame.is_branched_to() { + if self.reachable { + self.copy_branch_params_v2(&frame, consume_fuel_instr)?; + } self.push_frame_results(&frame)?; } if let Err(err) = self @@ -839,7 +841,6 @@ impl FuncTranslator { if end_of_then_reachable && has_results { let consume_fuel_instr = frame.consume_fuel_instr(); self.copy_branch_params(usize::from(len_results), consume_fuel_instr)?; - self.push_frame_results(&frame)?; let end_offset = self .labels .try_resolve_label(frame.label(), self.instrs.next_instr()) @@ -850,6 +851,7 @@ impl FuncTranslator { FuelCostsProvider::base, )?; } + self.push_frame_results(&frame)?; let next_instr = self.instrs.next_instr(); self.labels.try_pin_label(else_label, next_instr); self.labels.pin_label(frame.label(), next_instr).unwrap(); @@ -876,8 +878,8 @@ impl FuncTranslator { if end_of_else_reachable { let consume_fuel_instr: Option = frame.consume_fuel_instr(); self.copy_branch_params_v2(&frame, consume_fuel_instr)?; - self.push_frame_results(&frame)?; } + self.push_frame_results(&frame)?; self.labels .pin_label(frame.label(), self.instrs.next_instr()) .unwrap(); @@ -896,9 +898,11 @@ impl FuncTranslator { ElseReachability::OnlyElse => true, ElseReachability::Both => unreachable!(), }; - if end_is_reachable && frame.is_branched_to() { - let consume_fuel_instr = frame.consume_fuel_instr(); - self.copy_branch_params_v2(&frame, consume_fuel_instr)?; + if frame.is_branched_to() { + if end_is_reachable { + let consume_fuel_instr = frame.consume_fuel_instr(); + self.copy_branch_params_v2(&frame, consume_fuel_instr)?; + } self.push_frame_results(&frame)?; } self.labels From 8e51861ee67407f7ebc4823d9986c540570b8399 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 14 Jul 2025 19:54:54 +0200 Subject: [PATCH 286/343] fix reachability of divergent ifs Reachability was not calculated properly for ifs where either then or else block where known to be reachable at compile-time. --- .../wasmi/src/engine/translator/func2/mod.rs | 37 ++++++----- .../engine/translator/func2/stack/control.rs | 65 +++++++++++-------- 2 files changed, 59 insertions(+), 43 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index e8d86242d5..ab640d2d99 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -831,14 +831,18 @@ impl FuncTranslator { /// Translates the end of a Wasm `if` control frame. fn translate_end_if(&mut self, frame: IfControlFrame) -> Result<(), Error> { debug_assert!(!self.stack.is_control_empty()); + let is_end_of_then_reachable = self.reachable; let IfReachability::Both { else_label } = frame.reachability() else { - let reachability = frame.reachability().into(); - return self.translate_end_if_or_else_only(frame, reachability); + let is_end_reachable = match frame.reachability() { + IfReachability::OnlyThen => self.reachable, + IfReachability::OnlyElse => false, + IfReachability::Both { .. } => unreachable!(), + }; + return self.translate_end_if_or_else_only(frame, is_end_reachable); }; - let end_of_then_reachable = self.reachable; let len_results = frame.ty().len_results(self.engine()); let has_results = len_results >= 1; - if end_of_then_reachable && has_results { + if is_end_of_then_reachable && has_results { let consume_fuel_instr = frame.consume_fuel_instr(); self.copy_branch_params(usize::from(len_results), consume_fuel_instr)?; let end_offset = self @@ -862,13 +866,17 @@ impl FuncTranslator { /// Translates the end of a Wasm `else` control frame. fn translate_end_else(&mut self, frame: ElseControlFrame) -> Result<(), Error> { debug_assert!(!self.stack.is_control_empty()); - let reachability = frame.reachability(); - if matches!( - reachability, - ElseReachability::OnlyThen | ElseReachability::OnlyElse - ) { - return self.translate_end_if_or_else_only(frame, reachability); - } + match frame.reachability() { + ElseReachability::OnlyThen { + is_end_of_then_reachable, + } => { + return self.translate_end_if_or_else_only(frame, is_end_of_then_reachable); + } + ElseReachability::OnlyElse => { + return self.translate_end_if_or_else_only(frame, self.reachable); + } + _ => {} + }; let end_of_then_reachable = frame.is_end_of_then_reachable(); let end_of_else_reachable = self.reachable; let reachable = match (end_of_then_reachable, end_of_else_reachable) { @@ -891,13 +899,8 @@ impl FuncTranslator { fn translate_end_if_or_else_only( &mut self, frame: impl ControlFrameBase, - reachability: ElseReachability, + end_is_reachable: bool, ) -> Result<(), Error> { - let end_is_reachable = match reachability { - ElseReachability::OnlyThen => self.reachable, - ElseReachability::OnlyElse => true, - ElseReachability::Both => unreachable!(), - }; if frame.is_branched_to() { if end_is_reachable { let consume_fuel_instr = frame.consume_fuel_instr(); diff --git a/crates/wasmi/src/engine/translator/func2/stack/control.rs b/crates/wasmi/src/engine/translator/func2/stack/control.rs index fcd777db6f..819db1ef1f 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/control.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/control.rs @@ -177,7 +177,15 @@ impl ControlStack { let height = if_frame.height(); let label = if_frame.label(); let is_branched_to = if_frame.is_branched_to(); - let reachability = ElseReachability::from(if_frame.reachability); + let reachability = match if_frame.reachability { + IfReachability::Both { .. } => ElseReachability::Both { + is_end_of_then_reachable, + }, + IfReachability::OnlyThen => ElseReachability::OnlyThen { + is_end_of_then_reachable, + }, + IfReachability::OnlyElse => ElseReachability::OnlyElse, + }; self.frames.push(ControlFrame::from(ElseControlFrame { ty, height: StackHeight::from(height), @@ -185,12 +193,11 @@ impl ControlStack { consume_fuel, label, reachability, - is_end_of_then_reachable, })); self.expect_else = false; match reachability { - ElseReachability::OnlyThen | ElseReachability::OnlyElse => None, - ElseReachability::Both => { + ElseReachability::OnlyThen { .. } | ElseReachability::OnlyElse => None, + ElseReachability::Both { .. } => { let else_operands = self .else_operands .pop() @@ -714,12 +721,6 @@ pub struct ElseControlFrame { label: LabelRef, /// The reachability of the `then` and `else` blocks. reachability: ElseReachability, - /// Is `true` if code is reachable when entering the `else` block. - /// - /// # Note - /// - /// This means that the end of the `then` block was reachable. - is_end_of_then_reachable: bool, } /// The reachability of the `else` control flow frame. @@ -732,13 +733,27 @@ pub enum ElseReachability { /// This variant does not mean that necessarily both `then` and `else` /// blocks do exist and are non-empty. The `then` block might still be /// empty and the `then` block might still be missing. - Both, + Both { + /// Is `true` if code is reachable when entering the `else` block. + /// + /// # Note + /// + /// This means that the end of the `then` block was reachable. + is_end_of_then_reachable: bool, + }, /// Only the `then` block of the `if` is reachable. /// /// # Note /// /// This case happens only in case the `if` has a `true` constant condition. - OnlyThen, + OnlyThen { + /// Is `true` if code is reachable when entering the `else` block. + /// + /// # Note + /// + /// This means that the end of the `then` block was reachable. + is_end_of_then_reachable: bool, + }, /// Only the `else` block of the `if` is reachable. /// /// # Note @@ -747,16 +762,6 @@ pub enum ElseReachability { OnlyElse, } -impl From for ElseReachability { - fn from(reachability: IfReachability) -> Self { - match reachability { - IfReachability::Both { .. } => Self::Both, - IfReachability::OnlyThen => Self::OnlyThen, - IfReachability::OnlyElse => Self::OnlyElse, - } - } -} - impl ElseControlFrame { /// Returns the [`ElseReachability`] of the [`ElseReachability`]. pub fn reachability(&self) -> ElseReachability { @@ -770,7 +775,7 @@ impl ElseControlFrame { /// The `then` branch is unreachable if the `if` condition is a constant `false` value. pub fn is_then_reachable(&self) -> bool { match self.reachability { - ElseReachability::Both | ElseReachability::OnlyThen => true, + ElseReachability::Both { .. } | ElseReachability::OnlyThen { .. } => true, ElseReachability::OnlyElse => false, } } @@ -782,14 +787,22 @@ impl ElseControlFrame { /// The `else` branch is unreachable if the `if` condition is a constant `true` value. pub fn is_else_reachable(&self) -> bool { match self.reachability { - ElseReachability::Both | ElseReachability::OnlyElse => true, - ElseReachability::OnlyThen => false, + ElseReachability::Both { .. } | ElseReachability::OnlyElse => true, + ElseReachability::OnlyThen { .. } => false, } } /// Returns `true` if the end of the `then` branch is reachable. pub fn is_end_of_then_reachable(&self) -> bool { - self.is_end_of_then_reachable + match self.reachability { + ElseReachability::Both { + is_end_of_then_reachable, + } + | ElseReachability::OnlyThen { + is_end_of_then_reachable, + } => is_end_of_then_reachable, + ElseReachability::OnlyElse => false, + } } } From 1375f2becfc21d69cfc6e18ccbf9a459fc47c606 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 14 Jul 2025 20:30:46 +0200 Subject: [PATCH 287/343] apply clippy suggestion --- crates/wasmi/src/engine/translator/func2/instrs.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/func2/instrs.rs b/crates/wasmi/src/engine/translator/func2/instrs.rs index b2b36cf4d2..4b71ec8ead 100644 --- a/crates/wasmi/src/engine/translator/func2/instrs.rs +++ b/crates/wasmi/src/engine/translator/func2/instrs.rs @@ -318,7 +318,7 @@ impl InstrEncoder { operands: &[Operand], layout: &mut StackLayout, ) -> Result<(), Error> { - let mut remaining = &operands[..]; + let mut remaining = operands; let mut operand_to_reg = |operand: &Operand| -> Result { layout.operand_to_reg(*operand) }; let instr = loop { From fb512febbc5129e4e0f901401be0edcb1abcb102 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Tue, 15 Jul 2025 00:34:58 +0200 Subject: [PATCH 288/343] use new copy_branch_params_v2 API --- crates/wasmi/src/engine/translator/func2/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index ab640d2d99..fb756cd92a 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -844,7 +844,7 @@ impl FuncTranslator { let has_results = len_results >= 1; if is_end_of_then_reachable && has_results { let consume_fuel_instr = frame.consume_fuel_instr(); - self.copy_branch_params(usize::from(len_results), consume_fuel_instr)?; + self.copy_branch_params_v2(&frame, consume_fuel_instr)?; let end_offset = self .labels .try_resolve_label(frame.label(), self.instrs.next_instr()) From f03fcc89feb7250ddcc1efcc352a707246726220 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Tue, 15 Jul 2025 00:38:24 +0200 Subject: [PATCH 289/343] rename and re-doc copy_branch_params API --- crates/wasmi/src/engine/translator/func2/mod.rs | 10 +++++----- crates/wasmi/src/engine/translator/func2/visit.rs | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index fb756cd92a..e40f568638 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -351,18 +351,18 @@ impl FuncTranslator { self.fuel_costs.is_some() } - /// Convert all branch params up to `depth` to [`Operand::Temp`]. + /// Copy the top-most `len` operands to [`Operand::Temp`] values. /// /// # Note /// - /// - The top-most `depth` operands on the [`Stack`] will be [`Operand::Temp`] upon completion. + /// - The top-most `len` operands on the [`Stack`] will be [`Operand::Temp`] upon completion. /// - Does nothing if an [`Operand`] is already an [`Operand::Temp`]. - fn copy_branch_params( + fn move_operands_to_temp( &mut self, - depth: usize, + len: usize, consume_fuel: Option, ) -> Result<(), Error> { - for n in 0..depth { + for n in 0..len { let operand = self.stack.operand_to_temp(n); self.copy_operand_to_temp(operand, consume_fuel)?; } diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index 867241e805..71b3431304 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -109,7 +109,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { let len_params = block_ty.len_params(&self.engine); let continue_label = self.labels.new_label(); let consume_fuel = self.stack.consume_fuel_instr(); - self.copy_branch_params(usize::from(len_params), consume_fuel)?; + self.move_operands_to_temp(usize::from(len_params), consume_fuel)?; self.pin_label(continue_label); let consume_fuel = self.instrs.push_consume_fuel_instr()?; self.stack From cffcb70d10d92979961acc2b8d1cb8764ec0258b Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Tue, 15 Jul 2025 00:39:01 +0200 Subject: [PATCH 290/343] remove _v2 suffix from copy_branch_params_v2 --- crates/wasmi/src/engine/translator/func2/mod.rs | 10 +++++----- crates/wasmi/src/engine/translator/func2/visit.rs | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index e40f568638..668f27fe7b 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -397,7 +397,7 @@ impl FuncTranslator { /// /// - The top-most `depth` operands on the [`Stack`] will be [`Operand::Temp`] upon completion. /// - Does nothing if an [`Operand`] is already an [`Operand::Temp`]. - fn copy_branch_params_v2( + fn copy_branch_params( &mut self, target: &impl ControlFrameBase, consume_fuel_instr: Option, @@ -804,7 +804,7 @@ impl FuncTranslator { let consume_fuel_instr = frame.consume_fuel_instr(); if frame.is_branched_to() { if self.reachable { - self.copy_branch_params_v2(&frame, consume_fuel_instr)?; + self.copy_branch_params(&frame, consume_fuel_instr)?; } self.push_frame_results(&frame)?; } @@ -844,7 +844,7 @@ impl FuncTranslator { let has_results = len_results >= 1; if is_end_of_then_reachable && has_results { let consume_fuel_instr = frame.consume_fuel_instr(); - self.copy_branch_params_v2(&frame, consume_fuel_instr)?; + self.copy_branch_params(&frame, consume_fuel_instr)?; let end_offset = self .labels .try_resolve_label(frame.label(), self.instrs.next_instr()) @@ -885,7 +885,7 @@ impl FuncTranslator { }; if end_of_else_reachable { let consume_fuel_instr: Option = frame.consume_fuel_instr(); - self.copy_branch_params_v2(&frame, consume_fuel_instr)?; + self.copy_branch_params(&frame, consume_fuel_instr)?; } self.push_frame_results(&frame)?; self.labels @@ -904,7 +904,7 @@ impl FuncTranslator { if frame.is_branched_to() { if end_is_reachable { let consume_fuel_instr = frame.consume_fuel_instr(); - self.copy_branch_params_v2(&frame, consume_fuel_instr)?; + self.copy_branch_params(&frame, consume_fuel_instr)?; } self.push_frame_results(&frame)?; } diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index 71b3431304..e0edc196f5 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -169,7 +169,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { debug_assert!(frame.is_then_reachable() && frame.is_else_reachable()); if is_end_of_then_reachable { let consume_fuel_instr = frame.consume_fuel_instr(); - self.copy_branch_params_v2(&frame, consume_fuel_instr)?; + self.copy_branch_params(&frame, consume_fuel_instr)?; self.stack.trunc(frame.height()); frame.branch_to(); self.encode_br(frame.label())?; From 872df4bfe655cd558b9cabd0850302ced4b44dd7 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Tue, 15 Jul 2025 00:47:18 +0200 Subject: [PATCH 291/343] make new FuncTranslator encode ReturnMany --- .../wasmi/src/engine/translator/func2/mod.rs | 41 +++++++++++++------ 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index 668f27fe7b..cfee61d70e 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -748,22 +748,13 @@ impl FuncTranslator { Instruction::return_reg3_ext(v0, v1, v2) } _ => { - let len_copies = usize::from(len_results); - match self.try_form_regspan(len_copies)? { + let len_values = usize::from(len_results); + match self.try_form_regspan(len_values)? { Some(span) => { let values = BoundedRegSpan::new(span, len_results); Instruction::return_span(values) } - None => { - let Some(first_idx) = - self.copy_operands_to_temp(len_copies, consume_fuel)? - else { - unreachable!("`first_idx` must be `Some` since `len_copies` is >0") - }; - let result = self.layout.temp_to_reg(first_idx)?; - let values = BoundedRegSpan::new(RegSpan::new(result), len_results); - Instruction::return_span(values) - } + None => return self.encode_return_many(len_values, consume_fuel), } } }; @@ -773,6 +764,32 @@ impl FuncTranslator { Ok(instr) } + /// Encodes an [`Instruction::ReturnMany`] for `len` values. + /// + /// # Panics + /// + /// If `len` is not greater than or equal to 4. + fn encode_return_many( + &mut self, + len: usize, + consume_fuel_instr: Option, + ) -> Result { + self.stack.peek_n(len, &mut self.operands); + let [v0, v1, v2, rest @ ..] = &self.operands[..] else { + unreachable!("encode_return_many (pre-condition): len >= 4") + }; + let v0 = self.layout.operand_to_reg(*v0)?; + let v1 = self.layout.operand_to_reg(*v1)?; + let v2 = self.layout.operand_to_reg(*v2)?; + let return_instr = self.instrs.push_instr( + Instruction::return_many_ext(v0, v1, v2), + consume_fuel_instr, + FuelCostsProvider::base, + )?; + self.instrs.encode_register_list(rest, &mut self.layout)?; + Ok(return_instr) + } + /// Tries to form a [`RegSpan`] from the top-most `len` operands on the [`Stack`]. /// /// Returns `None` if forming a [`RegSpan`] was not possible. From f2ba04aed39459cecc65199dc3e522e92b2738bd Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Tue, 15 Jul 2025 00:48:01 +0200 Subject: [PATCH 292/343] remove now unused API --- .../wasmi/src/engine/translator/func2/mod.rs | 22 ------------------- 1 file changed, 22 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index cfee61d70e..e5aba07d18 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -369,28 +369,6 @@ impl FuncTranslator { Ok(()) } - /// Copy the top-most `len` [`Operand`]s into [`Operand::Temp`]s by copying if necessary. - /// - /// Returns the [`OperandIdx`] of the first [`Operand`]. - /// - /// # Note - /// - /// - This does _not_ manipulate the [`Stack`]. - /// - Does nothing if an [`Operand`] is already an [`Operand::Temp`]. - fn copy_operands_to_temp( - &mut self, - len: usize, - consume_fuel: Option, - ) -> Result, Error> { - let mut idx = None; - for n in 0..len { - let operand = self.stack.peek(n); - self.copy_operand_to_temp(operand, consume_fuel)?; - idx = Some(operand.index()); - } - Ok(idx) - } - /// Convert all branch params up to `depth` to [`Operand::Temp`]. /// /// # Note From 33f53f39c31d4725f5996034fd720983169d3c1b Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Tue, 15 Jul 2025 00:52:20 +0200 Subject: [PATCH 293/343] remove some unused APIs --- .../engine/translator/func2/stack/control.rs | 24 ------------------- .../engine/translator/func2/stack/locals.rs | 9 ------- .../engine/translator/func2/stack/operands.rs | 12 ---------- 3 files changed, 45 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/stack/control.rs b/crates/wasmi/src/engine/translator/func2/stack/control.rs index 819db1ef1f..3e61e92851 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/control.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/control.rs @@ -768,30 +768,6 @@ impl ElseControlFrame { self.reachability } - /// Returns `true` if the `then` branch is reachable. - /// - /// # Note - /// - /// The `then` branch is unreachable if the `if` condition is a constant `false` value. - pub fn is_then_reachable(&self) -> bool { - match self.reachability { - ElseReachability::Both { .. } | ElseReachability::OnlyThen { .. } => true, - ElseReachability::OnlyElse => false, - } - } - - /// Returns `true` if the `else` branch is reachable. - /// - /// # Note - /// - /// The `else` branch is unreachable if the `if` condition is a constant `true` value. - pub fn is_else_reachable(&self) -> bool { - match self.reachability { - ElseReachability::Both { .. } | ElseReachability::OnlyElse => true, - ElseReachability::OnlyThen { .. } => false, - } - } - /// Returns `true` if the end of the `then` branch is reachable. pub fn is_end_of_then_reachable(&self) -> bool { match self.reachability { diff --git a/crates/wasmi/src/engine/translator/func2/stack/locals.rs b/crates/wasmi/src/engine/translator/func2/stack/locals.rs index 588a0263a9..b38211c38c 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/locals.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/locals.rs @@ -89,15 +89,6 @@ impl LocalsRegistry { index } - /// Returns the first operand for this local on the stack if any. - /// - /// # Panics - /// - /// If `index` is out of bounds. - pub fn first_operand(&self, index: LocalIdx) -> Option { - self.first_operands[Self::local_idx_to_index(index)] - } - /// Replaces the first operand for this local on the stack and returns the old one. /// /// # Panics diff --git a/crates/wasmi/src/engine/translator/func2/stack/operands.rs b/crates/wasmi/src/engine/translator/func2/stack/operands.rs index f3a2b4a64f..1d7cc6eb13 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/operands.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/operands.rs @@ -116,18 +116,6 @@ impl OperandStack { self.max_height } - /// Truncates `self` to the target `height`. - /// - /// All operands above `height` are dropped. - /// - /// # Panic - /// - /// If `height` is greater than the current height of `self`. - pub fn trunc(&mut self, height: usize) { - assert!(height <= self.height()); - self.operands.truncate(height); - } - /// Updates the maximum stack height if needed. fn update_max_stack_height(&mut self) { self.max_height = core::cmp::max(self.max_height, self.height()); From 250f213e667c523513be814c1d4c881dcec89c8f Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Tue, 15 Jul 2025 22:32:37 +0200 Subject: [PATCH 294/343] add immediates buffer to FuncTranslator This is going to be useful for Wasm br_table translation targets. --- crates/wasmi/src/engine/translator/func2/mod.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index e5aba07d18..78ff33c64c 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -112,6 +112,8 @@ pub struct FuncTranslator { instrs: InstrEncoder, /// Temporary buffer for operands. operands: Vec, + /// Temporary buffer for immediate values. + immediates: Vec, } /// Heap allocated data structured used by the [`FuncTranslator`]. @@ -127,6 +129,8 @@ pub struct FuncTranslatorAllocations { instrs: InstrEncoderAllocations, /// Temporary buffer for operands. operands: Vec, + /// Temporary buffer for immediate values. + immediates: Vec, } impl Reset for FuncTranslatorAllocations { @@ -136,6 +140,7 @@ impl Reset for FuncTranslatorAllocations { self.labels.reset(); self.instrs.reset(); self.operands.clear(); + self.immediates.clear(); } } @@ -201,6 +206,7 @@ impl ReusableAllocations for FuncTranslator { labels: self.labels, instrs: self.instrs.into_allocations(), operands: self.operands, + immediates: self.immediates, } } } @@ -229,6 +235,7 @@ impl FuncTranslator { labels, instrs, operands, + immediates, } = alloc.into_reset(); let stack = Stack::new(&engine, stack); let instrs = InstrEncoder::new(&engine, instrs); @@ -243,6 +250,7 @@ impl FuncTranslator { labels, instrs, operands, + immediates, }; translator.init_func_body_block()?; translator.init_func_params()?; From a0cffac51937a7d06c7e7d0599d7b57f7489f64d Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Tue, 15 Jul 2025 22:33:06 +0200 Subject: [PATCH 295/343] add try_form_regspan_or_move utility method This is going to be useful for translation of Wasm's br_table. --- .../wasmi/src/engine/translator/func2/mod.rs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index 78ff33c64c..e25ded4f76 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -802,6 +802,24 @@ impl FuncTranslator { Ok(Some(RegSpan::new(start))) } + /// Tries to form a [`RegSpan`] from the top-most `len` operands on the [`Stack`] or copy to temporaries. + /// + /// Returns `None` if forming a [`RegSpan`] was not possible. + fn try_form_regspan_or_move( + &mut self, + len: usize, + consume_fuel_instr: Option, + ) -> Result { + if let Some(span) = self.try_form_regspan(len)? { + return Ok(span); + } + self.move_operands_to_temp(len, consume_fuel_instr)?; + let Some(span) = self.try_form_regspan(len)? else { + unreachable!("the top-most `len` operands are now temporaries thus `RegSpan` forming should succeed") + }; + Ok(span) + } + /// Translates the end of a Wasm `block` control frame. fn translate_end_block(&mut self, frame: BlockControlFrame) -> Result<(), Error> { let consume_fuel_instr = frame.consume_fuel_instr(); From 68f20c5b731691fa3331d9932a8ea01be7567a64 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Tue, 15 Jul 2025 22:33:25 +0200 Subject: [PATCH 296/343] remove invalid debug assertions in push_param --- crates/wasmi/src/engine/translator/func2/instrs.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/instrs.rs b/crates/wasmi/src/engine/translator/func2/instrs.rs index 4b71ec8ead..717936c17f 100644 --- a/crates/wasmi/src/engine/translator/func2/instrs.rs +++ b/crates/wasmi/src/engine/translator/func2/instrs.rs @@ -233,10 +233,6 @@ impl InstrEncoder { /// /// The parameter is associated to the last pushed [`Instruction`]. pub fn push_param(&mut self, instruction: Instruction) { - debug_assert!( - instruction.is_instruction_parameter(), - "non-parameter: {instruction:?}" - ); self.instrs.push(instruction); } From e77d2fae385baaa1e3393c9bf279c5996032a6ef Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Tue, 15 Jul 2025 22:33:35 +0200 Subject: [PATCH 297/343] implement br_table translation --- .../wasmi/src/engine/translator/func2/mod.rs | 90 +++++++++++++++++++ .../src/engine/translator/func2/visit.rs | 47 +++++++++- 2 files changed, 135 insertions(+), 2 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index e25ded4f76..a5c54cb62c 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -680,6 +680,96 @@ impl FuncTranslator { self.push_instr_with_result(result_ty, |result| make_instr(result, lhs, rhs), fuel_costs) } + /// Populate the `buffer` with the `table` targets including the `table` default target. + /// + /// Returns a shared slice to the `buffer` after it has been filled. + /// + /// # Note + /// + /// The `table` default target is pushed last to the `buffer`. + fn copy_targets_from_br_table( + table: &wasmparser::BrTable, + buffer: &mut Vec, + ) -> Result<(), Error> { + let default_target = table.default(); + buffer.clear(); + for target in table.targets() { + buffer.push(TypedVal::from(target?)); + } + buffer.push(TypedVal::from(default_target)); + Ok(()) + } + + /// Encodes a Wasm `br_table` that does not copy branching values. + /// + /// # Note + /// + /// Upon call the `immediates` buffer contains all `br_table` target values. + fn encode_br_table_0(&mut self, table: wasmparser::BrTable, index: Reg) -> Result<(), Error> { + debug_assert_eq!(self.immediates.len(), (table.len() + 1) as usize); + self.push_instr( + Instruction::branch_table_0(index, table.len() + 1), + FuelCostsProvider::base, + )?; + // Encode the `br_table` targets: + let targets = &self.immediates[..]; + for target in targets { + let Ok(depth) = usize::try_from(u32::from(*target)) else { + panic!("out of bounds `br_table` target does not fit `usize`: {target:?}"); + }; + let mut frame = self.stack.peek_control_mut(depth).control_frame(); + let offset = self + .labels + .try_resolve_label(frame.label(), self.instrs.next_instr())?; + self.instrs.push_param(Instruction::branch(offset)); + frame.branch_to(); + } + Ok(()) + } + + /// Encodes a Wasm `br_table` that has to copy `len_values` branching values. + /// + /// # Note + /// + /// Upon call the `immediates` buffer contains all `br_table` target values. + fn encode_br_table_n( + &mut self, + table: wasmparser::BrTable, + index: Reg, + len_values: u16, + ) -> Result<(), Error> { + debug_assert_eq!(self.immediates.len(), (table.len() + 1) as usize); + let consume_fuel_instr = self.stack.consume_fuel_instr(); + let values = self.try_form_regspan_or_move(usize::from(len_values), consume_fuel_instr)?; + self.push_instr( + Instruction::branch_table_span(index, table.len() + 1), + FuelCostsProvider::base, + )?; + self.instrs + .push_param(Instruction::register_span(BoundedRegSpan::new( + values, len_values, + ))); + // Encode the `br_table` targets: + let targets = &self.immediates[..]; + for target in targets { + let Ok(depth) = usize::try_from(u32::from(*target)) else { + panic!("out of bounds `br_table` target does not fit `usize`: {target:?}"); + }; + let mut frame = self.stack.peek_control_mut(depth).control_frame(); + let Some(results) = Self::frame_results_impl(&frame, &self.engine, &self.layout)? + else { + panic!("must have frame results since `br_table` requires to copy values"); + }; + let offset = self + .labels + .try_resolve_label(frame.label(), self.instrs.next_instr())?; + self.instrs + .push_param(Instruction::branch_table_target(results, offset)); + frame.branch_to(); + } + Ok(()) + } + /// Encodes a generic return instruction. fn encode_return(&mut self, consume_fuel: Option) -> Result { let len_results = self.func_type_with(FuncType::len_results); diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index e0edc196f5..a3fa77cb7d 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -261,8 +261,51 @@ impl<'a> VisitOperator<'a> for FuncTranslator { Ok(()) } - fn visit_br_table(&mut self, _targets: wasmparser::BrTable<'a>) -> Self::Output { - todo!() + fn visit_br_table(&mut self, table: wasmparser::BrTable<'a>) -> Self::Output { + bail_unreachable!(self); + let index = self.stack.pop(); + let default_target = table.default(); + if table.is_empty() { + // Case: the `br_table` only has a single target `t` which is equal to a `br t`. + return self.visit_br(default_target); + } + if let Operand::Immediate(index) = index { + // Case: the `br_table` index is a constant value, therefore always taking the same branch. + // Note: `usize::MAX` is used to fallback to the default target. + let chosen_index = usize::try_from(u32::from(index.val())).unwrap_or(usize::MAX); + let chosen_target = table + .targets() + .nth(chosen_index) + .transpose()? + .unwrap_or(default_target); + return self.visit_br(chosen_target); + } + Self::copy_targets_from_br_table(&table, &mut self.immediates)?; + let targets = &self.immediates[..]; + if targets + .iter() + .all(|&target| u32::from(target) == default_target) + { + // Case: all targets are the same and thus the `br_table` is equal to a `br`. + return self.visit_br(default_target); + } + // Note: The Wasm spec mandates that all `br_table` targets manipulate the + // Wasm value stack the same. This implies for Wasmi that all `br_table` + // targets have the same branch parameter arity. + let Ok(default_target) = usize::try_from(default_target) else { + panic!("out of bounds `default_target` does not fit into `usize`: {default_target}"); + }; + let index = self.layout.operand_to_reg(index)?; + let len_branch_params = self + .stack + .peek_control(default_target) + .len_branch_params(&self.engine); + match len_branch_params { + 0 => self.encode_br_table_0(table, index)?, + n => self.encode_br_table_n(table, index, n)?, + }; + self.reachable = false; + Ok(()) } fn visit_return(&mut self) -> Self::Output { From 54dbdda76e906dde5b8317219b613799896efa36 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Tue, 15 Jul 2025 23:13:49 +0200 Subject: [PATCH 298/343] add codegen for reinterpretation with local operands --- crates/wasmi/src/engine/translator/func2/mod.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index a5c54cb62c..c5dd64b55c 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -1329,7 +1329,13 @@ impl FuncTranslator { match self.stack.pop() { Operand::Local(input) => { debug_assert_eq!(input.ty(), ::TY); - todo!() // do we need a copy or should we allow to manipulate a local's type? + // TODO: improve performance by allowing type overwrites for local operands + let input = self.layout.local_to_reg(input.local_index())?; + self.push_instr_with_result( + ::TY, + |result| Instruction::copy(result, input), + FuelCostsProvider::base, + )?; } Operand::Temp(input) => { debug_assert_eq!(input.ty(), ::TY); From a04f85647576c566c84557de7e3ef448837f1651 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Tue, 15 Jul 2025 23:14:24 +0200 Subject: [PATCH 299/343] add ref.{func,null,is_null} operands translation --- .../src/engine/translator/func2/visit.rs | 64 ++++++++++++++++--- 1 file changed, 55 insertions(+), 9 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index a3fa77cb7d..d1fc38863e 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -9,11 +9,11 @@ use crate::{ }, BlockType, }, - ir, - ir::{Const16, Instruction}, - module, - module::{FuncIdx, WasmiValueType}, + ir::{self, Const16, Instruction}, + module::{self, FuncIdx, WasmiValueType}, Error, + ExternRef, + FuncRef, FuncType, }; use wasmparser::VisitOperator; @@ -1457,16 +1457,62 @@ impl<'a> VisitOperator<'a> for FuncTranslator { self.translate_select(Some(type_hint)) } - fn visit_ref_null(&mut self, _hty: wasmparser::HeapType) -> Self::Output { - todo!() + fn visit_ref_null(&mut self, ty: wasmparser::HeapType) -> Self::Output { + bail_unreachable!(self); + let type_hint = WasmiValueType::from(ty).into_inner(); + let null = match type_hint { + ValType::FuncRef => TypedVal::from(FuncRef::null()), + ValType::ExternRef => TypedVal::from(ExternRef::null()), + ty => panic!("expected a Wasm `reftype` but found: {ty:?}"), + }; + self.stack.push_immediate(null)?; + Ok(()) } fn visit_ref_is_null(&mut self) -> Self::Output { - todo!() + bail_unreachable!(self); + match self.stack.pop() { + Operand::Local(input) => { + // Note: `funcref` and `externref` both serialize to `UntypedValue` + // as `u64` so we can use `i64.eqz` translation for `ref.is_null` + // via reinterpretation of the value's type. + let input = self.layout.local_to_reg(input.local_index())?; + // TODO: improve performance by allowing type overwrites for local operands + self.push_instr_with_result( + ValType::I64, + |result| Instruction::copy(result, input), + FuelCostsProvider::base, + )?; + self.visit_i64_eqz() + } + Operand::Temp(input) => { + // Note: `funcref` and `externref` both serialize to `UntypedValue` + // as `u64` so we can use `i64.eqz` translation for `ref.is_null` + // via reinterpretation of the value's type. + self.stack.push_temp(ValType::I64, input.instr())?; + self.visit_i64_eqz() + } + Operand::Immediate(input) => { + let untyped = input.val().untyped(); + let is_null = match input.ty() { + ValType::FuncRef => FuncRef::from(untyped).is_null(), + ValType::ExternRef => ExternRef::from(untyped).is_null(), + invalid => panic!("`ref.is_null`: encountered invalid input type: {invalid:?}"), + }; + self.stack.push_immediate(i32::from(is_null))?; + Ok(()) + } + } } - fn visit_ref_func(&mut self, _function_index: u32) -> Self::Output { - todo!() + fn visit_ref_func(&mut self, function_index: u32) -> Self::Output { + bail_unreachable!(self); + self.push_instr_with_result( + ValType::FuncRef, + |result| Instruction::ref_func(result, function_index), + FuelCostsProvider::instance, + )?; + Ok(()) } fn visit_table_fill(&mut self, _table: u32) -> Self::Output { From 4b4f60be2a10b0e9289d5d0d070a22f6c69bf207 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Tue, 15 Jul 2025 23:35:13 +0200 Subject: [PATCH 300/343] fix frame_size calculation --- crates/wasmi/src/engine/translator/func2/layout/mod.rs | 5 +++++ crates/wasmi/src/engine/translator/func2/mod.rs | 8 ++++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/layout/mod.rs b/crates/wasmi/src/engine/translator/func2/layout/mod.rs index 6385f20bf7..717b851788 100644 --- a/crates/wasmi/src/engine/translator/func2/layout/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/layout/mod.rs @@ -125,6 +125,11 @@ impl StackLayout { pub fn consts(&self) -> ConstRegistryIter<'_> { self.consts.iter() } + + /// Returns the number of registered locals. + pub fn len_locals(&self) -> usize { + self.len_locals + } } impl AllocConst for StackLayout { diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index c5dd64b55c..046e940475 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -180,10 +180,10 @@ impl WasmTranslator<'_> for FuncTranslator { let Ok(max_height) = u16::try_from(self.stack.max_height()) else { return Err(Error::from(TranslationError::AllocatedTooManyRegisters)); }; - let Some(frame_size) = self - .func_type_with(FuncType::len_params) - .checked_add(max_height) - else { + let Ok(len_locals) = u16::try_from(self.layout.len_locals()) else { + return Err(Error::from(TranslationError::AllocatedTooManyRegisters)); + }; + let Some(frame_size) = len_locals.checked_add(max_height) else { return Err(Error::from(TranslationError::AllocatedTooManyRegisters)); }; self.update_branch_offsets()?; From 76fc88662ac0984384e6decbb670379c886a13a1 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 16 Jul 2025 11:41:16 +0200 Subject: [PATCH 301/343] simplify frame_size calculation --- crates/wasmi/src/engine/translator/func2/mod.rs | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index 046e940475..51e085f73b 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -177,13 +177,12 @@ impl WasmTranslator<'_> for FuncTranslator { mut self, finalize: impl FnOnce(CompiledFuncEntity), ) -> Result { - let Ok(max_height) = u16::try_from(self.stack.max_height()) else { - return Err(Error::from(TranslationError::AllocatedTooManyRegisters)); - }; - let Ok(len_locals) = u16::try_from(self.layout.len_locals()) else { - return Err(Error::from(TranslationError::AllocatedTooManyRegisters)); - }; - let Some(frame_size) = len_locals.checked_add(max_height) else { + let Some(frame_size) = self + .stack + .max_height() + .checked_add(self.layout.len_locals()) + .and_then(|x| u16::try_from(x).ok()) + else { return Err(Error::from(TranslationError::AllocatedTooManyRegisters)); }; self.update_branch_offsets()?; From 44e8340f21aef0d56e96d0a731a5506052480fc6 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 16 Jul 2025 11:57:39 +0200 Subject: [PATCH 302/343] implement translation of Wasm load operators --- .../wasmi/src/engine/translator/func2/mod.rs | 129 +++++++++++++++- .../src/engine/translator/func2/visit.rs | 140 ++++++++++++++---- 2 files changed, 239 insertions(+), 30 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index 51e085f73b..6e05bd6947 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -51,6 +51,9 @@ use crate::{ TranslationError, }, ir::{ + index, + Address, + Address32, BoundedRegSpan, BranchOffset, BranchOffset16, @@ -60,18 +63,21 @@ use crate::{ Const32, Instruction, IntoShiftAmount, + Offset16, + Offset64, + Offset64Lo, Reg, RegSpan, Sign, }, - module::{FuncIdx, FuncTypeIdx, ModuleHeader, TableIdx, WasmiValueType}, + module::{FuncIdx, FuncTypeIdx, MemoryIdx, ModuleHeader, TableIdx, WasmiValueType}, Engine, Error, FuncType, }; use alloc::vec::Vec; use core::mem; -use wasmparser::WasmFeatures; +use wasmparser::{MemArg, WasmFeatures}; /// Type concerned with translating from Wasm bytecode to Wasmi bytecode. #[derive(Debug)] @@ -1982,4 +1988,123 @@ impl FuncTranslator { self.stack.push_operand(lhs)?; Ok(true) } + + /// Translates a Wasm `load` instruction to Wasmi bytecode. + /// + /// # Note + /// + /// This chooses the right encoding for the given `load` instruction. + /// If `ptr+offset` is a constant value the address is pre-calculated. + /// + /// # Usage + /// + /// Used for translating the following Wasm operators to Wasmi bytecode: + /// + /// - `{i32, i64, f32, f64}.load` + /// - `i32.{load8_s, load8_u, load16_s, load16_u}` + /// - `i64.{load8_s, load8_u, load16_s, load16_u load32_s, load32_u}` + fn translate_load( + &mut self, + memarg: MemArg, + loaded_ty: ValType, + make_instr: fn(result: Reg, offset_lo: Offset64Lo) -> Instruction, + make_instr_offset16: fn(result: Reg, ptr: Reg, offset: Offset16) -> Instruction, + make_instr_at: fn(result: Reg, address: Address32) -> Instruction, + ) -> Result<(), Error> { + bail_unreachable!(self); + let (memory, offset) = Self::decode_memarg(memarg); + let ptr = self.stack.pop(); + let (ptr, offset) = match ptr { + Operand::Immediate(ptr) => { + let ptr = ptr.val(); + let Some(address) = self.effective_address(memory, ptr, offset) else { + return self.translate_trap(TrapCode::MemoryOutOfBounds); + }; + if let Ok(address) = Address32::try_from(address) { + self.push_instr_with_result( + loaded_ty, + |result| make_instr_at(result, address), + FuelCostsProvider::load, + )?; + if !memory.is_default() { + self.instrs.push_param(Instruction::memory_index(memory)); + } + return Ok(()); + } + // Case: we cannot use specialized encoding and thus have to fall back + // to the general case where `ptr` is zero and `offset` stores the + // `ptr+offset` address value. + let zero_ptr = self.layout.const_to_reg(0_u64)?; + (zero_ptr, u64::from(address)) + } + ptr => { + let ptr = self.layout.operand_to_reg(ptr)?; + (ptr, offset) + } + }; + if memory.is_default() { + if let Ok(offset) = Offset16::try_from(offset) { + self.push_instr_with_result( + loaded_ty, + |result| make_instr_offset16(result, ptr, offset), + FuelCostsProvider::load, + )?; + return Ok(()); + } + } + let (offset_hi, offset_lo) = Offset64::split(offset); + self.push_instr_with_result( + loaded_ty, + |result| make_instr(result, offset_lo), + FuelCostsProvider::load, + )?; + self.instrs + .push_param(Instruction::register_and_offset_hi(ptr, offset_hi)); + if !memory.is_default() { + self.instrs.push_param(Instruction::memory_index(memory)); + } + Ok(()) + } + + /// Returns the [`MemArg`] linear `memory` index and load/store `offset`. + /// + /// # Panics + /// + /// If the [`MemArg`] offset is not 32-bit. + fn decode_memarg(memarg: MemArg) -> (index::Memory, u64) { + let memory = index::Memory::from(memarg.memory); + (memory, memarg.offset) + } + + /// Returns the effective address `ptr+offset` if it is valid. + fn effective_address(&self, mem: index::Memory, ptr: TypedVal, offset: u64) -> Option
{ + let memory_type = *self + .module + .get_type_of_memory(MemoryIdx::from(u32::from(mem))); + let ptr = match memory_type.is_64() { + true => u64::from(ptr), + false => u64::from(u32::from(ptr)), + }; + let Some(address) = ptr.checked_add(offset) else { + // Case: address overflows any legal memory index. + return None; + }; + if let Some(max) = memory_type.maximum() { + // The memory's maximum size in bytes. + let max_size = max << memory_type.page_size_log2(); + if address > max_size { + // Case: address overflows the memory's maximum size. + return None; + } + } + if !memory_type.is_64() && address >= 1 << 32 { + // Case: address overflows the 32-bit memory index. + return None; + } + let Ok(address) = Address::try_from(address) else { + // Case: address is too big for the system to handle properly. + return None; + }; + Some(address) + } } diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index d1fc38863e..73904ec53e 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -492,60 +492,144 @@ impl<'a> VisitOperator<'a> for FuncTranslator { Ok(()) } - fn visit_i32_load(&mut self, _memarg: wasmparser::MemArg) -> Self::Output { - todo!() + fn visit_i32_load(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + self.translate_load( + memarg, + ValType::I32, + Instruction::load32, + Instruction::load32_offset16, + Instruction::load32_at, + ) } - fn visit_i64_load(&mut self, _memarg: wasmparser::MemArg) -> Self::Output { - todo!() + fn visit_i64_load(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + self.translate_load( + memarg, + ValType::I64, + Instruction::load64, + Instruction::load64_offset16, + Instruction::load64_at, + ) } - fn visit_f32_load(&mut self, _memarg: wasmparser::MemArg) -> Self::Output { - todo!() + fn visit_f32_load(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + self.translate_load( + memarg, + ValType::F32, + Instruction::load32, + Instruction::load32_offset16, + Instruction::load32_at, + ) } - fn visit_f64_load(&mut self, _memarg: wasmparser::MemArg) -> Self::Output { - todo!() + fn visit_f64_load(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + self.translate_load( + memarg, + ValType::F64, + Instruction::load64, + Instruction::load64_offset16, + Instruction::load64_at, + ) } - fn visit_i32_load8_s(&mut self, _memarg: wasmparser::MemArg) -> Self::Output { - todo!() + fn visit_i32_load8_s(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + self.translate_load( + memarg, + ValType::I32, + Instruction::i32_load8_s, + Instruction::i32_load8_s_offset16, + Instruction::i32_load8_s_at, + ) } - fn visit_i32_load8_u(&mut self, _memarg: wasmparser::MemArg) -> Self::Output { - todo!() + fn visit_i32_load8_u(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + self.translate_load( + memarg, + ValType::I32, + Instruction::i32_load8_u, + Instruction::i32_load8_u_offset16, + Instruction::i32_load8_u_at, + ) } - fn visit_i32_load16_s(&mut self, _memarg: wasmparser::MemArg) -> Self::Output { - todo!() + fn visit_i32_load16_s(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + self.translate_load( + memarg, + ValType::I32, + Instruction::i32_load16_s, + Instruction::i32_load16_s_offset16, + Instruction::i32_load16_s_at, + ) } - fn visit_i32_load16_u(&mut self, _memarg: wasmparser::MemArg) -> Self::Output { - todo!() + fn visit_i32_load16_u(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + self.translate_load( + memarg, + ValType::I32, + Instruction::i32_load16_u, + Instruction::i32_load16_u_offset16, + Instruction::i32_load16_u_at, + ) } - fn visit_i64_load8_s(&mut self, _memarg: wasmparser::MemArg) -> Self::Output { - todo!() + fn visit_i64_load8_s(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + self.translate_load( + memarg, + ValType::I64, + Instruction::i64_load8_s, + Instruction::i64_load8_s_offset16, + Instruction::i64_load8_s_at, + ) } - fn visit_i64_load8_u(&mut self, _memarg: wasmparser::MemArg) -> Self::Output { - todo!() + fn visit_i64_load8_u(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + self.translate_load( + memarg, + ValType::I64, + Instruction::i64_load8_u, + Instruction::i64_load8_u_offset16, + Instruction::i64_load8_u_at, + ) } - fn visit_i64_load16_s(&mut self, _memarg: wasmparser::MemArg) -> Self::Output { - todo!() + fn visit_i64_load16_s(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + self.translate_load( + memarg, + ValType::I64, + Instruction::i64_load16_s, + Instruction::i64_load16_s_offset16, + Instruction::i64_load16_s_at, + ) } - fn visit_i64_load16_u(&mut self, _memarg: wasmparser::MemArg) -> Self::Output { - todo!() + fn visit_i64_load16_u(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + self.translate_load( + memarg, + ValType::I64, + Instruction::i64_load16_u, + Instruction::i64_load16_u_offset16, + Instruction::i64_load16_u_at, + ) } - fn visit_i64_load32_s(&mut self, _memarg: wasmparser::MemArg) -> Self::Output { - todo!() + fn visit_i64_load32_s(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + self.translate_load( + memarg, + ValType::I64, + Instruction::i64_load32_s, + Instruction::i64_load32_s_offset16, + Instruction::i64_load32_s_at, + ) } - fn visit_i64_load32_u(&mut self, _memarg: wasmparser::MemArg) -> Self::Output { - todo!() + fn visit_i64_load32_u(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + self.translate_load( + memarg, + ValType::I64, + Instruction::i64_load32_u, + Instruction::i64_load32_u_offset16, + Instruction::i64_load32_u_at, + ) } fn visit_i32_store(&mut self, _memarg: wasmparser::MemArg) -> Self::Output { From a2fa548afa43e05b7d0e36cd791d37e5edb1139c Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 16 Jul 2025 13:40:23 +0200 Subject: [PATCH 303/343] implement Wasm integer store operators translation --- .../wasmi/src/engine/translator/func2/mod.rs | 188 +++++++++++++++++- .../wasmi/src/engine/translator/func2/op.rs | 160 +++++++++++++++ .../src/engine/translator/func2/visit.rs | 29 +-- 3 files changed, 362 insertions(+), 15 deletions(-) create mode 100644 crates/wasmi/src/engine/translator/func2/op.rs diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index 6e05bd6947..2f5b5284c5 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -4,6 +4,7 @@ mod utils; mod instrs; mod layout; +mod op; #[cfg(feature = "simd")] mod simd; mod stack; @@ -43,7 +44,7 @@ use crate::{ TryIntoCmpBranchInstr as _, }, labels::{LabelRef, LabelRegistry}, - utils::{Instr, WasmFloat, WasmInteger}, + utils::{Instr, WasmFloat, WasmInteger, Wrap}, WasmTranslator, }, BlockType, @@ -54,6 +55,7 @@ use crate::{ index, Address, Address32, + AnyConst16, BoundedRegSpan, BranchOffset, BranchOffset16, @@ -2066,6 +2068,190 @@ impl FuncTranslator { Ok(()) } + /// Translates Wasm integer `store` and `storeN` instructions to Wasmi bytecode. + /// + /// # Note + /// + /// This chooses the most efficient encoding for the given `store` instruction. + /// If `ptr+offset` is a constant value the pointer address is pre-calculated. + /// + /// # Usage + /// + /// Used for translating the following Wasm operators to Wasmi bytecode: + /// + /// - `{i32, i64}.{store, store8, store16, store32}` + fn translate_istore_wrap( + &mut self, + memarg: MemArg, + ) -> Result<(), Error> + where + T::Value: Copy + Wrap + From, + T::Param: TryFrom + Into, + { + bail_unreachable!(self); + let (ptr, value) = self.stack.pop2(); + self.encode_istore_wrap::(memarg, ptr, value) + } + + /// Encodes Wasm integer `store` and `storeN` instructions as Wasmi bytecode. + fn encode_istore_wrap( + &mut self, + memarg: MemArg, + ptr: Operand, + value: Operand, + ) -> Result<(), Error> + where + T::Value: Copy + Wrap + From, + T::Param: TryFrom + Into, + { + let (memory, offset) = Self::decode_memarg(memarg); + let (ptr, offset) = match ptr { + Operand::Immediate(ptr) => { + let ptr = ptr.val(); + let Some(address) = self.effective_address(memory, ptr, offset) else { + return self.translate_trap(TrapCode::MemoryOutOfBounds); + }; + if let Ok(address) = Address32::try_from(address) { + return self.encode_istore_wrap_at::(memory, address, value); + } + // Case: we cannot use specialized encoding and thus have to fall back + // to the general case where `ptr` is zero and `offset` stores the + // `ptr+offset` address value. + let zero_ptr = self.layout.const_to_reg(0_u64)?; + (zero_ptr, u64::from(address)) + } + ptr => { + let ptr = self.layout.operand_to_reg(ptr)?; + (ptr, offset) + } + }; + if memory.is_default() { + if let Some(_instr) = self.encode_istore_wrap_mem0::(ptr, offset, value)? { + return Ok(()); + } + } + let (offset_hi, offset_lo) = Offset64::split(offset); + let (instr, param) = { + match value { + Operand::Immediate(value) => { + let value = value.val(); + match T::Param::try_from(T::Value::from(value).wrap()).ok() { + Some(value) => ( + T::store_imm(ptr, offset_lo), + Instruction::imm16_and_offset_hi(value, offset_hi), + ), + None => ( + T::store(ptr, offset_lo), + Instruction::register_and_offset_hi( + self.layout.const_to_reg(value)?, + offset_hi, + ), + ), + } + } + value => { + let value = self.layout.operand_to_reg(value)?; + ( + T::store(ptr, offset_lo), + Instruction::register_and_offset_hi(value, offset_hi), + ) + } + } + }; + self.push_instr(instr, FuelCostsProvider::store)?; + self.instrs.push_param(param); + if !memory.is_default() { + self.instrs.push_param(Instruction::memory_index(memory)); + } + Ok(()) + } + + /// Encodes a Wasm integer `store` and `storeN` instructions as Wasmi bytecode. + /// + /// # Note + /// + /// This is used in cases where the `ptr` is a known constant value. + fn encode_istore_wrap_at( + &mut self, + memory: index::Memory, + address: Address32, + value: Operand, + ) -> Result<(), Error> + where + T::Value: Copy + From + Wrap, + T::Param: TryFrom, + { + match value { + Operand::Immediate(value) => { + let value = value.val(); + let wrapped = T::Value::from(value).wrap(); + if let Ok(value) = T::Param::try_from(wrapped) { + self.push_instr(T::store_at_imm(value, address), FuelCostsProvider::store)?; + } else { + let value = self.layout.const_to_reg(value)?; + self.push_instr(T::store_at(value, address), FuelCostsProvider::store)?; + } + } + value => { + let value = self.layout.operand_to_reg(value)?; + self.push_instr(T::store_at(value, address), FuelCostsProvider::store)?; + } + } + if !memory.is_default() { + self.instrs.push_param(Instruction::memory_index(memory)); + } + Ok(()) + } + + /// Encodes a Wasm integer `store` and `storeN` instructions as Wasmi bytecode. + /// + /// # Note + /// + /// This optimizes for cases where the Wasm linear memory that is operated on is known + /// to be the default memory. + /// Returns `Some` in case the optimized instructions have been encoded. + fn encode_istore_wrap_mem0( + &mut self, + ptr: Reg, + offset: u64, + value: Operand, + ) -> Result, Error> + where + T::Value: Copy + From + Wrap, + T::Param: TryFrom, + { + let Ok(offset16) = Offset16::try_from(offset) else { + return Ok(None); + }; + let instr = match value { + Operand::Immediate(value) => { + let value = value.val(); + let wrapped = T::Value::from(value).wrap(); + match T::Param::try_from(wrapped) { + Ok(value) => self.push_instr( + T::store_offset16_imm(ptr, offset16, value), + FuelCostsProvider::store, + )?, + Err(_) => { + let value = self.layout.const_to_reg(value)?; + self.push_instr( + T::store_offset16(ptr, offset16, value), + FuelCostsProvider::store, + )? + } + } + } + value => { + let value = self.layout.operand_to_reg(value)?; + self.push_instr( + T::store_offset16(ptr, offset16, value), + FuelCostsProvider::store, + )? + } + }; + Ok(Some(instr)) + } + /// Returns the [`MemArg`] linear `memory` index and load/store `offset`. /// /// # Panics diff --git a/crates/wasmi/src/engine/translator/func2/op.rs b/crates/wasmi/src/engine/translator/func2/op.rs new file mode 100644 index 0000000000..2fa817e748 --- /dev/null +++ b/crates/wasmi/src/engine/translator/func2/op.rs @@ -0,0 +1,160 @@ +use crate::ir::{Address32, Instruction, Offset16, Offset64Lo, Reg}; + +/// Trait implemented by all Wasm operators that can be translated as wrapping store instructions. +pub trait StoreWrapOperator { + /// The type of the value to the stored. + type Value; + /// The type of the wrapped value. + type Wrapped; + /// The type of the value as (at most) 16-bit encoded instruction parameter. + type Param; + + fn store(ptr: Reg, offset_lo: Offset64Lo) -> Instruction; + fn store_imm(ptr: Reg, offset_lo: Offset64Lo) -> Instruction; + fn store_offset16(ptr: Reg, offset: Offset16, value: Reg) -> Instruction; + fn store_offset16_imm(ptr: Reg, offset: Offset16, value: Self::Param) -> Instruction; + fn store_at(value: Reg, address: Address32) -> Instruction; + fn store_at_imm(value: Self::Param, address: Address32) -> Instruction; +} + +macro_rules! impl_store_wrap { + ( $( + impl StoreWrapOperator for $name:ident { + type Value = $value_ty:ty; + type Wrapped = $wrapped_ty:ty; + type Param = $param_ty:ty; + + fn store = $store:expr; + fn store_imm = $store_imm:expr; + fn store_offset16 = $store_offset16:expr; + fn store_offset16_imm = $store_offset16_imm:expr; + fn store_at = $store_at:expr; + fn store_at_imm = $store_at_imm:expr; + } + )* ) => { + $( + pub enum $name {} + impl StoreWrapOperator for $name { + type Value = $value_ty; + type Wrapped = $wrapped_ty; + type Param = $param_ty; + + fn store(ptr: Reg, offset_lo: Offset64Lo) -> Instruction { + $store(ptr, offset_lo) + } + + fn store_imm(ptr: Reg, offset_lo: Offset64Lo) -> Instruction { + $store_imm(ptr, offset_lo) + } + + fn store_offset16(ptr: Reg, offset: Offset16, value: Reg) -> Instruction { + $store_offset16(ptr, offset, value) + } + + fn store_offset16_imm(ptr: Reg, offset: Offset16, value: Self::Param) -> Instruction { + $store_offset16_imm(ptr, offset, value) + } + + fn store_at(value: Reg, address: Address32) -> Instruction { + $store_at(value, address) + } + + fn store_at_imm(value: Self::Param, address: Address32) -> Instruction { + $store_at_imm(value, address) + } + } + )* + }; +} +impl_store_wrap! { + impl StoreWrapOperator for I32Store { + type Value = i32; + type Wrapped = i32; + type Param = i16; + + fn store = Instruction::store32; + fn store_imm = Instruction::i32_store_imm16; + fn store_offset16 = Instruction::store32_offset16; + fn store_offset16_imm = Instruction::i32_store_offset16_imm16; + fn store_at = Instruction::store32_at; + fn store_at_imm = Instruction::i32_store_at_imm16; + } + + impl StoreWrapOperator for I64Store { + type Value = i64; + type Wrapped = i64; + type Param = i16; + + fn store = Instruction::store64; + fn store_imm = Instruction::i64_store_imm16; + fn store_offset16 = Instruction::store64_offset16; + fn store_offset16_imm = Instruction::i64_store_offset16_imm16; + fn store_at = Instruction::store64_at; + fn store_at_imm = Instruction::i64_store_at_imm16; + } + + impl StoreWrapOperator for I32Store8 { + type Value = i32; + type Wrapped = i8; + type Param = i8; + + fn store = Instruction::i32_store8; + fn store_imm = Instruction::i32_store8_imm; + fn store_offset16 = Instruction::i32_store8_offset16; + fn store_offset16_imm = Instruction::i32_store8_offset16_imm; + fn store_at = Instruction::i32_store8_at; + fn store_at_imm = Instruction::i32_store8_at_imm; + } + + impl StoreWrapOperator for I32Store16 { + type Value = i32; + type Wrapped = i16; + type Param = i16; + + fn store = Instruction::i32_store16; + fn store_imm = Instruction::i32_store16_imm; + fn store_offset16 = Instruction::i32_store16_offset16; + fn store_offset16_imm = Instruction::i32_store16_offset16_imm; + fn store_at = Instruction::i32_store16_at; + fn store_at_imm = Instruction::i32_store16_at_imm; + } + + impl StoreWrapOperator for I64Store8 { + type Value = i64; + type Wrapped = i8; + type Param = i8; + + fn store = Instruction::i64_store8; + fn store_imm = Instruction::i64_store8_imm; + fn store_offset16 = Instruction::i64_store8_offset16; + fn store_offset16_imm = Instruction::i64_store8_offset16_imm; + fn store_at = Instruction::i64_store8_at; + fn store_at_imm = Instruction::i64_store8_at_imm; + } + + impl StoreWrapOperator for I64Store16 { + type Value = i64; + type Wrapped = i16; + type Param = i16; + + fn store = Instruction::i64_store16; + fn store_imm = Instruction::i64_store16_imm; + fn store_offset16 = Instruction::i64_store16_offset16; + fn store_offset16_imm = Instruction::i64_store16_offset16_imm; + fn store_at = Instruction::i64_store16_at; + fn store_at_imm = Instruction::i64_store16_at_imm; + } + + impl StoreWrapOperator for I64Store32 { + type Value = i64; + type Wrapped = i32; + type Param = i16; + + fn store = Instruction::i64_store32; + fn store_imm = Instruction::i64_store32_imm16; + fn store_offset16 = Instruction::i64_store32_offset16; + fn store_offset16_imm = Instruction::i64_store32_offset16_imm16; + fn store_at = Instruction::i64_store32_at; + fn store_at_imm = Instruction::i64_store32_at_imm16; + } +} diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index 73904ec53e..ed11232ece 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -3,6 +3,7 @@ use crate::{ core::{wasm, FuelCostsProvider, Mutability, TrapCode, TypedVal, ValType, F32, F64}, engine::{ translator::func2::{ + op, stack::{AcquiredTarget, IfReachability}, ControlFrameBase, Operand, @@ -632,12 +633,12 @@ impl<'a> VisitOperator<'a> for FuncTranslator { ) } - fn visit_i32_store(&mut self, _memarg: wasmparser::MemArg) -> Self::Output { - todo!() + fn visit_i32_store(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + self.translate_istore_wrap::(memarg) } - fn visit_i64_store(&mut self, _memarg: wasmparser::MemArg) -> Self::Output { - todo!() + fn visit_i64_store(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + self.translate_istore_wrap::(memarg) } fn visit_f32_store(&mut self, _memarg: wasmparser::MemArg) -> Self::Output { @@ -648,24 +649,24 @@ impl<'a> VisitOperator<'a> for FuncTranslator { todo!() } - fn visit_i32_store8(&mut self, _memarg: wasmparser::MemArg) -> Self::Output { - todo!() + fn visit_i32_store8(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + self.translate_istore_wrap::(memarg) } - fn visit_i32_store16(&mut self, _memarg: wasmparser::MemArg) -> Self::Output { - todo!() + fn visit_i32_store16(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + self.translate_istore_wrap::(memarg) } - fn visit_i64_store8(&mut self, _memarg: wasmparser::MemArg) -> Self::Output { - todo!() + fn visit_i64_store8(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + self.translate_istore_wrap::(memarg) } - fn visit_i64_store16(&mut self, _memarg: wasmparser::MemArg) -> Self::Output { - todo!() + fn visit_i64_store16(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + self.translate_istore_wrap::(memarg) } - fn visit_i64_store32(&mut self, _memarg: wasmparser::MemArg) -> Self::Output { - todo!() + fn visit_i64_store32(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + self.translate_istore_wrap::(memarg) } fn visit_memory_size(&mut self, _mem: u32) -> Self::Output { From f8403d403aba671dc0297aaafad8b47da92d4cee Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 16 Jul 2025 13:50:08 +0200 Subject: [PATCH 304/343] implement translation for storing f{32,64} types --- .../wasmi/src/engine/translator/func2/mod.rs | 75 +++++++++++++++++++ .../src/engine/translator/func2/visit.rs | 18 ++++- 2 files changed, 89 insertions(+), 4 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index 2f5b5284c5..de53705718 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -2252,6 +2252,81 @@ impl FuncTranslator { Ok(Some(instr)) } + /// Translates a general Wasm `store` instruction to Wasmi bytecode. + /// + /// # Note + /// + /// This chooses the most efficient encoding for the given `store` instruction. + /// If `ptr+offset` is a constant value the pointer address is pre-calculated. + /// + /// # Usage + /// + /// Used for translating the following Wasm operators to Wasmi bytecode: + /// + /// - `{f32, f64, v128}.store` + fn translate_store( + &mut self, + memarg: MemArg, + store: fn(ptr: Reg, offset_lo: Offset64Lo) -> Instruction, + store_offset16: fn(ptr: Reg, offset: Offset16, value: Reg) -> Instruction, + store_at: fn(value: Reg, address: Address32) -> Instruction, + ) -> Result<(), Error> { + bail_unreachable!(self); + let (memory, offset) = Self::decode_memarg(memarg); + let (ptr, value) = self.stack.pop2(); + let (ptr, offset) = match ptr { + Operand::Immediate(ptr) => { + let Some(address) = self.effective_address(memory, ptr.val(), offset) else { + return self.translate_trap(TrapCode::MemoryOutOfBounds); + }; + if let Ok(address) = Address32::try_from(address) { + return self.encode_fstore_at(memory, address, value, store_at); + } + let zero_ptr = self.layout.const_to_reg(0_u64)?; + (zero_ptr, u64::from(address)) + } + ptr => { + let ptr = self.layout.operand_to_reg(ptr)?; + (ptr, offset) + } + }; + let (offset_hi, offset_lo) = Offset64::split(offset); + let value = self.layout.operand_to_reg(value)?; + if memory.is_default() { + if let Ok(offset) = Offset16::try_from(offset) { + self.push_instr(store_offset16(ptr, offset, value), FuelCostsProvider::store)?; + return Ok(()); + } + } + self.push_instr(store(ptr, offset_lo), FuelCostsProvider::store)?; + self.instrs + .push_param(Instruction::register_and_offset_hi(value, offset_hi)); + if !memory.is_default() { + self.instrs.push_param(Instruction::memory_index(memory)); + } + Ok(()) + } + + /// Encodes a Wasm `store` instruction with immediate address as Wasmi bytecode. + /// + /// # Note + /// + /// This is used in cases where the `ptr` is a known constant value. + fn encode_fstore_at( + &mut self, + memory: index::Memory, + address: Address32, + value: Operand, + make_instr_at: fn(value: Reg, address: Address32) -> Instruction, + ) -> Result<(), Error> { + let value = self.layout.operand_to_reg(value)?; + self.push_instr(make_instr_at(value, address), FuelCostsProvider::store)?; + if !memory.is_default() { + self.instrs.push_param(Instruction::memory_index(memory)); + } + Ok(()) + } + /// Returns the [`MemArg`] linear `memory` index and load/store `offset`. /// /// # Panics diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index ed11232ece..b1fb3c7932 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -641,12 +641,22 @@ impl<'a> VisitOperator<'a> for FuncTranslator { self.translate_istore_wrap::(memarg) } - fn visit_f32_store(&mut self, _memarg: wasmparser::MemArg) -> Self::Output { - todo!() + fn visit_f32_store(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + self.translate_store( + memarg, + Instruction::store32, + Instruction::store32_offset16, + Instruction::store32_at, + ) } - fn visit_f64_store(&mut self, _memarg: wasmparser::MemArg) -> Self::Output { - todo!() + fn visit_f64_store(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + self.translate_store( + memarg, + Instruction::store64, + Instruction::store64_offset16, + Instruction::store64_at, + ) } fn visit_i32_store8(&mut self, memarg: wasmparser::MemArg) -> Self::Output { From ebd6579c65c2986dbf78e87b7bc59425b5fc1597 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 16 Jul 2025 16:33:24 +0200 Subject: [PATCH 305/343] consider function local constants in frame size calculation --- crates/wasmi/src/engine/translator/func2/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index de53705718..a27cc67e94 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -189,6 +189,7 @@ impl WasmTranslator<'_> for FuncTranslator { .stack .max_height() .checked_add(self.layout.len_locals()) + .and_then(|frame_size| frame_size.checked_add(self.layout.consts().len())) .and_then(|x| u16::try_from(x).ok()) else { return Err(Error::from(TranslationError::AllocatedTooManyRegisters)); From aa6f650ee9cc30ab4e600c4608f5c9e190721da8 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 16 Jul 2025 17:54:12 +0200 Subject: [PATCH 306/343] put frame_size calculation into its own method --- .../wasmi/src/engine/translator/func2/mod.rs | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index a27cc67e94..b1b759ca0c 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -185,13 +185,7 @@ impl WasmTranslator<'_> for FuncTranslator { mut self, finalize: impl FnOnce(CompiledFuncEntity), ) -> Result { - let Some(frame_size) = self - .stack - .max_height() - .checked_add(self.layout.len_locals()) - .and_then(|frame_size| frame_size.checked_add(self.layout.consts().len())) - .and_then(|x| u16::try_from(x).ok()) - else { + let Some(frame_size) = self.frame_size() else { return Err(Error::from(TranslationError::AllocatedTooManyRegisters)); }; self.update_branch_offsets()?; @@ -285,6 +279,16 @@ impl FuncTranslator { Ok(()) } + /// Returns the frame size of the to-be-compiled function. + /// + /// Returns `None` if the frame size is out of bounds. + fn frame_size(&self) -> Option { + let frame_size = self.stack.max_height() + .checked_add(self.layout.len_locals())? + .checked_add(self.layout.consts().len())?; + u16::try_from(frame_size).ok() + } + /// Updates the branch offsets of all branch instructions inplace. /// /// # Panics From 0945419ed57f6d0ed1d718889499ffe5c53bf5c7 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 16 Jul 2025 21:37:31 +0200 Subject: [PATCH 307/343] apply rustfmt --- crates/wasmi/src/engine/translator/func2/mod.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index b1b759ca0c..e268c32c27 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -280,10 +280,12 @@ impl FuncTranslator { } /// Returns the frame size of the to-be-compiled function. - /// + /// /// Returns `None` if the frame size is out of bounds. fn frame_size(&self) -> Option { - let frame_size = self.stack.max_height() + let frame_size = self + .stack + .max_height() .checked_add(self.layout.len_locals())? .checked_add(self.layout.consts().len())?; u16::try_from(frame_size).ok() From 75c5f5b20864f45c7c6dd5dad6f2437d98eab7a7 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 16 Jul 2025 22:43:12 +0200 Subject: [PATCH 308/343] add memory.size operator translation --- .../wasmi/src/engine/translator/func2/visit.rs | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index b1fb3c7932..53c817d672 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -11,7 +11,7 @@ use crate::{ BlockType, }, ir::{self, Const16, Instruction}, - module::{self, FuncIdx, WasmiValueType}, + module::{self, FuncIdx, MemoryIdx, WasmiValueType}, Error, ExternRef, FuncRef, @@ -679,8 +679,19 @@ impl<'a> VisitOperator<'a> for FuncTranslator { self.translate_istore_wrap::(memarg) } - fn visit_memory_size(&mut self, _mem: u32) -> Self::Output { - todo!() + fn visit_memory_size(&mut self, mem: u32) -> Self::Output { + bail_unreachable!(self); + let index_ty = self + .module + .get_type_of_memory(MemoryIdx::from(mem)) + .index_ty() + .ty(); + self.push_instr_with_result( + index_ty, + |result| Instruction::memory_size(result, mem), + FuelCostsProvider::instance, + )?; + Ok(()) } fn visit_memory_grow(&mut self, _mem: u32) -> Self::Output { From 00c229fc661b06dae1ba49a7fe5758b7f7a67a07 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Thu, 17 Jul 2025 00:20:52 +0200 Subject: [PATCH 309/343] add memory.grow operator translation --- .../src/engine/translator/func2/visit.rs | 51 +++++++++++++++++-- 1 file changed, 48 insertions(+), 3 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index 53c817d672..2c2c63c582 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -1,6 +1,6 @@ use super::{ControlFrame, ControlFrameKind, FuncTranslator, LocalIdx}; use crate::{ - core::{wasm, FuelCostsProvider, Mutability, TrapCode, TypedVal, ValType, F32, F64}, + core::{wasm, FuelCostsProvider, IndexType, Mutability, TrapCode, TypedVal, ValType, F32, F64}, engine::{ translator::func2::{ op, @@ -17,6 +17,7 @@ use crate::{ FuncRef, FuncType, }; +use ir::Const32; use wasmparser::VisitOperator; macro_rules! impl_visit_operator { @@ -694,8 +695,52 @@ impl<'a> VisitOperator<'a> for FuncTranslator { Ok(()) } - fn visit_memory_grow(&mut self, _mem: u32) -> Self::Output { - todo!() + fn visit_memory_grow(&mut self, mem: u32) -> Self::Output { + bail_unreachable!(self); + let index_ty = self + .module + .get_type_of_memory(MemoryIdx::from(mem)) + .index_ty(); + let delta = self.stack.pop(); + if let Operand::Immediate(delta) = delta { + let delta = delta.val(); + let delta = match index_ty { + IndexType::I32 => u64::from(u32::from(delta)), + IndexType::I64 => u64::from(delta), + }; + if delta == 0 { + // Case: growing by 0 pages. + // + // Since `memory.grow` returns the `memory.size` before the + // operation a `memory.grow` with `delta` of 0 can be translated + // as `memory.size` instruction instead. + self.push_instr_with_result( + index_ty.ty(), + |result| Instruction::memory_size(result, mem), + FuelCostsProvider::instance, + )?; + return Ok(()); + } + if let Ok(delta) = >::try_from(delta) { + // Case: delta can be 32-bit encoded + self.push_instr_with_result( + index_ty.ty(), + |result| Instruction::memory_grow_imm(result, delta), + FuelCostsProvider::instance, + )?; + self.instrs.push_param(Instruction::memory_index(mem)); + return Ok(()); + } + } + // Case: fallback to generic `memory.grow` instruction + let delta = self.layout.operand_to_reg(delta)?; + self.push_instr_with_result( + index_ty.ty(), + |result| Instruction::memory_grow(result, delta), + FuelCostsProvider::instance, + )?; + self.instrs.push_param(Instruction::memory_index(mem)); + Ok(()) } fn visit_i32_const(&mut self, value: i32) -> Self::Output { From 4eae0f04727e84e05a54b9786438b8a2799047eb Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Thu, 17 Jul 2025 09:50:54 +0200 Subject: [PATCH 310/343] assert that cmp+branch fusion succeed at the end This is given due to prior checks asserting that fusion must succeed at this point. --- crates/wasmi/src/engine/translator/func2/mod.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index e268c32c27..a4100697dc 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -1270,7 +1270,10 @@ impl FuncTranslator { }, }; let offset = self.labels.try_resolve_label(label, instr)?; - cmp_instr.try_into_cmp_branch_instr(offset, &mut self.layout) + let fused = cmp_instr + .try_into_cmp_branch_instr(offset, &mut self.layout)? + .expect("cmp+branch fusion must succeed"); + Ok(Some(fused)) } /// Translates a unary Wasm instruction to Wasmi bytecode. From 70a4f9675ea0bcf2aa3b90ec5e7b1fbc48d40f04 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Thu, 17 Jul 2025 14:20:46 +0200 Subject: [PATCH 311/343] fix panic in try_fuse_branch_cmp The panic occurred because the final try_replace_instr was able to fail after having already registered a label user which never existed. Then in update_branch_offsets an invalid user's label was tried to be updated which obviously failed. --- .../wasmi/src/engine/translator/func2/mod.rs | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index a4100697dc..d89f7458e4 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -1216,18 +1216,35 @@ impl FuncTranslator { label: LabelRef, negate: bool, ) -> Result { + let Some(last_instr) = self.instrs.last_instr() else { + // Case: cannot fuse without a known last instruction + return Ok(false); + }; let Operand::Temp(condition) = condition else { + // Case: cannot fuse non-temporary operands + // - locals have observable behavior. + // - immediates cannot be the result of a previous instruction. return Ok(false); }; - debug_assert!(matches!(condition.ty(), ValType::I32 | ValType::I64)); let Some(origin) = condition.instr() else { + // Case: cannot fuse temporary operands without origin instruction return Ok(false); }; + if last_instr != origin { + // Case: cannot fuse if last instruction does not match origin instruction + return Ok(false); + } + debug_assert!(matches!(condition.ty(), ValType::I32 | ValType::I64)); let fused_instr = self.try_make_fused_branch_cmp_instr(origin, condition, label, negate)?; let Some(fused_instr) = fused_instr else { + // Case: not possible to perform fusion with last instruction return Ok(false); }; - self.instrs.try_replace_instr(origin, fused_instr) + assert!( + self.instrs.try_replace_instr(origin, fused_instr)?, + "op-code fusion must suceed at this point", + ); + Ok(true) } /// Try to return a fused cmp+branch [`Instruction`] from the given parameters. From f7426433fd9c4ea21a4d606b9ee69bad2e826ee1 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Thu, 17 Jul 2025 14:33:32 +0200 Subject: [PATCH 312/343] deduplicate fuse_nez and fuze_eqz implementations --- .../wasmi/src/engine/translator/func2/mod.rs | 99 ++++++++----------- 1 file changed, 41 insertions(+), 58 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index d89f7458e4..ffd1d82359 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -39,8 +39,8 @@ use crate::{ translator::{ comparator::{ CompareResult as _, - LogicalizeCmpInstr as _, - NegateCmpInstr as _, + LogicalizeCmpInstr, + NegateCmpInstr, TryIntoCmpBranchInstr as _, }, labels::{LabelRef, LabelRegistry}, @@ -1928,45 +1928,7 @@ impl FuncTranslator { /// - `Ok(false)` if instruction fusion could not be applied. /// - `Err(_)` if an error occurred. pub fn fuse_eqz(&mut self, lhs: Operand, rhs: T) -> Result { - if !rhs.is_zero() { - // Case: cannot fuse with non-zero `rhs` - return Ok(false); - } - let lhs_reg = match lhs { - Operand::Immediate(_) => { - // Case: const-eval opt should take place instead since both operands are const - return Ok(false); - } - operand => self.layout.operand_to_reg(operand)?, - }; - let Some(last_instr) = self.instrs.last_instr() else { - // Case: cannot fuse without registered last instruction - return Ok(false); - }; - let last_instruction = *self.instrs.get(last_instr); - let Some(result) = last_instruction.compare_result() else { - // Case: cannot fuse non-cmp instructions - return Ok(false); - }; - if matches!(self.layout.stack_space(result), StackSpace::Local) { - // Case: cannot fuse cmp instructions with local result - // Note: local results have observable side effects which must not change - return Ok(false); - } - if result != lhs_reg { - // Case: the `cmp` instruction does not feed into the `eqz` and cannot be fused - return Ok(false); - } - let Some(negated) = last_instruction.negate_cmp_instr() else { - // Case: the `cmp` instruction cannot be negated - return Ok(false); - }; - if !self.instrs.try_replace_instr(last_instr, negated)? { - // Case: could not replace the `cmp` instruction with the fused one - return Ok(false); - } - self.stack.push_operand(lhs)?; - Ok(true) + self.fuse_commutative_cmp_with(lhs, rhs, NegateCmpInstr::negate_cmp_instr) } /// Tries to fuse a Wasm `i32.ne` instruction with 0 `rhs` value. @@ -1977,40 +1939,61 @@ impl FuncTranslator { /// - `Ok(false)` if instruction fusion could not be applied. /// - `Err(_)` if an error occurred. pub fn fuse_nez(&mut self, lhs: Operand, rhs: T) -> Result { + self.fuse_commutative_cmp_with(lhs, rhs, LogicalizeCmpInstr::logicalize_cmp_instr) + } + + /// Tries to fuse a `i{32,64}`.{eq,ne}` instruction with `rhs` of zero. + /// + /// Generically applies `f` onto the fused last instruction. + /// + /// Returns + /// + /// - `Ok(true)` if the intruction fusion was successful. + /// - `Ok(false)` if instruction fusion could not be applied. + /// - `Err(_)` if an error occurred. + pub fn fuse_commutative_cmp_with( + &mut self, + lhs: Operand, + rhs: T, + f: fn(&Instruction) -> Option, + ) -> Result { if !rhs.is_zero() { // Case: cannot fuse with non-zero `rhs` return Ok(false); } - let lhs_reg = match lhs { - Operand::Immediate(_) => { - // Case: const-eval opt should take place instead since both operands are const - return Ok(false); - } - operand => self.layout.operand_to_reg(operand)?, - }; let Some(last_instr) = self.instrs.last_instr() else { // Case: cannot fuse without registered last instruction return Ok(false); }; - let last_instruction = *self.instrs.get(last_instr); - let Some(result) = last_instruction.compare_result() else { - // Case: cannot fuse non-cmp instructions + let Operand::Temp(lhs_opd) = lhs else { + // Case: cannot fuse non-temporary operands + // - locals have observable behavior. + // - immediates cannot be the result of a previous instruction. return Ok(false); }; - if matches!(self.layout.stack_space(result), StackSpace::Local) { - // Case: cannot fuse cmp instructions with local result - // Note: local results have observable side effects which must not change + let Some(origin) = lhs_opd.instr() else { + // Case: `lhs` has no origin instruciton, thus not possible to fuse. + return Ok(false); + }; + if origin != last_instr { + // Case: `lhs`'s origin instruction does not match the last instruction return Ok(false); } + let lhs_reg = self.layout.temp_to_reg(lhs_opd.operand_index())?; + let last_instruction = self.instrs.get(last_instr); + let Some(result) = last_instruction.compare_result() else { + // Case: cannot fuse non-cmp instructions + return Ok(false); + }; if result != lhs_reg { - // Case: the `cmp` instruction does not feed into the `nez` and cannot be fused + // Case: the `cmp` instruction does not feed into the `eqz` and cannot be fused return Ok(false); } - let Some(logicalized) = last_instruction.logicalize_cmp_instr() else { - // Case: the `cmp` instruction cannot be logicalized + let Some(negated) = f(last_instruction) else { + // Case: the `cmp` instruction cannot be negated return Ok(false); }; - if !self.instrs.try_replace_instr(last_instr, logicalized)? { + if !self.instrs.try_replace_instr(last_instr, negated)? { // Case: could not replace the `cmp` instruction with the fused one return Ok(false); } From a3ec2abf7678f10958c43b81bab5a58a590c7b14 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Fri, 18 Jul 2025 00:52:57 +0200 Subject: [PATCH 313/343] fix translation of `if` with return values and no `else` --- .../wasmi/src/engine/translator/func2/mod.rs | 23 +++++++- .../engine/translator/func2/stack/control.rs | 59 +++++++++++-------- .../src/engine/translator/func2/stack/mod.rs | 11 +++- 3 files changed, 64 insertions(+), 29 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index ffd1d82359..602bcf5dd4 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -980,10 +980,27 @@ impl FuncTranslator { FuelCostsProvider::base, )?; } + self.labels + .try_pin_label(else_label, self.instrs.next_instr()); + self.operands.clear(); + self.operands.extend(self.stack.pop_else_providers()); + if has_results { + // We haven't visited the `else` block and thus the `else` + // providers are still on the auxiliary stack and need to + // be popped. We use them to restore the stack to the state + // when entering the `if` block so that we can properly copy + // the `else` results to were they are expected. + let consume_fuel_instr = self.instrs.push_consume_fuel_instr()?; + self.stack.trunc(frame.height()); + for else_operand in self.operands.iter().copied() { + self.stack.push_operand(else_operand)?; + } + self.copy_branch_params(&frame, consume_fuel_instr)?; + } self.push_frame_results(&frame)?; - let next_instr = self.instrs.next_instr(); - self.labels.try_pin_label(else_label, next_instr); - self.labels.pin_label(frame.label(), next_instr).unwrap(); + self.labels + .pin_label(frame.label(), self.instrs.next_instr()) + .unwrap(); self.reachable = true; Ok(()) } diff --git a/crates/wasmi/src/engine/translator/func2/stack/control.rs b/crates/wasmi/src/engine/translator/func2/stack/control.rs index 3e61e92851..a42049996f 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/control.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/control.rs @@ -38,7 +38,13 @@ pub struct ControlStack { /// Special operand stack to memorize operands for `else` control frames. else_operands: ElseOperands, /// This is `true` if an `if` with else providers was just popped from the stack. - expect_else: bool, + /// + /// # Note + /// + /// This means that its associated `else` operands need to be taken care of by + /// either pushing back an `else` control frame or by manually popping them off + /// the control stack. + orphaned_else_operands: bool, } /// Duplicated operands for Wasm `else` control frames. @@ -71,19 +77,13 @@ impl ElseOperands { let start = self.ends.last().copied().unwrap_or(0); Some(self.operands.drain(start..end)) } - - /// Drops the top-most Wasm `else` operands from `self`. - pub fn drop(&mut self) { - self.ends.pop().expect("tried to drop empty else operands"); - let start = self.ends.last().copied().unwrap_or(0); - self.operands.truncate(start); - } } impl Reset for ControlStack { fn reset(&mut self) { self.frames.clear(); self.else_operands.reset(); + self.orphaned_else_operands = false; } } @@ -100,7 +100,7 @@ impl ControlStack { /// Pushes a new unreachable Wasm control frame onto the [`ControlStack`]. pub fn push_unreachable(&mut self, kind: ControlFrameKind) { - self.try_drop_else_providers(); + debug_assert!(!self.orphaned_else_operands); self.frames.push(ControlFrame::from(kind)) } @@ -112,7 +112,7 @@ impl ControlStack { label: LabelRef, consume_fuel: Option, ) { - self.try_drop_else_providers(); + debug_assert!(!self.orphaned_else_operands); self.frames.push(ControlFrame::from(BlockControlFrame { ty, height: StackHeight::from(height), @@ -130,7 +130,7 @@ impl ControlStack { label: LabelRef, consume_fuel: Option, ) { - self.try_drop_else_providers(); + debug_assert!(!self.orphaned_else_operands); self.frames.push(ControlFrame::from(LoopControlFrame { ty, height: StackHeight::from(height), @@ -150,7 +150,7 @@ impl ControlStack { reachability: IfReachability, else_operands: impl IntoIterator, ) { - self.try_drop_else_providers(); + debug_assert!(!self.orphaned_else_operands); self.frames.push(ControlFrame::from(IfControlFrame { ty, height: StackHeight::from(height), @@ -178,14 +178,19 @@ impl ControlStack { let label = if_frame.label(); let is_branched_to = if_frame.is_branched_to(); let reachability = match if_frame.reachability { - IfReachability::Both { .. } => ElseReachability::Both { - is_end_of_then_reachable, - }, + IfReachability::Both { .. } => { + debug_assert!(self.orphaned_else_operands); + self.orphaned_else_operands = false; + ElseReachability::Both { + is_end_of_then_reachable, + } + } IfReachability::OnlyThen => ElseReachability::OnlyThen { is_end_of_then_reachable, }, IfReachability::OnlyElse => ElseReachability::OnlyElse, }; + debug_assert!(!self.orphaned_else_operands); self.frames.push(ControlFrame::from(ElseControlFrame { ty, height: StackHeight::from(height), @@ -194,7 +199,7 @@ impl ControlStack { label, reachability, })); - self.expect_else = false; + self.orphaned_else_operands = false; match reachability { ElseReachability::OnlyThen { .. } | ElseReachability::OnlyElse => None, ElseReachability::Both { .. } => { @@ -209,9 +214,9 @@ impl ControlStack { /// Pops the top-most [`ControlFrame`] and returns it if any. pub fn pop(&mut self) -> Option { - self.try_drop_else_providers(); + debug_assert!(!self.orphaned_else_operands); let frame = self.frames.pop()?; - self.expect_else = match &frame { + self.orphaned_else_operands = match &frame { ControlFrame::If(frame) => { matches!(frame.reachability, IfReachability::Both { .. }) } @@ -220,14 +225,18 @@ impl ControlStack { Some(frame) } - /// Drops the top-most else operands if `expect_else` is `true`. + /// Pops the top-most `else` operands from the control stack. /// - /// Otherwise do nothing. - pub fn try_drop_else_providers(&mut self) { - if self.expect_else { - self.else_operands.drop(); - } - self.expect_else = false; + /// # Panics (Debug) + /// + /// If the `else` operands are not in orphaned state. + pub fn pop_else_providers(&mut self) -> Drain<'_, Operand> { + debug_assert!(self.orphaned_else_operands); + let Some(else_operands) = self.else_operands.pop() else { + panic!("missing `else` operands") + }; + self.orphaned_else_operands = false; + else_operands } /// Returns a shared reference to the [`ControlFrame`] at `depth` if any. diff --git a/crates/wasmi/src/engine/translator/func2/stack/mod.rs b/crates/wasmi/src/engine/translator/func2/stack/mod.rs index c897cde3f3..542c918427 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/mod.rs @@ -35,7 +35,7 @@ use crate::{ Engine, Error, }; -use alloc::vec::Vec; +use alloc::vec::{Drain, Vec}; #[cfg(doc)] use crate::ir::Instruction; @@ -312,6 +312,15 @@ impl Stack { .unwrap_or_else(|| panic!("tried to pop control from empty control stack")) } + /// Pops the top-most `else` operands from the control stack. + /// + /// # Panics (Debug) + /// + /// If the `else` operands are not in orphaned state. + pub fn pop_else_providers(&mut self) -> Drain<'_, Operand> { + self.controls.pop_else_providers() + } + /// Returns a shared reference to the [`ControlFrame`] at `depth`. /// /// # Panics From f879f357fac3c75e7b4642760f668b8f3ab2dc24 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Fri, 18 Jul 2025 13:42:57 +0200 Subject: [PATCH 314/343] add OperandStack::push_operand --- .../src/engine/translator/func2/stack/mod.rs | 20 +++---------------- .../engine/translator/func2/stack/operands.rs | 16 +++++++++++++++ 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/stack/mod.rs b/crates/wasmi/src/engine/translator/func2/stack/mod.rs index 542c918427..b7084d222e 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/mod.rs @@ -274,18 +274,8 @@ impl Stack { self.controls .push_else(if_frame, consume_fuel, is_end_of_then_reachable); if let Some(else_operands) = else_operands { - for operand in else_operands { - match operand { - Operand::Local(op) => { - self.operands.push_local(op.local_index())?; - } - Operand::Temp(op) => { - self.operands.push_temp(op.ty(), op.instr())?; - } - Operand::Immediate(op) => { - self.operands.push_immediate(op.val())?; - } - } + for else_operand in else_operands { + self.operands.push_operand(else_operand)?; } } Ok(()) @@ -354,11 +344,7 @@ impl Stack { /// - If too many operands have been pushed onto the [`Stack`]. /// - If the local with `local_idx` does not exist. pub fn push_operand(&mut self, operand: Operand) -> Result { - match operand { - Operand::Local(operand) => self.push_local(operand.local_index()), - Operand::Temp(operand) => self.push_temp(operand.ty(), operand.instr()), - Operand::Immediate(operand) => self.push_immediate(operand.val()), - } + self.operands.push_operand(operand) } /// Pushes a local variable with index `local_idx` to the [`Stack`]. diff --git a/crates/wasmi/src/engine/translator/func2/stack/operands.rs b/crates/wasmi/src/engine/translator/func2/stack/operands.rs index 1d7cc6eb13..a5d59659bb 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/operands.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/operands.rs @@ -131,6 +131,22 @@ impl OperandStack { OperandIdx::from(self.height() - depth - 1) } + /// Pushes the [`Operand`] back to the [`OperandStack`]. + /// + /// Returns the new [`OperandIdx`]. + /// + /// # Errors + /// + /// - If too many operands have been pushed onto the [`OperandStack`]. + /// - If the local with `local_idx` does not exist. + pub fn push_operand(&mut self, operand: Operand) -> Result { + match operand { + Operand::Local(operand) => self.push_local(operand.local_index()), + Operand::Temp(operand) => self.push_temp(operand.ty(), operand.instr()), + Operand::Immediate(operand) => self.push_immediate(operand.val()), + } + } + /// Pushes a local variable with index `local_idx` to the [`OperandStack`]. /// /// # Errors From 0bee66389cf883a921e39e3fb90ea76553951fdd Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Fri, 18 Jul 2025 13:43:33 +0200 Subject: [PATCH 315/343] refactor Stack `else` operands API --- crates/wasmi/src/engine/translator/func2/mod.rs | 7 +------ .../wasmi/src/engine/translator/func2/stack/mod.rs | 13 +++++++++---- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index 602bcf5dd4..c019a61e69 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -982,8 +982,7 @@ impl FuncTranslator { } self.labels .try_pin_label(else_label, self.instrs.next_instr()); - self.operands.clear(); - self.operands.extend(self.stack.pop_else_providers()); + self.stack.push_else_providers(&frame)?; if has_results { // We haven't visited the `else` block and thus the `else` // providers are still on the auxiliary stack and need to @@ -991,10 +990,6 @@ impl FuncTranslator { // when entering the `if` block so that we can properly copy // the `else` results to were they are expected. let consume_fuel_instr = self.instrs.push_consume_fuel_instr()?; - self.stack.trunc(frame.height()); - for else_operand in self.operands.iter().copied() { - self.stack.push_operand(else_operand)?; - } self.copy_branch_params(&frame, consume_fuel_instr)?; } self.push_frame_results(&frame)?; diff --git a/crates/wasmi/src/engine/translator/func2/stack/mod.rs b/crates/wasmi/src/engine/translator/func2/stack/mod.rs index b7084d222e..2287f05045 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/mod.rs @@ -35,7 +35,7 @@ use crate::{ Engine, Error, }; -use alloc::vec::{Drain, Vec}; +use alloc::vec::Vec; #[cfg(doc)] use crate::ir::Instruction; @@ -270,6 +270,7 @@ impl Stack { consume_fuel: Option, ) -> Result<(), Error> { debug_assert!(self.is_fuel_metering_enabled() == consume_fuel.is_some()); + // self.trunc(if_frame.height()); let else_operands = self.controls .push_else(if_frame, consume_fuel, is_end_of_then_reachable); @@ -302,13 +303,17 @@ impl Stack { .unwrap_or_else(|| panic!("tried to pop control from empty control stack")) } - /// Pops the top-most `else` operands from the control stack. + /// Pushes the top-most `else` operands from the control stack onto the operand stack. /// /// # Panics (Debug) /// /// If the `else` operands are not in orphaned state. - pub fn pop_else_providers(&mut self) -> Drain<'_, Operand> { - self.controls.pop_else_providers() + pub fn push_else_providers(&mut self, frame: &impl ControlFrameBase) -> Result<(), Error> { + self.trunc(frame.height()); + for else_operand in self.controls.pop_else_providers() { + self.operands.push_operand(else_operand)?; + } + Ok(()) } /// Returns a shared reference to the [`ControlFrame`] at `depth`. From 00f9ca665595a41b4db6b8abb9707ddadbd65b1a Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Fri, 18 Jul 2025 13:44:20 +0200 Subject: [PATCH 316/343] rename `else` operands API methods --- crates/wasmi/src/engine/translator/func2/mod.rs | 2 +- crates/wasmi/src/engine/translator/func2/stack/control.rs | 2 +- crates/wasmi/src/engine/translator/func2/stack/mod.rs | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index c019a61e69..38acdaef17 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -982,7 +982,7 @@ impl FuncTranslator { } self.labels .try_pin_label(else_label, self.instrs.next_instr()); - self.stack.push_else_providers(&frame)?; + self.stack.push_else_operands(&frame)?; if has_results { // We haven't visited the `else` block and thus the `else` // providers are still on the auxiliary stack and need to diff --git a/crates/wasmi/src/engine/translator/func2/stack/control.rs b/crates/wasmi/src/engine/translator/func2/stack/control.rs index a42049996f..23f94c4b29 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/control.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/control.rs @@ -230,7 +230,7 @@ impl ControlStack { /// # Panics (Debug) /// /// If the `else` operands are not in orphaned state. - pub fn pop_else_providers(&mut self) -> Drain<'_, Operand> { + pub fn pop_else_operands(&mut self) -> Drain<'_, Operand> { debug_assert!(self.orphaned_else_operands); let Some(else_operands) = self.else_operands.pop() else { panic!("missing `else` operands") diff --git a/crates/wasmi/src/engine/translator/func2/stack/mod.rs b/crates/wasmi/src/engine/translator/func2/stack/mod.rs index 2287f05045..e9212f3d9b 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/mod.rs @@ -308,9 +308,9 @@ impl Stack { /// # Panics (Debug) /// /// If the `else` operands are not in orphaned state. - pub fn push_else_providers(&mut self, frame: &impl ControlFrameBase) -> Result<(), Error> { + pub fn push_else_operands(&mut self, frame: &impl ControlFrameBase) -> Result<(), Error> { self.trunc(frame.height()); - for else_operand in self.controls.pop_else_providers() { + for else_operand in self.controls.pop_else_operands() { self.operands.push_operand(else_operand)?; } Ok(()) From daa5878f88200822c15a87aea4da2c4062108379 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Fri, 18 Jul 2025 14:09:38 +0200 Subject: [PATCH 317/343] remove unnecessary IfControlFrame APIs --- .../engine/translator/func2/stack/control.rs | 20 ------------------- .../src/engine/translator/func2/visit.rs | 3 +-- 2 files changed, 1 insertion(+), 22 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/stack/control.rs b/crates/wasmi/src/engine/translator/func2/stack/control.rs index 23f94c4b29..694bdf32fd 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/control.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/control.rs @@ -623,26 +623,6 @@ impl IfControlFrame { self.reachability } - /// Returns the label to the `else` branch of the [`IfControlFrame`]. - pub fn else_label(&self) -> Option { - match self.reachability { - IfReachability::Both { else_label } => Some(else_label), - IfReachability::OnlyThen | IfReachability::OnlyElse => None, - } - } - - /// Returns `true` if the `then` branch is reachable. - /// - /// # Note - /// - /// The `then` branch is unreachable if the `if` condition is a constant `false` value. - pub fn is_then_reachable(&self) -> bool { - match self.reachability { - IfReachability::Both { .. } | IfReachability::OnlyThen => true, - IfReachability::OnlyElse => false, - } - } - /// Returns `true` if the `else` branch is reachable. /// /// # Note diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index 2c2c63c582..4942a3d9cb 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -167,8 +167,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { // - Copy `if` branch parameters. // - Branch from end of `then` to end of `if`. let is_end_of_then_reachable = self.reachable; - if let Some(else_label) = frame.else_label() { - debug_assert!(frame.is_then_reachable() && frame.is_else_reachable()); + if let IfReachability::Both { else_label } = frame.reachability() { if is_end_of_then_reachable { let consume_fuel_instr = frame.consume_fuel_instr(); self.copy_branch_params(&frame, consume_fuel_instr)?; From cb81cbd1d2041fa7654b82e85ea9fe2a08f769df Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Fri, 18 Jul 2025 14:24:57 +0200 Subject: [PATCH 318/343] remove misplaced stack.trunc call --- crates/wasmi/src/engine/translator/func2/visit.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index 4942a3d9cb..3adfc95045 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -171,7 +171,6 @@ impl<'a> VisitOperator<'a> for FuncTranslator { if is_end_of_then_reachable { let consume_fuel_instr = frame.consume_fuel_instr(); self.copy_branch_params(&frame, consume_fuel_instr)?; - self.stack.trunc(frame.height()); frame.branch_to(); self.encode_br(frame.label())?; } From b6408e6da287919a1aa05cfcc883c66fd08c89cb Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Fri, 18 Jul 2025 14:25:09 +0200 Subject: [PATCH 319/343] refactor how else operands are handled --- .../engine/translator/func2/stack/control.rs | 25 ++++--------------- .../src/engine/translator/func2/stack/mod.rs | 18 ++++++------- 2 files changed, 13 insertions(+), 30 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/stack/control.rs b/crates/wasmi/src/engine/translator/func2/stack/control.rs index 694bdf32fd..6a1db3082f 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/control.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/control.rs @@ -172,25 +172,21 @@ impl ControlStack { if_frame: IfControlFrame, consume_fuel: Option, is_end_of_then_reachable: bool, - ) -> Option> { + ) { + debug_assert!(!self.orphaned_else_operands); let ty = if_frame.ty(); let height = if_frame.height(); let label = if_frame.label(); let is_branched_to = if_frame.is_branched_to(); let reachability = match if_frame.reachability { - IfReachability::Both { .. } => { - debug_assert!(self.orphaned_else_operands); - self.orphaned_else_operands = false; - ElseReachability::Both { - is_end_of_then_reachable, - } - } + IfReachability::Both { .. } => ElseReachability::Both { + is_end_of_then_reachable, + }, IfReachability::OnlyThen => ElseReachability::OnlyThen { is_end_of_then_reachable, }, IfReachability::OnlyElse => ElseReachability::OnlyElse, }; - debug_assert!(!self.orphaned_else_operands); self.frames.push(ControlFrame::from(ElseControlFrame { ty, height: StackHeight::from(height), @@ -199,17 +195,6 @@ impl ControlStack { label, reachability, })); - self.orphaned_else_operands = false; - match reachability { - ElseReachability::OnlyThen { .. } | ElseReachability::OnlyElse => None, - ElseReachability::Both { .. } => { - let else_operands = self - .else_operands - .pop() - .unwrap_or_else(|| panic!("missing operands for `else` control frame")); - Some(else_operands) - } - } } /// Pops the top-most [`ControlFrame`] and returns it if any. diff --git a/crates/wasmi/src/engine/translator/func2/stack/mod.rs b/crates/wasmi/src/engine/translator/func2/stack/mod.rs index e9212f3d9b..574b78b815 100644 --- a/crates/wasmi/src/engine/translator/func2/stack/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/stack/mod.rs @@ -270,15 +270,9 @@ impl Stack { consume_fuel: Option, ) -> Result<(), Error> { debug_assert!(self.is_fuel_metering_enabled() == consume_fuel.is_some()); - // self.trunc(if_frame.height()); - let else_operands = - self.controls - .push_else(if_frame, consume_fuel, is_end_of_then_reachable); - if let Some(else_operands) = else_operands { - for else_operand in else_operands { - self.operands.push_operand(else_operand)?; - } - } + self.push_else_operands(&if_frame)?; + self.controls + .push_else(if_frame, consume_fuel, is_end_of_then_reachable); Ok(()) } @@ -308,7 +302,11 @@ impl Stack { /// # Panics (Debug) /// /// If the `else` operands are not in orphaned state. - pub fn push_else_operands(&mut self, frame: &impl ControlFrameBase) -> Result<(), Error> { + pub fn push_else_operands(&mut self, frame: &IfControlFrame) -> Result<(), Error> { + match frame.reachability() { + IfReachability::Both { .. } => {} + IfReachability::OnlyThen | IfReachability::OnlyElse => return Ok(()), + }; self.trunc(frame.height()); for else_operand in self.controls.pop_else_operands() { self.operands.push_operand(else_operand)?; From 3d19f469831a7b262e881633e8a4b852a261facc Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Fri, 18 Jul 2025 17:32:16 +0200 Subject: [PATCH 320/343] add FuncTranslator::push_param convenience method --- .../wasmi/src/engine/translator/func2/mod.rs | 36 ++++++++++--------- .../src/engine/translator/func2/visit.rs | 8 ++--- 2 files changed, 23 insertions(+), 21 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index 38acdaef17..0ded8a4e58 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -694,6 +694,12 @@ impl FuncTranslator { self.push_instr_with_result(result_ty, |result| make_instr(result, lhs, rhs), fuel_costs) } + /// Pushes an instruction parameter `param` to the list of instructions. + fn push_param(&mut self, param: Instruction) -> Result<(), Error> { + self.instrs.push_param(param); + Ok(()) + } + /// Populate the `buffer` with the `table` targets including the `table` default target. /// /// Returns a shared slice to the `buffer` after it has been filled. @@ -759,10 +765,9 @@ impl FuncTranslator { Instruction::branch_table_span(index, table.len() + 1), FuelCostsProvider::base, )?; - self.instrs - .push_param(Instruction::register_span(BoundedRegSpan::new( - values, len_values, - ))); + self.push_param(Instruction::register_span(BoundedRegSpan::new( + values, len_values, + )))?; // Encode the `br_table` targets: let targets = &self.immediates[..]; for target in targets { @@ -1910,8 +1915,7 @@ impl FuncTranslator { mem::swap(&mut true_val, &mut false_val); } }; - self.instrs - .push_param(Instruction::register2_ext(true_val, false_val)); + self.push_param(Instruction::register2_ext(true_val, false_val))?; Ok(()) } @@ -2051,7 +2055,7 @@ impl FuncTranslator { FuelCostsProvider::load, )?; if !memory.is_default() { - self.instrs.push_param(Instruction::memory_index(memory)); + self.push_param(Instruction::memory_index(memory))?; } return Ok(()); } @@ -2082,10 +2086,9 @@ impl FuncTranslator { |result| make_instr(result, offset_lo), FuelCostsProvider::load, )?; - self.instrs - .push_param(Instruction::register_and_offset_hi(ptr, offset_hi)); + self.push_param(Instruction::register_and_offset_hi(ptr, offset_hi))?; if !memory.is_default() { - self.instrs.push_param(Instruction::memory_index(memory)); + self.push_param(Instruction::memory_index(memory))?; } Ok(()) } @@ -2181,9 +2184,9 @@ impl FuncTranslator { } }; self.push_instr(instr, FuelCostsProvider::store)?; - self.instrs.push_param(param); + self.push_param(param)?; if !memory.is_default() { - self.instrs.push_param(Instruction::memory_index(memory)); + self.push_param(Instruction::memory_index(memory))?; } Ok(()) } @@ -2220,7 +2223,7 @@ impl FuncTranslator { } } if !memory.is_default() { - self.instrs.push_param(Instruction::memory_index(memory)); + self.push_param(Instruction::memory_index(memory))?; } Ok(()) } @@ -2321,10 +2324,9 @@ impl FuncTranslator { } } self.push_instr(store(ptr, offset_lo), FuelCostsProvider::store)?; - self.instrs - .push_param(Instruction::register_and_offset_hi(value, offset_hi)); + self.push_param(Instruction::register_and_offset_hi(value, offset_hi))?; if !memory.is_default() { - self.instrs.push_param(Instruction::memory_index(memory)); + self.push_param(Instruction::memory_index(memory))?; } Ok(()) } @@ -2344,7 +2346,7 @@ impl FuncTranslator { let value = self.layout.operand_to_reg(value)?; self.push_instr(make_instr_at(value, address), FuelCostsProvider::store)?; if !memory.is_default() { - self.instrs.push_param(Instruction::memory_index(memory)); + self.push_param(Instruction::memory_index(memory))?; } Ok(()) } diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index 3adfc95045..ac33615c5d 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -377,7 +377,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { _ => unreachable!(), }; let call_instr = self.push_instr(instr, FuelCostsProvider::call)?; - self.instrs.push_param(indirect_params); + self.push_param(indirect_params)?; self.stack.pop_n(len_params, &mut self.operands); self.instrs .encode_register_list(&self.operands, &mut self.layout)?; @@ -726,7 +726,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { |result| Instruction::memory_grow_imm(result, delta), FuelCostsProvider::instance, )?; - self.instrs.push_param(Instruction::memory_index(mem)); + self.push_param(Instruction::memory_index(mem))?; return Ok(()); } } @@ -737,7 +737,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { |result| Instruction::memory_grow(result, delta), FuelCostsProvider::instance, )?; - self.instrs.push_param(Instruction::memory_index(mem)); + self.push_param(Instruction::memory_index(mem))?; Ok(()) } @@ -1737,7 +1737,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { _ => unreachable!(), }; self.push_instr(instr, FuelCostsProvider::call)?; - self.instrs.push_param(indirect_params); + self.push_param(indirect_params)?; self.stack.pop_n(len_params, &mut self.operands); self.instrs .encode_register_list(&self.operands, &mut self.layout)?; From c1d88f6c04c37d3628a882ca4ae3e6b90fce3ad5 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Fri, 18 Jul 2025 17:32:38 +0200 Subject: [PATCH 321/343] add FuncTranslator::make_operand16 utility method --- .../wasmi/src/engine/translator/func2/mod.rs | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index 0ded8a4e58..8bc9a254b1 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -1414,6 +1414,29 @@ impl FuncTranslator { } } + /// Creates a new 16-bit encoded [`Operand16`] from the given `operand`. + pub fn make_operand16(&mut self, operand: Operand) -> Result, Error> + where + T: From + TryInto>, + { + let reg = match operand { + Operand::Local(operand) => self.layout.local_to_reg(operand.local_index())?, + Operand::Temp(operand) => self.layout.temp_to_reg(operand.operand_index())?, + Operand::Immediate(operand) => { + let operand = operand.val(); + let opd16 = match T::from(operand).try_into() { + Ok(rhs) => Operand16::Immediate(rhs), + Err(_) => { + let rhs = self.layout.const_to_reg(operand)?; + Operand16::Reg(rhs) + } + }; + return Ok(opd16); + } + }; + Ok(Operand16::Reg(reg)) + } + /// Converts the `provider` to a 16-bit index-type constant value. /// /// # Note From 073b7da74757c8e826a0b353d09bb3ecbcafdc34 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Fri, 18 Jul 2025 17:32:50 +0200 Subject: [PATCH 322/343] add Wasm memory.init translation --- .../wasmi/src/engine/translator/func2/visit.rs | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index ac33615c5d..cb6d5d58ed 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -7,6 +7,7 @@ use crate::{ stack::{AcquiredTarget, IfReachability}, ControlFrameBase, Operand, + Operand16, }, BlockType, }, @@ -1573,8 +1574,20 @@ impl<'a> VisitOperator<'a> for FuncTranslator { self.translate_unary(Instruction::i64_trunc_sat_f64_u, wasm::i64_trunc_sat_f64_u) } - fn visit_memory_init(&mut self, _data_index: u32, _mem: u32) -> Self::Output { - todo!() + fn visit_memory_init(&mut self, data_index: u32, mem: u32) -> Self::Output { + bail_unreachable!(self); + let (dst, src, len) = self.stack.pop3(); + let dst = self.layout.operand_to_reg(dst)?; + let src = self.layout.operand_to_reg(src)?; + let len = self.make_operand16::(len)?; + let instr = match len { + Operand16::Immediate(len) => Instruction::memory_init_imm(dst, src, len), + Operand16::Reg(len) => Instruction::memory_init(dst, src, len), + }; + self.push_instr(instr, FuelCostsProvider::instance)?; + self.push_param(Instruction::memory_index(mem))?; + self.push_param(Instruction::data_index(data_index))?; + Ok(()) } fn visit_data_drop(&mut self, _data_index: u32) -> Self::Output { From 4cf72bea50ddbc6277a0b9d548b70245b8b0b6c2 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Fri, 18 Jul 2025 17:32:58 +0200 Subject: [PATCH 323/343] add Wasm data.drop translation --- crates/wasmi/src/engine/translator/func2/visit.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index cb6d5d58ed..0f6179ff9d 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -1590,8 +1590,13 @@ impl<'a> VisitOperator<'a> for FuncTranslator { Ok(()) } - fn visit_data_drop(&mut self, _data_index: u32) -> Self::Output { - todo!() + fn visit_data_drop(&mut self, data_index: u32) -> Self::Output { + bail_unreachable!(self); + self.push_instr( + Instruction::data_drop(data_index), + FuelCostsProvider::instance, + )?; + Ok(()) } fn visit_memory_copy(&mut self, _dst_mem: u32, _src_mem: u32) -> Self::Output { From b4986741a17b0deb192769db25186fc3d427493d Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Fri, 18 Jul 2025 18:07:50 +0200 Subject: [PATCH 324/343] rename Operand16 -> Input16 --- .../wasmi/src/engine/translator/func2/mod.rs | 72 +++++++++---------- .../src/engine/translator/func2/utils.rs | 9 ++- .../src/engine/translator/func2/visit.rs | 8 +-- 3 files changed, 45 insertions(+), 44 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index 8bc9a254b1..3a99312e7e 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -31,7 +31,7 @@ use self::{ StackAllocations, TempOperand, }, - utils::{Operand16, Reset, ReusableAllocations}, + utils::{Input, Input16, Reset, ReusableAllocations}, }; use crate::{ core::{FuelCostsProvider, IndexType, TrapCode, Typed, TypedVal, UntypedVal, ValType}, @@ -1401,23 +1401,23 @@ impl FuncTranslator { } /// Creates a new 16-bit encoded [`Operand16`] from the given `value`. - pub fn make_imm16(&mut self, value: T) -> Result, Error> + pub fn make_imm16(&mut self, value: T) -> Result, Error> where T: Into + Copy + TryInto>, { match value.try_into() { - Ok(rhs) => Ok(Operand16::Immediate(rhs)), + Ok(rhs) => Ok(Input::Immediate(rhs)), Err(_) => { let rhs = self.layout.const_to_reg(value)?; - Ok(Operand16::Reg(rhs)) + Ok(Input::Reg(rhs)) } } } - /// Creates a new 16-bit encoded [`Operand16`] from the given `operand`. - pub fn make_operand16(&mut self, operand: Operand) -> Result, Error> + /// Creates a new 16-bit encoded [`Input16`] from the given `operand`. + pub fn make_input16(&mut self, operand: Operand) -> Result, Error> where - T: From + TryInto>, + T: From + Into + TryInto>, { let reg = match operand { Operand::Local(operand) => self.layout.local_to_reg(operand.local_index())?, @@ -1425,16 +1425,16 @@ impl FuncTranslator { Operand::Immediate(operand) => { let operand = operand.val(); let opd16 = match T::from(operand).try_into() { - Ok(rhs) => Operand16::Immediate(rhs), + Ok(rhs) => Input::Immediate(rhs), Err(_) => { let rhs = self.layout.const_to_reg(operand)?; - Operand16::Reg(rhs) + Input::Reg(rhs) } }; return Ok(opd16); } }; - Ok(Operand16::Reg(reg)) + Ok(Input::Reg(reg)) } /// Converts the `provider` to a 16-bit index-type constant value. @@ -1447,29 +1447,29 @@ impl FuncTranslator { &mut self, operand: Operand, index_type: IndexType, - ) -> Result, Error> { + ) -> Result, Error> { let value = match operand { Operand::Immediate(value) => value.val(), operand => { debug_assert_eq!(operand.ty(), index_type.ty()); let reg = self.layout.operand_to_reg(operand)?; - return Ok(Operand16::Reg(reg)); + return Ok(Input::Reg(reg)); } }; match index_type { IndexType::I64 => { if let Ok(value) = Const16::try_from(u64::from(value)) { - return Ok(Operand16::Immediate(value)); + return Ok(Input::Immediate(value)); } } IndexType::I32 => { if let Ok(value) = Const16::try_from(u32::from(value)) { - return Ok(Operand16::Immediate(>::cast(value))); + return Ok(Input::Immediate(>::cast(value))); } } } let reg = self.layout.const_to_reg(value)?; - Ok(Operand16::Reg(reg)) + Ok(Input::Reg(reg)) } /// Evaluates `consteval(lhs, rhs)` and pushed either its result or tranlates a `trap`. @@ -1544,8 +1544,8 @@ impl FuncTranslator { self.push_instr_with_result( ::TY, |result| match rhs16 { - Operand16::Immediate(rhs) => make_ri(result, lhs, rhs), - Operand16::Reg(rhs) => make_rr(result, lhs, rhs), + Input::Immediate(rhs) => make_ri(result, lhs, rhs), + Input::Reg(rhs) => make_rr(result, lhs, rhs), }, FuelCostsProvider::base, ) @@ -1591,8 +1591,8 @@ impl FuncTranslator { self.push_instr_with_result( ::TY, |result| match rhs16 { - Operand16::Immediate(rhs) => make_instr_imm16_rhs(result, lhs, rhs), - Operand16::Reg(rhs) => make_instr(result, lhs, rhs), + Input::Immediate(rhs) => make_instr_imm16_rhs(result, lhs, rhs), + Input::Reg(rhs) => make_instr(result, lhs, rhs), }, FuelCostsProvider::base, ) @@ -1604,8 +1604,8 @@ impl FuncTranslator { self.push_instr_with_result( ::TY, |result| match lhs16 { - Operand16::Immediate(lhs) => make_instr_imm16_lhs(result, lhs, rhs), - Operand16::Reg(lhs) => make_instr(result, lhs, rhs), + Input::Immediate(lhs) => make_instr_imm16_lhs(result, lhs, rhs), + Input::Reg(lhs) => make_instr(result, lhs, rhs), }, FuelCostsProvider::base, ) @@ -1644,8 +1644,8 @@ impl FuncTranslator { self.push_instr_with_result( ::TY, |result| match rhs16 { - Operand16::Immediate(rhs) => make_instr_imm16_rhs(result, lhs, rhs), - Operand16::Reg(rhs) => make_instr(result, lhs, rhs), + Input::Immediate(rhs) => make_instr_imm16_rhs(result, lhs, rhs), + Input::Reg(rhs) => make_instr(result, lhs, rhs), }, FuelCostsProvider::base, ) @@ -1657,8 +1657,8 @@ impl FuncTranslator { self.push_instr_with_result( ::TY, |result| match lhs16 { - Operand16::Immediate(lhs) => make_instr_imm16_lhs(result, lhs, rhs), - Operand16::Reg(lhs) => make_instr(result, lhs, rhs), + Input::Immediate(lhs) => make_instr_imm16_lhs(result, lhs, rhs), + Input::Reg(lhs) => make_instr(result, lhs, rhs), }, FuelCostsProvider::base, ) @@ -1694,17 +1694,17 @@ impl FuncTranslator { let lhs = self.layout.operand_to_reg(lhs)?; let rhs = T::from(rhs.val()); let rhs16 = match rhs.wrapping_neg().try_into() { - Ok(rhs) => Operand16::Immediate(rhs), + Ok(rhs) => Input::Immediate(rhs), Err(_) => { let rhs = self.layout.const_to_reg(rhs)?; - Operand16::Reg(rhs) + Input::Reg(rhs) } }; self.push_instr_with_result( ::TY, |result| match rhs16 { - Operand16::Immediate(rhs) => make_add_ri(result, lhs, rhs), - Operand16::Reg(rhs) => make_sub_rr(result, lhs, rhs), + Input::Immediate(rhs) => make_add_ri(result, lhs, rhs), + Input::Reg(rhs) => make_sub_rr(result, lhs, rhs), }, FuelCostsProvider::base, ) @@ -1716,8 +1716,8 @@ impl FuncTranslator { self.push_instr_with_result( ::TY, |result| match lhs16 { - Operand16::Immediate(lhs) => make_sub_ir(result, lhs, rhs), - Operand16::Reg(lhs) => make_sub_rr(result, lhs, rhs), + Input::Immediate(lhs) => make_sub_ir(result, lhs, rhs), + Input::Reg(lhs) => make_sub_rr(result, lhs, rhs), }, FuelCostsProvider::base, ) @@ -1778,8 +1778,8 @@ impl FuncTranslator { self.push_instr_with_result( ::TY, |result| match lhs16 { - Operand16::Immediate(lhs) => make_instr_imm16_lhs(result, lhs, rhs), - Operand16::Reg(lhs) => make_instr(result, lhs, rhs), + Input::Immediate(lhs) => make_instr_imm16_lhs(result, lhs, rhs), + Input::Reg(lhs) => make_instr(result, lhs, rhs), }, FuelCostsProvider::base, ) @@ -1951,10 +1951,8 @@ impl FuncTranslator { let table_type = *self.module.get_type_of_table(TableIdx::from(table_index)); let index = self.make_index16(index, table_type.index_ty())?; let instr = match index { - Operand16::Reg(index) => Instruction::call_indirect_params(index, table_index), - Operand16::Immediate(index) => { - Instruction::call_indirect_params_imm16(index, table_index) - } + Input::Reg(index) => Instruction::call_indirect_params(index, table_index), + Input::Immediate(index) => Instruction::call_indirect_params_imm16(index, table_index), }; Ok(instr) } diff --git a/crates/wasmi/src/engine/translator/func2/utils.rs b/crates/wasmi/src/engine/translator/func2/utils.rs index ccff936792..65c3ee9021 100644 --- a/crates/wasmi/src/engine/translator/func2/utils.rs +++ b/crates/wasmi/src/engine/translator/func2/utils.rs @@ -51,10 +51,13 @@ pub trait ReusableAllocations { fn into_allocations(self) -> Self::Allocations; } -/// A 16-bit encoded operand for a Wasmi instruction. -pub enum Operand16 { +/// A 16-bit encoded input to Wasmi instruction. +pub type Input16 = Input>; + +/// A concrete input to a Wasmi instruction. +pub enum Input { /// A [`Reg`] operand. Reg(Reg), /// A 16-bit encoded immediate value operand. - Immediate(Const16), + Immediate(T), } diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index 0f6179ff9d..f55e54f3fa 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -6,8 +6,8 @@ use crate::{ op, stack::{AcquiredTarget, IfReachability}, ControlFrameBase, + Input, Operand, - Operand16, }, BlockType, }, @@ -1579,10 +1579,10 @@ impl<'a> VisitOperator<'a> for FuncTranslator { let (dst, src, len) = self.stack.pop3(); let dst = self.layout.operand_to_reg(dst)?; let src = self.layout.operand_to_reg(src)?; - let len = self.make_operand16::(len)?; + let len = self.make_input16::(len)?; let instr = match len { - Operand16::Immediate(len) => Instruction::memory_init_imm(dst, src, len), - Operand16::Reg(len) => Instruction::memory_init(dst, src, len), + Input::Immediate(len) => Instruction::memory_init_imm(dst, src, len), + Input::Reg(len) => Instruction::memory_init(dst, src, len), }; self.push_instr(instr, FuelCostsProvider::instance)?; self.push_param(Instruction::memory_index(mem))?; From da2d025353d4d112a5af371f9acf169c53e8ac8c Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Fri, 18 Jul 2025 18:14:09 +0200 Subject: [PATCH 325/343] add make_input utility method --- .../wasmi/src/engine/translator/func2/mod.rs | 32 ++++++++++++------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index 3a99312e7e..094a019987 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -1417,22 +1417,30 @@ impl FuncTranslator { /// Creates a new 16-bit encoded [`Input16`] from the given `operand`. pub fn make_input16(&mut self, operand: Operand) -> Result, Error> where - T: From + Into + TryInto>, + T: From + Into + TryInto> + Copy, { + self.make_input(operand, |this, imm| { + let opd16 = match T::from(imm).try_into() { + Ok(rhs) => Input::Immediate(rhs), + Err(_) => { + let rhs = this.layout.const_to_reg(imm)?; + Input::Reg(rhs) + } + }; + Ok(opd16) + }) + } + + /// Create a new generic [`Input`] from the given `operand`. + fn make_input( + &mut self, + operand: Operand, + f: impl FnOnce(&mut Self, TypedVal) -> Result, Error>, + ) -> Result, Error> { let reg = match operand { Operand::Local(operand) => self.layout.local_to_reg(operand.local_index())?, Operand::Temp(operand) => self.layout.temp_to_reg(operand.operand_index())?, - Operand::Immediate(operand) => { - let operand = operand.val(); - let opd16 = match T::from(operand).try_into() { - Ok(rhs) => Input::Immediate(rhs), - Err(_) => { - let rhs = self.layout.const_to_reg(operand)?; - Input::Reg(rhs) - } - }; - return Ok(opd16); - } + Operand::Immediate(operand) => return f(self, operand.val()), }; Ok(Input::Reg(reg)) } From de5055b58aee7070e2f2129dcdccdfff5e53a3ce Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Fri, 18 Jul 2025 18:14:20 +0200 Subject: [PATCH 326/343] add Wasm memory.copy translation --- .../src/engine/translator/func2/visit.rs | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index f55e54f3fa..d3a049a334 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -1599,8 +1599,23 @@ impl<'a> VisitOperator<'a> for FuncTranslator { Ok(()) } - fn visit_memory_copy(&mut self, _dst_mem: u32, _src_mem: u32) -> Self::Output { - todo!() + fn visit_memory_copy(&mut self, dst_mem: u32, src_mem: u32) -> Self::Output { + bail_unreachable!(self); + let (dst, src, len) = self.stack.pop3(); + let dst_memory_type = *self.module.get_type_of_memory(MemoryIdx::from(dst_mem)); + let src_memory_type = *self.module.get_type_of_memory(MemoryIdx::from(src_mem)); + let min_index_ty = dst_memory_type.index_ty().min(&src_memory_type.index_ty()); + let dst = self.layout.operand_to_reg(dst)?; + let src = self.layout.operand_to_reg(src)?; + let len = self.make_index16(len, min_index_ty)?; + let instr = match len { + Input::Reg(len) => Instruction::memory_copy(dst, src, len), + Input::Immediate(len) => Instruction::memory_copy_imm(dst, src, len), + }; + self.push_instr(instr, FuelCostsProvider::instance)?; + self.push_param(Instruction::memory_index(dst_mem))?; + self.push_param(Instruction::memory_index(src_mem))?; + Ok(()) } fn visit_memory_fill(&mut self, _mem: u32) -> Self::Output { From aefc644544b004814638bef75cbd88cd75aea09a Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Fri, 18 Jul 2025 18:14:29 +0200 Subject: [PATCH 327/343] add Wasm memory.fill translation --- .../src/engine/translator/func2/visit.rs | 27 +++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index d3a049a334..14ebc5f2bc 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -1618,8 +1618,31 @@ impl<'a> VisitOperator<'a> for FuncTranslator { Ok(()) } - fn visit_memory_fill(&mut self, _mem: u32) -> Self::Output { - todo!() + fn visit_memory_fill(&mut self, mem: u32) -> Self::Output { + bail_unreachable!(self); + let memory_type = *self.module.get_type_of_memory(MemoryIdx::from(mem)); + let (dst, value, len) = self.stack.pop3(); + let dst = self.layout.operand_to_reg(dst)?; + let value = self.make_input(value, |_, value| { + let byte = u32::from(value) as u8; + Ok(Input::Immediate(byte)) + })?; + let len = self.make_index16(len, memory_type.index_ty())?; + let instr: Instruction = match (value, len) { + (Input::Reg(value), Input::Reg(len)) => Instruction::memory_fill(dst, value, len), + (Input::Reg(value), Input::Immediate(len)) => { + Instruction::memory_fill_exact(dst, value, len) + } + (Input::Immediate(value), Input::Reg(len)) => { + Instruction::memory_fill_imm(dst, value, len) + } + (Input::Immediate(value), Input::Immediate(len)) => { + Instruction::memory_fill_imm_exact(dst, value, len) + } + }; + self.push_instr(instr, FuelCostsProvider::instance)?; + self.push_param(Instruction::memory_index(mem))?; + Ok(()) } fn visit_table_init(&mut self, _elem_index: u32, _table: u32) -> Self::Output { From 4dc0d19e30969ecab6acc516ec1cc8b4f94e4da0 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Fri, 18 Jul 2025 18:16:49 +0200 Subject: [PATCH 328/343] fix broken intra doc link --- crates/wasmi/src/engine/translator/func2/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index 094a019987..1f25b8b8a3 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -1400,7 +1400,7 @@ impl FuncTranslator { Ok(()) } - /// Creates a new 16-bit encoded [`Operand16`] from the given `value`. + /// Creates a new 16-bit encoded [`Input16`] from the given `value`. pub fn make_imm16(&mut self, value: T) -> Result, Error> where T: Into + Copy + TryInto>, From 36c1e2404d590d06c4c08e6eb47e32c3cf8d87ac Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Fri, 18 Jul 2025 19:18:42 +0200 Subject: [PATCH 329/343] add FuncTranslator::make_index32 utility --- .../wasmi/src/engine/translator/func2/mod.rs | 36 ++++++++++++++++++- .../src/engine/translator/func2/utils.rs | 5 ++- 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index 1f25b8b8a3..a4d36cf507 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -31,7 +31,7 @@ use self::{ StackAllocations, TempOperand, }, - utils::{Input, Input16, Reset, ReusableAllocations}, + utils::{Input, Input16, Input32, Reset, ReusableAllocations}, }; use crate::{ core::{FuelCostsProvider, IndexType, TrapCode, Typed, TypedVal, UntypedVal, ValType}, @@ -1480,6 +1480,40 @@ impl FuncTranslator { Ok(Input::Reg(reg)) } + /// Converts the `provider` to a 32-bit index-type constant value. + /// + /// # Note + /// + /// - Turns immediates that cannot be 32-bit encoded into function local constants. + /// - The behavior is different whether `memory64` is enabled or disabled. + pub(super) fn make_index32( + &mut self, + operand: Operand, + index_type: IndexType, + ) -> Result, Error> { + let value = match operand { + Operand::Immediate(value) => value.val(), + operand => { + debug_assert_eq!(operand.ty(), index_type.ty()); + let reg = self.layout.operand_to_reg(operand)?; + return Ok(Input::Reg(reg)); + } + }; + match index_type { + IndexType::I64 => { + if let Ok(value) = Const32::try_from(u64::from(value)) { + return Ok(Input::Immediate(value)); + } + } + IndexType::I32 => { + let value32 = Const32::from(u32::from(value)); + return Ok(Input::Immediate(>::cast(value32))); + } + } + let reg = self.layout.const_to_reg(value)?; + Ok(Input::Reg(reg)) + } + /// Evaluates `consteval(lhs, rhs)` and pushed either its result or tranlates a `trap`. fn translate_binary_consteval_fallible( &mut self, diff --git a/crates/wasmi/src/engine/translator/func2/utils.rs b/crates/wasmi/src/engine/translator/func2/utils.rs index 65c3ee9021..7211653781 100644 --- a/crates/wasmi/src/engine/translator/func2/utils.rs +++ b/crates/wasmi/src/engine/translator/func2/utils.rs @@ -1,4 +1,4 @@ -use crate::ir::{Const16, Reg}; +use crate::ir::{Const16, Const32, Reg}; /// Bail out early in case the current code is unreachable. /// @@ -54,6 +54,9 @@ pub trait ReusableAllocations { /// A 16-bit encoded input to Wasmi instruction. pub type Input16 = Input>; +/// A 32-bit encoded input to Wasmi instruction. +pub type Input32 = Input>; + /// A concrete input to a Wasmi instruction. pub enum Input { /// A [`Reg`] operand. From 26f8820351fc6a353d5d2804255c59dfef1710ba Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Fri, 18 Jul 2025 19:19:06 +0200 Subject: [PATCH 330/343] add Wasm table.init translation --- .../wasmi/src/engine/translator/func2/visit.rs | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index 14ebc5f2bc..583ff8d34e 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -12,7 +12,7 @@ use crate::{ BlockType, }, ir::{self, Const16, Instruction}, - module::{self, FuncIdx, MemoryIdx, WasmiValueType}, + module::{self, FuncIdx, MemoryIdx, TableIdx, WasmiValueType}, Error, ExternRef, FuncRef, @@ -1645,8 +1645,20 @@ impl<'a> VisitOperator<'a> for FuncTranslator { Ok(()) } - fn visit_table_init(&mut self, _elem_index: u32, _table: u32) -> Self::Output { - todo!() + fn visit_table_init(&mut self, elem_index: u32, table: u32) -> Self::Output { + bail_unreachable!(self); + let (dst, src, len) = self.stack.pop3(); + let dst = self.layout.operand_to_reg(dst)?; + let src = self.layout.operand_to_reg(src)?; + let len = self.make_input16::(len)?; + let instr = match len { + Input::Reg(len) => Instruction::table_init(dst, src, len), + Input::Immediate(len) => Instruction::table_init_imm(dst, src, len), + }; + self.push_instr(instr, FuelCostsProvider::instance)?; + self.push_param(Instruction::table_index(table))?; + self.push_param(Instruction::elem_index(elem_index))?; + Ok(()) } fn visit_elem_drop(&mut self, _elem_index: u32) -> Self::Output { From 49b8b8c8a8fd0b6647baf04d8c05518cda76dbd7 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Fri, 18 Jul 2025 19:19:16 +0200 Subject: [PATCH 331/343] add Wasm elem.drop translation --- crates/wasmi/src/engine/translator/func2/visit.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index 583ff8d34e..0d5d6be8ea 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -1661,8 +1661,13 @@ impl<'a> VisitOperator<'a> for FuncTranslator { Ok(()) } - fn visit_elem_drop(&mut self, _elem_index: u32) -> Self::Output { - todo!() + fn visit_elem_drop(&mut self, elem_index: u32) -> Self::Output { + bail_unreachable!(self); + self.push_instr( + Instruction::elem_drop(elem_index), + FuelCostsProvider::instance, + )?; + Ok(()) } fn visit_table_copy(&mut self, _dst_table: u32, _src_table: u32) -> Self::Output { From 676cf9a9bd29fea3f10cc9f951e915155fda5160 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Fri, 18 Jul 2025 19:19:24 +0200 Subject: [PATCH 332/343] add Wasm table.copy translation --- .../src/engine/translator/func2/visit.rs | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index 0d5d6be8ea..2e080c8c0d 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -1670,8 +1670,23 @@ impl<'a> VisitOperator<'a> for FuncTranslator { Ok(()) } - fn visit_table_copy(&mut self, _dst_table: u32, _src_table: u32) -> Self::Output { - todo!() + fn visit_table_copy(&mut self, dst_table: u32, src_table: u32) -> Self::Output { + bail_unreachable!(self); + let (dst, src, len) = self.stack.pop3(); + let dst_table_type = *self.module.get_type_of_table(TableIdx::from(dst_table)); + let src_table_type = *self.module.get_type_of_table(TableIdx::from(src_table)); + let min_index_ty = dst_table_type.index_ty().min(&src_table_type.index_ty()); + let dst = self.layout.operand_to_reg(dst)?; + let src = self.layout.operand_to_reg(src)?; + let len = self.make_index16(len, min_index_ty)?; + let instr = match len { + Input::Reg(len) => Instruction::table_copy(dst, src, len), + Input::Immediate(len) => Instruction::table_copy_imm(dst, src, len), + }; + self.push_instr(instr, FuelCostsProvider::instance)?; + self.push_param(Instruction::table_index(dst_table))?; + self.push_param(Instruction::table_index(src_table))?; + Ok(()) } fn visit_typed_select(&mut self, ty: wasmparser::ValType) -> Self::Output { From ffb1f3a9fc22f9863b1337290b64dde043100d7c Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Fri, 18 Jul 2025 19:19:30 +0200 Subject: [PATCH 333/343] add Wasm table.fill translation --- .../wasmi/src/engine/translator/func2/visit.rs | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index 2e080c8c0d..c74e78a573 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -1752,8 +1752,20 @@ impl<'a> VisitOperator<'a> for FuncTranslator { Ok(()) } - fn visit_table_fill(&mut self, _table: u32) -> Self::Output { - todo!() + fn visit_table_fill(&mut self, table: u32) -> Self::Output { + bail_unreachable!(self); + let (dst, value, len) = self.stack.pop3(); + let table_type = *self.module.get_type_of_table(TableIdx::from(table)); + let dst = self.layout.operand_to_reg(dst)?; + let value = self.layout.operand_to_reg(value)?; + let len = self.make_index16(len, table_type.index_ty())?; + let instr = match len { + Input::Reg(len) => Instruction::table_fill(dst, len, value), + Input::Immediate(len) => Instruction::table_fill_imm(dst, len, value), + }; + self.push_instr(instr, FuelCostsProvider::instance)?; + self.push_param(Instruction::table_index(table))?; + Ok(()) } fn visit_table_get(&mut self, _table: u32) -> Self::Output { From b4e6e8c94755530ae7d176c0f23572f9e743ced5 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Fri, 18 Jul 2025 19:19:36 +0200 Subject: [PATCH 334/343] add Wasm table.get translation --- .../src/engine/translator/func2/visit.rs | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index c74e78a573..a6446ba958 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -1768,8 +1768,23 @@ impl<'a> VisitOperator<'a> for FuncTranslator { Ok(()) } - fn visit_table_get(&mut self, _table: u32) -> Self::Output { - todo!() + fn visit_table_get(&mut self, table: u32) -> Self::Output { + bail_unreachable!(self); + let table_type = *self.module.get_type_of_table(TableIdx::from(table)); + let index = self.stack.pop(); + let item_ty = table_type.element(); + let index_ty = table_type.index_ty(); + let index = self.make_index32(index, index_ty)?; + self.push_instr_with_result( + item_ty, + |result| match index { + Input::Reg(index) => Instruction::table_get(result, index), + Input::Immediate(index) => Instruction::table_get_imm(result, index), + }, + FuelCostsProvider::instance, + )?; + self.push_param(Instruction::table_index(table))?; + Ok(()) } fn visit_table_set(&mut self, _table: u32) -> Self::Output { From 40fdbc0ea9b7c877ae896aec17e2150c8c11ae8b Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Fri, 18 Jul 2025 19:19:41 +0200 Subject: [PATCH 335/343] add Wasm table.set translation --- .../wasmi/src/engine/translator/func2/visit.rs | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index a6446ba958..c594f09e57 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -1787,8 +1787,20 @@ impl<'a> VisitOperator<'a> for FuncTranslator { Ok(()) } - fn visit_table_set(&mut self, _table: u32) -> Self::Output { - todo!() + fn visit_table_set(&mut self, table: u32) -> Self::Output { + bail_unreachable!(self); + let table_type = *self.module.get_type_of_table(TableIdx::from(table)); + let index_ty = table_type.index_ty(); + let (index, value) = self.stack.pop2(); + let index = self.make_index32(index, index_ty)?; + let value = self.layout.operand_to_reg(value)?; + let instr = match index { + Input::Reg(index) => Instruction::table_set(index, value), + Input::Immediate(index) => Instruction::table_set_at(value, index), + }; + self.push_instr(instr, FuelCostsProvider::instance)?; + self.push_param(Instruction::table_index(table))?; + Ok(()) } fn visit_table_grow(&mut self, _table: u32) -> Self::Output { From cb91c48e1372a1a2c38eafc943b8b8891785906c Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Fri, 18 Jul 2025 19:19:50 +0200 Subject: [PATCH 336/343] add Wasm table.grow translation --- .../src/engine/translator/func2/visit.rs | 34 +++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index c594f09e57..3345705ccf 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -1803,8 +1803,38 @@ impl<'a> VisitOperator<'a> for FuncTranslator { Ok(()) } - fn visit_table_grow(&mut self, _table: u32) -> Self::Output { - todo!() + fn visit_table_grow(&mut self, table: u32) -> Self::Output { + bail_unreachable!(self); + let table_type = *self.module.get_type_of_table(TableIdx::from(table)); + let index_ty = table_type.index_ty(); + let (value, delta) = self.stack.pop2(); + let delta = self.make_index16(delta, index_ty)?; + if let Input::Immediate(delta) = delta { + if u64::from(delta) == 0 { + // Case: growing by 0 elements. + // + // Since `table.grow` returns the `table.size` before the + // operation a `table.grow` with `delta` of 0 can be translated + // as `table.size` instruction instead. + self.push_instr_with_result( + index_ty.ty(), + |result| Instruction::table_size(result, table), + FuelCostsProvider::instance, + )?; + return Ok(()); + } + } + let value = self.layout.operand_to_reg(value)?; + self.push_instr_with_result( + index_ty.ty(), + |result| match delta { + Input::Reg(delta) => Instruction::table_grow(result, delta, value), + Input::Immediate(delta) => Instruction::table_grow_imm(result, delta, value), + }, + FuelCostsProvider::instance, + )?; + self.push_param(Instruction::table_index(table))?; + Ok(()) } fn visit_table_size(&mut self, _table: u32) -> Self::Output { From 79b10e12e371723204b10469b4b8dda323fd3ced Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Fri, 18 Jul 2025 19:19:55 +0200 Subject: [PATCH 337/343] add Wasm table.size translation --- crates/wasmi/src/engine/translator/func2/visit.rs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index 3345705ccf..5aa41576ec 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -1837,8 +1837,16 @@ impl<'a> VisitOperator<'a> for FuncTranslator { Ok(()) } - fn visit_table_size(&mut self, _table: u32) -> Self::Output { - todo!() + fn visit_table_size(&mut self, table: u32) -> Self::Output { + bail_unreachable!(self); + let table_type = *self.module.get_type_of_table(TableIdx::from(table)); + let index_ty = table_type.index_ty(); + self.push_instr_with_result( + index_ty.ty(), + |result| Instruction::table_size(result, table), + FuelCostsProvider::instance, + )?; + Ok(()) } fn visit_return_call(&mut self, function_index: u32) -> Self::Output { From 6e422525a1a471c09302c4383369e146151695aa Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Fri, 18 Jul 2025 19:49:17 +0200 Subject: [PATCH 338/343] add wide-arithmetic operators translation --- .../wasmi/src/engine/translator/func2/mod.rs | 120 ++++++++++++++++++ .../src/engine/translator/func2/visit.rs | 8 +- 2 files changed, 124 insertions(+), 4 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/mod.rs b/crates/wasmi/src/engine/translator/func2/mod.rs index a4d36cf507..78f98e6034 100644 --- a/crates/wasmi/src/engine/translator/func2/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/mod.rs @@ -63,6 +63,7 @@ use crate::{ ComparatorAndOffset, Const16, Const32, + FixedRegSpan, Instruction, IntoShiftAmount, Offset16, @@ -2455,4 +2456,123 @@ impl FuncTranslator { }; Some(address) } + + /// Translates a Wasm `i64.binop128` instruction from the `wide-arithmetic` proposal. + fn translate_i64_binop128( + &mut self, + make_instr: fn(results: [Reg; 2], lhs_lo: Reg) -> Instruction, + const_eval: fn(lhs_lo: i64, lhs_hi: i64, rhs_lo: i64, rhs_hi: i64) -> (i64, i64), + ) -> Result<(), Error> { + bail_unreachable!(self); + let (rhs_lo, rhs_hi) = self.stack.pop2(); + let (lhs_lo, lhs_hi) = self.stack.pop2(); + if let ( + Operand::Immediate(lhs_lo), + Operand::Immediate(lhs_hi), + Operand::Immediate(rhs_lo), + Operand::Immediate(rhs_hi), + ) = (lhs_lo, lhs_hi, rhs_lo, rhs_hi) + { + let (result_lo, result_hi) = const_eval( + lhs_lo.val().into(), + lhs_hi.val().into(), + rhs_lo.val().into(), + rhs_hi.val().into(), + ); + self.stack.push_immediate(result_lo)?; + self.stack.push_immediate(result_hi)?; + return Ok(()); + } + let rhs_lo = self.layout.operand_to_reg(rhs_lo)?; + let rhs_hi = self.layout.operand_to_reg(rhs_hi)?; + let lhs_lo = self.layout.operand_to_reg(lhs_lo)?; + let lhs_hi = self.layout.operand_to_reg(lhs_hi)?; + let result_lo = self.stack.push_temp(ValType::I64, None)?; + let result_hi = self.stack.push_temp(ValType::I64, None)?; + let result_lo = self.layout.temp_to_reg(result_lo)?; + let result_hi = self.layout.temp_to_reg(result_hi)?; + self.push_instr( + make_instr([result_lo, result_hi], lhs_lo), + FuelCostsProvider::base, + )?; + self.push_param(Instruction::register3_ext(lhs_hi, rhs_lo, rhs_hi))?; + Ok(()) + } + + /// Translates a Wasm `i64.mul_wide_sx` instruction from the `wide-arithmetic` proposal. + fn translate_i64_mul_wide_sx( + &mut self, + make_instr: fn(results: FixedRegSpan<2>, lhs: Reg, rhs: Reg) -> Instruction, + const_eval: fn(lhs: i64, rhs: i64) -> (i64, i64), + signed: bool, + ) -> Result<(), Error> { + bail_unreachable!(self); + let (lhs, rhs) = self.stack.pop2(); + let (lhs, rhs) = match (lhs, rhs) { + (Operand::Immediate(lhs), Operand::Immediate(rhs)) => { + let (result_lo, result_hi) = const_eval(lhs.val().into(), rhs.val().into()); + self.stack.push_immediate(result_lo)?; + self.stack.push_immediate(result_hi)?; + return Ok(()); + } + (lhs, Operand::Immediate(rhs)) => { + let rhs = rhs.val(); + if self.try_opt_i64_mul_wide_sx(lhs, rhs, signed)? { + return Ok(()); + } + let lhs = self.layout.operand_to_reg(lhs)?; + let rhs = self.layout.const_to_reg(rhs)?; + (lhs, rhs) + } + (Operand::Immediate(lhs), rhs) => { + let lhs = lhs.val(); + if self.try_opt_i64_mul_wide_sx(rhs, lhs, signed)? { + return Ok(()); + } + let lhs = self.layout.const_to_reg(lhs)?; + let rhs = self.layout.operand_to_reg(rhs)?; + (lhs, rhs) + } + (lhs, rhs) => { + let lhs = self.layout.operand_to_reg(lhs)?; + let rhs = self.layout.operand_to_reg(rhs)?; + (lhs, rhs) + } + }; + let result0 = self.stack.push_temp(ValType::I64, None)?; + let _result1 = self.stack.push_temp(ValType::I64, None)?; + let result0 = self.layout.temp_to_reg(result0)?; + let Ok(results) = >::new(RegSpan::new(result0)) else { + return Err(Error::from(TranslationError::AllocatedTooManyRegisters)); + }; + self.push_instr(make_instr(results, lhs, rhs), FuelCostsProvider::base)?; + Ok(()) + } + + /// Try to optimize a `i64.mul_wide_sx` instruction with one [`Reg`] and one immediate input. + /// + /// - Returns `Ok(true)` if the optimiation was applied successfully. + /// - Returns `Ok(false)` if no optimization was applied. + fn try_opt_i64_mul_wide_sx( + &mut self, + lhs: Operand, + rhs: TypedVal, + signed: bool, + ) -> Result { + let rhs = i64::from(rhs); + if rhs == 0 { + // Case: `mul(x, 0)` or `mul(0, x)` always evaluates to 0. + self.stack.push_immediate(0_i64)?; // lo-bits + self.stack.push_immediate(0_i64)?; // hi-bits + return Ok(true); + } + if rhs == 1 && !signed { + // Case: `mul(x, 1)` or `mul(1, x)` always evaluates to just `x`. + // This is only valid if `x` is not a singed (negative) value. + self.stack.push_operand(lhs)?; // lo-bits + self.stack.push_immediate(0_i64)?; // hi-bits + return Ok(true); + } + Ok(false) + } } diff --git a/crates/wasmi/src/engine/translator/func2/visit.rs b/crates/wasmi/src/engine/translator/func2/visit.rs index 5aa41576ec..d7a87f0388 100644 --- a/crates/wasmi/src/engine/translator/func2/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/visit.rs @@ -1911,18 +1911,18 @@ impl<'a> VisitOperator<'a> for FuncTranslator { } fn visit_i64_add128(&mut self) -> Self::Output { - todo!() + self.translate_i64_binop128(Instruction::i64_add128, wasm::i64_add128) } fn visit_i64_sub128(&mut self) -> Self::Output { - todo!() + self.translate_i64_binop128(Instruction::i64_sub128, wasm::i64_sub128) } fn visit_i64_mul_wide_s(&mut self) -> Self::Output { - todo!() + self.translate_i64_mul_wide_sx(Instruction::i64_mul_wide_s, wasm::i64_mul_wide_s, true) } fn visit_i64_mul_wide_u(&mut self) -> Self::Output { - todo!() + self.translate_i64_mul_wide_sx(Instruction::i64_mul_wide_u, wasm::i64_mul_wide_u, false) } } From e1fdedece425ce0f1868f405754b462fe067fb5c Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Sat, 19 Jul 2025 14:16:49 +0200 Subject: [PATCH 339/343] add Wasm simd splat translation --- .../src/engine/translator/func2/simd/mod.rs | 38 +++++++++++++++++++ .../src/engine/translator/func2/simd/visit.rs | 13 ++++--- 2 files changed, 45 insertions(+), 6 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/simd/mod.rs b/crates/wasmi/src/engine/translator/func2/simd/mod.rs index 10f88caa8a..bd55baf79b 100644 --- a/crates/wasmi/src/engine/translator/func2/simd/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/simd/mod.rs @@ -1,3 +1,41 @@ use super::FuncTranslator; mod visit; + +use crate::{ + core::{simd::IntoLaneIdx, FuelCostsProvider, Typed, TypedVal, ValType, V128}, + engine::translator::{func2::Operand, utils::Wrap}, + ir::{ + Instruction, + Reg, + }, + Error, +}; + +impl FuncTranslator { + /// Generically translate any of the Wasm `simd` splat instructions. + fn translate_simd_splat( + &mut self, + make_instr: fn(result: Reg, value: Reg) -> Instruction, + const_eval: fn(Wrapped) -> V128, + ) -> Result<(), Error> + where + T: From + Wrap, + { + bail_unreachable!(self); + let value = self.stack.pop(); + if let Operand::Immediate(value) = value { + let value = T::from(value.val()).wrap(); + let result = const_eval(value); + self.stack.push_immediate(result)?; + return Ok(()); + }; + let value = self.layout.operand_to_reg(value)?; + self.push_instr_with_result( + ValType::V128, + |result| make_instr(result, value), + FuelCostsProvider::simd, + )?; + Ok(()) + } +} diff --git a/crates/wasmi/src/engine/translator/func2/simd/visit.rs b/crates/wasmi/src/engine/translator/func2/simd/visit.rs index 7845b92636..5c76de80e2 100644 --- a/crates/wasmi/src/engine/translator/func2/simd/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/simd/visit.rs @@ -1,4 +1,5 @@ use super::FuncTranslator; +use crate::{core::simd, ir::Instruction}; use wasmparser::VisitSimdOperator; impl VisitSimdOperator<'_> for FuncTranslator { @@ -159,27 +160,27 @@ impl VisitSimdOperator<'_> for FuncTranslator { } fn visit_i8x16_splat(&mut self) -> Self::Output { - todo!() + self.translate_simd_splat::(Instruction::i8x16_splat, simd::i8x16_splat) } fn visit_i16x8_splat(&mut self) -> Self::Output { - todo!() + self.translate_simd_splat::(Instruction::i16x8_splat, simd::i16x8_splat) } fn visit_i32x4_splat(&mut self) -> Self::Output { - todo!() + self.translate_simd_splat::(Instruction::i32x4_splat, simd::i32x4_splat) } fn visit_i64x2_splat(&mut self) -> Self::Output { - todo!() + self.translate_simd_splat::(Instruction::i64x2_splat, simd::i64x2_splat) } fn visit_f32x4_splat(&mut self) -> Self::Output { - todo!() + self.translate_simd_splat::(Instruction::f32x4_splat, simd::f32x4_splat) } fn visit_f64x2_splat(&mut self) -> Self::Output { - todo!() + self.translate_simd_splat::(Instruction::f64x2_splat, simd::f64x2_splat) } fn visit_i8x16_eq(&mut self) -> Self::Output { From b5bd22845b9f1952013490d9c88dc06483de31a4 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Sat, 19 Jul 2025 14:18:06 +0200 Subject: [PATCH 340/343] reorder VisitSimdOperator trait methods --- .../src/engine/translator/func2/simd/visit.rs | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/simd/visit.rs b/crates/wasmi/src/engine/translator/func2/simd/visit.rs index 5c76de80e2..55d2d92419 100644 --- a/crates/wasmi/src/engine/translator/func2/simd/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/simd/visit.rs @@ -99,55 +99,55 @@ impl VisitSimdOperator<'_> for FuncTranslator { todo!() } - fn visit_i8x16_extract_lane_s(&mut self, _lane: u8) -> Self::Output { + fn visit_i8x16_extract_lane_s(&mut self, lane: u8) -> Self::Output { todo!() } - fn visit_i8x16_extract_lane_u(&mut self, _lane: u8) -> Self::Output { + fn visit_i8x16_extract_lane_u(&mut self, lane: u8) -> Self::Output { todo!() } - fn visit_i8x16_replace_lane(&mut self, _lane: u8) -> Self::Output { + fn visit_i16x8_extract_lane_s(&mut self, lane: u8) -> Self::Output { todo!() } - fn visit_i16x8_extract_lane_s(&mut self, _lane: u8) -> Self::Output { + fn visit_i16x8_extract_lane_u(&mut self, lane: u8) -> Self::Output { todo!() } - fn visit_i16x8_extract_lane_u(&mut self, _lane: u8) -> Self::Output { + fn visit_i32x4_extract_lane(&mut self, lane: u8) -> Self::Output { todo!() } - fn visit_i16x8_replace_lane(&mut self, _lane: u8) -> Self::Output { + fn visit_i64x2_extract_lane(&mut self, lane: u8) -> Self::Output { todo!() } - fn visit_i32x4_extract_lane(&mut self, _lane: u8) -> Self::Output { + fn visit_f32x4_extract_lane(&mut self, lane: u8) -> Self::Output { todo!() } - fn visit_i32x4_replace_lane(&mut self, _lane: u8) -> Self::Output { + fn visit_f64x2_extract_lane(&mut self, lane: u8) -> Self::Output { todo!() } - fn visit_i64x2_extract_lane(&mut self, _lane: u8) -> Self::Output { + fn visit_i8x16_replace_lane(&mut self, _lane: u8) -> Self::Output { todo!() } - fn visit_i64x2_replace_lane(&mut self, _lane: u8) -> Self::Output { + fn visit_i16x8_replace_lane(&mut self, _lane: u8) -> Self::Output { todo!() } - fn visit_f32x4_extract_lane(&mut self, _lane: u8) -> Self::Output { + fn visit_i32x4_replace_lane(&mut self, _lane: u8) -> Self::Output { todo!() } - fn visit_f32x4_replace_lane(&mut self, _lane: u8) -> Self::Output { + fn visit_i64x2_replace_lane(&mut self, _lane: u8) -> Self::Output { todo!() } - fn visit_f64x2_extract_lane(&mut self, _lane: u8) -> Self::Output { + fn visit_f32x4_replace_lane(&mut self, _lane: u8) -> Self::Output { todo!() } From 59388af278cd05585c324aa56393996c678fef41 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Sat, 19 Jul 2025 14:18:31 +0200 Subject: [PATCH 341/343] add Wasm simd extract_lane translation --- .../src/engine/translator/func2/simd/mod.rs | 29 +++++++++++ .../src/engine/translator/func2/simd/visit.rs | 48 +++++++++++++++---- 2 files changed, 69 insertions(+), 8 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/simd/mod.rs b/crates/wasmi/src/engine/translator/func2/simd/mod.rs index bd55baf79b..7aa96d2dc4 100644 --- a/crates/wasmi/src/engine/translator/func2/simd/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/simd/mod.rs @@ -38,4 +38,33 @@ impl FuncTranslator { )?; Ok(()) } + + /// Generically translate any of the Wasm `simd` extract lane instructions. + fn translate_extract_lane( + &mut self, + lane: u8, + make_instr: fn(result: Reg, input: Reg, lane: T::LaneIdx) -> Instruction, + const_eval: fn(input: V128, lane: T::LaneIdx) -> R, + ) -> Result<(), Error> + where + R: Into + Typed, + { + bail_unreachable!(self); + let Ok(lane) = ::try_from(lane) else { + panic!("encountered out of bounds lane index: {lane}") + }; + let input = self.stack.pop(); + if let Operand::Immediate(input) = input { + let result = const_eval(input.val().into(), lane); + self.stack.push_immediate(result)?; + return Ok(()); + }; + let input = self.layout.operand_to_reg(input)?; + self.push_instr_with_result( + ::TY, + |result| make_instr(result, input, lane), + FuelCostsProvider::simd, + )?; + Ok(()) + } } diff --git a/crates/wasmi/src/engine/translator/func2/simd/visit.rs b/crates/wasmi/src/engine/translator/func2/simd/visit.rs index 55d2d92419..cff228b155 100644 --- a/crates/wasmi/src/engine/translator/func2/simd/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/simd/visit.rs @@ -100,35 +100,67 @@ impl VisitSimdOperator<'_> for FuncTranslator { } fn visit_i8x16_extract_lane_s(&mut self, lane: u8) -> Self::Output { - todo!() + self.translate_extract_lane::( + lane, + Instruction::i8x16_extract_lane_s, + simd::i8x16_extract_lane_s, + ) } fn visit_i8x16_extract_lane_u(&mut self, lane: u8) -> Self::Output { - todo!() + self.translate_extract_lane::( + lane, + Instruction::i8x16_extract_lane_u, + simd::i8x16_extract_lane_u, + ) } fn visit_i16x8_extract_lane_s(&mut self, lane: u8) -> Self::Output { - todo!() + self.translate_extract_lane::( + lane, + Instruction::i16x8_extract_lane_s, + simd::i16x8_extract_lane_s, + ) } fn visit_i16x8_extract_lane_u(&mut self, lane: u8) -> Self::Output { - todo!() + self.translate_extract_lane::( + lane, + Instruction::i16x8_extract_lane_u, + simd::i16x8_extract_lane_u, + ) } fn visit_i32x4_extract_lane(&mut self, lane: u8) -> Self::Output { - todo!() + self.translate_extract_lane::( + lane, + Instruction::i32x4_extract_lane, + simd::i32x4_extract_lane, + ) } fn visit_i64x2_extract_lane(&mut self, lane: u8) -> Self::Output { - todo!() + self.translate_extract_lane::( + lane, + Instruction::i64x2_extract_lane, + simd::i64x2_extract_lane, + ) } fn visit_f32x4_extract_lane(&mut self, lane: u8) -> Self::Output { - todo!() + self.translate_extract_lane::( + lane, + Instruction::f32x4_extract_lane, + simd::f32x4_extract_lane, + ) } fn visit_f64x2_extract_lane(&mut self, lane: u8) -> Self::Output { - todo!() + self.translate_extract_lane::( + lane, + Instruction::f64x2_extract_lane, + simd::f64x2_extract_lane, + ) } fn visit_i8x16_replace_lane(&mut self, _lane: u8) -> Self::Output { From b61660b7e37328586dd6d7e69a47d4678ce4b0c9 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Sat, 19 Jul 2025 16:15:49 +0200 Subject: [PATCH 342/343] add Typed impls for i8,i16,u8,u16 primitives --- crates/core/src/typed.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crates/core/src/typed.rs b/crates/core/src/typed.rs index 94ab475ff3..93669c0e39 100644 --- a/crates/core/src/typed.rs +++ b/crates/core/src/typed.rs @@ -16,6 +16,10 @@ macro_rules! impl_typed_for { } impl_typed_for! { bool => ValType::I32; + i8 => ValType::I32; + u8 => ValType::I32; + i16 => ValType::I32; + u16 => ValType::I32; i32 => ValType::I32; u32 => ValType::I32; i64 => ValType::I64; From 9f040e72c13e8984ba256847437077e20508298f Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Sat, 19 Jul 2025 16:17:28 +0200 Subject: [PATCH 343/343] add translations for most simd and relaxed-simd operators --- .../src/engine/translator/func2/simd/mod.rs | 321 ++++++- .../src/engine/translator/func2/simd/visit.rs | 890 ++++++++++++------ 2 files changed, 945 insertions(+), 266 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func2/simd/mod.rs b/crates/wasmi/src/engine/translator/func2/simd/mod.rs index 7aa96d2dc4..22b87ca045 100644 --- a/crates/wasmi/src/engine/translator/func2/simd/mod.rs +++ b/crates/wasmi/src/engine/translator/func2/simd/mod.rs @@ -3,14 +3,25 @@ use super::FuncTranslator; mod visit; use crate::{ - core::{simd::IntoLaneIdx, FuelCostsProvider, Typed, TypedVal, ValType, V128}, - engine::translator::{func2::Operand, utils::Wrap}, + core::{simd::IntoLaneIdx, FuelCostsProvider, TrapCode, Typed, TypedVal, ValType, V128}, + engine::translator::{ + func2::Operand, + utils::{Instr, Wrap}, + }, ir::{ + index, + index::Memory, + Address32, Instruction, + IntoShiftAmount, + Offset64, + Offset64Lo, + Offset8, Reg, }, Error, }; +use wasmparser::MemArg; impl FuncTranslator { /// Generically translate any of the Wasm `simd` splat instructions. @@ -67,4 +78,310 @@ impl FuncTranslator { )?; Ok(()) } + + /// Generically translate a Wasm unary instruction. + fn translate_simd_unary( + &mut self, + make_instr: fn(result: Reg, input: Reg) -> Instruction, + const_eval: fn(input: V128) -> T, + ) -> Result<(), Error> + where + T: Into, + { + bail_unreachable!(self); + let input = self.stack.pop(); + if let Operand::Immediate(input) = input { + // Case: the input is an immediate so we can const-eval the result. + let result = const_eval(input.val().into()); + self.stack.push_immediate(result)?; + return Ok(()); + }; + let input = self.layout.operand_to_reg(input)?; + self.push_instr_with_result( + ValType::V128, + |result| make_instr(result, input), + FuelCostsProvider::simd, + )?; + Ok(()) + } + + /// Generically translate a Wasm binary instruction. + fn translate_simd_binary( + &mut self, + make_instr: fn(result: Reg, lhs: Reg, rhs: Reg) -> Instruction, + const_eval: fn(lhs: V128, rhs: V128) -> V128, + ) -> Result<(), Error> { + bail_unreachable!(self); + let (lhs, rhs) = self.stack.pop2(); + if let (Operand::Immediate(lhs), Operand::Immediate(rhs)) = (lhs, rhs) { + // Case: both inputs are immediates so we can const-eval the result. + let result = const_eval(lhs.val().into(), rhs.val().into()); + self.stack.push_immediate(result)?; + return Ok(()); + } + let lhs = self.layout.operand_to_reg(lhs)?; + let rhs = self.layout.operand_to_reg(rhs)?; + self.push_instr_with_result( + ValType::V128, + |result| make_instr(result, lhs, rhs), + FuelCostsProvider::simd, + )?; + Ok(()) + } + + /// Generically translate a Wasm ternary instruction. + fn translate_simd_ternary( + &mut self, + make_instr: fn(result: Reg, a: Reg, b: Reg) -> Instruction, + const_eval: fn(lhas: V128, b: V128, c: V128) -> V128, + ) -> Result<(), Error> { + bail_unreachable!(self); + let (a, b, c) = self.stack.pop3(); + if let (Operand::Immediate(lhs), Operand::Immediate(rhs), Operand::Immediate(c)) = (a, b, c) + { + // Case: all inputs are immediates so we can const-eval the result. + let result = const_eval(lhs.val().into(), rhs.val().into(), c.val().into()); + self.stack.push_immediate(result)?; + return Ok(()); + } + let lhs = self.layout.operand_to_reg(a)?; + let rhs = self.layout.operand_to_reg(b)?; + let selector = self.layout.operand_to_reg(c)?; + self.push_instr_with_result( + ValType::V128, + |result| make_instr(result, lhs, rhs), + FuelCostsProvider::simd, + )?; + self.push_param(Instruction::register(selector))?; + Ok(()) + } + + /// Generically translate a Wasm SIMD shift instruction. + fn translate_simd_shift( + &mut self, + make_instr: fn(result: Reg, lhs: Reg, rhs: Reg) -> Instruction, + make_instr_imm: fn( + result: Reg, + lhs: Reg, + rhs: ::Output, + ) -> Instruction, + const_eval: fn(lhs: V128, rhs: u32) -> V128, + ) -> Result<(), Error> + where + T: IntoShiftAmount>, + { + bail_unreachable!(self); + let (lhs, rhs) = self.stack.pop2(); + if let (Operand::Immediate(lhs), Operand::Immediate(rhs)) = (lhs, rhs) { + // Case: both inputs are immediates so we can const-eval the result. + let result = const_eval(lhs.val().into(), rhs.val().into()); + self.stack.push_immediate(result)?; + return Ok(()); + } + if let Operand::Immediate(rhs) = rhs { + let Some(rhs) = T::into_shift_amount(rhs.val().into()) else { + // Case: the shift operation is a no-op + self.stack.push_operand(lhs)?; + return Ok(()); + }; + let lhs = self.layout.operand_to_reg(lhs)?; + self.push_instr_with_result( + ValType::V128, + |result| make_instr_imm(result, lhs, rhs), + FuelCostsProvider::simd, + )?; + return Ok(()); + } + let lhs = self.layout.operand_to_reg(lhs)?; + let rhs = self.layout.operand_to_reg(rhs)?; + self.push_instr_with_result( + ValType::V128, + |result| make_instr(result, lhs, rhs), + FuelCostsProvider::simd, + )?; + Ok(()) + } + + fn translate_v128_load_lane( + &mut self, + memarg: MemArg, + lane: u8, + make_instr: fn(result: Reg, offset_lo: Offset64Lo) -> Instruction, + make_instr_at: fn(result: Reg, address: Address32) -> Instruction, + ) -> Result<(), Error> { + bail_unreachable!(self); + let (memory, offset) = Self::decode_memarg(memarg); + let Ok(lane) = ::try_from(lane) else { + panic!("encountered out of bounds lane: {lane}"); + }; + let (ptr, x) = self.stack.pop2(); + let x = self.layout.operand_to_reg(x)?; + let (ptr, offset) = match ptr { + Operand::Immediate(ptr) => { + let Some(address) = self.effective_address(memory, ptr.val(), offset) else { + return self.translate_trap(TrapCode::MemoryOutOfBounds); + }; + if let Ok(address) = Address32::try_from(address) { + return self.translate_v128_load_lane_at::( + memory, + x, + lane, + address, + make_instr_at, + ); + } + let zero_ptr = self.layout.const_to_reg(0_u64)?; + (zero_ptr, u64::from(address)) + } + ptr => { + let ptr = self.layout.operand_to_reg(ptr)?; + (ptr, offset) + } + }; + let (offset_hi, offset_lo) = Offset64::split(offset); + self.push_instr_with_result( + ::TY, + |result| make_instr(result, offset_lo), + FuelCostsProvider::load, + )?; + self.push_param(Instruction::register_and_offset_hi(ptr, offset_hi))?; + self.push_param(Instruction::register_and_lane(x, lane))?; + if !memory.is_default() { + self.push_param(Instruction::memory_index(memory))?; + } + Ok(()) + } + + fn translate_v128_load_lane_at( + &mut self, + memory: Memory, + x: Reg, + lane: LaneType, + address: Address32, + make_instr_at: fn(result: Reg, address: Address32) -> Instruction, + ) -> Result<(), Error> + where + LaneType: Into, + { + self.push_instr_with_result( + ::TY, + |result| make_instr_at(result, address), + FuelCostsProvider::load, + )?; + self.push_param(Instruction::register_and_lane(x, lane))?; + if !memory.is_default() { + self.push_param(Instruction::memory_index(memory))?; + } + Ok(()) + } + + #[allow(clippy::type_complexity)] + fn translate_v128_store_lane( + &mut self, + memarg: MemArg, + lane: u8, + make_instr: fn(ptr: Reg, offset_lo: Offset64Lo) -> Instruction, + make_instr_offset8: fn( + ptr: Reg, + value: Reg, + offset: Offset8, + lane: T::LaneIdx, + ) -> Instruction, + make_instr_at: fn(value: Reg, address: Address32) -> Instruction, + translate_imm: fn( + &mut Self, + memarg: MemArg, + ptr: Operand, + lane: T::LaneIdx, + value: V128, + ) -> Result<(), Error>, + ) -> Result<(), Error> { + bail_unreachable!(self); + let Ok(lane) = ::try_from(lane) else { + panic!("encountered out of bounds lane index: {lane}"); + }; + let (ptr, v128) = self.stack.pop2(); + let v128 = match v128 { + Operand::Immediate(v128) => { + // Case: with `v128` being an immediate value we can extract its + // lane value and translate as a more efficient non-SIMD operation. + return translate_imm(self, memarg, ptr, lane, V128::from(v128.val())); + } + v128 => self.layout.operand_to_reg(v128)?, + }; + let (memory, offset) = Self::decode_memarg(memarg); + let (ptr, offset) = match ptr { + Operand::Immediate(ptr) => { + let Some(address) = self.effective_address(memory, ptr.val(), offset) else { + return self.translate_trap(TrapCode::MemoryOutOfBounds); + }; + if let Ok(address) = Address32::try_from(address) { + return self.translate_v128_store_lane_at::( + memory, + address, + v128, + lane, + make_instr_at, + ); + } + // Case: we cannot use specialized encoding and thus have to fall back + // to the general case where `ptr` is zero and `offset` stores the + // `ptr+offset` address value. + let zero_ptr = self.layout.const_to_reg(0_u64)?; + (zero_ptr, u64::from(address)) + } + ptr => { + let ptr = self.layout.operand_to_reg(ptr)?; + (ptr, offset) + } + }; + if let Ok(Some(_)) = + self.translate_v128_store_lane_mem0(memory, ptr, offset, v128, lane, make_instr_offset8) + { + return Ok(()); + } + let (offset_hi, offset_lo) = Offset64::split(offset); + let instr = make_instr(ptr, offset_lo); + let param = Instruction::register_and_offset_hi(v128, offset_hi); + let param2 = Instruction::lane_and_memory_index(lane, memory); + self.push_instr(instr, FuelCostsProvider::store)?; + self.push_param(param)?; + self.push_param(param2)?; + Ok(()) + } + + fn translate_v128_store_lane_at( + &mut self, + memory: index::Memory, + address: Address32, + value: Reg, + lane: T::LaneIdx, + make_instr_at: fn(value: Reg, address: Address32) -> Instruction, + ) -> Result<(), Error> { + self.push_instr(make_instr_at(value, address), FuelCostsProvider::store)?; + self.push_param(Instruction::lane_and_memory_index(lane, memory))?; + Ok(()) + } + + fn translate_v128_store_lane_mem0( + &mut self, + memory: Memory, + ptr: Reg, + offset: u64, + value: Reg, + lane: LaneType, + make_instr_offset8: fn(Reg, Reg, Offset8, LaneType) -> Instruction, + ) -> Result, Error> { + if !memory.is_default() { + return Ok(None); + } + let Ok(offset8) = Offset8::try_from(offset) else { + return Ok(None); + }; + let instr = self.push_instr( + make_instr_offset8(ptr, value, offset8, lane), + FuelCostsProvider::store, + )?; + Ok(Some(instr)) + } } diff --git a/crates/wasmi/src/engine/translator/func2/simd/visit.rs b/crates/wasmi/src/engine/translator/func2/simd/visit.rs index cff228b155..422673c2e5 100644 --- a/crates/wasmi/src/engine/translator/func2/simd/visit.rs +++ b/crates/wasmi/src/engine/translator/func2/simd/visit.rs @@ -1,102 +1,284 @@ use super::FuncTranslator; -use crate::{core::simd, ir::Instruction}; -use wasmparser::VisitSimdOperator; +use crate::{ + core::{simd, simd::ImmLaneIdx32, FuelCostsProvider, ValType, V128}, + engine::translator::func2::Operand, + ir::{Instruction, Reg}, +}; +use core::array; +use wasmparser::{MemArg, VisitSimdOperator}; + +/// Used to swap operands of binary [`Instruction`] constructor. +macro_rules! swap_ops { + ($fn_name:path) => { + |result: Reg, lhs, rhs| -> Instruction { $fn_name(result, rhs, lhs) } + }; +} impl VisitSimdOperator<'_> for FuncTranslator { - fn visit_v128_load(&mut self, _memarg: wasmparser::MemArg) -> Self::Output { - todo!() - } - - fn visit_v128_load8x8_s(&mut self, _memarg: wasmparser::MemArg) -> Self::Output { - todo!() + fn visit_v128_load(&mut self, memarg: MemArg) -> Self::Output { + self.translate_load( + memarg, + ValType::V128, + Instruction::v128_load, + Instruction::v128_load_offset16, + Instruction::v128_load_at, + ) } - fn visit_v128_load8x8_u(&mut self, _memarg: wasmparser::MemArg) -> Self::Output { - todo!() + fn visit_v128_load8x8_s(&mut self, memarg: MemArg) -> Self::Output { + self.translate_load( + memarg, + ValType::V128, + Instruction::v128_load8x8_s, + Instruction::v128_load8x8_s_offset16, + Instruction::v128_load8x8_s_at, + ) } - fn visit_v128_load16x4_s(&mut self, _memarg: wasmparser::MemArg) -> Self::Output { - todo!() + fn visit_v128_load8x8_u(&mut self, memarg: MemArg) -> Self::Output { + self.translate_load( + memarg, + ValType::V128, + Instruction::v128_load8x8_u, + Instruction::v128_load8x8_u_offset16, + Instruction::v128_load8x8_u_at, + ) } - fn visit_v128_load16x4_u(&mut self, _memarg: wasmparser::MemArg) -> Self::Output { - todo!() + fn visit_v128_load16x4_s(&mut self, memarg: MemArg) -> Self::Output { + self.translate_load( + memarg, + ValType::V128, + Instruction::v128_load16x4_s, + Instruction::v128_load16x4_s_offset16, + Instruction::v128_load16x4_s_at, + ) } - fn visit_v128_load32x2_s(&mut self, _memarg: wasmparser::MemArg) -> Self::Output { - todo!() + fn visit_v128_load16x4_u(&mut self, memarg: MemArg) -> Self::Output { + self.translate_load( + memarg, + ValType::V128, + Instruction::v128_load16x4_u, + Instruction::v128_load16x4_u_offset16, + Instruction::v128_load16x4_u_at, + ) } - fn visit_v128_load32x2_u(&mut self, _memarg: wasmparser::MemArg) -> Self::Output { - todo!() + fn visit_v128_load32x2_s(&mut self, memarg: MemArg) -> Self::Output { + self.translate_load( + memarg, + ValType::V128, + Instruction::v128_load32x2_s, + Instruction::v128_load32x2_s_offset16, + Instruction::v128_load32x2_s_at, + ) } - fn visit_v128_load8_splat(&mut self, _memarg: wasmparser::MemArg) -> Self::Output { - todo!() + fn visit_v128_load32x2_u(&mut self, memarg: MemArg) -> Self::Output { + self.translate_load( + memarg, + ValType::V128, + Instruction::v128_load32x2_u, + Instruction::v128_load32x2_u_offset16, + Instruction::v128_load32x2_u_at, + ) } - fn visit_v128_load16_splat(&mut self, _memarg: wasmparser::MemArg) -> Self::Output { - todo!() + fn visit_v128_load8_splat(&mut self, memarg: MemArg) -> Self::Output { + self.translate_load( + memarg, + ValType::V128, + Instruction::v128_load8_splat, + Instruction::v128_load8_splat_offset16, + Instruction::v128_load8_splat_at, + ) } - fn visit_v128_load32_splat(&mut self, _memarg: wasmparser::MemArg) -> Self::Output { - todo!() + fn visit_v128_load16_splat(&mut self, memarg: MemArg) -> Self::Output { + self.translate_load( + memarg, + ValType::V128, + Instruction::v128_load16_splat, + Instruction::v128_load16_splat_offset16, + Instruction::v128_load16_splat_at, + ) } - fn visit_v128_load64_splat(&mut self, _memarg: wasmparser::MemArg) -> Self::Output { - todo!() + fn visit_v128_load32_splat(&mut self, memarg: MemArg) -> Self::Output { + self.translate_load( + memarg, + ValType::V128, + Instruction::v128_load32_splat, + Instruction::v128_load32_splat_offset16, + Instruction::v128_load32_splat_at, + ) } - fn visit_v128_load32_zero(&mut self, _memarg: wasmparser::MemArg) -> Self::Output { - todo!() + fn visit_v128_load64_splat(&mut self, memarg: MemArg) -> Self::Output { + self.translate_load( + memarg, + ValType::V128, + Instruction::v128_load64_splat, + Instruction::v128_load64_splat_offset16, + Instruction::v128_load64_splat_at, + ) } - fn visit_v128_load64_zero(&mut self, _memarg: wasmparser::MemArg) -> Self::Output { - todo!() + fn visit_v128_load32_zero(&mut self, memarg: MemArg) -> Self::Output { + self.translate_load( + memarg, + ValType::V128, + Instruction::v128_load32_zero, + Instruction::v128_load32_zero_offset16, + Instruction::v128_load32_zero_at, + ) } - fn visit_v128_store(&mut self, _memarg: wasmparser::MemArg) -> Self::Output { - todo!() + fn visit_v128_load64_zero(&mut self, memarg: MemArg) -> Self::Output { + self.translate_load( + memarg, + ValType::V128, + Instruction::v128_load64_zero, + Instruction::v128_load64_zero_offset16, + Instruction::v128_load64_zero_at, + ) } - fn visit_v128_load8_lane(&mut self, _memarg: wasmparser::MemArg, _lane: u8) -> Self::Output { - todo!() + fn visit_v128_store(&mut self, memarg: MemArg) -> Self::Output { + self.translate_store( + memarg, + Instruction::v128_store, + Instruction::v128_store_offset16, + Instruction::v128_store_at, + ) } - fn visit_v128_load16_lane(&mut self, _memarg: wasmparser::MemArg, _lane: u8) -> Self::Output { - todo!() + fn visit_v128_load8_lane(&mut self, memarg: MemArg, lane: u8) -> Self::Output { + self.translate_v128_load_lane::( + memarg, + lane, + Instruction::v128_load8_lane, + Instruction::v128_load8_lane_at, + ) } - fn visit_v128_load32_lane(&mut self, _memarg: wasmparser::MemArg, _lane: u8) -> Self::Output { - todo!() + fn visit_v128_load16_lane(&mut self, memarg: MemArg, lane: u8) -> Self::Output { + self.translate_v128_load_lane::( + memarg, + lane, + Instruction::v128_load16_lane, + Instruction::v128_load16_lane_at, + ) } - fn visit_v128_load64_lane(&mut self, _memarg: wasmparser::MemArg, _lane: u8) -> Self::Output { - todo!() + fn visit_v128_load32_lane(&mut self, memarg: MemArg, lane: u8) -> Self::Output { + self.translate_v128_load_lane::( + memarg, + lane, + Instruction::v128_load32_lane, + Instruction::v128_load32_lane_at, + ) } - fn visit_v128_store8_lane(&mut self, _memarg: wasmparser::MemArg, _lane: u8) -> Self::Output { - todo!() + fn visit_v128_load64_lane(&mut self, memarg: MemArg, lane: u8) -> Self::Output { + self.translate_v128_load_lane::( + memarg, + lane, + Instruction::v128_load64_lane, + Instruction::v128_load64_lane_at, + ) } - fn visit_v128_store16_lane(&mut self, _memarg: wasmparser::MemArg, _lane: u8) -> Self::Output { - todo!() + fn visit_v128_store8_lane(&mut self, memarg: MemArg, lane: u8) -> Self::Output { + self.translate_v128_store_lane::( + memarg, + lane, + Instruction::v128_store8_lane, + Instruction::v128_store8_lane_offset8, + Instruction::v128_store8_lane_at, + |_this, _memarg, _ptr, lane, v128| { + let _value = simd::i8x16_extract_lane_s(v128, lane); + todo!() + }, + ) } - fn visit_v128_store32_lane(&mut self, _memarg: wasmparser::MemArg, _lane: u8) -> Self::Output { - todo!() + fn visit_v128_store16_lane(&mut self, memarg: MemArg, lane: u8) -> Self::Output { + self.translate_v128_store_lane::( + memarg, + lane, + Instruction::v128_store16_lane, + Instruction::v128_store16_lane_offset8, + Instruction::v128_store16_lane_at, + |_this, _memarg, _ptr, lane, v128| { + let _value = simd::i16x8_extract_lane_s(v128, lane); + todo!() + }, + ) } - fn visit_v128_store64_lane(&mut self, _memarg: wasmparser::MemArg, _lane: u8) -> Self::Output { - todo!() + fn visit_v128_store32_lane(&mut self, memarg: MemArg, lane: u8) -> Self::Output { + self.translate_v128_store_lane::( + memarg, + lane, + Instruction::v128_store32_lane, + Instruction::v128_store32_lane_offset8, + Instruction::v128_store32_lane_at, + |_this, _memarg, _ptr, lane, v128| { + let _value = simd::i32x4_extract_lane(v128, lane); + todo!() + }, + ) } - fn visit_v128_const(&mut self, _value: wasmparser::V128) -> Self::Output { - todo!() + fn visit_v128_store64_lane(&mut self, memarg: MemArg, lane: u8) -> Self::Output { + self.translate_v128_store_lane::( + memarg, + lane, + Instruction::v128_store64_lane, + Instruction::v128_store64_lane_offset8, + Instruction::v128_store64_lane_at, + |_this, _memarg, _ptr, lane, v128| { + let _value = simd::i64x2_extract_lane(v128, lane); + todo!() + }, + ) } - fn visit_i8x16_shuffle(&mut self, _lanes: [u8; 16]) -> Self::Output { - todo!() + fn visit_v128_const(&mut self, value: wasmparser::V128) -> Self::Output { + bail_unreachable!(self); + let v128 = V128::from(value.i128() as u128); + self.stack.push_immediate(v128)?; + Ok(()) + } + + fn visit_i8x16_shuffle(&mut self, lanes: [u8; 16]) -> Self::Output { + bail_unreachable!(self); + let selector: [ImmLaneIdx32; 16] = array::from_fn(|i| { + let Ok(lane) = ImmLaneIdx32::try_from(lanes[i]) else { + panic!("encountered out of bounds lane at index {i}: {}", lanes[i]) + }; + lane + }); + let (lhs, rhs) = self.stack.pop2(); + if let (Operand::Immediate(lhs), Operand::Immediate(rhs)) = (lhs, rhs) { + let result = simd::i8x16_shuffle(lhs.val().into(), rhs.val().into(), selector); + self.stack.push_immediate(result)?; + return Ok(()); + } + let lhs = self.layout.operand_to_reg(lhs)?; + let rhs = self.layout.operand_to_reg(rhs)?; + let selector = self + .layout + .const_to_reg(V128::from(u128::from_ne_bytes(lanes)))?; + self.push_instr_with_result( + ValType::V128, + |result| Instruction::i8x16_shuffle(result, lhs, rhs), + FuelCostsProvider::simd, + )?; + self.push_param(Instruction::register(selector))?; + Ok(()) } fn visit_i8x16_extract_lane_s(&mut self, lane: u8) -> Self::Output { @@ -188,7 +370,7 @@ impl VisitSimdOperator<'_> for FuncTranslator { } fn visit_i8x16_swizzle(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::i8x16_swizzle, simd::i8x16_swizzle) } fn visit_i8x16_splat(&mut self) -> Self::Output { @@ -216,846 +398,1026 @@ impl VisitSimdOperator<'_> for FuncTranslator { } fn visit_i8x16_eq(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::i8x16_eq, simd::i8x16_eq) } fn visit_i8x16_ne(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::i8x16_ne, simd::i8x16_ne) } fn visit_i8x16_lt_s(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::i8x16_lt_s, simd::i8x16_lt_s) } fn visit_i8x16_lt_u(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::i8x16_lt_u, simd::i8x16_lt_u) } fn visit_i8x16_gt_s(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(swap_ops!(Instruction::i8x16_lt_s), simd::i8x16_gt_s) } fn visit_i8x16_gt_u(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(swap_ops!(Instruction::i8x16_lt_u), simd::i8x16_gt_u) } fn visit_i8x16_le_s(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::i8x16_le_s, simd::i8x16_le_s) } fn visit_i8x16_le_u(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::i8x16_le_u, simd::i8x16_le_u) } fn visit_i8x16_ge_s(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(swap_ops!(Instruction::i8x16_le_s), simd::i8x16_ge_s) } fn visit_i8x16_ge_u(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(swap_ops!(Instruction::i8x16_le_u), simd::i8x16_ge_u) } fn visit_i16x8_eq(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::i16x8_eq, simd::i16x8_eq) } fn visit_i16x8_ne(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::i16x8_ne, simd::i16x8_ne) } fn visit_i16x8_lt_s(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::i16x8_lt_s, simd::i16x8_lt_s) } fn visit_i16x8_lt_u(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::i16x8_lt_u, simd::i16x8_lt_u) } fn visit_i16x8_gt_s(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(swap_ops!(Instruction::i16x8_lt_s), simd::i16x8_gt_s) } fn visit_i16x8_gt_u(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(swap_ops!(Instruction::i16x8_lt_u), simd::i16x8_gt_u) } fn visit_i16x8_le_s(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::i16x8_le_s, simd::i16x8_le_s) } fn visit_i16x8_le_u(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::i16x8_le_u, simd::i16x8_le_u) } fn visit_i16x8_ge_s(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(swap_ops!(Instruction::i16x8_le_s), simd::i16x8_ge_s) } fn visit_i16x8_ge_u(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(swap_ops!(Instruction::i16x8_le_u), simd::i16x8_ge_u) } fn visit_i32x4_eq(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::i32x4_eq, simd::i32x4_eq) } fn visit_i32x4_ne(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::i32x4_ne, simd::i32x4_ne) } fn visit_i32x4_lt_s(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::i32x4_lt_s, simd::i32x4_lt_s) } fn visit_i32x4_lt_u(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::i32x4_lt_u, simd::i32x4_lt_u) } fn visit_i32x4_gt_s(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(swap_ops!(Instruction::i32x4_lt_s), simd::i32x4_gt_s) } fn visit_i32x4_gt_u(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(swap_ops!(Instruction::i32x4_lt_u), simd::i32x4_gt_u) } fn visit_i32x4_le_s(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::i32x4_le_s, simd::i32x4_le_s) } fn visit_i32x4_le_u(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::i32x4_le_u, simd::i32x4_le_u) } fn visit_i32x4_ge_s(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(swap_ops!(Instruction::i32x4_le_s), simd::i32x4_ge_s) } fn visit_i32x4_ge_u(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(swap_ops!(Instruction::i32x4_le_u), simd::i32x4_ge_u) } fn visit_i64x2_eq(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::i64x2_eq, simd::i64x2_eq) } fn visit_i64x2_ne(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::i64x2_ne, simd::i64x2_ne) } fn visit_i64x2_lt_s(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::i64x2_lt_s, simd::i64x2_lt_s) } fn visit_i64x2_gt_s(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(swap_ops!(Instruction::i64x2_lt_s), simd::i64x2_gt_s) } fn visit_i64x2_le_s(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::i64x2_le_s, simd::i64x2_le_s) } fn visit_i64x2_ge_s(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(swap_ops!(Instruction::i64x2_le_s), simd::i64x2_ge_s) } fn visit_f32x4_eq(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::f32x4_eq, simd::f32x4_eq) } fn visit_f32x4_ne(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::f32x4_ne, simd::f32x4_ne) } fn visit_f32x4_lt(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::f32x4_lt, simd::f32x4_lt) } fn visit_f32x4_gt(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(swap_ops!(Instruction::f32x4_lt), simd::f32x4_gt) } fn visit_f32x4_le(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::f32x4_le, simd::f32x4_le) } fn visit_f32x4_ge(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(swap_ops!(Instruction::f32x4_le), simd::f32x4_ge) } fn visit_f64x2_eq(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::f64x2_eq, simd::f64x2_eq) } fn visit_f64x2_ne(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::f64x2_ne, simd::f64x2_ne) } fn visit_f64x2_lt(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::f64x2_lt, simd::f64x2_lt) } fn visit_f64x2_gt(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(swap_ops!(Instruction::f64x2_lt), simd::f64x2_gt) } fn visit_f64x2_le(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::f64x2_le, simd::f64x2_le) } fn visit_f64x2_ge(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(swap_ops!(Instruction::f64x2_le), simd::f64x2_ge) } fn visit_v128_not(&mut self) -> Self::Output { - todo!() + self.translate_simd_unary(Instruction::v128_not, simd::v128_not) } fn visit_v128_and(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::v128_and, simd::v128_and) } fn visit_v128_andnot(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::v128_andnot, simd::v128_andnot) } fn visit_v128_or(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::v128_or, simd::v128_or) } fn visit_v128_xor(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::v128_xor, simd::v128_xor) } fn visit_v128_bitselect(&mut self) -> Self::Output { - todo!() + self.translate_simd_ternary(Instruction::v128_bitselect, simd::v128_bitselect) } fn visit_v128_any_true(&mut self) -> Self::Output { - todo!() + self.translate_simd_unary(Instruction::v128_any_true, simd::v128_any_true) } fn visit_i8x16_abs(&mut self) -> Self::Output { - todo!() + self.translate_simd_unary(Instruction::i8x16_abs, simd::i8x16_abs) } fn visit_i8x16_neg(&mut self) -> Self::Output { - todo!() + self.translate_simd_unary(Instruction::i8x16_neg, simd::i8x16_neg) } fn visit_i8x16_popcnt(&mut self) -> Self::Output { - todo!() + self.translate_simd_unary(Instruction::i8x16_popcnt, simd::i8x16_popcnt) } fn visit_i8x16_all_true(&mut self) -> Self::Output { - todo!() + self.translate_simd_unary(Instruction::i8x16_all_true, simd::i8x16_all_true) } fn visit_i8x16_bitmask(&mut self) -> Self::Output { - todo!() + self.translate_simd_unary(Instruction::i8x16_bitmask, simd::i8x16_bitmask) } fn visit_i8x16_narrow_i16x8_s(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary( + Instruction::i8x16_narrow_i16x8_s, + simd::i8x16_narrow_i16x8_s, + ) } fn visit_i8x16_narrow_i16x8_u(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary( + Instruction::i8x16_narrow_i16x8_u, + simd::i8x16_narrow_i16x8_u, + ) } fn visit_i8x16_shl(&mut self) -> Self::Output { - todo!() + self.translate_simd_shift::( + Instruction::i8x16_shl, + Instruction::i8x16_shl_by, + simd::i8x16_shl, + ) } fn visit_i8x16_shr_s(&mut self) -> Self::Output { - todo!() + self.translate_simd_shift::( + Instruction::i8x16_shr_s, + Instruction::i8x16_shr_s_by, + simd::i8x16_shr_s, + ) } fn visit_i8x16_shr_u(&mut self) -> Self::Output { - todo!() + self.translate_simd_shift::( + Instruction::i8x16_shr_u, + Instruction::i8x16_shr_u_by, + simd::i8x16_shr_u, + ) } fn visit_i8x16_add(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::i8x16_add, simd::i8x16_add) } fn visit_i8x16_add_sat_s(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::i8x16_add_sat_s, simd::i8x16_add_sat_s) } fn visit_i8x16_add_sat_u(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::i8x16_add_sat_u, simd::i8x16_add_sat_u) } fn visit_i8x16_sub(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::i8x16_sub, simd::i8x16_sub) } fn visit_i8x16_sub_sat_s(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::i8x16_sub_sat_s, simd::i8x16_sub_sat_s) } fn visit_i8x16_sub_sat_u(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::i8x16_sub_sat_u, simd::i8x16_sub_sat_u) } fn visit_i8x16_min_s(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::i8x16_min_s, simd::i8x16_min_s) } fn visit_i8x16_min_u(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::i8x16_min_u, simd::i8x16_min_u) } fn visit_i8x16_max_s(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::i8x16_max_s, simd::i8x16_max_s) } fn visit_i8x16_max_u(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::i8x16_max_u, simd::i8x16_max_u) } fn visit_i8x16_avgr_u(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::i8x16_avgr_u, simd::i8x16_avgr_u) } fn visit_i16x8_extadd_pairwise_i8x16_s(&mut self) -> Self::Output { - todo!() + self.translate_simd_unary( + Instruction::i16x8_extadd_pairwise_i8x16_s, + simd::i16x8_extadd_pairwise_i8x16_s, + ) } fn visit_i16x8_extadd_pairwise_i8x16_u(&mut self) -> Self::Output { - todo!() + self.translate_simd_unary( + Instruction::i16x8_extadd_pairwise_i8x16_u, + simd::i16x8_extadd_pairwise_i8x16_u, + ) } fn visit_i16x8_abs(&mut self) -> Self::Output { - todo!() + self.translate_simd_unary(Instruction::i16x8_abs, simd::i16x8_abs) } fn visit_i16x8_neg(&mut self) -> Self::Output { - todo!() + self.translate_simd_unary(Instruction::i16x8_neg, simd::i16x8_neg) } fn visit_i16x8_q15mulr_sat_s(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::i16x8_q15mulr_sat_s, simd::i16x8_q15mulr_sat_s) } fn visit_i16x8_all_true(&mut self) -> Self::Output { - todo!() + self.translate_simd_unary(Instruction::i16x8_all_true, simd::i16x8_all_true) } fn visit_i16x8_bitmask(&mut self) -> Self::Output { - todo!() + self.translate_simd_unary(Instruction::i16x8_bitmask, simd::i16x8_bitmask) } fn visit_i16x8_narrow_i32x4_s(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary( + Instruction::i16x8_narrow_i32x4_s, + simd::i16x8_narrow_i32x4_s, + ) } fn visit_i16x8_narrow_i32x4_u(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary( + Instruction::i16x8_narrow_i32x4_u, + simd::i16x8_narrow_i32x4_u, + ) } fn visit_i16x8_extend_low_i8x16_s(&mut self) -> Self::Output { - todo!() + self.translate_simd_unary( + Instruction::i16x8_extend_low_i8x16_s, + simd::i16x8_extend_low_i8x16_s, + ) } fn visit_i16x8_extend_high_i8x16_s(&mut self) -> Self::Output { - todo!() + self.translate_simd_unary( + Instruction::i16x8_extend_high_i8x16_s, + simd::i16x8_extend_high_i8x16_s, + ) } fn visit_i16x8_extend_low_i8x16_u(&mut self) -> Self::Output { - todo!() + self.translate_simd_unary( + Instruction::i16x8_extend_low_i8x16_u, + simd::i16x8_extend_low_i8x16_u, + ) } fn visit_i16x8_extend_high_i8x16_u(&mut self) -> Self::Output { - todo!() + self.translate_simd_unary( + Instruction::i16x8_extend_high_i8x16_u, + simd::i16x8_extend_high_i8x16_u, + ) } fn visit_i16x8_shl(&mut self) -> Self::Output { - todo!() + self.translate_simd_shift::( + Instruction::i16x8_shl, + Instruction::i16x8_shl_by, + simd::i16x8_shl, + ) } fn visit_i16x8_shr_s(&mut self) -> Self::Output { - todo!() + self.translate_simd_shift::( + Instruction::i16x8_shr_s, + Instruction::i16x8_shr_s_by, + simd::i16x8_shr_s, + ) } fn visit_i16x8_shr_u(&mut self) -> Self::Output { - todo!() + self.translate_simd_shift::( + Instruction::i16x8_shr_u, + Instruction::i16x8_shr_u_by, + simd::i16x8_shr_u, + ) } fn visit_i16x8_add(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::i16x8_add, simd::i16x8_add) } fn visit_i16x8_add_sat_s(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::i16x8_add_sat_s, simd::i16x8_add_sat_s) } fn visit_i16x8_add_sat_u(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::i16x8_add_sat_u, simd::i16x8_add_sat_u) } fn visit_i16x8_sub(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::i16x8_sub, simd::i16x8_sub) } fn visit_i16x8_sub_sat_s(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::i16x8_sub_sat_s, simd::i16x8_sub_sat_s) } fn visit_i16x8_sub_sat_u(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::i16x8_sub_sat_u, simd::i16x8_sub_sat_u) } fn visit_i16x8_mul(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::i16x8_mul, simd::i16x8_mul) } fn visit_i16x8_min_s(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::i16x8_min_s, simd::i16x8_min_s) } fn visit_i16x8_min_u(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::i16x8_min_u, simd::i16x8_min_u) } fn visit_i16x8_max_s(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::i16x8_max_s, simd::i16x8_max_s) } fn visit_i16x8_max_u(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::i16x8_max_u, simd::i16x8_max_u) } fn visit_i16x8_avgr_u(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::i16x8_avgr_u, simd::i16x8_avgr_u) } fn visit_i16x8_extmul_low_i8x16_s(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary( + Instruction::i16x8_extmul_low_i8x16_s, + simd::i16x8_extmul_low_i8x16_s, + ) } fn visit_i16x8_extmul_high_i8x16_s(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary( + Instruction::i16x8_extmul_high_i8x16_s, + simd::i16x8_extmul_high_i8x16_s, + ) } fn visit_i16x8_extmul_low_i8x16_u(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary( + Instruction::i16x8_extmul_low_i8x16_u, + simd::i16x8_extmul_low_i8x16_u, + ) } fn visit_i16x8_extmul_high_i8x16_u(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary( + Instruction::i16x8_extmul_high_i8x16_u, + simd::i16x8_extmul_high_i8x16_u, + ) } fn visit_i32x4_extadd_pairwise_i16x8_s(&mut self) -> Self::Output { - todo!() + self.translate_simd_unary( + Instruction::i32x4_extadd_pairwise_i16x8_s, + simd::i32x4_extadd_pairwise_i16x8_s, + ) } fn visit_i32x4_extadd_pairwise_i16x8_u(&mut self) -> Self::Output { - todo!() + self.translate_simd_unary( + Instruction::i32x4_extadd_pairwise_i16x8_u, + simd::i32x4_extadd_pairwise_i16x8_u, + ) } fn visit_i32x4_abs(&mut self) -> Self::Output { - todo!() + self.translate_simd_unary(Instruction::i32x4_abs, simd::i32x4_abs) } fn visit_i32x4_neg(&mut self) -> Self::Output { - todo!() + self.translate_simd_unary(Instruction::i32x4_neg, simd::i32x4_neg) } fn visit_i32x4_all_true(&mut self) -> Self::Output { - todo!() + self.translate_simd_unary(Instruction::i32x4_all_true, simd::i32x4_all_true) } fn visit_i32x4_bitmask(&mut self) -> Self::Output { - todo!() + self.translate_simd_unary(Instruction::i32x4_bitmask, simd::i32x4_bitmask) } fn visit_i32x4_extend_low_i16x8_s(&mut self) -> Self::Output { - todo!() + self.translate_simd_unary( + Instruction::i32x4_extend_low_i16x8_s, + simd::i32x4_extend_low_i16x8_s, + ) } fn visit_i32x4_extend_high_i16x8_s(&mut self) -> Self::Output { - todo!() + self.translate_simd_unary( + Instruction::i32x4_extend_high_i16x8_s, + simd::i32x4_extend_high_i16x8_s, + ) } fn visit_i32x4_extend_low_i16x8_u(&mut self) -> Self::Output { - todo!() + self.translate_simd_unary( + Instruction::i32x4_extend_low_i16x8_u, + simd::i32x4_extend_low_i16x8_u, + ) } fn visit_i32x4_extend_high_i16x8_u(&mut self) -> Self::Output { - todo!() + self.translate_simd_unary( + Instruction::i32x4_extend_high_i16x8_u, + simd::i32x4_extend_high_i16x8_u, + ) } fn visit_i32x4_shl(&mut self) -> Self::Output { - todo!() + self.translate_simd_shift::( + Instruction::i32x4_shl, + Instruction::i32x4_shl_by, + simd::i32x4_shl, + ) } fn visit_i32x4_shr_s(&mut self) -> Self::Output { - todo!() + self.translate_simd_shift::( + Instruction::i32x4_shr_s, + Instruction::i32x4_shr_s_by, + simd::i32x4_shr_s, + ) } fn visit_i32x4_shr_u(&mut self) -> Self::Output { - todo!() + self.translate_simd_shift::( + Instruction::i32x4_shr_u, + Instruction::i32x4_shr_u_by, + simd::i32x4_shr_u, + ) } fn visit_i32x4_add(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::i32x4_add, simd::i32x4_add) } fn visit_i32x4_sub(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::i32x4_sub, simd::i32x4_sub) } fn visit_i32x4_mul(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::i32x4_mul, simd::i32x4_mul) } fn visit_i32x4_min_s(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::i32x4_min_s, simd::i32x4_min_s) } fn visit_i32x4_min_u(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::i32x4_min_u, simd::i32x4_min_u) } fn visit_i32x4_max_s(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::i32x4_max_s, simd::i32x4_max_s) } fn visit_i32x4_max_u(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::i32x4_max_u, simd::i32x4_max_u) } fn visit_i32x4_dot_i16x8_s(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::i32x4_dot_i16x8_s, simd::i32x4_dot_i16x8_s) } fn visit_i32x4_extmul_low_i16x8_s(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary( + Instruction::i32x4_extmul_low_i16x8_s, + simd::i32x4_extmul_low_i16x8_s, + ) } fn visit_i32x4_extmul_high_i16x8_s(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary( + Instruction::i32x4_extmul_high_i16x8_s, + simd::i32x4_extmul_high_i16x8_s, + ) } fn visit_i32x4_extmul_low_i16x8_u(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary( + Instruction::i32x4_extmul_low_i16x8_u, + simd::i32x4_extmul_low_i16x8_u, + ) } fn visit_i32x4_extmul_high_i16x8_u(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary( + Instruction::i32x4_extmul_high_i16x8_u, + simd::i32x4_extmul_high_i16x8_u, + ) } fn visit_i64x2_abs(&mut self) -> Self::Output { - todo!() + self.translate_simd_unary(Instruction::i64x2_abs, simd::i64x2_abs) } fn visit_i64x2_neg(&mut self) -> Self::Output { - todo!() + self.translate_simd_unary(Instruction::i64x2_neg, simd::i64x2_neg) } fn visit_i64x2_all_true(&mut self) -> Self::Output { - todo!() + self.translate_simd_unary(Instruction::i64x2_all_true, simd::i64x2_all_true) } fn visit_i64x2_bitmask(&mut self) -> Self::Output { - todo!() + self.translate_simd_unary(Instruction::i64x2_bitmask, simd::i64x2_bitmask) } fn visit_i64x2_extend_low_i32x4_s(&mut self) -> Self::Output { - todo!() + self.translate_simd_unary( + Instruction::i64x2_extend_low_i32x4_s, + simd::i64x2_extend_low_i32x4_s, + ) } fn visit_i64x2_extend_high_i32x4_s(&mut self) -> Self::Output { - todo!() + self.translate_simd_unary( + Instruction::i64x2_extend_high_i32x4_s, + simd::i64x2_extend_high_i32x4_s, + ) } fn visit_i64x2_extend_low_i32x4_u(&mut self) -> Self::Output { - todo!() + self.translate_simd_unary( + Instruction::i64x2_extend_low_i32x4_u, + simd::i64x2_extend_low_i32x4_u, + ) } fn visit_i64x2_extend_high_i32x4_u(&mut self) -> Self::Output { - todo!() + self.translate_simd_unary( + Instruction::i64x2_extend_high_i32x4_u, + simd::i64x2_extend_high_i32x4_u, + ) } fn visit_i64x2_shl(&mut self) -> Self::Output { - todo!() + self.translate_simd_shift::( + Instruction::i64x2_shl, + Instruction::i64x2_shl_by, + simd::i64x2_shl, + ) } fn visit_i64x2_shr_s(&mut self) -> Self::Output { - todo!() + self.translate_simd_shift::( + Instruction::i64x2_shr_s, + Instruction::i64x2_shr_s_by, + simd::i64x2_shr_s, + ) } fn visit_i64x2_shr_u(&mut self) -> Self::Output { - todo!() + self.translate_simd_shift::( + Instruction::i64x2_shr_u, + Instruction::i64x2_shr_u_by, + simd::i64x2_shr_u, + ) } fn visit_i64x2_add(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::i64x2_add, simd::i64x2_add) } fn visit_i64x2_sub(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::i64x2_sub, simd::i64x2_sub) } fn visit_i64x2_mul(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::i64x2_mul, simd::i64x2_mul) } fn visit_i64x2_extmul_low_i32x4_s(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary( + Instruction::i64x2_extmul_low_i32x4_s, + simd::i64x2_extmul_low_i32x4_s, + ) } fn visit_i64x2_extmul_high_i32x4_s(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary( + Instruction::i64x2_extmul_high_i32x4_s, + simd::i64x2_extmul_high_i32x4_s, + ) } fn visit_i64x2_extmul_low_i32x4_u(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary( + Instruction::i64x2_extmul_low_i32x4_u, + simd::i64x2_extmul_low_i32x4_u, + ) } fn visit_i64x2_extmul_high_i32x4_u(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary( + Instruction::i64x2_extmul_high_i32x4_u, + simd::i64x2_extmul_high_i32x4_u, + ) } fn visit_f32x4_ceil(&mut self) -> Self::Output { - todo!() + self.translate_simd_unary(Instruction::f32x4_ceil, simd::f32x4_ceil) } fn visit_f32x4_floor(&mut self) -> Self::Output { - todo!() + self.translate_simd_unary(Instruction::f32x4_floor, simd::f32x4_floor) } fn visit_f32x4_trunc(&mut self) -> Self::Output { - todo!() + self.translate_simd_unary(Instruction::f32x4_trunc, simd::f32x4_trunc) } fn visit_f32x4_nearest(&mut self) -> Self::Output { - todo!() + self.translate_simd_unary(Instruction::f32x4_nearest, simd::f32x4_nearest) } fn visit_f32x4_abs(&mut self) -> Self::Output { - todo!() + self.translate_simd_unary(Instruction::f32x4_abs, simd::f32x4_abs) } fn visit_f32x4_neg(&mut self) -> Self::Output { - todo!() + self.translate_simd_unary(Instruction::f32x4_neg, simd::f32x4_neg) } fn visit_f32x4_sqrt(&mut self) -> Self::Output { - todo!() + self.translate_simd_unary(Instruction::f32x4_sqrt, simd::f32x4_sqrt) } fn visit_f32x4_add(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::f32x4_add, simd::f32x4_add) } fn visit_f32x4_sub(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::f32x4_sub, simd::f32x4_sub) } fn visit_f32x4_mul(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::f32x4_mul, simd::f32x4_mul) } fn visit_f32x4_div(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::f32x4_div, simd::f32x4_div) } fn visit_f32x4_min(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::f32x4_min, simd::f32x4_min) } fn visit_f32x4_max(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::f32x4_max, simd::f32x4_max) } fn visit_f32x4_pmin(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::f32x4_pmin, simd::f32x4_pmin) } fn visit_f32x4_pmax(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::f32x4_pmax, simd::f32x4_pmax) } fn visit_f64x2_ceil(&mut self) -> Self::Output { - todo!() + self.translate_simd_unary(Instruction::f64x2_ceil, simd::f64x2_ceil) } fn visit_f64x2_floor(&mut self) -> Self::Output { - todo!() + self.translate_simd_unary(Instruction::f64x2_floor, simd::f64x2_floor) } fn visit_f64x2_trunc(&mut self) -> Self::Output { - todo!() + self.translate_simd_unary(Instruction::f64x2_trunc, simd::f64x2_trunc) } fn visit_f64x2_nearest(&mut self) -> Self::Output { - todo!() + self.translate_simd_unary(Instruction::f64x2_nearest, simd::f64x2_nearest) } fn visit_f64x2_abs(&mut self) -> Self::Output { - todo!() + self.translate_simd_unary(Instruction::f64x2_abs, simd::f64x2_abs) } fn visit_f64x2_neg(&mut self) -> Self::Output { - todo!() + self.translate_simd_unary(Instruction::f64x2_neg, simd::f64x2_neg) } fn visit_f64x2_sqrt(&mut self) -> Self::Output { - todo!() + self.translate_simd_unary(Instruction::f64x2_sqrt, simd::f64x2_sqrt) } fn visit_f64x2_add(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::f64x2_add, simd::f64x2_add) } fn visit_f64x2_sub(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::f64x2_sub, simd::f64x2_sub) } fn visit_f64x2_mul(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::f64x2_mul, simd::f64x2_mul) } fn visit_f64x2_div(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::f64x2_div, simd::f64x2_div) } fn visit_f64x2_min(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::f64x2_min, simd::f64x2_min) } fn visit_f64x2_max(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::f64x2_max, simd::f64x2_max) } fn visit_f64x2_pmin(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::f64x2_pmin, simd::f64x2_pmin) } fn visit_f64x2_pmax(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary(Instruction::f64x2_pmax, simd::f64x2_pmax) } fn visit_i32x4_trunc_sat_f32x4_s(&mut self) -> Self::Output { - todo!() + self.translate_simd_unary( + Instruction::i32x4_trunc_sat_f32x4_s, + simd::i32x4_trunc_sat_f32x4_s, + ) } fn visit_i32x4_trunc_sat_f32x4_u(&mut self) -> Self::Output { - todo!() + self.translate_simd_unary( + Instruction::i32x4_trunc_sat_f32x4_u, + simd::i32x4_trunc_sat_f32x4_u, + ) } fn visit_f32x4_convert_i32x4_s(&mut self) -> Self::Output { - todo!() + self.translate_simd_unary( + Instruction::f32x4_convert_i32x4_s, + simd::f32x4_convert_i32x4_s, + ) } fn visit_f32x4_convert_i32x4_u(&mut self) -> Self::Output { - todo!() + self.translate_simd_unary( + Instruction::f32x4_convert_i32x4_u, + simd::f32x4_convert_i32x4_u, + ) } fn visit_i32x4_trunc_sat_f64x2_s_zero(&mut self) -> Self::Output { - todo!() + self.translate_simd_unary( + Instruction::i32x4_trunc_sat_f64x2_s_zero, + simd::i32x4_trunc_sat_f64x2_s_zero, + ) } fn visit_i32x4_trunc_sat_f64x2_u_zero(&mut self) -> Self::Output { - todo!() + self.translate_simd_unary( + Instruction::i32x4_trunc_sat_f64x2_u_zero, + simd::i32x4_trunc_sat_f64x2_u_zero, + ) } fn visit_f64x2_convert_low_i32x4_s(&mut self) -> Self::Output { - todo!() + self.translate_simd_unary( + Instruction::f64x2_convert_low_i32x4_s, + simd::f64x2_convert_low_i32x4_s, + ) } fn visit_f64x2_convert_low_i32x4_u(&mut self) -> Self::Output { - todo!() + self.translate_simd_unary( + Instruction::f64x2_convert_low_i32x4_u, + simd::f64x2_convert_low_i32x4_u, + ) } fn visit_f32x4_demote_f64x2_zero(&mut self) -> Self::Output { - todo!() + self.translate_simd_unary( + Instruction::f32x4_demote_f64x2_zero, + simd::f32x4_demote_f64x2_zero, + ) } fn visit_f64x2_promote_low_f32x4(&mut self) -> Self::Output { - todo!() + self.translate_simd_unary( + Instruction::f64x2_promote_low_f32x4, + simd::f64x2_promote_low_f32x4, + ) } fn visit_i8x16_relaxed_swizzle(&mut self) -> Self::Output { - todo!() + self.visit_i8x16_swizzle() } fn visit_i32x4_relaxed_trunc_f32x4_s(&mut self) -> Self::Output { - todo!() + self.visit_i32x4_trunc_sat_f32x4_s() } fn visit_i32x4_relaxed_trunc_f32x4_u(&mut self) -> Self::Output { - todo!() + self.visit_i32x4_trunc_sat_f32x4_u() } fn visit_i32x4_relaxed_trunc_f64x2_s_zero(&mut self) -> Self::Output { - todo!() + self.visit_i32x4_trunc_sat_f64x2_s_zero() } fn visit_i32x4_relaxed_trunc_f64x2_u_zero(&mut self) -> Self::Output { - todo!() + self.visit_i32x4_trunc_sat_f64x2_u_zero() } fn visit_f32x4_relaxed_madd(&mut self) -> Self::Output { - todo!() + self.translate_simd_ternary(Instruction::f32x4_relaxed_madd, simd::f32x4_relaxed_madd) } fn visit_f32x4_relaxed_nmadd(&mut self) -> Self::Output { - todo!() + self.translate_simd_ternary(Instruction::f32x4_relaxed_nmadd, simd::f32x4_relaxed_nmadd) } fn visit_f64x2_relaxed_madd(&mut self) -> Self::Output { - todo!() + self.translate_simd_ternary(Instruction::f64x2_relaxed_madd, simd::f64x2_relaxed_madd) } fn visit_f64x2_relaxed_nmadd(&mut self) -> Self::Output { - todo!() + self.translate_simd_ternary(Instruction::f64x2_relaxed_nmadd, simd::f64x2_relaxed_nmadd) } fn visit_i8x16_relaxed_laneselect(&mut self) -> Self::Output { - todo!() + self.visit_v128_bitselect() } fn visit_i16x8_relaxed_laneselect(&mut self) -> Self::Output { - todo!() + self.visit_v128_bitselect() } fn visit_i32x4_relaxed_laneselect(&mut self) -> Self::Output { - todo!() + self.visit_v128_bitselect() } fn visit_i64x2_relaxed_laneselect(&mut self) -> Self::Output { - todo!() + self.visit_v128_bitselect() } fn visit_f32x4_relaxed_min(&mut self) -> Self::Output { - todo!() + self.visit_f32x4_min() } fn visit_f32x4_relaxed_max(&mut self) -> Self::Output { - todo!() + self.visit_f32x4_max() } fn visit_f64x2_relaxed_min(&mut self) -> Self::Output { - todo!() + self.visit_f64x2_min() } fn visit_f64x2_relaxed_max(&mut self) -> Self::Output { - todo!() + self.visit_f64x2_max() } fn visit_i16x8_relaxed_q15mulr_s(&mut self) -> Self::Output { - todo!() + self.visit_i16x8_q15mulr_sat_s() } fn visit_i16x8_relaxed_dot_i8x16_i7x16_s(&mut self) -> Self::Output { - todo!() + self.translate_simd_binary( + Instruction::i16x8_relaxed_dot_i8x16_i7x16_s, + simd::i16x8_relaxed_dot_i8x16_i7x16_s, + ) } fn visit_i32x4_relaxed_dot_i8x16_i7x16_add_s(&mut self) -> Self::Output { - todo!() + self.translate_simd_ternary( + Instruction::i32x4_relaxed_dot_i8x16_i7x16_add_s, + simd::i32x4_relaxed_dot_i8x16_i7x16_add_s, + ) } }