From 70a37939d367e83ab62002bad64fb11e763f3d2f Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Fri, 6 Dec 2024 13:23:55 -0700 Subject: [PATCH] Support executing Pulley in Wasmtime (#9744) * Support executing Pulley in Wasmtime This commit is the initial implementation of executing the Pulley interpreter from the `wasmtime` crate. This gives access to all of the `wasmtime` crate's runtime APIs backed by execution of bytecode in Pulley. This builds on the previous PRs I've been making for support in Pulley to culminate in testing on CI in this PR. This PR handles some final tidbits related to producing a runnable image that can be interpreted by the `wasmtime` crate such as: * Pulley compilation artifacts are no longer marked as natively executable, just read-only. * Pulley compilation artifacts include wasm-to-array trampolines like normal platforms (removes a pulley special-case). * Dispatch of host calls from Pulley to the Wasmtime runtime are implemented. * Pulley's list of panicking wasm features is slimmed down as most are covered by "this lowering isn't supported" errors. * Execution of wasm code now has an `if` to see whether Pulley is enabled within a `Store` or not. * Traps and basic "unwinding" of the pulley stack are implemented (e.g. a "pulley version" of `setjmp` and `longjmp`, sort of) * Halting the interpreter has been refactored to help shrink the size of `ControlFlow` and handle metadata with each done state. Some minor refactorings are also included here and there along with a few fixes here and there necessary to get tests passing. The next major part of this commit is updates to our `wast` test suite and executing all `*.wast` files. Pulley is now executed by default for all files as a new execution engine. This means that all platforms in CI will start executing Pulley tests. At this time almost all tests are flagged as "expected to fail" but there are a small handful of allow-listed tests which are expected to pass. This exact list will change over time as CLIF lowerings are implemented and the interpreter is extended. Follow-up PRs will extend the testing strategy further such as: * Extending `#[wasmtime_test]` to include Pulley in addition to Winch. * Getting testing set up on CI for 32-bit platforms. prtest:full * Fix pulley fuzz build * Fix clippy lints * Shuffle around some `#[cfg]`'d code * Remove unused imports * Update feature sets testing MIRI Enable pulley for wasmtime/wasmtime-cli and also enable all features for wasmtime-environ * Round up pulley's page size to 64k * Skip pulley tests on s390x for now * Add a safety rail for matching a pulley target to the host * Fix more pulley tests on s390x * Review comments * Fix fuzz build --- .github/workflows/main.yml | 4 +- .../src/isa/pulley_shared/inst/args.rs | 4 +- .../src/isa/pulley_shared/inst/emit.rs | 10 +- .../codegen/src/isa/pulley_shared/lower.isle | 8 +- crates/cranelift/src/compiler.rs | 11 +- crates/cranelift/src/func_environ.rs | 20 +- crates/cranelift/src/obj.rs | 18 +- crates/environ/src/compile/mod.rs | 3 + .../environ/src/compile/module_artifacts.rs | 17 +- crates/environ/src/component/info.rs | 21 +- crates/environ/src/ext.rs | 18 + crates/environ/src/lib.rs | 2 + crates/environ/src/module_artifacts.rs | 4 - crates/fuzzing/src/generators/config.rs | 2 +- crates/misc/component-test-util/src/lib.rs | 21 +- crates/test-macros/src/lib.rs | 4 +- crates/wasmtime/src/compile.rs | 106 +++--- crates/wasmtime/src/config.rs | 28 +- crates/wasmtime/src/engine.rs | 23 +- .../src/runtime/component/func/host.rs | 24 +- crates/wasmtime/src/runtime/func.rs | 16 +- crates/wasmtime/src/runtime/func/typed.rs | 4 +- crates/wasmtime/src/runtime/instance.rs | 10 +- .../wasmtime/src/runtime/module/registry.rs | 1 - crates/wasmtime/src/runtime/store.rs | 21 +- crates/wasmtime/src/runtime/trap.rs | 1 - crates/wasmtime/src/runtime/vm.rs | 8 + crates/wasmtime/src/runtime/vm/component.rs | 16 +- crates/wasmtime/src/runtime/vm/interpreter.rs | 324 ++++++++++++++++++ .../src/runtime/vm/interpreter_disabled.rs | 49 +++ .../wasmtime/src/runtime/vm/traphandlers.rs | 152 +++++++- .../src/runtime/vm/traphandlers/signals.rs | 100 ------ crates/wasmtime/src/runtime/vm/vmcontext.rs | 31 +- crates/wast-util/src/lib.rs | 208 ++++++++--- pulley/examples/objdump.rs | 4 +- pulley/fuzz/src/interp.rs | 7 +- pulley/src/interp.rs | 201 +++++++---- pulley/src/opcode.rs | 16 - pulley/tests/all/interp.rs | 12 +- tests/all/pulley.rs | 25 +- tests/disas/pulley/epoch-simple.wat | 2 +- tests/wast.rs | 19 +- 42 files changed, 1120 insertions(+), 455 deletions(-) create mode 100644 crates/environ/src/ext.rs create mode 100644 crates/wasmtime/src/runtime/vm/interpreter.rs create mode 100644 crates/wasmtime/src/runtime/vm/interpreter_disabled.rs diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index af864e99fb2c..8fe8bd484bff 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1046,9 +1046,9 @@ jobs: strategy: matrix: crate: - - "wasmtime" + - "wasmtime --features pulley" - "wasmtime-cli" - - "wasmtime-environ" + - "wasmtime-environ --all-features" - "pulley-interpreter --all-features" needs: determine if: needs.determine.outputs.test-miri && github.repository == 'bytecodealliance/wasmtime' diff --git a/cranelift/codegen/src/isa/pulley_shared/inst/args.rs b/cranelift/codegen/src/isa/pulley_shared/inst/args.rs index e70dfbaf2249..0d6dc6161104 100644 --- a/cranelift/codegen/src/isa/pulley_shared/inst/args.rs +++ b/cranelift/codegen/src/isa/pulley_shared/inst/args.rs @@ -180,7 +180,9 @@ impl Amode { + frame_layout.outgoing_args_size; i64::from(sp_offset) - offset } - StackAMode::Slot(offset) => *offset, + StackAMode::Slot(offset) => { + offset + i64::from(state.frame_layout().outgoing_args_size) + } StackAMode::OutgoingArg(offset) => *offset, }, } diff --git a/cranelift/codegen/src/isa/pulley_shared/inst/emit.rs b/cranelift/codegen/src/isa/pulley_shared/inst/emit.rs index 6131cee2c461..42095c0d6e99 100644 --- a/cranelift/codegen/src/isa/pulley_shared/inst/emit.rs +++ b/cranelift/codegen/src/isa/pulley_shared/inst/emit.rs @@ -549,9 +549,15 @@ fn pulley_emit

( *start_offset = sink.cur_offset(); } - Inst::PushFrame => enc::push_frame(sink), + Inst::PushFrame => { + sink.add_trap(ir::TrapCode::STACK_OVERFLOW); + enc::push_frame(sink); + } Inst::PopFrame => enc::pop_frame(sink), - Inst::StackAlloc32 { amt } => enc::stack_alloc32(sink, *amt), + Inst::StackAlloc32 { amt } => { + sink.add_trap(ir::TrapCode::STACK_OVERFLOW); + enc::stack_alloc32(sink, *amt); + } Inst::StackFree32 { amt } => enc::stack_free32(sink, *amt), Inst::Zext8 { dst, src } => enc::zext8(sink, dst, src), diff --git a/cranelift/codegen/src/isa/pulley_shared/lower.isle b/cranelift/codegen/src/isa/pulley_shared/lower.isle index 00de3b70838c..6f6a4fff5edc 100644 --- a/cranelift/codegen/src/isa/pulley_shared/lower.isle +++ b/cranelift/codegen/src/isa/pulley_shared/lower.isle @@ -59,7 +59,7 @@ ;;;; Rules for `trapz` and `trapnz` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -(rule (lower (trapz a @ (value_type (fits_in_64 ty)) code)) +(rule (lower (trapz a @ (value_type (ty_32_or_64 ty)) code)) (let ((zero Reg (pulley_xconst8 0))) (side_effect (pulley_trap_if (IntCC.Equal) (ty_to_operand_size ty) @@ -67,7 +67,7 @@ zero code)))) -(rule (lower (trapnz a @ (value_type (fits_in_64 ty)) code)) +(rule (lower (trapnz a @ (value_type (ty_32_or_64 ty)) code)) (let ((zero Reg (pulley_xconst8 0))) (side_effect (pulley_trap_if (IntCC.NotEqual) (ty_to_operand_size ty) @@ -77,14 +77,14 @@ ;; Fold `(trap[n]z (icmp ...))` together. -(rule 1 (lower (trapz (icmp cc a b @ (value_type (fits_in_64 ty))) code)) +(rule 1 (lower (trapz (icmp cc a b @ (value_type (ty_32_or_64 ty))) code)) (side_effect (pulley_trap_if (intcc_complement cc) (ty_to_operand_size ty) a b code))) -(rule 1 (lower (trapnz (icmp cc a b @ (value_type (fits_in_64 ty))) code)) +(rule 1 (lower (trapnz (icmp cc a b @ (value_type (ty_32_or_64 ty))) code)) (side_effect (pulley_trap_if cc (ty_to_operand_size ty) a diff --git a/crates/cranelift/src/compiler.rs b/crates/cranelift/src/compiler.rs index cbe2bfa9e7d2..ea87148ca01a 100644 --- a/crates/cranelift/src/compiler.rs +++ b/crates/cranelift/src/compiler.rs @@ -30,7 +30,7 @@ use wasmtime_environ::{ AddressMapSection, BuiltinFunctionIndex, CacheStore, CompileError, DefinedFuncIndex, FlagValue, FunctionBodyData, FunctionLoc, HostCall, ModuleTranslation, ModuleTypesBuilder, PtrSize, RelocationTarget, StackMapInformation, StaticModuleIndex, TrapEncodingBuilder, TrapSentinel, - Tunables, VMOffsets, WasmFuncType, WasmFunctionInfo, WasmValType, + TripleExt, Tunables, VMOffsets, WasmFuncType, WasmFunctionInfo, WasmValType, }; #[cfg(feature = "component-model")] @@ -152,12 +152,7 @@ impl Compiler { // `call` instruction where the name is `colocated: false`. This will // force a pulley-specific relocation to get emitted in addition to // using the `call_indirect_host` instruction. - let is_pulley = match self.isa.triple().architecture { - target_lexicon::Architecture::Pulley32 => true, - target_lexicon::Architecture::Pulley64 => true, - _ => false, - }; - if is_pulley { + if self.isa.triple().is_pulley() { let mut new_signature = signature.clone(); new_signature .params @@ -246,7 +241,7 @@ impl wasmtime_environ::Compiler for Compiler { // abort for the whole program since the runtime limits configured by // the embedder should cause wasm to trap before it reaches that // (ensuring the host has enough space as well for its functionality). - if !func_env.is_pulley() { + if !isa.triple().is_pulley() { let vmctx = context .func .create_global_value(ir::GlobalValueData::VMContext); diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs index 53db72ca394c..b4cb0f879b19 100644 --- a/crates/cranelift/src/func_environ.rs +++ b/crates/cranelift/src/func_environ.rs @@ -22,9 +22,9 @@ use wasmparser::{Operator, WasmFeatures}; use wasmtime_environ::{ BuiltinFunctionIndex, DataIndex, ElemIndex, EngineOrModuleTypeIndex, FuncIndex, GlobalIndex, IndexType, Memory, MemoryIndex, Module, ModuleInternedTypeIndex, ModuleTranslation, - ModuleTypesBuilder, PtrSize, Table, TableIndex, Tunables, TypeConvert, TypeIndex, VMOffsets, - WasmCompositeInnerType, WasmFuncType, WasmHeapTopType, WasmHeapType, WasmRefType, WasmResult, - WasmValType, + ModuleTypesBuilder, PtrSize, Table, TableIndex, TripleExt, Tunables, TypeConvert, TypeIndex, + VMOffsets, WasmCompositeInnerType, WasmFuncType, WasmHeapTopType, WasmHeapType, WasmRefType, + WasmResult, WasmValType, }; use wasmtime_environ::{FUNCREF_INIT_BIT, FUNCREF_MASK}; @@ -1187,16 +1187,6 @@ impl<'module_environment> FuncEnvironment<'module_environment> { i32::from(self.offsets.ptr.vm_func_ref_type_index()), ) } - - /// Returns whether the current compilation target is for the Pulley - /// interpreter. - pub fn is_pulley(&self) -> bool { - match self.isa.triple().architecture { - target_lexicon::Architecture::Pulley32 => true, - target_lexicon::Architecture::Pulley64 => true, - _ => false, - } - } } struct Call<'a, 'func, 'module_env> { @@ -3419,7 +3409,7 @@ impl FuncEnvironment<'_> { /// being targetted since the Pulley runtime doesn't catch segfaults for /// itself. pub fn clif_memory_traps_enabled(&self) -> bool { - self.tunables.signals_based_traps && !self.is_pulley() + self.tunables.signals_based_traps && !self.isa.triple().is_pulley() } /// Returns whether it's acceptable to have CLIF instructions natively trap, @@ -3429,7 +3419,7 @@ impl FuncEnvironment<'_> { /// unconditionally since Pulley doesn't use hardware-based traps in its /// runtime. pub fn clif_instruction_traps_enabled(&self) -> bool { - self.tunables.signals_based_traps || self.is_pulley() + self.tunables.signals_based_traps || self.isa.triple().is_pulley() } } diff --git a/crates/cranelift/src/obj.rs b/crates/cranelift/src/obj.rs index a55c08b88441..e64be78367a3 100644 --- a/crates/cranelift/src/obj.rs +++ b/crates/cranelift/src/obj.rs @@ -22,11 +22,11 @@ use cranelift_control::ControlPlane; use gimli::write::{Address, EhFrame, EndianVec, FrameTable, Writer}; use gimli::RunTimeEndian; use object::write::{Object, SectionId, StandardSegment, Symbol, SymbolId, SymbolSection}; -use object::{Architecture, SectionKind, SymbolFlags, SymbolKind, SymbolScope}; +use object::{Architecture, SectionFlags, SectionKind, SymbolFlags, SymbolKind, SymbolScope}; use std::collections::HashMap; use std::ops::Range; -use wasmtime_environ::obj::LibCall; -use wasmtime_environ::{Compiler, Unsigned}; +use wasmtime_environ::obj::{self, LibCall}; +use wasmtime_environ::{Compiler, TripleExt, Unsigned}; const TEXT_SECTION_NAME: &[u8] = b".text"; @@ -83,6 +83,18 @@ impl<'a> ModuleTextBuilder<'a> { SectionKind::Text, ); + // If this target is Pulley then flag the text section as not needing the + // executable bit in virtual memory which means that the runtime won't + // try to call `Mmap::make_exectuable`, which makes Pulley more + // portable. + if compiler.triple().is_pulley() { + let section = obj.section_mut(text_section); + assert!(matches!(section.flags, SectionFlags::None)); + section.flags = SectionFlags::Elf { + sh_flags: obj::SH_WASMTIME_NOT_EXECUTED, + }; + } + Self { compiler, obj, diff --git a/crates/environ/src/compile/mod.rs b/crates/environ/src/compile/mod.rs index 642e20c2debe..db4bf9548518 100644 --- a/crates/environ/src/compile/mod.rs +++ b/crates/environ/src/compile/mod.rs @@ -338,6 +338,9 @@ pub trait Compiler: Send + Sync { // 64 KB is the maximal page size (i.e. memory translation granule size) // supported by the architecture and is used on some platforms. (_, Architecture::Aarch64(..)) => 0x10000, + // Conservatively assume the max-of-all-supported-hosts for pulley + // and round up to 64k. + (_, Architecture::Pulley32 | Architecture::Pulley64) => 0x10000, _ => 0x1000, } } diff --git a/crates/environ/src/compile/module_artifacts.rs b/crates/environ/src/compile/module_artifacts.rs index bc1d22634c5b..7d9d132eddd6 100644 --- a/crates/environ/src/compile/module_artifacts.rs +++ b/crates/environ/src/compile/module_artifacts.rs @@ -28,9 +28,6 @@ pub struct ObjectBuilder<'a> { /// will go. data: SectionId, - /// The target triple for this compilation. - triple: target_lexicon::Triple, - /// The section identifier for function name information, or otherwise where /// the `name` custom section of wasm is copied into. /// @@ -46,11 +43,7 @@ pub struct ObjectBuilder<'a> { impl<'a> ObjectBuilder<'a> { /// Creates a new builder for the `obj` specified. - pub fn new( - mut obj: Object<'a>, - tunables: &'a Tunables, - triple: target_lexicon::Triple, - ) -> ObjectBuilder<'a> { + pub fn new(mut obj: Object<'a>, tunables: &'a Tunables) -> ObjectBuilder<'a> { let data = obj.add_section( obj.segment_name(StandardSegment::Data).to_vec(), obj::ELF_WASM_DATA.as_bytes().to_vec(), @@ -60,7 +53,6 @@ impl<'a> ObjectBuilder<'a> { obj, tunables, data, - triple, names: None, dwarf: None, } @@ -225,12 +217,6 @@ impl<'a> ObjectBuilder<'a> { self.push_debuginfo(&mut dwarf, &debuginfo); } - let is_pulley = matches!( - self.triple.architecture, - target_lexicon::Architecture::Pulley32 | target_lexicon::Architecture::Pulley64 - ); - assert!(!is_pulley || wasm_to_array_trampolines.is_empty()); - Ok(CompiledModuleInfo { module, funcs, @@ -240,7 +226,6 @@ impl<'a> ObjectBuilder<'a> { has_unparsed_debuginfo, code_section_offset: debuginfo.wasm_file.code_section_offset, has_wasm_debuginfo: self.tunables.parse_wasm_debuginfo, - is_pulley, dwarf, }, }) diff --git a/crates/environ/src/component/info.rs b/crates/environ/src/component/info.rs index e1b6f2edf58b..8c5442a6d243 100644 --- a/crates/environ/src/component/info.rs +++ b/crates/environ/src/component/info.rs @@ -452,19 +452,30 @@ pub struct CanonicalOptions { } /// Possible encodings of strings within the component model. -// -// Note that the `repr(u8)` is load-bearing here since this is used in an -// `extern "C" fn()` function argument which is called from cranelift-compiled -// code so we must know the representation of this. #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Hash)] #[allow(missing_docs, reason = "self-describing variants")] -#[repr(u8)] pub enum StringEncoding { Utf8, Utf16, CompactUtf16, } +impl StringEncoding { + /// Decodes the `u8` provided back into a `StringEncoding`, if it's valid. + pub fn from_u8(val: u8) -> Option { + if val == StringEncoding::Utf8 as u8 { + return Some(StringEncoding::Utf8); + } + if val == StringEncoding::Utf16 as u8 { + return Some(StringEncoding::Utf16); + } + if val == StringEncoding::CompactUtf16 as u8 { + return Some(StringEncoding::CompactUtf16); + } + None + } +} + /// Possible transcoding operations that must be provided by the host. /// /// Note that each transcoding operation may have a unique signature depending diff --git a/crates/environ/src/ext.rs b/crates/environ/src/ext.rs new file mode 100644 index 000000000000..8311c4789a96 --- /dev/null +++ b/crates/environ/src/ext.rs @@ -0,0 +1,18 @@ +use target_lexicon::{Architecture, Triple}; + +/// Extension methods for `target_lexicon::Triple`. +pub trait TripleExt { + /// Helper for returning whether this target is for pulley, wasmtime's + /// interpreter. + fn is_pulley(&self) -> bool; +} + +impl TripleExt for Triple { + fn is_pulley(&self) -> bool { + match self.architecture { + Architecture::Pulley32 => true, + Architecture::Pulley64 => true, + _ => false, + } + } +} diff --git a/crates/environ/src/lib.rs b/crates/environ/src/lib.rs index 555e5586678a..91c887b55a52 100644 --- a/crates/environ/src/lib.rs +++ b/crates/environ/src/lib.rs @@ -19,6 +19,7 @@ mod address_map; mod builtin; mod demangling; mod error; +mod ext; mod gc; mod hostcall; mod module; @@ -33,6 +34,7 @@ mod tunables; mod types; mod vmoffsets; +pub use self::ext::*; pub use crate::address_map::*; pub use crate::builtin::*; pub use crate::demangling::*; diff --git a/crates/environ/src/module_artifacts.rs b/crates/environ/src/module_artifacts.rs index 6707da994ebe..1d51f1215cdb 100644 --- a/crates/environ/src/module_artifacts.rs +++ b/crates/environ/src/module_artifacts.rs @@ -107,10 +107,6 @@ pub struct Metadata { /// weren't found in the original wasm module itself. pub has_wasm_debuginfo: bool, - /// Whether this artifact contains Pulley bytecode (instead of machine code) - /// or not. - pub is_pulley: bool, - /// Dwarf sections and the offsets at which they're stored in the /// ELF_WASMTIME_DWARF pub dwarf: Vec<(u8, Range)>, diff --git a/crates/fuzzing/src/generators/config.rs b/crates/fuzzing/src/generators/config.rs index fd7180853e16..8fbdb737aa2b 100644 --- a/crates/fuzzing/src/generators/config.rs +++ b/crates/fuzzing/src/generators/config.rs @@ -233,7 +233,7 @@ impl Config { InstanceAllocationStrategy::Pooling(_) ), compiler: match self.wasmtime.compiler_strategy { - CompilerStrategy::Cranelift => wasmtime_wast_util::Compiler::Cranelift, + CompilerStrategy::Cranelift => wasmtime_wast_util::Compiler::CraneliftNative, CompilerStrategy::Winch => wasmtime_wast_util::Compiler::Winch, }, } diff --git a/crates/misc/component-test-util/src/lib.rs b/crates/misc/component-test-util/src/lib.rs index bc7993d69cd8..eff4a4299e5c 100644 --- a/crates/misc/component-test-util/src/lib.rs +++ b/crates/misc/component-test-util/src/lib.rs @@ -132,16 +132,23 @@ forward_impls! { /// Helper method to apply `wast_config` to `config`. pub fn apply_wast_config(config: &mut Config, wast_config: &wasmtime_wast_util::WastConfig) { + use wasmtime_wast_util::{Collector, Compiler}; + config.strategy(match wast_config.compiler { - wasmtime_wast_util::Compiler::Cranelift => wasmtime::Strategy::Cranelift, - wasmtime_wast_util::Compiler::Winch => wasmtime::Strategy::Winch, + Compiler::CraneliftNative | Compiler::CraneliftPulley => wasmtime::Strategy::Cranelift, + Compiler::Winch => wasmtime::Strategy::Winch, }); - config.collector(match wast_config.collector { - wasmtime_wast_util::Collector::Auto => wasmtime::Collector::Auto, - wasmtime_wast_util::Collector::Null => wasmtime::Collector::Null, - wasmtime_wast_util::Collector::DeferredReferenceCounting => { - wasmtime::Collector::DeferredReferenceCounting + if let Compiler::CraneliftPulley = wast_config.compiler { + if cfg!(target_pointer_width = "32") { + config.target("pulley32").unwrap(); + } else { + config.target("pulley64").unwrap(); } + } + config.collector(match wast_config.collector { + Collector::Auto => wasmtime::Collector::Auto, + Collector::Null => wasmtime::Collector::Null, + Collector::DeferredReferenceCounting => wasmtime::Collector::DeferredReferenceCounting, }); } diff --git a/crates/test-macros/src/lib.rs b/crates/test-macros/src/lib.rs index 7c125bbc653c..5a9c8c89a12c 100644 --- a/crates/test-macros/src/lib.rs +++ b/crates/test-macros/src/lib.rs @@ -55,7 +55,7 @@ impl TestConfig { self.strategies.retain(|s| *s != Compiler::Winch); Ok(()) } else if meta.path.is_ident("Cranelift") { - self.strategies.retain(|s| *s != Compiler::Cranelift); + self.strategies.retain(|s| *s != Compiler::CraneliftNative); Ok(()) } else { Err(meta.error("Unknown strategy")) @@ -97,7 +97,7 @@ impl TestConfig { impl Default for TestConfig { fn default() -> Self { Self { - strategies: vec![Compiler::Cranelift, Compiler::Winch], + strategies: vec![Compiler::CraneliftNative, Compiler::Winch], flags: Default::default(), test_attribute: None, } diff --git a/crates/wasmtime/src/compile.rs b/crates/wasmtime/src/compile.rs index 86339b44dba3..1222fb9607c4 100644 --- a/crates/wasmtime/src/compile.rs +++ b/crates/wasmtime/src/compile.rs @@ -80,8 +80,7 @@ pub(crate) fn build_artifacts( .context("failed to parse WebAssembly module")?; let functions = mem::take(&mut translation.function_body_inputs); - let compile_inputs = - CompileInputs::for_module(engine.compiler().triple(), &types, &translation, functions); + let compile_inputs = CompileInputs::for_module(&types, &translation, functions); let unlinked_compile_outputs = compile_inputs.compile(engine)?; let (compiled_funcs, function_indices) = unlinked_compile_outputs.pre_link(); @@ -324,9 +323,6 @@ struct CompileOutput { /// The collection of things we need to compile for a Wasm module or component. #[derive(Default)] struct CompileInputs<'a> { - // Whether or not we need to compile wasm-to-native trampolines. - need_wasm_to_array_trampolines: bool, - inputs: Vec>, } @@ -337,18 +333,11 @@ impl<'a> CompileInputs<'a> { /// Create the `CompileInputs` for a core Wasm module. fn for_module( - triple: &target_lexicon::Triple, types: &'a ModuleTypesBuilder, translation: &'a ModuleTranslation<'a>, functions: PrimaryMap>, ) -> Self { - let mut ret = CompileInputs { - need_wasm_to_array_trampolines: !matches!( - triple.architecture, - target_lexicon::Architecture::Pulley32 | target_lexicon::Architecture::Pulley64 - ), - inputs: vec![], - }; + let mut ret = CompileInputs { inputs: vec![] }; let module_index = StaticModuleIndex::from_u32(0); ret.collect_inputs_in_translations(types, [(module_index, translation, functions)]); @@ -370,14 +359,7 @@ impl<'a> CompileInputs<'a> { ), >, ) -> Self { - let triple = engine.compiler().triple(); - let mut ret = CompileInputs { - need_wasm_to_array_trampolines: !matches!( - triple.architecture, - target_lexicon::Architecture::Pulley32 | target_lexicon::Architecture::Pulley64 - ), - inputs: vec![], - }; + let mut ret = CompileInputs { inputs: vec![] }; ret.collect_inputs_in_translations(types.module_types_builder(), module_translations); let tunables = engine.tunables(); @@ -518,28 +500,25 @@ impl<'a> CompileInputs<'a> { } } - if self.need_wasm_to_array_trampolines { - let mut trampoline_types_seen = HashSet::new(); - for (_func_type_index, trampoline_type_index) in types.trampoline_types() { - let is_new = trampoline_types_seen.insert(trampoline_type_index); - if !is_new { - continue; - } - let trampoline_func_ty = types[trampoline_type_index].unwrap_func(); - self.push_input(move |compiler| { - let trampoline = - compiler.compile_wasm_to_array_trampoline(trampoline_func_ty)?; - Ok(CompileOutput { - key: CompileKey::wasm_to_array_trampoline(trampoline_type_index), - symbol: format!( - "signatures[{}]::wasm_to_array_trampoline", - trampoline_type_index.as_u32() - ), - function: CompiledFunction::Function(trampoline), - info: None, - }) - }); + let mut trampoline_types_seen = HashSet::new(); + for (_func_type_index, trampoline_type_index) in types.trampoline_types() { + let is_new = trampoline_types_seen.insert(trampoline_type_index); + if !is_new { + continue; } + let trampoline_func_ty = types[trampoline_type_index].unwrap_func(); + self.push_input(move |compiler| { + let trampoline = compiler.compile_wasm_to_array_trampoline(trampoline_func_ty)?; + Ok(CompileOutput { + key: CompileKey::wasm_to_array_trampoline(trampoline_type_index), + symbol: format!( + "signatures[{}]::wasm_to_array_trampoline", + trampoline_type_index.as_u32() + ), + function: CompiledFunction::Function(trampoline), + info: None, + }) + }); } } @@ -738,8 +717,7 @@ impl FunctionIndices { )?; } - let mut obj = - wasmtime_environ::ObjectBuilder::new(obj, tunables, compiler.triple().clone()); + let mut obj = wasmtime_environ::ObjectBuilder::new(obj, tunables); let mut artifacts = Artifacts::default(); // Remove this as it's not needed by anything below and we'll debug @@ -831,29 +809,23 @@ impl FunctionIndices { }) .collect(); - let wasm_to_array_trampolines = match engine.compiler().triple().architecture { - target_lexicon::Architecture::Pulley32 - | target_lexicon::Architecture::Pulley64 => vec![], - _ => { - let unique_and_sorted_trampoline_sigs = translation - .module - .types - .iter() - .map(|(_, ty)| *ty) - .filter(|idx| types[*idx].is_func()) - .map(|idx| types.trampoline_type(idx)) - .collect::>(); - unique_and_sorted_trampoline_sigs - .iter() - .map(|idx| { - let trampoline = types.trampoline_type(*idx); - let key = CompileKey::wasm_to_array_trampoline(trampoline); - let compiled = wasm_to_array_trampolines[&key]; - (*idx, symbol_ids_and_locs[compiled.unwrap_function()].1) - }) - .collect() - } - }; + let unique_and_sorted_trampoline_sigs = translation + .module + .types + .iter() + .map(|(_, ty)| *ty) + .filter(|idx| types[*idx].is_func()) + .map(|idx| types.trampoline_type(idx)) + .collect::>(); + let wasm_to_array_trampolines = unique_and_sorted_trampoline_sigs + .iter() + .map(|idx| { + let trampoline = types.trampoline_type(*idx); + let key = CompileKey::wasm_to_array_trampoline(trampoline); + let compiled = wasm_to_array_trampolines[&key]; + (*idx, symbol_ids_and_locs[compiled.unwrap_function()].1) + }) + .collect(); obj.append(translation, funcs, wasm_to_array_trampolines) }) diff --git a/crates/wasmtime/src/config.rs b/crates/wasmtime/src/config.rs index f20ef72aec4c..9aeaa1406caf 100644 --- a/crates/wasmtime/src/config.rs +++ b/crates/wasmtime/src/config.rs @@ -1927,35 +1927,17 @@ impl Config { #[cfg(any(feature = "cranelift", feature = "winch"))] match self.compiler_config.strategy { None | Some(Strategy::Cranelift) => match self.compiler_target().architecture { - // Pulley doesn't support most of wasm at this time and there's - // lots of panicking bits and pieces within the backend. This - // doesn't fully cover all panicking cases but it's at least a - // starting place to have a ratchet. As the pulley backend is - // developed this'll get filtered down over time. + // Pulley is just starting and most errors are because of + // unsupported lowerings which is a first-class error. Some + // errors are panics though due to unimplemented bits in ABI + // code and those causes are listed here. target_lexicon::Architecture::Pulley32 | target_lexicon::Architecture::Pulley64 => { - WasmFeatures::SATURATING_FLOAT_TO_INT - | WasmFeatures::SIGN_EXTENSION - | WasmFeatures::REFERENCE_TYPES - | WasmFeatures::MULTI_VALUE - | WasmFeatures::BULK_MEMORY - | WasmFeatures::SIMD + WasmFeatures::SIMD | WasmFeatures::RELAXED_SIMD - | WasmFeatures::THREADS - | WasmFeatures::SHARED_EVERYTHING_THREADS | WasmFeatures::TAIL_CALL | WasmFeatures::FLOATS - | WasmFeatures::MULTI_MEMORY - | WasmFeatures::EXCEPTIONS | WasmFeatures::MEMORY64 - | WasmFeatures::EXTENDED_CONST - | WasmFeatures::FUNCTION_REFERENCES - | WasmFeatures::MEMORY_CONTROL - | WasmFeatures::GC - | WasmFeatures::CUSTOM_PAGE_SIZES - | WasmFeatures::LEGACY_EXCEPTIONS | WasmFeatures::GC_TYPES - | WasmFeatures::STACK_SWITCHING - | WasmFeatures::WIDE_ARITHMETIC } // Other Cranelift backends are either 100% missing or complete diff --git a/crates/wasmtime/src/engine.rs b/crates/wasmtime/src/engine.rs index 535bdda9e669..8062ef934911 100644 --- a/crates/wasmtime/src/engine.rs +++ b/crates/wasmtime/src/engine.rs @@ -255,7 +255,8 @@ impl Engine { fn _check_compatible_with_native_host(&self) -> Result<(), String> { #[cfg(any(feature = "cranelift", feature = "winch"))] { - use target_lexicon::{Architecture, PointerWidth, Triple}; + use target_lexicon::Triple; + use wasmtime_environ::TripleExt; let compiler = self.compiler(); @@ -268,16 +269,18 @@ impl Engine { return true; } - // Otherwise if there's a mismatch the only allowed - // configuration at this time is that any target can run Pulley, - // Wasmtime's interpreter. This only works though if the - // pointer-width of pulley matches the pointer-width of the - // host, so check that here. - match host.pointer_width() { - Ok(PointerWidth::U32) => target.architecture == Architecture::Pulley32, - Ok(PointerWidth::U64) => target.architecture == Architecture::Pulley64, - _ => false, + // If there's a mismatch and the target is a compatible pulley + // target, then that's also ok to run. + if cfg!(feature = "pulley") + && target.is_pulley() + && target.pointer_width() == host.pointer_width() + && target.endianness() == host.endianness() + { + return true; } + + // ... otherwise everything else is considered not a match. + false }; if !target_matches_host() { diff --git a/crates/wasmtime/src/runtime/component/func/host.rs b/crates/wasmtime/src/runtime/component/func/host.rs index d55ac2ce5237..1461554ad8a9 100644 --- a/crates/wasmtime/src/runtime/component/func/host.rs +++ b/crates/wasmtime/src/runtime/component/func/host.rs @@ -41,11 +41,11 @@ impl HostFunc { extern "C" fn entrypoint( cx: *mut VMOpaqueContext, data: *mut u8, - ty: TypeFuncIndex, - flags: InstanceFlags, + ty: u32, + flags: *mut u8, memory: *mut VMMemoryDefinition, realloc: *mut VMFuncRef, - string_encoding: StringEncoding, + string_encoding: u8, storage: *mut MaybeUninit, storage_len: usize, ) -> bool @@ -61,11 +61,11 @@ impl HostFunc { instance, types, store, - ty, - flags, + TypeFuncIndex::from_u32(ty), + InstanceFlags::from_raw(flags), memory, realloc, - string_encoding, + StringEncoding::from_u8(string_encoding).unwrap(), core::slice::from_raw_parts_mut(storage, storage_len), |store, args| (*data)(store, args), ) @@ -424,11 +424,11 @@ fn validate_inbounds_dynamic(abi: &CanonicalAbiInfo, memory: &[u8], ptr: &ValRaw extern "C" fn dynamic_entrypoint( cx: *mut VMOpaqueContext, data: *mut u8, - ty: TypeFuncIndex, - flags: InstanceFlags, + ty: u32, + flags: *mut u8, memory: *mut VMMemoryDefinition, realloc: *mut VMFuncRef, - string_encoding: StringEncoding, + string_encoding: u8, storage: *mut MaybeUninit, storage_len: usize, ) -> bool @@ -442,11 +442,11 @@ where instance, types, store, - ty, - flags, + TypeFuncIndex::from_u32(ty), + InstanceFlags::from_raw(flags), memory, realloc, - string_encoding, + StringEncoding::from_u8(string_encoding).unwrap(), core::slice::from_raw_parts_mut(storage, storage_len), |store, params, results| (*data)(store, params, results), ) diff --git a/crates/wasmtime/src/runtime/func.rs b/crates/wasmtime/src/runtime/func.rs index 2a8748de2615..ce335651641e 100644 --- a/crates/wasmtime/src/runtime/func.rs +++ b/crates/wasmtime/src/runtime/func.rs @@ -1,7 +1,7 @@ use crate::prelude::*; use crate::runtime::vm::{ - ExportFunction, SendSyncPtr, StoreBox, VMArrayCallHostFuncContext, VMContext, VMFuncRef, - VMFunctionImport, VMOpaqueContext, + ExportFunction, InterpreterRef, SendSyncPtr, StoreBox, VMArrayCallHostFuncContext, VMContext, + VMFuncRef, VMFunctionImport, VMOpaqueContext, }; use crate::runtime::Uninhabited; use crate::store::{AutoAssertNoGc, StoreData, StoreOpaque, Stored}; @@ -1068,10 +1068,12 @@ impl Func { func_ref: NonNull, params_and_returns: *mut [ValRaw], ) -> Result<()> { - invoke_wasm_and_catch_traps(store, |caller| { - func_ref - .as_ref() - .array_call(caller.cast::(), params_and_returns) + invoke_wasm_and_catch_traps(store, |caller, vm| { + func_ref.as_ref().array_call( + vm, + VMOpaqueContext::from_vmcontext(caller), + params_and_returns, + ) }) } @@ -1592,7 +1594,7 @@ impl Func { /// can pass to the called wasm function, if desired. pub(crate) fn invoke_wasm_and_catch_traps( store: &mut StoreContextMut<'_, T>, - closure: impl FnMut(*mut VMContext) -> bool, + closure: impl FnMut(*mut VMContext, Option>) -> bool, ) -> Result<()> { unsafe { let exit = enter_wasm(store); diff --git a/crates/wasmtime/src/runtime/func/typed.rs b/crates/wasmtime/src/runtime/func/typed.rs index 5d233b8a1945..26f5e2dfdb2f 100644 --- a/crates/wasmtime/src/runtime/func/typed.rs +++ b/crates/wasmtime/src/runtime/func/typed.rs @@ -211,7 +211,7 @@ where // the memory go away, so the size matters here for performance. let mut captures = (func, storage); - let result = invoke_wasm_and_catch_traps(store, |caller| { + let result = invoke_wasm_and_catch_traps(store, |caller, vm| { let (func_ref, storage) = &mut captures; let storage_len = mem::size_of_val::>(storage) / mem::size_of::(); let storage: *mut Storage<_, _> = storage; @@ -219,7 +219,7 @@ where let storage = core::ptr::slice_from_raw_parts_mut(storage, storage_len); func_ref .as_ref() - .array_call(VMOpaqueContext::from_vmcontext(caller), storage) + .array_call(vm, VMOpaqueContext::from_vmcontext(caller), storage) }); let (_, storage) = captures; diff --git a/crates/wasmtime/src/runtime/instance.rs b/crates/wasmtime/src/runtime/instance.rs index ab8e12d16d01..5e3117fe4383 100644 --- a/crates/wasmtime/src/runtime/instance.rs +++ b/crates/wasmtime/src/runtime/instance.rs @@ -361,10 +361,12 @@ impl Instance { let f = instance.get_exported_func(start); let caller_vmctx = instance.vmctx(); unsafe { - super::func::invoke_wasm_and_catch_traps(store, |_default_caller| { - f.func_ref - .as_ref() - .array_call(VMOpaqueContext::from_vmcontext(caller_vmctx), &mut []) + super::func::invoke_wasm_and_catch_traps(store, |_default_caller, vm| { + f.func_ref.as_ref().array_call( + vm, + VMOpaqueContext::from_vmcontext(caller_vmctx), + &mut [], + ) })?; } Ok(()) diff --git a/crates/wasmtime/src/runtime/module/registry.rs b/crates/wasmtime/src/runtime/module/registry.rs index 194ff9206aa9..84d0822c31d7 100644 --- a/crates/wasmtime/src/runtime/module/registry.rs +++ b/crates/wasmtime/src/runtime/module/registry.rs @@ -256,7 +256,6 @@ type GlobalRegistry = BTreeMap)>; /// Find which registered region of code contains the given program counter, and /// what offset that PC is within that module's code. -#[cfg(all(feature = "signals-based-traps", not(miri)))] pub fn lookup_code(pc: usize) -> Option<(Arc, usize)> { let all_modules = global_code().read(); let (_end, (start, module)) = all_modules.range(pc..).next()?; diff --git a/crates/wasmtime/src/runtime/store.rs b/crates/wasmtime/src/runtime/store.rs index 7651c7900a88..cbae0ccd64ce 100644 --- a/crates/wasmtime/src/runtime/store.rs +++ b/crates/wasmtime/src/runtime/store.rs @@ -84,8 +84,8 @@ use crate::prelude::*; use crate::runtime::vm::mpk::{self, ProtectionKey, ProtectionMask}; use crate::runtime::vm::{ Backtrace, ExportGlobal, GcRootsList, GcStore, InstanceAllocationRequest, InstanceAllocator, - InstanceHandle, ModuleRuntimeInfo, OnDemandInstanceAllocator, SignalHandler, StoreBox, - StorePtr, VMContext, VMFuncRef, VMGcRef, VMRuntimeLimits, + InstanceHandle, Interpreter, InterpreterRef, ModuleRuntimeInfo, OnDemandInstanceAllocator, + SignalHandler, StoreBox, StorePtr, VMContext, VMFuncRef, VMGcRef, VMRuntimeLimits, }; use crate::trampoline::VMHostGlobalContext; use crate::type_registry::RegisteredType; @@ -103,6 +103,7 @@ use core::ops::{Deref, DerefMut, Range}; use core::pin::Pin; use core::ptr; use core::task::{Context, Poll}; +use wasmtime_environ::TripleExt; mod context; pub use self::context::*; @@ -387,6 +388,11 @@ pub struct StoreOpaque { component_calls: crate::runtime::vm::component::CallContexts, #[cfg(feature = "component-model")] host_resource_data: crate::component::HostResourceData, + + /// State related to the Pulley interpreter if that's enabled and configured + /// for this store's `Engine`. This is `None` if pulley was disabled at + /// compile time or if it's not being used by the `Engine`. + interpreter: Option, } #[cfg(feature = "async")] @@ -575,6 +581,11 @@ impl Store { component_calls: Default::default(), #[cfg(feature = "component-model")] host_resource_data: Default::default(), + interpreter: if cfg!(feature = "pulley") && engine.target().is_pulley() { + Some(Interpreter::new()) + } else { + None + }, }, limiter: None, call_hook: None, @@ -1972,7 +1983,6 @@ impl StoreOpaque { /// with spectre mitigations enabled since the hardware fault address is /// always zero in these situations which means that the trapping context /// doesn't have enough information to report the fault address. - #[cfg(all(feature = "signals-based-traps", not(miri)))] pub(crate) fn wasm_fault( &self, pc: usize, @@ -2133,6 +2143,11 @@ at https://bytecodealliance.org/security. } } } + + pub(crate) fn interpreter(&mut self) -> Option> { + let i = self.interpreter.as_mut()?; + Some(i.as_interpreter_ref()) + } } impl StoreContextMut<'_, T> { diff --git a/crates/wasmtime/src/runtime/trap.rs b/crates/wasmtime/src/runtime/trap.rs index 358a96519abb..2b95ac6c3110 100644 --- a/crates/wasmtime/src/runtime/trap.rs +++ b/crates/wasmtime/src/runtime/trap.rs @@ -93,7 +93,6 @@ pub(crate) fn from_runtime_box( // otherwise the information about what the wasm was doing when the // error was generated would be lost. crate::runtime::vm::TrapReason::User(error) => (error, None), - #[cfg(all(feature = "signals-based-traps", not(miri)))] crate::runtime::vm::TrapReason::Jit { pc, faulting_addr, diff --git a/crates/wasmtime/src/runtime/vm.rs b/crates/wasmtime/src/runtime/vm.rs index 8a8db87e169c..4abefb6da3c5 100644 --- a/crates/wasmtime/src/runtime/vm.rs +++ b/crates/wasmtime/src/runtime/vm.rs @@ -45,6 +45,13 @@ pub mod debug_builtins; pub mod libcalls; pub mod mpk; +#[cfg(feature = "pulley")] +pub(crate) mod interpreter; +#[cfg(not(feature = "pulley"))] +pub(crate) mod interpreter_disabled; +#[cfg(not(feature = "pulley"))] +pub(crate) use interpreter_disabled as interpreter; + #[cfg(feature = "debug-builtins")] pub use wasmtime_jit_debug::gdb_jit_int::GdbJitImageRegistration; @@ -63,6 +70,7 @@ pub use crate::runtime::vm::instance::{ InstanceLimits, PoolConcurrencyLimitError, PoolingInstanceAllocator, PoolingInstanceAllocatorConfig, }; +pub use crate::runtime::vm::interpreter::*; pub use crate::runtime::vm::memory::{ Memory, RuntimeLinearMemory, RuntimeMemoryCreator, SharedMemory, }; diff --git a/crates/wasmtime/src/runtime/vm/component.rs b/crates/wasmtime/src/runtime/vm/component.rs index 263045b57091..813416e134b0 100644 --- a/crates/wasmtime/src/runtime/vm/component.rs +++ b/crates/wasmtime/src/runtime/vm/component.rs @@ -109,11 +109,11 @@ pub struct ComponentInstance { pub type VMLoweringCallee = extern "C" fn( vmctx: *mut VMOpaqueContext, data: *mut u8, - ty: TypeFuncIndex, - flags: InstanceFlags, + ty: u32, + flags: *mut u8, opt_memory: *mut VMMemoryDefinition, opt_realloc: *mut VMFuncRef, - string_encoding: StringEncoding, + string_encoding: u8, args_and_results: *mut mem::MaybeUninit, nargs_and_results: usize, ) -> bool; @@ -802,6 +802,16 @@ pub struct InstanceFlags(SendSyncPtr); #[allow(missing_docs)] impl InstanceFlags { + /// Wraps the given pointer as an `InstanceFlags` + /// + /// # Unsafety + /// + /// This is a raw pointer argument which needs to be valid for the lifetime + /// that `InstanceFlags` is used. + pub unsafe fn from_raw(ptr: *mut u8) -> InstanceFlags { + InstanceFlags(SendSyncPtr::new(NonNull::new(ptr.cast()).unwrap())) + } + #[inline] pub unsafe fn may_leave(&self) -> bool { *(*self.as_raw()).as_i32() & FLAG_MAY_LEAVE != 0 diff --git a/crates/wasmtime/src/runtime/vm/interpreter.rs b/crates/wasmtime/src/runtime/vm/interpreter.rs new file mode 100644 index 000000000000..c12c21e74efa --- /dev/null +++ b/crates/wasmtime/src/runtime/vm/interpreter.rs @@ -0,0 +1,324 @@ +use crate::prelude::*; +use crate::runtime::vm::vmcontext::VMArrayCallNative; +use crate::runtime::vm::{tls, TrapRegisters, TrapTest, VMContext, VMOpaqueContext}; +use crate::ValRaw; +use core::ptr::NonNull; +use pulley_interpreter::interp::{DoneReason, RegType, Val, Vm, XRegVal}; +use pulley_interpreter::{Reg, XReg}; +use wasmtime_environ::{BuiltinFunctionIndex, HostCall}; + +/// Interpreter state stored within a `Store`. +#[repr(transparent)] +pub struct Interpreter { + /// Pulley VM state, stored behind a `Box` to make the storage in + /// `Store` only pointer-sized (that way if you enable pulley but don't + /// use it it's low-overhead). + pulley: Box, +} + +impl Interpreter { + /// Creates a new interpreter ready to interpret code. + pub fn new() -> Interpreter { + Interpreter { + pulley: Box::new(Vm::new()), + } + } + + /// Returns the `InterpreterRef` structure which can be used to actually + /// execute interpreted code. + pub fn as_interpreter_ref(&mut self) -> InterpreterRef<'_> { + InterpreterRef(&mut self.pulley) + } +} + +/// Wrapper around `&mut pulley_interpreter::Vm` to enable compiling this to a +/// zero-sized structure when pulley is disabled at compile time. +#[repr(transparent)] +pub struct InterpreterRef<'a>(&'a mut Vm); + +#[derive(Clone, Copy)] +struct Setjmp { + sp: *mut u8, + fp: *mut u8, + lr: *mut u8, +} + +impl InterpreterRef<'_> { + /// Invokes interpreted code. + /// + /// The `bytecode` pointer should previously have been produced by Cranelift + /// and `callee` / `caller` / `args_and_results` are normal array-call + /// arguments being passed around. + pub unsafe fn call( + mut self, + mut bytecode: NonNull, + callee: *mut VMOpaqueContext, + caller: *mut VMOpaqueContext, + args_and_results: *mut [ValRaw], + ) -> bool { + // Initialize argument registers with the ABI arguments. + let args = [ + XRegVal::new_ptr(callee).into(), + XRegVal::new_ptr(caller).into(), + XRegVal::new_ptr(args_and_results.cast::()).into(), + XRegVal::new_u64(args_and_results.len() as u64).into(), + ]; + self.0.call_start(&args); + + // Fake a "poor man's setjmp" for now by saving some critical context to + // get restored when a trap happens. This pseudo-implements the stack + // unwinding necessary for a trap. + // + // See more comments in `trap` below about how this isn't actually + // correct as it's not saving all callee-save state. + let setjmp = Setjmp { + sp: self.0[XReg::sp].get_ptr(), + fp: self.0[XReg::fp].get_ptr(), + lr: self.0[XReg::lr].get_ptr(), + }; + + // Run the interpreter as much as possible until it finishes, and then + // handle each finish condition differently. + let ret = loop { + match self.0.call_run(bytecode) { + // If the VM returned entirely then read the return value and + // return that (it indicates whether a trap happened or not. + DoneReason::ReturnToHost(()) => { + match self.0.call_end([RegType::XReg]).next().unwrap() { + #[allow( + clippy::cast_possible_truncation, + reason = "intentionally reading the lower bits only" + )] + Val::XReg(xreg) => break (xreg.get_u32() as u8) != 0, + _ => unreachable!(), + } + } + // If the VM wants to call out to the host then dispatch that + // here based on `sig`. Once that returns we can resume + // execution at `resume`. + DoneReason::CallIndirectHost { id, resume } => { + self.call_indirect_host(id); + bytecode = resume; + } + // If the VM trapped then process that here and return `false`. + DoneReason::Trap(pc) => { + self.trap(pc, setjmp); + break false; + } + } + }; + + debug_assert!(self.0[XReg::sp].get_ptr() == setjmp.sp); + debug_assert!(self.0[XReg::fp].get_ptr() == setjmp.fp); + debug_assert!(self.0[XReg::lr].get_ptr() == setjmp.lr); + ret + } + + /// Handles an interpreter trap. This will initialize the trap state stored + /// in TLS via the `test_if_trap` helper below by reading the pc/fp of the + /// interpreter and seeing if that's a valid opcode to trap at. + fn trap(&mut self, pc: NonNull, setjmp: Setjmp) { + let result = tls::with(|s| { + let s = s.unwrap(); + s.test_if_trap( + TrapRegisters { + pc: pc.as_ptr() as usize, + fp: self.0[XReg::fp].get_ptr::() as usize, + }, + None, + |_| false, + ) + }); + + match result { + // This shouldn't be possible, so this is a fatal error if it + // happens. + TrapTest::NotWasm => panic!("pulley trap at {pc:?} without trap code registered"), + + // Not possible with our closure above returning `false`. + TrapTest::HandledByEmbedder => unreachable!(), + + // Trap was handled, yay! We don't use `jmp_buf`. + TrapTest::Trap { jmp_buf: _ } => {} + } + + // Perform a "longjmp" by restoring the "setjmp" context saved when this + // started. + // + // FIXME: this is not restoring callee-save state. For example if + // there's more than one Pulley activation on the stack that means that + // the previous one is expecting the callee (the host) to preserve all + // callee-save registers. That's not restored here which means with + // multiple activations we're effectively corrupting callee-save + // registers. + // + // One fix for this is to possibly update the `SystemV` ABI on pulley to + // have no callee-saved registers and make everything caller-saved. That + // would force all trampolines to save all state which is basically + // what we want as they'll naturally restore state if we later return to + // them. + let Setjmp { sp, fp, lr } = setjmp; + self.0[XReg::sp].set_ptr(sp); + self.0[XReg::fp].set_ptr(fp); + self.0[XReg::lr].set_ptr(lr); + } + + /// Handles the `call_indirect_host` instruction, dispatching the `sig` + /// number here which corresponds to `wasmtime_environ::HostCall`. + #[allow( + clippy::cast_possible_truncation, + clippy::cast_sign_loss, + reason = "macro-generated code" + )] + unsafe fn call_indirect_host(&mut self, id: u8) { + let id = u32::from(id); + let fnptr = self.0[XReg::x0].get_ptr::(); + let mut arg_reg = 1; + + /// Helper macro to invoke a builtin. + /// + /// Used as: + /// + /// `call(@builtin(ty1, ty2, ...) -> retty)` - invoke a core or + /// component builtin with the macro-defined signature. + /// + /// `call(@host Ty(ty1, ty2, ...) -> retty)` - invoke a host function + /// with the type `Ty`. The other types in the macro are checked by + /// rustc to match the actual `Ty` definition in Rust. + macro_rules! call { + (@builtin($($param:ident),*) $(-> $result:ident)?) => {{ + type T = unsafe extern "C" fn($(call!(@ty $param)),*) $(-> call!(@ty $result))?; + call!(@host T($($param),*) $(-> $result)?); + }}; + (@host $ty:ident($($param:ident),*) $(-> $result:ident)?) => {{ + // Convert the pointer from pulley to a native function pointer. + union GetNative { + fnptr: *mut u8, + host: $ty, + } + let host = GetNative { fnptr }.host; + + // Decode each argument according to this macro, pulling + // arguments from successive registers. + let ret = host($({ + let reg = XReg::new(arg_reg).unwrap(); + arg_reg += 1; + call!(@get $param reg) + }),*); + let _ = arg_reg; // silence last dead arg_reg increment warning + + // Store the return value, if one is here, in x0. + $( + let dst = XReg::x0; + call!(@set $result dst ret); + )? + let _ = ret; // silence warning if no return value + + // Return from the outer `call_indirect_host` host function as + // it's been processed. + return; + }}; + + // Conversion from macro-defined types to Rust host types. + (@ty bool) => (bool); + (@ty u8) => (u8); + (@ty u32) => (u32); + (@ty i32) => (i32); + (@ty u64) => (u64); + (@ty i64) => (i64); + (@ty vmctx) => (*mut VMContext); + (@ty pointer) => (*mut u8); + (@ty ptr_u8) => (*mut u8); + (@ty ptr_u16) => (*mut u16); + (@ty ptr_size) => (*mut usize); + (@ty size) => (usize); + + // Conversion from a pulley register value to the macro-defined + // type. + (@get u8 $reg:ident) => (self.0[$reg].get_i32() as u8); + (@get u32 $reg:ident) => (self.0[$reg].get_u32()); + (@get i32 $reg:ident) => (self.0[$reg].get_i32()); + (@get i64 $reg:ident) => (self.0[$reg].get_i64()); + (@get vmctx $reg:ident) => (self.0[$reg].get_ptr()); + (@get pointer $reg:ident) => (self.0[$reg].get_ptr()); + (@get ptr $reg:ident) => (self.0[$reg].get_ptr()); + (@get ptr_u8 $reg:ident) => (self.0[$reg].get_ptr()); + (@get ptr_u16 $reg:ident) => (self.0[$reg].get_ptr()); + (@get ptr_size $reg:ident) => (self.0[$reg].get_ptr()); + (@get size $reg:ident) => (self.0[$reg].get_ptr::() as usize); + + // Conversion from a Rust value back into a macro-defined type, + // stored in a pulley register. + (@set bool $reg:ident $val:ident) => (self.0[$reg].set_i32(i32::from($val))); + (@set i32 $reg:ident $val:ident) => (self.0[$reg].set_i32($val)); + (@set u64 $reg:ident $val:ident) => (self.0[$reg].set_u64($val)); + (@set i64 $reg:ident $val:ident) => (self.0[$reg].set_i64($val)); + (@set pointer $reg:ident $val:ident) => (self.0[$reg].set_ptr($val)); + (@set size $reg:ident $val:ident) => (self.0[$reg].set_ptr($val as *mut u8)); + } + + // With the helper macro above structure this into: + // + // foreach [core, component] + // * dispatch the call-the-host function pointer type + // * dispatch all builtins by their index. + // + // The hope is that this is relatively easy for LLVM to optimize since + // it's a bunch of: + // + // if id == 0 { ...; return; } + // if id == 1 { ...; return; } + // if id == 2 { ...; return; } + // ... + // + + if id == const { HostCall::ArrayCall.index() } { + call!(@host VMArrayCallNative(ptr, ptr, ptr, size) -> bool); + } + + macro_rules! core { + ( + $( + $( #[cfg($attr:meta)] )? + $name:ident($($pname:ident: $param:ident ),* ) $(-> $result:ident)?; + )* + ) => { + $( + $( #[cfg($attr)] )? + if id == const { HostCall::Builtin(BuiltinFunctionIndex::$name()).index() } { + call!(@builtin($($param),*) $(-> $result)?); + } + )* + } + } + wasmtime_environ::foreach_builtin_function!(core); + + #[cfg(feature = "component-model")] + { + use crate::runtime::vm::component::VMLoweringCallee; + use wasmtime_environ::component::ComponentBuiltinFunctionIndex; + + if id == const { HostCall::ComponentLowerImport.index() } { + call!(@host VMLoweringCallee(ptr, ptr, u32, ptr, ptr, ptr, u8, ptr, size) -> bool); + } + + macro_rules! component { + ( + $( + $name:ident($($pname:ident: $param:ident ),* ) $(-> $result:ident)?; + )* + ) => { + $( + if id == const { HostCall::ComponentBuiltin(ComponentBuiltinFunctionIndex::$name()).index() } { + call!(@builtin($($param),*) $(-> $result)?); + } + )* + } + } + wasmtime_environ::foreach_builtin_component_function!(component); + } + + // if we got this far then something has gone seriously wrong. + unreachable!() + } +} diff --git a/crates/wasmtime/src/runtime/vm/interpreter_disabled.rs b/crates/wasmtime/src/runtime/vm/interpreter_disabled.rs new file mode 100644 index 000000000000..d410c2cb1bbc --- /dev/null +++ b/crates/wasmtime/src/runtime/vm/interpreter_disabled.rs @@ -0,0 +1,49 @@ +//! Stubs for when pulley is disabled at compile time. +//! +//! Note that this is structured so that these structures are all zero-sized and +//! `Option` is also zero-sized so there should be no runtime cost for +//! having these structures plumbed around. + +use crate::runtime::vm::VMOpaqueContext; +use crate::runtime::Uninhabited; +use crate::ValRaw; +use core::marker; +use core::mem; +use core::ptr::NonNull; + +pub struct Interpreter { + empty: Uninhabited, +} + +const _: () = assert!(mem::size_of::() == 0); +const _: () = assert!(mem::size_of::>() == 0); + +impl Interpreter { + pub fn new() -> Interpreter { + unreachable!() + } + + pub fn as_interpreter_ref(&mut self) -> InterpreterRef<'_> { + match self.empty {} + } +} + +pub struct InterpreterRef<'a> { + empty: Uninhabited, + _marker: marker::PhantomData<&'a mut Interpreter>, +} + +const _: () = assert!(mem::size_of::>() == 0); +const _: () = assert!(mem::size_of::>>() == 0); + +impl InterpreterRef<'_> { + pub unsafe fn call( + self, + _bytecode: NonNull, + _callee: *mut VMOpaqueContext, + _caller: *mut VMOpaqueContext, + _args_and_results: *mut [ValRaw], + ) -> bool { + match self.empty {} + } +} diff --git a/crates/wasmtime/src/runtime/vm/traphandlers.rs b/crates/wasmtime/src/runtime/vm/traphandlers.rs index 5eb3128dd5fd..c5fb8d608157 100644 --- a/crates/wasmtime/src/runtime/vm/traphandlers.rs +++ b/crates/wasmtime/src/runtime/vm/traphandlers.rs @@ -16,9 +16,10 @@ mod signals; pub use self::signals::*; use crate::prelude::*; +use crate::runtime::module::lookup_code; use crate::runtime::store::StoreOpaque; use crate::runtime::vm::sys::traphandlers; -use crate::runtime::vm::{Instance, VMContext, VMRuntimeLimits}; +use crate::runtime::vm::{Instance, InterpreterRef, VMContext, VMRuntimeLimits}; use crate::{StoreContextMut, WasmBacktrace}; use core::cell::Cell; use core::ops::Range; @@ -30,6 +31,26 @@ pub use self::tls::{tls_eager_initialize, AsyncWasmCallState, PreviousAsyncWasmC pub use traphandlers::SignalHandler; +pub(crate) struct TrapRegisters { + pub pc: usize, + pub fp: usize, +} + +/// Return value from `test_if_trap`. +pub(crate) enum TrapTest { + /// Not a wasm trap, need to delegate to whatever process handler is next. + NotWasm, + /// This trap was handled by the embedder via custom embedding APIs. + #[cfg_attr(miri, expect(dead_code, reason = "using #[cfg] too unergonomic"))] + HandledByEmbedder, + /// This is a wasm trap, it needs to be handled. + #[cfg_attr(miri, expect(dead_code, reason = "using #[cfg] too unergonomic"))] + Trap { + /// How to longjmp back to the original wasm frame. + jmp_buf: *const u8, + }, +} + fn lazy_per_thread_init() { traphandlers::lazy_per_thread_init(); } @@ -283,7 +304,6 @@ pub enum TrapReason { User(Error), /// A trap raised from Cranelift-generated code. - #[cfg(all(feature = "signals-based-traps", not(miri)))] Jit { /// The program counter where this trap originated. /// @@ -334,17 +354,41 @@ pub unsafe fn catch_traps( mut closure: F, ) -> Result<(), Box> where - F: FnMut(*mut VMContext) -> bool, + F: FnMut(*mut VMContext, Option>) -> bool, { let caller = store.0.default_caller(); + let result = CallThreadState::new(store.0, caller).with(|cx| match store.0.interpreter() { + // In interpreted mode directly invoke the host closure since we won't + // be using host-based `setjmp`/`longjmp` as that's not going to save + // the context we want. + Some(r) => { + cx.jmp_buf + .set(CallThreadState::JMP_BUF_INTERPRETER_SENTINEL); + closure(caller, Some(r)) + } - let result = CallThreadState::new(store.0, caller).with(|cx| { - traphandlers::wasmtime_setjmp( + // In native mode, however, defer to C to do the `setjmp` since Rust + // doesn't understand `setjmp`. + // + // Note that here we pass a function pointer to C to catch longjmp + // within, here it's `call_closure`, and that passes `None` for the + // interpreter since this branch is only ever taken if the interpreter + // isn't present. + None => traphandlers::wasmtime_setjmp( cx.jmp_buf.as_ptr(), - call_closure::, + { + extern "C" fn call_closure(payload: *mut u8, caller: *mut VMContext) -> bool + where + F: FnMut(*mut VMContext, Option>) -> bool, + { + unsafe { (*(payload as *mut F))(caller, None) } + } + + call_closure:: + }, &mut closure as *mut F as *mut u8, caller, - ) + ), }); return match result { @@ -357,13 +401,6 @@ where #[cfg(all(feature = "std", panic = "unwind"))] Err((UnwindReason::Panic(panic), _, _)) => std::panic::resume_unwind(panic), }; - - extern "C" fn call_closure(payload: *mut u8, caller: *mut VMContext) -> bool - where - F: FnMut(*mut VMContext) -> bool, - { - unsafe { (*(payload as *mut F))(caller) } - } } // Module to hide visibility of the `CallThreadState::prev` field and force @@ -416,6 +453,8 @@ mod call_thread_state { } impl CallThreadState { + pub const JMP_BUF_INTERPRETER_SENTINEL: *mut u8 = 1 as *mut u8; + #[inline] pub(super) fn new(store: &mut StoreOpaque, caller: *mut VMContext) -> CallThreadState { let limits = unsafe { *Instance::from_vmctx(caller, |i| i.runtime_limits()) }; @@ -580,6 +619,91 @@ impl CallThreadState { Some(this) }) } + + /// Trap handler using our thread-local state. + /// + /// * `regs` - some special program registers at the time that the trap + /// happened, for example `pc`. + /// * `faulting_addr` - the system-provided address that the a fault, if + /// any, happened at. This is used when debug-asserting that all segfaults + /// are known to live within a `Store` in a valid range. + /// * `call_handler` - a closure used to invoke the platform-specific + /// signal handler for each instance, if available. + /// + /// Attempts to handle the trap if it's a wasm trap. Returns a `TrapTest` + /// which indicates what this could be, such as: + /// + /// * `TrapTest::NotWasm` - not a wasm fault, this should get forwarded to + /// the next platform-specific fault handler. + /// * `TrapTest::HandledByEmbedder` - the embedder `call_handler` handled + /// this signal, nothing else to do. + /// * `TrapTest::Trap` - this is a wasm trap an the stack needs to be + /// unwound now. + pub(crate) fn test_if_trap( + &self, + regs: TrapRegisters, + faulting_addr: Option, + call_handler: impl Fn(&SignalHandler) -> bool, + ) -> TrapTest { + // If we haven't even started to handle traps yet, bail out. + if self.jmp_buf.get().is_null() { + return TrapTest::NotWasm; + } + + // First up see if any instance registered has a custom trap handler, + // in which case run them all. If anything handles the trap then we + // return that the trap was handled. + let _ = &call_handler; + #[cfg(all(feature = "signals-based-traps", not(miri)))] + if let Some(handler) = self.signal_handler { + if unsafe { call_handler(&*handler) } { + return TrapTest::HandledByEmbedder; + } + } + + // If this fault wasn't in wasm code, then it's not our problem + let Some((code, text_offset)) = lookup_code(regs.pc) else { + return TrapTest::NotWasm; + }; + + // If the fault was at a location that was not marked as potentially + // trapping, then that's a bug in Cranelift/Winch/etc. Don't try to + // catch the trap and pretend this isn't wasm so the program likely + // aborts. + let Some(trap) = code.lookup_trap_code(text_offset) else { + return TrapTest::NotWasm; + }; + + // If all that passed then this is indeed a wasm trap, so return the + // `jmp_buf` passed to `wasmtime_longjmp` to resume. + self.set_jit_trap(regs, faulting_addr, trap); + TrapTest::Trap { + jmp_buf: self.take_jmp_buf(), + } + } + + pub(crate) fn take_jmp_buf(&self) -> *const u8 { + self.jmp_buf.replace(ptr::null()) + } + + pub(crate) fn set_jit_trap( + &self, + TrapRegisters { pc, fp, .. }: TrapRegisters, + faulting_addr: Option, + trap: wasmtime_environ::Trap, + ) { + let backtrace = self.capture_backtrace(self.limits, Some((pc, fp))); + let coredump = self.capture_coredump(self.limits, Some((pc, fp))); + self.unwind.set(Some(( + UnwindReason::Trap(TrapReason::Jit { + pc, + faulting_addr, + trap, + }), + backtrace, + coredump, + ))) + } } // A private inner module for managing the TLS state that we require across diff --git a/crates/wasmtime/src/runtime/vm/traphandlers/signals.rs b/crates/wasmtime/src/runtime/vm/traphandlers/signals.rs index 1a9e8da94d0e..ac5d6a3409a7 100644 --- a/crates/wasmtime/src/runtime/vm/traphandlers/signals.rs +++ b/crates/wasmtime/src/runtime/vm/traphandlers/signals.rs @@ -6,30 +6,8 @@ //! thise module serves as a shared entrypoint for initialization entrypoints //! (`init_traps`) and testing if a trapping opcode is wasm (`test_if_trap`). -use crate::runtime::module::lookup_code; use crate::sync::RwLock; use crate::vm::sys::traphandlers::TrapHandler; -use crate::vm::traphandlers::{CallThreadState, SignalHandler, TrapReason, UnwindReason}; -use core::ptr; - -pub(crate) struct TrapRegisters { - pub pc: usize, - pub fp: usize, -} - -/// Return value from `test_if_trap`. -pub(crate) enum TrapTest { - /// Not a wasm trap, need to delegate to whatever process handler is next. - NotWasm, - /// This trap was handled by the embedder via custom embedding APIs. - HandledByEmbedder, - /// This is a wasm trap, it needs to be handled. - #[cfg_attr(miri, allow(dead_code))] - Trap { - /// How to longjmp back to the original wasm frame. - jmp_buf: *const u8, - }, -} /// Platform-specific trap-handler state. /// @@ -84,81 +62,3 @@ pub unsafe fn deinit_traps() { let mut lock = TRAP_HANDLER.write(); let _ = lock.take(); } - -impl CallThreadState { - /// Trap handler using our thread-local state. - /// - /// * `pc` - the program counter the trap happened at - /// * `call_handler` - a closure used to invoke the platform-specific - /// signal handler for each instance, if available. - /// - /// Attempts to handle the trap if it's a wasm trap. Returns a few - /// different things: - /// - /// * null - the trap didn't look like a wasm trap and should continue as a - /// trap - /// * 1 as a pointer - the trap was handled by a custom trap handler on an - /// instance, and the trap handler should quickly return. - /// * a different pointer - a jmp_buf buffer to longjmp to, meaning that - /// the wasm trap was successfully handled. - pub(crate) fn test_if_trap( - &self, - regs: TrapRegisters, - faulting_addr: Option, - call_handler: impl Fn(&SignalHandler) -> bool, - ) -> TrapTest { - // If we haven't even started to handle traps yet, bail out. - if self.jmp_buf.get().is_null() { - return TrapTest::NotWasm; - } - - // First up see if any instance registered has a custom trap handler, - // in which case run them all. If anything handles the trap then we - // return that the trap was handled. - if let Some(handler) = self.signal_handler { - if unsafe { call_handler(&*handler) } { - return TrapTest::HandledByEmbedder; - } - } - - // If this fault wasn't in wasm code, then it's not our problem - let Some((code, text_offset)) = lookup_code(regs.pc) else { - return TrapTest::NotWasm; - }; - - let Some(trap) = code.lookup_trap_code(text_offset) else { - return TrapTest::NotWasm; - }; - - self.set_jit_trap(regs, faulting_addr, trap); - - // If all that passed then this is indeed a wasm trap, so return the - // `jmp_buf` passed to `wasmtime_longjmp` to resume. - TrapTest::Trap { - jmp_buf: self.take_jmp_buf(), - } - } - - pub(crate) fn take_jmp_buf(&self) -> *const u8 { - self.jmp_buf.replace(ptr::null()) - } - - pub(crate) fn set_jit_trap( - &self, - TrapRegisters { pc, fp, .. }: TrapRegisters, - faulting_addr: Option, - trap: wasmtime_environ::Trap, - ) { - let backtrace = self.capture_backtrace(self.limits, Some((pc, fp))); - let coredump = self.capture_coredump(self.limits, Some((pc, fp))); - self.unwind.set(Some(( - UnwindReason::Trap(TrapReason::Jit { - pc, - faulting_addr, - trap, - }), - backtrace, - coredump, - ))) - } -} diff --git a/crates/wasmtime/src/runtime/vm/vmcontext.rs b/crates/wasmtime/src/runtime/vm/vmcontext.rs index 8b14680e22c0..61c78620b1b7 100644 --- a/crates/wasmtime/src/runtime/vm/vmcontext.rs +++ b/crates/wasmtime/src/runtime/vm/vmcontext.rs @@ -5,7 +5,7 @@ mod vm_host_func_context; pub use self::vm_host_func_context::VMArrayCallHostFuncContext; use crate::prelude::*; -use crate::runtime::vm::{GcStore, VMGcRef}; +use crate::runtime::vm::{GcStore, InterpreterRef, VMGcRef}; use crate::store::StoreOpaque; use core::cell::UnsafeCell; use core::ffi::c_void; @@ -720,8 +720,35 @@ impl VMFuncRef { /// # Unsafety /// /// This method is unsafe because it can be called with any pointers. They - /// must all be valid for this wasm function call to proceed. + /// must all be valid for this wasm function call to proceed. For example + /// the `caller` must be valid machine code if `pulley` is `None` or it must + /// be valid bytecode if `pulley` is `Some`. Additionally `args_and_results` + /// must be large enough to handle all the arguments/results for this call. + /// + /// Note that the unsafety invariants to maintain here are not currently + /// exhaustively documented. pub unsafe fn array_call( + &self, + pulley: Option>, + caller: *mut VMOpaqueContext, + args_and_results: *mut [ValRaw], + ) -> bool { + match pulley { + Some(vm) => self.array_call_interpreted(vm, caller, args_and_results), + None => self.array_call_native(caller, args_and_results), + } + } + + unsafe fn array_call_interpreted( + &self, + vm: InterpreterRef<'_>, + caller: *mut VMOpaqueContext, + args_and_results: *mut [ValRaw], + ) -> bool { + vm.call(self.array_call.cast(), self.vmctx, caller, args_and_results) + } + + unsafe fn array_call_native( &self, caller: *mut VMOpaqueContext, args_and_results: *mut [ValRaw], diff --git a/crates/wast-util/src/lib.rs b/crates/wast-util/src/lib.rs index e88c18cb0594..cc82f83e4a24 100644 --- a/crates/wast-util/src/lib.rs +++ b/crates/wast-util/src/lib.rs @@ -200,6 +200,15 @@ macro_rules! define_test_config { pub struct TestConfig { $(pub $option: Option,)* } + + impl TestConfig { + $( + pub fn $option(&self) -> bool { + self.$option.unwrap_or(false) + } + )* + } + } } @@ -229,25 +238,81 @@ pub struct WastConfig { pub collector: Collector, } +/// Different compilers that can be tested in Wasmtime. #[derive(PartialEq, Debug, Copy, Clone)] pub enum Compiler { - Cranelift, + /// Cranelift backend. + /// + /// This tests the Cranelift code generator for native platforms. This + /// notably excludes Pulley since that's listed separately below even though + /// Pulley is a backend of Cranelift. This is only used for native code + /// generation such as x86_64. + CraneliftNative, + + /// Winch backend. + /// + /// This tests the Winch backend for native platforms. Currently Winch + /// primarily supports x86_64. Winch, + + /// Pulley interpreter. + /// + /// This tests the Cranelift pulley backend plus the pulley execution + /// environment of the output bytecode. Note that this is separate from + /// `Cranelift` above to be able to test both on platforms where Cranelift + /// has native codegen support. + CraneliftPulley, } impl Compiler { + /// Returns whether this compiler is known to fail for the provided + /// `TestConfig`. + /// + /// This function will determine if the configuration of the test provided + /// is known to guarantee fail. This effectively tracks the proposal support + /// for each compiler backend/runtime and tests whether `config` enables or + /// disables features that aren't supported. + /// + /// Note that this is closely aligned with + /// `Config::compiler_panicking_wasm_features`. pub fn should_fail(&self, config: &TestConfig) -> bool { match self { - Compiler::Cranelift => {} + // Currently Cranelift supports all wasm proposals that wasmtime + // tests. + Compiler::CraneliftNative => {} + + // Winch doesn't have quite the full breadth of support that + // Cranelift has quite yet. Compiler::Winch => { - // A few proposals that winch has no support for. - if config.gc == Some(true) - || config.threads == Some(true) - || config.tail_call == Some(true) - || config.function_references == Some(true) - || config.gc == Some(true) - || config.relaxed_simd == Some(true) - || config.gc_types == Some(true) + if config.gc() + || config.threads() + || config.tail_call() + || config.function_references() + || config.gc() + || config.relaxed_simd() + || config.gc_types() + { + return true; + } + } + + // Pulley is just getting started, it implements almost no proposals + // yet. + Compiler::CraneliftPulley => { + // Unsupported proposals + if config.memory64() + || config.custom_page_sizes() + || config.multi_memory() + || config.threads() + || config.gc() + || config.function_references() + || config.relaxed_simd() + || config.reference_types() + || config.tail_call() + || config.extended_const() + || config.wide_arithmetic() + || config.simd() + || config.gc_types() { return true; } @@ -256,6 +321,27 @@ impl Compiler { false } + + /// Returns whether this complier configuration supports the current host + /// architecture. + pub fn supports_host(&self) -> bool { + match self { + Compiler::CraneliftNative => { + cfg!(target_arch = "x86_64") + || cfg!(target_arch = "aarch64") + || cfg!(target_arch = "riscv64") + || cfg!(target_arch = "s390x") + } + Compiler::Winch => { + cfg!(target_arch = "x86_64") + } + Compiler::CraneliftPulley => { + // FIXME(#9747) pulley needs more refactoring to support a + // big-endian host. + cfg!(target_endian = "little") + } + } + } } #[derive(PartialEq, Debug, Copy, Clone)] @@ -269,10 +355,7 @@ impl WastTest { /// Returns whether this test exercises the GC types and might want to use /// multiple different garbage collectors. pub fn test_uses_gc_types(&self) -> bool { - self.config - .gc - .or(self.config.function_references) - .unwrap_or(false) + self.config.gc() || self.config.function_references() } /// Returns the optional spec proposal that this test is associated with. @@ -283,8 +366,74 @@ impl WastTest { /// Returns whether this test should fail under the specified extra /// configuration. pub fn should_fail(&self, config: &WastConfig) -> bool { - // Winch only supports x86_64 at this time. - if config.compiler == Compiler::Winch && !cfg!(target_arch = "x86_64") { + if !config.compiler.supports_host() { + return true; + } + + // Some tests are known to fail with the pooling allocator + if config.pooling { + let unsupported = [ + // allocates too much memory for the pooling configuration here + "misc_testsuite/memory64/more-than-4gb.wast", + // shared memories + pooling allocator aren't supported yet + "misc_testsuite/memory-combos.wast", + "misc_testsuite/threads/LB.wast", + "misc_testsuite/threads/LB_atomic.wast", + "misc_testsuite/threads/MP.wast", + "misc_testsuite/threads/MP_atomic.wast", + "misc_testsuite/threads/MP_wait.wast", + "misc_testsuite/threads/SB.wast", + "misc_testsuite/threads/SB_atomic.wast", + "misc_testsuite/threads/atomics_notify.wast", + "misc_testsuite/threads/atomics_wait_address.wast", + "misc_testsuite/threads/wait_notify.wast", + "spec_testsuite/proposals/threads/atomic.wast", + "spec_testsuite/proposals/threads/exports.wast", + "spec_testsuite/proposals/threads/memory.wast", + ]; + + if unsupported.iter().any(|part| self.path.ends_with(part)) { + return true; + } + } + + // Pulley is in a bit of a special state at this time where it supports + // only a subset of the initial MVP of WebAssembly. That means that no + // test technically passes by default but a few do happen to use just + // the right subset of wasm that we can pass it. For now maintain an + // allow-list of tests that are known to pass in Pulley. As tests are + // fixed they should get added to this list. Over time this list will + // instead get inverted to "these tests are known to fail" once Pulley + // implements more proposals. + if config.compiler == Compiler::CraneliftPulley { + let supported = [ + "custom-page-sizes/custom-page-sizes-invalid.wast", + "exception-handling/exports.wast", + "extended-const/data.wast", + "misc_testsuite/component-model/adapter.wast", + "misc_testsuite/component-model/aliasing.wast", + "misc_testsuite/component-model/import.wast", + "misc_testsuite/component-model/instance.wast", + "misc_testsuite/component-model/linking.wast", + "misc_testsuite/component-model/nested.wast", + "misc_testsuite/component-model/types.wast", + "misc_testsuite/elem-ref-null.wast", + "misc_testsuite/elem_drop.wast", + "misc_testsuite/empty.wast", + "misc_testsuite/fib.wast", + "misc_testsuite/func-400-params.wast", + "misc_testsuite/gc/more-rec-groups-than-types.wast", + "misc_testsuite/gc/rec-group-funcs.wast", + "misc_testsuite/rs2wasm-add-func.wast", + "misc_testsuite/stack_overflow.wast", + "misc_testsuite/winch/misc.wast", + "threads/exports.wast", + ]; + + if supported.iter().any(|part| self.path.ends_with(part)) { + return false; + } + return true; } @@ -427,33 +576,6 @@ impl WastTest { } } - // Some tests are known to fail with the pooling allocator - if config.pooling { - let unsupported = [ - // allocates too much memory for the pooling configuration here - "misc_testsuite/memory64/more-than-4gb.wast", - // shared memories + pooling allocator aren't supported yet - "misc_testsuite/memory-combos.wast", - "misc_testsuite/threads/LB.wast", - "misc_testsuite/threads/LB_atomic.wast", - "misc_testsuite/threads/MP.wast", - "misc_testsuite/threads/MP_atomic.wast", - "misc_testsuite/threads/MP_wait.wast", - "misc_testsuite/threads/SB.wast", - "misc_testsuite/threads/SB_atomic.wast", - "misc_testsuite/threads/atomics_notify.wast", - "misc_testsuite/threads/atomics_wait_address.wast", - "misc_testsuite/threads/wait_notify.wast", - "spec_testsuite/proposals/threads/atomic.wast", - "spec_testsuite/proposals/threads/exports.wast", - "spec_testsuite/proposals/threads/memory.wast", - ]; - - if unsupported.iter().any(|part| self.path.ends_with(part)) { - return true; - } - } - false } } diff --git a/pulley/examples/objdump.rs b/pulley/examples/objdump.rs index 7e9c7f8ed154..ad7b57b0dd51 100644 --- a/pulley/examples/objdump.rs +++ b/pulley/examples/objdump.rs @@ -5,7 +5,7 @@ //! cargo run --example objdump -F disas -p pulley-interpreter foo.cwasm use anyhow::{bail, Result}; -use object::{File, Object as _, ObjectSection, ObjectSymbol, SectionKind, SymbolKind}; +use object::{File, Object as _, ObjectSection, ObjectSymbol, SymbolKind}; use pulley_interpreter::decode::Decoder; use pulley_interpreter::disas::Disassembler; @@ -14,7 +14,7 @@ fn main() -> Result<()> { let image = File::parse(&cwasm[..])?; - let text = match image.sections().find(|s| s.kind() == SectionKind::Text) { + let text = match image.sections().find(|s| s.name().ok() == Some(".text")) { Some(section) => section.data()?, None => bail!("no text section"), }; diff --git a/pulley/fuzz/src/interp.rs b/pulley/fuzz/src/interp.rs index 4cf59b973c65..7e434709830a 100644 --- a/pulley/fuzz/src/interp.rs +++ b/pulley/fuzz/src/interp.rs @@ -1,5 +1,5 @@ use pulley_interpreter::{ - interp::Vm, + interp::{DoneReason, Vm}, op::{self, ExtendedOp, Op}, *, }; @@ -30,8 +30,8 @@ pub fn interp(ops: Vec) { let args = &[]; let rets = &[]; match vm.call(NonNull::from(&encoded[0]), args, rets.into_iter().copied()) { - Ok(rets) => assert_eq!(rets.count(), 0), - Err(pc) => { + DoneReason::ReturnToHost(rets) => assert_eq!(rets.count(), 0), + DoneReason::Trap(pc) => { let pc = pc.as_ptr() as usize; let start = &encoded[0] as *const u8 as usize; @@ -47,6 +47,7 @@ pub fn interp(ops: Vec) { assert_eq!(encoded[index + 1], a); assert_eq!(encoded[index + 2], b); } + DoneReason::CallIndirectHost { .. } => unreachable!(), }; } } diff --git a/pulley/src/interp.rs b/pulley/src/interp.rs index 55d718b70a58..d7507a73e5e7 100644 --- a/pulley/src/interp.rs +++ b/pulley/src/interp.rs @@ -4,7 +4,6 @@ use crate::decode::*; use crate::encode::Encode; use crate::imms::*; use crate::regs::*; -use crate::ExtendedOpcode; use alloc::string::ToString; use alloc::{vec, vec::Vec}; use core::fmt; @@ -77,7 +76,28 @@ impl Vm { func: NonNull, args: &[Val], rets: impl IntoIterator + 'a, - ) -> Result + 'a, NonNull> { + ) -> DoneReason + 'a> { + self.call_start(args); + + match self.call_run(func) { + DoneReason::ReturnToHost(()) => DoneReason::ReturnToHost(self.call_end(rets)), + DoneReason::Trap(pc) => DoneReason::Trap(pc), + DoneReason::CallIndirectHost { id, resume } => { + DoneReason::CallIndirectHost { id, resume } + } + } + } + + /// Peforms the initial part of [`Vm::call`] in setting up the `args` + /// provided in registers according to Pulley's ABI. + /// + /// # Unsafety + /// + /// All the same unsafety as `call` and additiionally, you must + /// invoke `call_run` and then `call_end` after calling `call_start`. + /// If you don't want to wrangle these invocations, use `call` instead + /// of `call_{start,run,end}`. + pub unsafe fn call_start<'a>(&'a mut self, args: &[Val]) { // NB: make sure this method stays in sync with // `PulleyMachineDeps::compute_arg_locs`! @@ -101,14 +121,45 @@ impl Vm { }, } } + } - self.run(func)?; + /// Peforms the internal part of [`Vm::call`] where bytecode is actually + /// executed. + /// + /// # Unsafety + /// + /// In addition to all the invariants documented for `call`, you + /// may only invoke `call_run` after invoking `call_start` to + /// initialize this call's arguments. + pub unsafe fn call_run(&mut self, pc: NonNull) -> DoneReason<()> { + self.state.debug_assert_done_reason_none(); + let interpreter = Interpreter { + state: &mut self.state, + pc: UnsafeBytecodeStream::new(pc), + }; + let done = interpreter.run(); + self.state.done_decode(done) + } + + /// Peforms the tail end of [`Vm::call`] by returning the values as + /// determined by `rets` according to Pulley's ABI. + /// + /// # Unsafety + /// + /// In addition to the invariants documented for `call`, this may + /// only be called after `call_run`. + pub unsafe fn call_end<'a>( + &'a mut self, + rets: impl IntoIterator + 'a, + ) -> impl Iterator + 'a { + // NB: make sure this method stays in sync with + // `PulleyMachineDeps::compute_arg_locs`! let mut x_rets = (0..16).map(|x| XReg::new_unchecked(x)); let mut f_rets = (0..16).map(|f| FReg::new_unchecked(f)); let mut v_rets = (0..16).map(|v| VReg::new_unchecked(v)); - Ok(rets.into_iter().map(move |ty| match ty { + rets.into_iter().map(move |ty| match ty { RegType::XReg => match x_rets.next() { Some(reg) => Val::XReg(self.state[reg]), None => todo!("stack slots"), @@ -121,43 +172,7 @@ impl Vm { Some(reg) => Val::VReg(self.state[reg]), None => todo!("stack slots"), }, - })) - } - - unsafe fn run(&mut self, pc: NonNull) -> Result<(), NonNull> { - let interpreter = Interpreter { - state: &mut self.state, - pc: UnsafeBytecodeStream::new(pc), - }; - match interpreter.run() { - Done::ReturnToHost => self.return_to_host(), - Done::Trap(pc) => self.trap(pc), - Done::HostCall => self.host_call(), - } - } - - #[cold] - #[inline(never)] - fn return_to_host(&self) -> Result<(), NonNull> { - Ok(()) - } - - #[cold] - #[inline(never)] - fn trap(&self, pc: NonNull) -> Result<(), NonNull> { - // We are given the VM's PC upon having executed a trap instruction, - // which is actually pointing to the next instruction after the - // trap. Back the PC up to point exactly at the trap. - let trap_pc = unsafe { - NonNull::new_unchecked(pc.as_ptr().byte_sub(ExtendedOpcode::ENCODED_SIZE_OF_TRAP)) - }; - Err(trap_pc) - } - - #[cold] - #[inline(never)] - fn host_call(&self) -> Result<(), NonNull> { - todo!() + }) } } @@ -526,6 +541,7 @@ pub struct MachineState { f_regs: [FRegVal; FReg::RANGE.end as usize], v_regs: [VRegVal; VReg::RANGE.end as usize], stack: Vec, + done_reason: Option>, } unsafe impl Send for MachineState {} @@ -538,6 +554,7 @@ impl fmt::Debug for MachineState { f_regs, v_regs, stack: _, + done_reason: _, } = self; struct RegMap<'a, R>(&'a [R], fn(u8) -> alloc::string::String); @@ -571,6 +588,20 @@ impl fmt::Debug for MachineState { macro_rules! index_reg { ($reg_ty:ty,$value_ty:ty,$field:ident) => { + impl Index<$reg_ty> for Vm { + type Output = $value_ty; + + fn index(&self, reg: $reg_ty) -> &Self::Output { + &self.state[reg] + } + } + + impl IndexMut<$reg_ty> for Vm { + fn index_mut(&mut self, reg: $reg_ty) -> &mut Self::Output { + &mut self.state[reg] + } + } + impl Index<$reg_ty> for MachineState { type Output = $value_ty; @@ -599,6 +630,7 @@ impl MachineState { f_regs: Default::default(), v_regs: Default::default(), stack, + done_reason: None, }; // Take care to construct SP such that we preserve pointer provenance @@ -615,20 +647,72 @@ impl MachineState { } } -/// The reason the interpreter loop terminated. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -enum Done { - /// A `ret` instruction was executed and the call stack was empty. This is - /// how the loop normally ends. - ReturnToHost, +/// Inner private module to prevent creation of the `Done` structure outside of +/// this module. +mod done { + use super::{Interpreter, MachineState}; + use core::ptr::NonNull; + + /// Zero-sized sentinel indicating that pulley execution has halted. + /// + /// The reason for halting is stored in `MachineState`. + #[derive(Copy, Clone, Debug, PartialEq, Eq)] + pub struct Done { + _priv: (), + } + + /// Reason that the pulley interpreter has ceased execution. + pub enum DoneReason { + /// A trap happened at this bytecode instruction. + Trap(NonNull), + /// The `call_indirect_host` instruction was executed. + CallIndirectHost { + /// The payload of `call_indirect_host`. + id: u8, + /// Where to resume execution after the host has finished. + resume: NonNull, + }, + /// Pulley has finished and the provided value is being returned. + ReturnToHost(T), + } + + impl MachineState { + pub(super) fn debug_assert_done_reason_none(&mut self) { + debug_assert!(self.done_reason.is_none()); + } + + pub(super) fn done_decode(&mut self, Done { _priv }: Done) -> DoneReason<()> { + self.done_reason.take().unwrap() + } + } + + impl Interpreter<'_> { + /// Finishes execution by recording `DoneReason::Trap`. + pub fn done_trap(&mut self, pc: NonNull) -> Done { + self.state.done_reason = Some(DoneReason::Trap(pc)); + Done { _priv: () } + } - /// A `trap` instruction was executed at the given PC. - Trap(NonNull), + /// Finishes execution by recording `DoneReason::CallIndirectHost`. + pub fn done_call_indirect_host(&mut self, id: u8) -> Done { + self.state.done_reason = Some(DoneReason::CallIndirectHost { + id, + resume: self.pc.as_ptr(), + }); + Done { _priv: () } + } - #[allow(dead_code)] - HostCall, + /// Finishes execution by recording `DoneReason::ReturnToHost`. + pub fn done_return_to_host(&mut self) -> Done { + self.state.done_reason = Some(DoneReason::ReturnToHost(())); + Done { _priv: () } + } + } } +use done::Done; +pub use done::DoneReason; + struct Interpreter<'a> { state: &'a mut MachineState, pc: UnsafeBytecodeStream, @@ -676,7 +760,7 @@ impl Interpreter<'_> { let sp_raw = sp as usize; let base_raw = self.state.stack.as_ptr() as usize; if sp_raw < base_raw { - return ControlFlow::Break(Done::Trap(pc)); + return ControlFlow::Break(self.done_trap(pc)); } self.set_sp_unchecked(sp); ControlFlow::Continue(()) @@ -731,7 +815,7 @@ impl OpVisitor for Interpreter<'_> { fn ret(&mut self) -> ControlFlow { let lr = self.state[XReg::lr]; if lr == XRegVal::HOST_RETURN_ADDR { - ControlFlow::Break(Done::ReturnToHost) + ControlFlow::Break(self.done_return_to_host()) } else { let return_addr = lr.get_ptr(); self.pc = unsafe { UnsafeBytecodeStream::new(NonNull::new_unchecked(return_addr)) }; @@ -1288,7 +1372,8 @@ impl OpVisitor for Interpreter<'_> { // SAFETY: part of the contract of the interpreter is only dealing with // valid bytecode, so this offset should be safe. self.pc = unsafe { self.pc.offset(idx * 4) }; - let rel = unwrap_uninhabited(PcRelOffset::decode(&mut self.pc)); + let mut tmp = self.pc; + let rel = unwrap_uninhabited(PcRelOffset::decode(&mut tmp)); self.pc_rel_jump(rel, 0) } @@ -1350,11 +1435,11 @@ impl ExtendedOpVisitor for Interpreter<'_> { } fn trap(&mut self) -> ControlFlow { - ControlFlow::Break(Done::Trap(self.pc.as_ptr())) + let trap_pc = self.current_pc::(); + ControlFlow::Break(self.done_trap(trap_pc)) } - fn call_indirect_host(&mut self, sig: u8) -> ControlFlow { - let _ = sig; // TODO: should stash this somewhere - ControlFlow::Break(Done::ReturnToHost) + fn call_indirect_host(&mut self, id: u8) -> ControlFlow { + ControlFlow::Break(self.done_call_indirect_host(id)) } } diff --git a/pulley/src/opcode.rs b/pulley/src/opcode.rs index de1d8420eb04..0ff21217e316 100644 --- a/pulley/src/opcode.rs +++ b/pulley/src/opcode.rs @@ -84,9 +84,6 @@ macro_rules! define_extended_opcode { for_each_extended_op!(define_extended_opcode); impl ExtendedOpcode { - #[cfg_attr(not(feature = "interp"), allow(unused))] - pub(crate) const ENCODED_SIZE_OF_TRAP: usize = 3; - /// Create a new `ExtendedOpcode` from the given bytes. /// /// Returns `None` if `bytes` is not a valid extended opcode. @@ -108,16 +105,3 @@ impl ExtendedOpcode { core::mem::transmute(byte) } } - -#[cfg(all(test, feature = "encode"))] -mod tests { - use super::*; - use alloc::vec::Vec; - - #[test] - fn encoded_size_of_trap() { - let mut buf = Vec::new(); - crate::encode::trap(&mut buf); - assert_eq!(ExtendedOpcode::ENCODED_SIZE_OF_TRAP, buf.len()); - } -} diff --git a/pulley/tests/all/interp.rs b/pulley/tests/all/interp.rs index 8015cfed1412..886add4be222 100644 --- a/pulley/tests/all/interp.rs +++ b/pulley/tests/all/interp.rs @@ -1,7 +1,10 @@ //! Interpreter tests. use interp::Val; -use pulley_interpreter::{interp::Vm, *}; +use pulley_interpreter::{ + interp::{DoneReason, Vm}, + *, +}; use std::{cell::UnsafeCell, fmt::Debug, ptr::NonNull}; fn encoded(ops: &[Op]) -> Vec { @@ -16,8 +19,11 @@ fn encoded(ops: &[Op]) -> Vec { unsafe fn run(vm: &mut Vm, ops: &[Op]) -> Result<(), NonNull> { let _ = env_logger::try_init(); let ops = encoded(ops); - let _ = vm.call(NonNull::from(&ops[..]).cast(), &[], [])?; - Ok(()) + match vm.call(NonNull::from(&ops[..]).cast(), &[], []) { + DoneReason::ReturnToHost(_) => Ok(()), + DoneReason::Trap(pc) => Err(pc), + DoneReason::CallIndirectHost { .. } => unimplemented!(), + } } unsafe fn assert_one( diff --git a/tests/all/pulley.rs b/tests/all/pulley.rs index f3695283c2d9..ef24819b5590 100644 --- a/tests/all/pulley.rs +++ b/tests/all/pulley.rs @@ -15,10 +15,25 @@ fn pulley_config() -> Config { config } +// Pulley is known to not support big-endian platforms at this time, so assert +// that big-endian platforms do indeed fail and success is only on little-endian +// platforms. When pulley has support for big-endian this will get deleted. +fn assert_result_expected(r: Result) { + match r { + Ok(_) => { + assert!(!cfg!(target_endian = "big")); + } + Err(e) => { + assert!(cfg!(target_endian = "big"), "bad error: {e:?}"); + } + } +} + #[test] fn can_compile_pulley_module() -> Result<()> { let engine = Engine::new(&pulley_config())?; - Module::new(&engine, "(module)")?; + assert_result_expected(Module::new(&engine, "(module)")); + Ok(()) } @@ -61,16 +76,16 @@ fn pulley_wrong_architecture_is_rejected() -> Result<()> { #[cfg(not(miri))] fn can_run_on_cli() -> Result<()> { use crate::cli_tests::run_wasmtime; - run_wasmtime(&[ + assert_result_expected(run_wasmtime(&[ "--target", pulley_target(), "tests/all/cli_tests/empty-module.wat", - ])?; - run_wasmtime(&[ + ])); + assert_result_expected(run_wasmtime(&[ "run", "--target", pulley_target(), "tests/all/cli_tests/empty-module.wat", - ])?; + ])); Ok(()) } diff --git a/tests/disas/pulley/epoch-simple.wat b/tests/disas/pulley/epoch-simple.wat index 6eace319937b..1f503e82902e 100644 --- a/tests/disas/pulley/epoch-simple.wat +++ b/tests/disas/pulley/epoch-simple.wat @@ -15,5 +15,5 @@ ;; br_if x7, 0x8 // target = 0x1b ;; 19: pop_frame ;; ret -;; 1b: call 0xa // target = 0x25 +;; 1b: call 0x83 // target = 0x9e ;; 20: jump 0xfffffffffffffff9 // target = 0x19 diff --git a/tests/wast.rs b/tests/wast.rs index cfbd12a06899..68abc4923726 100644 --- a/tests/wast.rs +++ b/tests/wast.rs @@ -22,7 +22,16 @@ fn main() { // run this test in. for test in tests { let test_uses_gc_types = test.test_uses_gc_types(); - for compiler in [Compiler::Cranelift, Compiler::Winch] { + for compiler in [ + Compiler::CraneliftNative, + Compiler::Winch, + Compiler::CraneliftPulley, + ] { + // Skip compilers that have no support for this host. + if !compiler.supports_host() { + continue; + } + for pooling in [true, false] { let collectors: &[_] = if !pooling && test_uses_gc_types { &[Collector::DeferredReferenceCounting, Collector::Null] @@ -104,12 +113,12 @@ fn run_wast(test: &WastTest, config: WastConfig) -> anyhow::Result<()> { // `crates/wast-util/src/lib.rs` file. let should_fail = test.should_fail(&config); - let multi_memory = test_config.multi_memory.unwrap_or(false); - let test_hogs_memory = test_config.hogs_memory.unwrap_or(false); - let relaxed_simd = test_config.relaxed_simd.unwrap_or(false); + let multi_memory = test_config.multi_memory(); + let test_hogs_memory = test_config.hogs_memory(); + let relaxed_simd = test_config.relaxed_simd(); let is_cranelift = match config.compiler { - Compiler::Cranelift => true, + Compiler::CraneliftNative | Compiler::CraneliftPulley => true, _ => false, };