1use crate::{
22 epoch,
23 host::HostState,
24 instance_wrapper::{EntryPoint, InstanceWrapper, MemoryWrapper},
25 util::{self, replace_strategy_if_broken},
26};
27
28use parking_lot::Mutex;
29use sc_allocator::{AllocationStats, FreeingBumpHeapAllocator};
30use sc_executor_common::{
31 error::{Error, Result, WasmError},
32 runtime_blob::RuntimeBlob,
33 util::checked_range,
34 wasm_runtime::{HeapAllocStrategy, WasmInstance, WasmModule},
35};
36use sp_runtime_interface::unpack_ptr_and_len;
37use sp_wasm_interface::{HostFunctions, Pointer, WordSize};
38use std::{
39 path::{Path, PathBuf},
40 sync::{
41 atomic::{AtomicBool, Ordering},
42 Arc,
43 },
44 time::Duration,
45};
46use wasmtime::{AsContext, Cache, CacheConfig, Engine, Memory};
47
48const MAX_INSTANCE_COUNT: u32 = 64;
49
50#[derive(Default)]
51pub(crate) struct StoreData {
52 pub(crate) host_state: Option<HostState>,
54 pub(crate) memory: Option<Memory>,
56 pub(crate) limits: wasmtime::StoreLimits,
58 pub(crate) memory_max_pages: Option<u32>,
60}
61
62impl StoreData {
63 pub fn host_state_mut(&mut self) -> Option<&mut HostState> {
65 self.host_state.as_mut()
66 }
67
68 pub fn memory(&self) -> Memory {
70 self.memory.expect("memory is always set; qed")
71 }
72}
73
74pub(crate) type Store = wasmtime::Store<StoreData>;
75
76enum Strategy {
77 RecreateInstance(InstanceCreator),
78}
79
80struct InstanceCreator {
81 engine: Engine,
82 instance_pre: Arc<wasmtime::InstancePre<StoreData>>,
83 instance_counter: Arc<InstanceCounter>,
84 heap_alloc_strategy: HeapAllocStrategy,
85}
86
87impl InstanceCreator {
88 fn instantiate(&mut self) -> Result<InstanceWrapper> {
89 InstanceWrapper::new(
90 &self.engine,
91 &self.instance_pre,
92 self.instance_counter.clone(),
93 self.heap_alloc_strategy,
94 )
95 }
96}
97
98pub(crate) struct ReleaseInstanceHandle {
100 counter: Arc<InstanceCounter>,
101}
102
103impl Drop for ReleaseInstanceHandle {
104 fn drop(&mut self) {
105 {
106 let mut counter = self.counter.counter.lock();
107 *counter = counter.saturating_sub(1);
108 }
109
110 self.counter.wait_for_instance.notify_one();
111 }
112}
113
114#[derive(Default)]
121pub(crate) struct InstanceCounter {
122 counter: Mutex<u32>,
123 wait_for_instance: parking_lot::Condvar,
124}
125
126impl InstanceCounter {
127 pub fn acquire_instance(self: Arc<Self>) -> ReleaseInstanceHandle {
134 let mut counter = self.counter.lock();
135
136 while *counter >= MAX_INSTANCE_COUNT {
137 self.wait_for_instance.wait(&mut counter);
138 }
139 *counter += 1;
140
141 ReleaseInstanceHandle { counter: self.clone() }
142 }
143}
144
145pub struct WasmtimeRuntime {
148 engine: Engine,
149 instance_pre: Arc<wasmtime::InstancePre<StoreData>>,
150 instantiation_strategy: InternalInstantiationStrategy,
151 instance_counter: Arc<InstanceCounter>,
152 epoch_interruption: bool,
154}
155
156impl WasmModule for WasmtimeRuntime {
157 fn new_instance(
158 &self,
159 heap_alloc_strategy: HeapAllocStrategy,
160 ) -> Result<Box<dyn WasmInstance>> {
161 let strategy = match self.instantiation_strategy {
162 InternalInstantiationStrategy::Builtin => Strategy::RecreateInstance(InstanceCreator {
163 engine: self.engine.clone(),
164 instance_pre: self.instance_pre.clone(),
165 instance_counter: self.instance_counter.clone(),
166 heap_alloc_strategy,
167 }),
168 };
169
170 Ok(Box::new(WasmtimeInstance { strategy, epoch_interruption: self.epoch_interruption }))
171 }
172}
173
174pub struct WasmtimeInstance {
177 strategy: Strategy,
178 epoch_interruption: bool,
180}
181
182impl WasmtimeInstance {
183 fn call_impl(
184 &mut self,
185 method: &str,
186 data: &[u8],
187 timeout: Option<Duration>,
188 allocation_stats: &mut Option<AllocationStats>,
189 ) -> Result<Vec<u8>> {
190 let epoch_deadline = match (self.epoch_interruption, timeout) {
191 (true, Some(timeout)) => Some(epoch::deadline_ticks(timeout)),
192 (true, None) => Some(epoch::MAX_DEADLINE_TICKS),
194 (false, Some(_)) => return Err(Error::ExecutionTimeoutUnsupported),
195 (false, None) => None,
196 };
197
198 match &mut self.strategy {
199 Strategy::RecreateInstance(ref mut instance_creator) => {
200 let mut instance_wrapper = instance_creator.instantiate()?;
201 if let Some(ticks) = epoch_deadline {
202 instance_wrapper.store_mut().set_epoch_deadline(ticks);
203 }
204 let heap_base = instance_wrapper.extract_heap_base()?;
205 let entrypoint = instance_wrapper.resolve_entrypoint(method)?;
206 let allocator = FreeingBumpHeapAllocator::new(heap_base);
207
208 perform_call(data, &mut instance_wrapper, entrypoint, allocator, allocation_stats)
209 },
210 }
211 }
212}
213
214impl WasmInstance for WasmtimeInstance {
215 fn call_with_allocation_stats(
216 &mut self,
217 method: &str,
218 data: &[u8],
219 ) -> (Result<Vec<u8>>, Option<AllocationStats>) {
220 let mut allocation_stats = None;
221 let result = self.call_impl(method, data, None, &mut allocation_stats);
222 (result, allocation_stats)
223 }
224
225 fn call_with_timeout(
226 &mut self,
227 method: &str,
228 data: &[u8],
229 timeout: Duration,
230 ) -> Result<Vec<u8>> {
231 self.call_impl(method, data, Some(timeout), &mut None)
232 }
233
234 fn set_heap_alloc_strategy(&mut self, heap_alloc_strategy: HeapAllocStrategy) {
235 match &mut self.strategy {
236 Strategy::RecreateInstance(ref mut creator) => {
237 creator.heap_alloc_strategy = heap_alloc_strategy;
238 },
239 }
240 }
241}
242
243fn setup_wasmtime_caching(
247 cache_path: &Path,
248 config: &mut wasmtime::Config,
249) -> std::result::Result<(), String> {
250 use std::fs;
251
252 let wasmtime_cache_root = cache_path.join("wasmtime");
253 fs::create_dir_all(&wasmtime_cache_root)
254 .map_err(|err| format!("cannot create the dirs to cache: {}", err))?;
255
256 let mut cache_config = CacheConfig::new();
257 cache_config.with_directory(cache_path);
258
259 let cache =
260 Cache::new(cache_config).map_err(|err| format!("failed to initiate Cache: {err:?}"))?;
261
262 config.cache(Some(cache));
263
264 Ok(())
265}
266
267fn common_config(semantics: &Semantics) -> std::result::Result<wasmtime::Config, WasmError> {
268 let mut config = wasmtime::Config::new();
269 config.cranelift_opt_level(wasmtime::OptLevel::SpeedAndSize);
270 config.cranelift_nan_canonicalization(semantics.canonicalize_nans);
271
272 let profiler = match std::env::var_os("WASMTIME_PROFILING_STRATEGY") {
273 Some(os_string) if os_string == "jitdump" => wasmtime::ProfilingStrategy::JitDump,
274 Some(os_string) if os_string == "perfmap" => wasmtime::ProfilingStrategy::PerfMap,
275 None => wasmtime::ProfilingStrategy::None,
276 Some(_) => {
277 static UNKNOWN_PROFILING_STRATEGY: AtomicBool = AtomicBool::new(false);
279 if !UNKNOWN_PROFILING_STRATEGY.swap(true, Ordering::Relaxed) {
281 log::warn!("WASMTIME_PROFILING_STRATEGY is set to unknown value, ignored.");
282 }
283 wasmtime::ProfilingStrategy::None
284 },
285 };
286 config.profiler(profiler);
287
288 let native_stack_max = match semantics.deterministic_stack_limit {
289 Some(DeterministicStackLimit { native_stack_max, .. }) => native_stack_max,
290
291 None => 1024 * 1024,
296 };
297
298 config.max_wasm_stack(native_stack_max as usize);
299
300 config.parallel_compilation(semantics.parallel_compilation);
301 config.epoch_interruption(semantics.epoch_interruption);
302
303 config.wasm_reference_types(semantics.wasm_reference_types);
306 config.wasm_simd(semantics.wasm_simd);
307 config.wasm_relaxed_simd(semantics.wasm_simd);
308 config.wasm_bulk_memory(semantics.wasm_bulk_memory);
309 config.wasm_multi_value(semantics.wasm_multi_value);
310 config.wasm_multi_memory(false);
311 config.wasm_threads(false);
312 config.wasm_memory64(false);
313 config.wasm_tail_call(false);
314 config.wasm_extended_const(false);
315
316 let (use_pooling, use_cow) = match semantics.instantiation_strategy {
317 InstantiationStrategy::PoolingCopyOnWrite => (true, true),
318 InstantiationStrategy::Pooling => (true, false),
319 InstantiationStrategy::RecreateInstanceCopyOnWrite => (false, true),
320 InstantiationStrategy::RecreateInstance => (false, false),
321 };
322
323 const WASM_PAGE_SIZE: u64 = 65536;
324
325 config.memory_init_cow(use_cow);
326 config.memory_guaranteed_dense_image_size(u64::MAX);
330
331 if use_pooling {
332 const MAX_WASM_PAGES: u64 = 0x10000;
333
334 let mut pooling_config = wasmtime::PoolingAllocationConfig::default();
335 pooling_config
336 .max_unused_warm_slots(4)
337 .max_core_instance_size(512 * 1024)
345 .table_elements(8192)
346 .max_memory_size(MAX_WASM_PAGES as usize * WASM_PAGE_SIZE as usize)
352 .total_tables(MAX_INSTANCE_COUNT)
353 .total_memories(MAX_INSTANCE_COUNT)
354 .total_core_instances(MAX_INSTANCE_COUNT);
357
358 config.allocation_strategy(wasmtime::InstanceAllocationStrategy::Pooling(pooling_config));
359 }
360
361 Ok(config)
362}
363
364#[derive(Clone)]
390pub struct DeterministicStackLimit {
391 pub logical_max: u32,
396 pub native_stack_max: u32,
407}
408
409#[non_exhaustive]
419#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug)]
420pub enum InstantiationStrategy {
421 PoolingCopyOnWrite,
426
427 RecreateInstanceCopyOnWrite,
430
431 Pooling,
434
435 RecreateInstance,
437}
438
439enum InternalInstantiationStrategy {
440 Builtin,
441}
442
443#[derive(Clone)]
444pub struct Semantics {
445 pub instantiation_strategy: InstantiationStrategy,
447
448 pub deterministic_stack_limit: Option<DeterministicStackLimit>,
459
460 pub canonicalize_nans: bool,
473
474 pub parallel_compilation: bool,
476
477 pub heap_alloc_strategy: HeapAllocStrategy,
479
480 pub wasm_multi_value: bool,
482
483 pub wasm_bulk_memory: bool,
485
486 pub wasm_reference_types: bool,
488
489 pub wasm_simd: bool,
491
492 pub epoch_interruption: bool,
497}
498
499#[derive(Clone)]
500pub struct Config {
501 pub allow_missing_func_imports: bool,
506
507 pub cache_path: Option<PathBuf>,
509
510 pub semantics: Semantics,
512}
513
514enum CodeSupplyMode<'a> {
515 Fresh(RuntimeBlob),
517
518 Precompiled(&'a Path),
526
527 PrecompiledBytes(&'a [u8]),
532}
533
534pub fn create_runtime<H>(
540 blob: RuntimeBlob,
541 config: Config,
542) -> std::result::Result<WasmtimeRuntime, WasmError>
543where
544 H: HostFunctions,
545{
546 unsafe { do_create_runtime::<H>(CodeSupplyMode::Fresh(blob), config) }
548}
549
550pub unsafe fn create_runtime_from_artifact<H>(
567 compiled_artifact_path: &Path,
568 config: Config,
569) -> std::result::Result<WasmtimeRuntime, WasmError>
570where
571 H: HostFunctions,
572{
573 do_create_runtime::<H>(CodeSupplyMode::Precompiled(compiled_artifact_path), config)
574}
575
576pub unsafe fn create_runtime_from_artifact_bytes<H>(
592 compiled_artifact_bytes: &[u8],
593 config: Config,
594) -> std::result::Result<WasmtimeRuntime, WasmError>
595where
596 H: HostFunctions,
597{
598 do_create_runtime::<H>(CodeSupplyMode::PrecompiledBytes(compiled_artifact_bytes), config)
599}
600
601unsafe fn do_create_runtime<H>(
606 code_supply_mode: CodeSupplyMode<'_>,
607 mut config: Config,
608) -> std::result::Result<WasmtimeRuntime, WasmError>
609where
610 H: HostFunctions,
611{
612 replace_strategy_if_broken(&mut config.semantics.instantiation_strategy);
613
614 let mut wasmtime_config = common_config(&config.semantics)?;
615 if let Some(ref cache_path) = config.cache_path {
616 if let Err(reason) = setup_wasmtime_caching(cache_path, &mut wasmtime_config) {
617 log::warn!(
618 "failed to setup wasmtime cache. Performance may degrade significantly: {}.",
619 reason,
620 );
621 }
622 }
623
624 let engine = Engine::new(&wasmtime_config)
625 .map_err(|e| WasmError::Other(format!("cannot create the wasmtime engine: {:#}", e)))?;
626
627 let (module, instantiation_strategy) = match code_supply_mode {
628 CodeSupplyMode::Fresh(blob) => {
629 let blob = prepare_blob_for_compilation(blob, &config.semantics)?;
630 let serialized_blob = blob.clone().serialize();
631
632 let module = wasmtime::Module::new(&engine, &serialized_blob)
633 .map_err(|e| WasmError::Other(format!("cannot create module: {:#}", e)))?;
634
635 match config.semantics.instantiation_strategy {
636 InstantiationStrategy::Pooling |
637 InstantiationStrategy::PoolingCopyOnWrite |
638 InstantiationStrategy::RecreateInstance |
639 InstantiationStrategy::RecreateInstanceCopyOnWrite => {
640 (module, InternalInstantiationStrategy::Builtin)
641 },
642 }
643 },
644 CodeSupplyMode::Precompiled(compiled_artifact_path) => {
645 let module = wasmtime::Module::deserialize_file(&engine, compiled_artifact_path)
650 .map_err(|e| WasmError::Other(format!("cannot deserialize module: {:#}", e)))?;
651
652 (module, InternalInstantiationStrategy::Builtin)
653 },
654 CodeSupplyMode::PrecompiledBytes(compiled_artifact_bytes) => {
655 let module = wasmtime::Module::deserialize(&engine, compiled_artifact_bytes)
660 .map_err(|e| WasmError::Other(format!("cannot deserialize module: {:#}", e)))?;
661
662 (module, InternalInstantiationStrategy::Builtin)
663 },
664 };
665
666 let mut linker = wasmtime::Linker::new(&engine);
667 crate::imports::prepare_imports::<H>(&mut linker, &module, config.allow_missing_func_imports)?;
668
669 let instance_pre = linker
670 .instantiate_pre(&module)
671 .map_err(|e| WasmError::Other(format!("cannot preinstantiate module: {:#}", e)))?;
672
673 let epoch_interruption = config.semantics.epoch_interruption;
674 if epoch_interruption {
675 epoch::spawn_ticker(&engine)?;
676 }
677
678 Ok(WasmtimeRuntime {
679 engine,
680 instance_pre: Arc::new(instance_pre),
681 instantiation_strategy,
682 instance_counter: Default::default(),
683 epoch_interruption,
684 })
685}
686
687fn prepare_blob_for_compilation(
688 mut blob: RuntimeBlob,
689 semantics: &Semantics,
690) -> std::result::Result<RuntimeBlob, WasmError> {
691 if let Some(DeterministicStackLimit { logical_max, .. }) = semantics.deterministic_stack_limit {
692 blob = blob.inject_stack_depth_metering(logical_max)?;
693 }
694
695 blob.convert_memory_import_into_export()?;
700 blob.clear_memory_max_limit()?;
701
702 Ok(blob)
703}
704
705pub fn prepare_runtime_artifact(
708 blob: RuntimeBlob,
709 semantics: &Semantics,
710) -> std::result::Result<Vec<u8>, WasmError> {
711 let mut semantics = semantics.clone();
712 replace_strategy_if_broken(&mut semantics.instantiation_strategy);
713
714 let blob = prepare_blob_for_compilation(blob, &semantics)?;
715
716 let engine = Engine::new(&common_config(&semantics)?)
717 .map_err(|e| WasmError::Other(format!("cannot create the engine: {:#}", e)))?;
718
719 engine
720 .precompile_module(&blob.serialize())
721 .map_err(|e| WasmError::Other(format!("cannot precompile module: {:#}", e)))
722}
723
724fn perform_call(
725 data: &[u8],
726 instance_wrapper: &mut InstanceWrapper,
727 entrypoint: EntryPoint,
728 mut allocator: FreeingBumpHeapAllocator,
729 allocation_stats: &mut Option<AllocationStats>,
730) -> Result<Vec<u8>> {
731 let (data_ptr, data_len) = inject_input_data(instance_wrapper, &mut allocator, data)?;
732
733 let host_state = HostState::new(allocator);
734
735 instance_wrapper.store_mut().data_mut().host_state = Some(host_state);
737
738 let ret = entrypoint
739 .call(instance_wrapper.store_mut(), data_ptr, data_len)
740 .map(unpack_ptr_and_len);
741
742 let host_state = instance_wrapper.store_mut().data_mut().host_state.take().expect(
744 "the host state is always set before calling into WASM so it can't be None here; qed",
745 );
746 *allocation_stats = Some(host_state.allocation_stats());
747
748 let (output_ptr, output_len) = ret?;
749 let output = extract_output_data(instance_wrapper, output_ptr, output_len)?;
750
751 Ok(output)
752}
753
754fn inject_input_data(
755 instance: &mut InstanceWrapper,
756 allocator: &mut FreeingBumpHeapAllocator,
757 data: &[u8],
758) -> Result<(Pointer<u8>, WordSize)> {
759 let mut ctx = instance.store_mut();
760 let memory = ctx.data().memory();
761 let data_len = data.len() as WordSize;
762 let data_ptr = allocator.allocate(&mut MemoryWrapper(&memory, &mut ctx), data_len)?;
763 util::write_memory_from(instance.store_mut(), data_ptr, data)?;
764 Ok((data_ptr, data_len))
765}
766
767fn extract_output_data(
768 instance: &InstanceWrapper,
769 output_ptr: u32,
770 output_len: u32,
771) -> Result<Vec<u8>> {
772 let ctx = instance.store();
773
774 let memory_size = ctx.as_context().data().memory().data_size(ctx);
780 if checked_range(output_ptr as usize, output_len as usize, memory_size).is_none() {
781 Err(Error::OutputExceedsBounds)?
782 }
783 let mut output = vec![0; output_len as usize];
784
785 util::read_memory_into(ctx, Pointer::new(output_ptr), &mut output)?;
786 Ok(output)
787}