wasmtime_runtime/
table.rs

1//! Memory management for tables.
2//!
3//! `Table` is to WebAssembly tables what `LinearMemory` is to WebAssembly linear memories.
4
5use crate::vmcontext::{VMCallerCheckedFuncRef, VMTableDefinition};
6use crate::{Store, VMExternRef};
7use anyhow::{bail, format_err, Error, Result};
8use std::convert::{TryFrom, TryInto};
9use std::ops::Range;
10use std::ptr;
11use wasmtime_environ::{TablePlan, Trap, WasmType, FUNCREF_INIT_BIT, FUNCREF_MASK};
12
13/// An element going into or coming out of a table.
14///
15/// Table elements are stored as pointers and are default-initialized with `ptr::null_mut`.
16#[derive(Clone)]
17pub enum TableElement {
18    /// A `funcref`.
19    FuncRef(*mut VMCallerCheckedFuncRef),
20    /// An `exrernref`.
21    ExternRef(Option<VMExternRef>),
22    /// An uninitialized funcref value. This should never be exposed
23    /// beyond the `wasmtime` crate boundary; the upper-level code
24    /// (which has access to the info needed for lazy initialization)
25    /// will replace it when fetched.
26    UninitFunc,
27}
28
29#[derive(Copy, Clone, PartialEq, Eq)]
30pub enum TableElementType {
31    Func,
32    Extern,
33}
34
35// The usage of `*mut VMCallerCheckedFuncRef` is safe w.r.t. thread safety, this
36// just relies on thread-safety of `VMExternRef` itself.
37unsafe impl Send for TableElement where VMExternRef: Send {}
38unsafe impl Sync for TableElement where VMExternRef: Sync {}
39
40impl TableElement {
41    /// Consumes the given raw table element value into a table element.
42    ///
43    /// # Safety
44    ///
45    /// This is unsafe as it will *not* clone any externref, leaving the reference count unchanged.
46    ///
47    /// This should only be used if the raw pointer is no longer in use.
48    unsafe fn from_table_value(ty: TableElementType, ptr: usize) -> Self {
49        match (ty, ptr) {
50            (TableElementType::Func, 0) => Self::UninitFunc,
51            (TableElementType::Func, ptr) => Self::FuncRef((ptr & FUNCREF_MASK) as _),
52            (TableElementType::Extern, 0) => Self::ExternRef(None),
53            (TableElementType::Extern, ptr) => {
54                Self::ExternRef(Some(VMExternRef::from_raw(ptr as *mut u8)))
55            }
56        }
57    }
58
59    /// Clones a table element from the underlying table element.
60    ///
61    /// # Safety
62    ///
63    /// This is unsafe as it will clone any externref, incrementing the reference count.
64    unsafe fn clone_from_table_value(ty: TableElementType, ptr: usize) -> Self {
65        match (ty, ptr) {
66            (TableElementType::Func, 0) => Self::UninitFunc,
67            (TableElementType::Func, ptr) => Self::FuncRef((ptr & FUNCREF_MASK) as _),
68            (TableElementType::Extern, 0) => Self::ExternRef(None),
69            (TableElementType::Extern, ptr) => {
70                Self::ExternRef(Some(VMExternRef::clone_from_raw(ptr as *mut u8)))
71            }
72        }
73    }
74
75    /// Consumes a table element into a raw table element value. This
76    /// includes any tag bits or other storage details that we
77    /// maintain in the table slot.
78    ///
79    /// # Safety
80    ///
81    /// This is unsafe as it will consume any underlying externref into a raw pointer without modifying
82    /// the reference count.
83    ///
84    /// Use `from_raw` to properly drop any table elements stored as raw pointers.
85    unsafe fn into_table_value(self) -> usize {
86        match self {
87            Self::UninitFunc => 0,
88            Self::FuncRef(e) => (e as usize) | FUNCREF_INIT_BIT,
89            Self::ExternRef(e) => e.map_or(0, |e| e.into_raw() as usize),
90        }
91    }
92
93    /// Consumes a table element into a pointer/reference, as it
94    /// exists outside the table itself. This strips off any tag bits
95    /// or other information that only lives inside the table.
96    ///
97    /// Can only be done to an initialized table element; lazy init
98    /// must occur first. (In other words, lazy values do not survive
99    /// beyond the table, as every table read path initializes them.)
100    ///
101    /// # Safety
102    ///
103    /// The same warnings as for `into_table_values()` apply.
104    pub(crate) unsafe fn into_ref_asserting_initialized(self) -> usize {
105        match self {
106            Self::FuncRef(e) => e as usize,
107            Self::ExternRef(e) => e.map_or(0, |e| e.into_raw() as usize),
108            Self::UninitFunc => panic!("Uninitialized table element value outside of table slot"),
109        }
110    }
111
112    /// Indicates whether this value is the "uninitialized element"
113    /// value.
114    pub(crate) fn is_uninit(&self) -> bool {
115        match self {
116            Self::UninitFunc => true,
117            _ => false,
118        }
119    }
120}
121
122impl From<*mut VMCallerCheckedFuncRef> for TableElement {
123    fn from(f: *mut VMCallerCheckedFuncRef) -> TableElement {
124        TableElement::FuncRef(f)
125    }
126}
127
128impl From<Option<VMExternRef>> for TableElement {
129    fn from(x: Option<VMExternRef>) -> TableElement {
130        TableElement::ExternRef(x)
131    }
132}
133
134impl From<VMExternRef> for TableElement {
135    fn from(x: VMExternRef) -> TableElement {
136        TableElement::ExternRef(Some(x))
137    }
138}
139
140/// Represents an instance's table.
141pub enum Table {
142    /// A "static" table where storage space is managed externally, currently
143    /// used with the pooling allocator.
144    Static {
145        /// Where data for this table is stored. The length of this list is the
146        /// maximum size of the table.
147        data: &'static mut [usize],
148        /// The current size of the table.
149        size: u32,
150        /// The type of this table.
151        ty: TableElementType,
152    },
153    /// A "dynamic" table where table storage space is dynamically allocated via
154    /// `malloc` (aka Rust's `Vec`).
155    Dynamic {
156        /// Dynamically managed storage space for this table. The length of this
157        /// vector is the current size of the table.
158        elements: Vec<usize>,
159        /// The type of this table.
160        ty: TableElementType,
161        /// Maximum size that `elements` can grow to.
162        maximum: Option<u32>,
163    },
164}
165
166fn wasm_to_table_type(ty: WasmType) -> Result<TableElementType> {
167    match ty {
168        WasmType::FuncRef => Ok(TableElementType::Func),
169        WasmType::ExternRef => Ok(TableElementType::Extern),
170        ty => bail!("invalid table element type {:?}", ty),
171    }
172}
173
174impl Table {
175    /// Create a new dynamic (movable) table instance for the specified table plan.
176    pub fn new_dynamic(plan: &TablePlan, store: &mut dyn Store) -> Result<Self> {
177        Self::limit_new(plan, store)?;
178        let elements = vec![0; plan.table.minimum as usize];
179        let ty = wasm_to_table_type(plan.table.wasm_ty)?;
180        let maximum = plan.table.maximum;
181
182        Ok(Table::Dynamic {
183            elements,
184            ty,
185            maximum,
186        })
187    }
188
189    /// Create a new static (immovable) table instance for the specified table plan.
190    pub fn new_static(
191        plan: &TablePlan,
192        data: &'static mut [usize],
193        store: &mut dyn Store,
194    ) -> Result<Self> {
195        Self::limit_new(plan, store)?;
196        let size = plan.table.minimum;
197        let ty = wasm_to_table_type(plan.table.wasm_ty)?;
198        if data.len() < (plan.table.minimum as usize) {
199            bail!(
200                "initial table size of {} exceeds the pooling allocator's \
201                 configured maximum table size of {} elements",
202                plan.table.minimum,
203                data.len(),
204            );
205        }
206        let data = match plan.table.maximum {
207            Some(max) if (max as usize) < data.len() => &mut data[..max as usize],
208            _ => data,
209        };
210
211        Ok(Table::Static { data, size, ty })
212    }
213
214    fn limit_new(plan: &TablePlan, store: &mut dyn Store) -> Result<()> {
215        if !store.table_growing(0, plan.table.minimum, plan.table.maximum)? {
216            bail!(
217                "table minimum size of {} elements exceeds table limits",
218                plan.table.minimum
219            );
220        }
221        Ok(())
222    }
223
224    /// Returns the type of the elements in this table.
225    pub fn element_type(&self) -> TableElementType {
226        match self {
227            Table::Static { ty, .. } => *ty,
228            Table::Dynamic { ty, .. } => *ty,
229        }
230    }
231
232    /// Returns whether or not the underlying storage of the table is "static".
233    #[cfg(feature = "pooling-allocator")]
234    pub(crate) fn is_static(&self) -> bool {
235        if let Table::Static { .. } = self {
236            true
237        } else {
238            false
239        }
240    }
241
242    /// Returns the number of allocated elements.
243    pub fn size(&self) -> u32 {
244        match self {
245            Table::Static { size, .. } => *size,
246            Table::Dynamic { elements, .. } => elements.len().try_into().unwrap(),
247        }
248    }
249
250    /// Returns the maximum number of elements at runtime.
251    ///
252    /// Returns `None` if the table is unbounded.
253    ///
254    /// The runtime maximum may not be equal to the maximum from the table's Wasm type
255    /// when it is being constrained by an instance allocator.
256    pub fn maximum(&self) -> Option<u32> {
257        match self {
258            Table::Static { data, .. } => Some(data.len() as u32),
259            Table::Dynamic { maximum, .. } => maximum.clone(),
260        }
261    }
262
263    /// Fill `table[dst..]` with values from `items`
264    ///
265    /// Returns a trap error on out-of-bounds accesses.
266    pub fn init_funcs(
267        &mut self,
268        dst: u32,
269        items: impl ExactSizeIterator<Item = *mut VMCallerCheckedFuncRef>,
270    ) -> Result<(), Trap> {
271        assert!(self.element_type() == TableElementType::Func);
272
273        let elements = match self
274            .elements_mut()
275            .get_mut(usize::try_from(dst).unwrap()..)
276            .and_then(|s| s.get_mut(..items.len()))
277        {
278            Some(elements) => elements,
279            None => return Err(Trap::TableOutOfBounds),
280        };
281
282        for (item, slot) in items.zip(elements) {
283            unsafe {
284                *slot = TableElement::FuncRef(item).into_table_value();
285            }
286        }
287        Ok(())
288    }
289
290    /// Fill `table[dst..dst + len]` with `val`.
291    ///
292    /// Returns a trap error on out-of-bounds accesses.
293    pub fn fill(&mut self, dst: u32, val: TableElement, len: u32) -> Result<(), Trap> {
294        let start = dst as usize;
295        let end = start
296            .checked_add(len as usize)
297            .ok_or_else(|| Trap::TableOutOfBounds)?;
298
299        if end > self.size() as usize {
300            return Err(Trap::TableOutOfBounds);
301        }
302
303        debug_assert!(self.type_matches(&val));
304
305        let ty = self.element_type();
306        if let Some((last, elements)) = self.elements_mut()[start..end].split_last_mut() {
307            for e in elements {
308                Self::set_raw(ty, e, val.clone());
309            }
310
311            Self::set_raw(ty, last, val);
312        }
313
314        Ok(())
315    }
316
317    /// Grow table by the specified amount of elements.
318    ///
319    /// Returns the previous size of the table if growth is successful.
320    ///
321    /// Returns `None` if table can't be grown by the specified amount of
322    /// elements, or if the `init_value` is the wrong kind of table element.
323    ///
324    /// # Unsafety
325    ///
326    /// Resizing the table can reallocate its internal elements buffer. This
327    /// table's instance's `VMContext` has raw pointers to the elements buffer
328    /// that are used by Wasm, and they need to be fixed up before we call into
329    /// Wasm again. Failure to do so will result in use-after-free inside Wasm.
330    ///
331    /// Generally, prefer using `InstanceHandle::table_grow`, which encapsulates
332    /// this unsafety.
333    pub unsafe fn grow(
334        &mut self,
335        delta: u32,
336        init_value: TableElement,
337        store: &mut dyn Store,
338    ) -> Result<Option<u32>, Error> {
339        let old_size = self.size();
340        let new_size = match old_size.checked_add(delta) {
341            Some(s) => s,
342            None => return Ok(None),
343        };
344
345        if !store.table_growing(old_size, new_size, self.maximum())? {
346            return Ok(None);
347        }
348
349        if let Some(max) = self.maximum() {
350            if new_size > max {
351                store.table_grow_failed(&format_err!("Table maximum size exceeded"));
352                return Ok(None);
353            }
354        }
355
356        debug_assert!(self.type_matches(&init_value));
357
358        // First resize the storage and then fill with the init value
359        match self {
360            Table::Static { size, data, .. } => {
361                debug_assert!(data[*size as usize..new_size as usize]
362                    .iter()
363                    .all(|x| *x == 0));
364                *size = new_size;
365            }
366            Table::Dynamic { elements, .. } => {
367                elements.resize(new_size as usize, 0);
368            }
369        }
370
371        self.fill(old_size, init_value, delta)
372            .expect("table should not be out of bounds");
373
374        Ok(Some(old_size))
375    }
376
377    /// Get reference to the specified element.
378    ///
379    /// Returns `None` if the index is out of bounds.
380    pub fn get(&self, index: u32) -> Option<TableElement> {
381        self.elements()
382            .get(index as usize)
383            .map(|p| unsafe { TableElement::clone_from_table_value(self.element_type(), *p) })
384    }
385
386    /// Set reference to the specified element.
387    ///
388    /// # Errors
389    ///
390    /// Returns an error if `index` is out of bounds or if this table type does
391    /// not match the element type.
392    pub fn set(&mut self, index: u32, elem: TableElement) -> Result<(), ()> {
393        if !self.type_matches(&elem) {
394            return Err(());
395        }
396
397        let ty = self.element_type();
398        let e = self.elements_mut().get_mut(index as usize).ok_or(())?;
399        Self::set_raw(ty, e, elem);
400        Ok(())
401    }
402
403    /// Copy `len` elements from `src_table[src_index..]` into `dst_table[dst_index..]`.
404    ///
405    /// # Errors
406    ///
407    /// Returns an error if the range is out of bounds of either the source or
408    /// destination tables.
409    pub unsafe fn copy(
410        dst_table: *mut Self,
411        src_table: *mut Self,
412        dst_index: u32,
413        src_index: u32,
414        len: u32,
415    ) -> Result<(), Trap> {
416        // https://webassembly.github.io/bulk-memory-operations/core/exec/instructions.html#exec-table-copy
417
418        if src_index
419            .checked_add(len)
420            .map_or(true, |n| n > (*src_table).size())
421            || dst_index
422                .checked_add(len)
423                .map_or(true, |m| m > (*dst_table).size())
424        {
425            return Err(Trap::TableOutOfBounds);
426        }
427
428        debug_assert!(
429            (*dst_table).element_type() == (*src_table).element_type(),
430            "table element type mismatch"
431        );
432
433        let src_range = src_index as usize..src_index as usize + len as usize;
434        let dst_range = dst_index as usize..dst_index as usize + len as usize;
435
436        // Check if the tables are the same as we cannot mutably borrow and also borrow the same `RefCell`
437        if ptr::eq(dst_table, src_table) {
438            (*dst_table).copy_elements_within(dst_range, src_range);
439        } else {
440            Self::copy_elements(&mut *dst_table, &*src_table, dst_range, src_range);
441        }
442
443        Ok(())
444    }
445
446    /// Return a `VMTableDefinition` for exposing the table to compiled wasm code.
447    pub fn vmtable(&mut self) -> VMTableDefinition {
448        match self {
449            Table::Static { data, size, .. } => VMTableDefinition {
450                base: data.as_mut_ptr().cast(),
451                current_elements: *size,
452            },
453            Table::Dynamic { elements, .. } => VMTableDefinition {
454                base: elements.as_mut_ptr().cast(),
455                current_elements: elements.len().try_into().unwrap(),
456            },
457        }
458    }
459
460    fn type_matches(&self, val: &TableElement) -> bool {
461        match (&val, self.element_type()) {
462            (TableElement::FuncRef(_), TableElementType::Func) => true,
463            (TableElement::ExternRef(_), TableElementType::Extern) => true,
464            _ => false,
465        }
466    }
467
468    fn elements(&self) -> &[usize] {
469        match self {
470            Table::Static { data, size, .. } => &data[..*size as usize],
471            Table::Dynamic { elements, .. } => &elements[..],
472        }
473    }
474
475    fn elements_mut(&mut self) -> &mut [usize] {
476        match self {
477            Table::Static { data, size, .. } => &mut data[..*size as usize],
478            Table::Dynamic { elements, .. } => &mut elements[..],
479        }
480    }
481
482    fn set_raw(ty: TableElementType, elem: &mut usize, val: TableElement) {
483        unsafe {
484            let old = *elem;
485            *elem = val.into_table_value();
486
487            // Drop the old element
488            let _ = TableElement::from_table_value(ty, old);
489        }
490    }
491
492    fn copy_elements(
493        dst_table: &mut Self,
494        src_table: &Self,
495        dst_range: Range<usize>,
496        src_range: Range<usize>,
497    ) {
498        // This can only be used when copying between different tables
499        debug_assert!(!ptr::eq(dst_table, src_table));
500
501        let ty = dst_table.element_type();
502
503        match ty {
504            TableElementType::Func => {
505                // `funcref` are `Copy`, so just do a mempcy
506                dst_table.elements_mut()[dst_range]
507                    .copy_from_slice(&src_table.elements()[src_range]);
508            }
509            TableElementType::Extern => {
510                // We need to clone each `externref`
511                let dst = dst_table.elements_mut();
512                let src = src_table.elements();
513                for (s, d) in src_range.zip(dst_range) {
514                    let elem = unsafe { TableElement::clone_from_table_value(ty, src[s]) };
515                    Self::set_raw(ty, &mut dst[d], elem);
516                }
517            }
518        }
519    }
520
521    fn copy_elements_within(&mut self, dst_range: Range<usize>, src_range: Range<usize>) {
522        let ty = self.element_type();
523        let dst = self.elements_mut();
524        match ty {
525            TableElementType::Func => {
526                // `funcref` are `Copy`, so just do a memmove
527                dst.copy_within(src_range, dst_range.start);
528            }
529            TableElementType::Extern => {
530                // We need to clone each `externref` while handling overlapping
531                // ranges
532                if dst_range.start <= src_range.start {
533                    for (s, d) in src_range.zip(dst_range) {
534                        let elem = unsafe { TableElement::clone_from_table_value(ty, dst[s]) };
535                        Self::set_raw(ty, &mut dst[d], elem);
536                    }
537                } else {
538                    for (s, d) in src_range.rev().zip(dst_range.rev()) {
539                        let elem = unsafe { TableElement::clone_from_table_value(ty, dst[s]) };
540                        Self::set_raw(ty, &mut dst[d], elem);
541                    }
542                }
543            }
544        }
545    }
546}
547
548impl Drop for Table {
549    fn drop(&mut self) {
550        let ty = self.element_type();
551
552        // funcref tables can skip this
553        if let TableElementType::Func = ty {
554            return;
555        }
556
557        // Properly drop any table elements stored in the table
558        for element in self.elements() {
559            drop(unsafe { TableElement::from_table_value(ty, *element) });
560        }
561    }
562}
563
564// The default table representation is an empty funcref table that cannot grow.
565impl Default for Table {
566    fn default() -> Self {
567        Table::Static {
568            data: &mut [],
569            size: 0,
570            ty: TableElementType::Func,
571        }
572    }
573}
wasmtime_runtime/table.rs

wasmtime_runtime/
table.rs
