wasmtime_jit/code_memory.rs
1//! Memory management for executable code.
2
3use crate::subslice_range;
4use crate::unwind::UnwindRegistration;
5use anyhow::{anyhow, bail, Context, Result};
6use object::read::{File, Object, ObjectSection};
7use object::ObjectSymbol;
8use std::mem;
9use std::mem::ManuallyDrop;
10use std::ops::Range;
11use wasmtime_environ::obj;
12use wasmtime_environ::FunctionLoc;
13use wasmtime_jit_icache_coherence as icache_coherence;
14use wasmtime_runtime::libcalls;
15use wasmtime_runtime::{MmapVec, VMTrampoline};
16
17/// Management of executable memory within a `MmapVec`
18///
19/// This type consumes ownership of a region of memory and will manage the
20/// executable permissions of the contained JIT code as necessary.
21pub struct CodeMemory {
22 // NB: these are `ManuallyDrop` because `unwind_registration` must be
23 // dropped first since it refers to memory owned by `mmap`.
24 mmap: ManuallyDrop<MmapVec>,
25 unwind_registration: ManuallyDrop<Option<UnwindRegistration>>,
26 published: bool,
27 enable_branch_protection: bool,
28
29 relocations: Vec<(usize, obj::LibCall)>,
30
31 // Ranges within `self.mmap` of where the particular sections lie.
32 text: Range<usize>,
33 unwind: Range<usize>,
34 trap_data: Range<usize>,
35 wasm_data: Range<usize>,
36 address_map_data: Range<usize>,
37 func_name_data: Range<usize>,
38 info_data: Range<usize>,
39 dwarf: Range<usize>,
40}
41
42impl Drop for CodeMemory {
43 fn drop(&mut self) {
44 // Drop `unwind_registration` before `self.mmap`
45 unsafe {
46 ManuallyDrop::drop(&mut self.unwind_registration);
47 ManuallyDrop::drop(&mut self.mmap);
48 }
49 }
50}
51
52fn _assert() {
53 fn _assert_send_sync<T: Send + Sync>() {}
54 _assert_send_sync::<CodeMemory>();
55}
56
57impl CodeMemory {
58 /// Creates a new `CodeMemory` by taking ownership of the provided
59 /// `MmapVec`.
60 ///
61 /// The returned `CodeMemory` manages the internal `MmapVec` and the
62 /// `publish` method is used to actually make the memory executable.
63 pub fn new(mmap: MmapVec) -> Result<Self> {
64 let obj = File::parse(&mmap[..])
65 .with_context(|| "failed to parse internal compilation artifact")?;
66
67 let mut relocations = Vec::new();
68 let mut text = 0..0;
69 let mut unwind = 0..0;
70 let mut enable_branch_protection = None;
71 let mut trap_data = 0..0;
72 let mut wasm_data = 0..0;
73 let mut address_map_data = 0..0;
74 let mut func_name_data = 0..0;
75 let mut info_data = 0..0;
76 let mut dwarf = 0..0;
77 for section in obj.sections() {
78 let data = section.data()?;
79 let name = section.name()?;
80 let range = subslice_range(data, &mmap);
81
82 // Double-check that sections are all aligned properly.
83 if section.align() != 0 && data.len() != 0 {
84 if (data.as_ptr() as u64 - mmap.as_ptr() as u64) % section.align() != 0 {
85 bail!(
86 "section `{}` isn't aligned to {:#x}",
87 section.name().unwrap_or("ERROR"),
88 section.align()
89 );
90 }
91 }
92
93 match name {
94 obj::ELF_WASM_BTI => match data.len() {
95 1 => enable_branch_protection = Some(data[0] != 0),
96 _ => bail!("invalid `{name}` section"),
97 },
98 ".text" => {
99 text = range;
100
101 // The text section might have relocations for things like
102 // libcalls which need to be applied, so handle those here.
103 //
104 // Note that only a small subset of possible relocations are
105 // handled. Only those required by the compiler side of
106 // things are processed.
107 for (offset, reloc) in section.relocations() {
108 assert_eq!(reloc.kind(), object::RelocationKind::Absolute);
109 assert_eq!(reloc.encoding(), object::RelocationEncoding::Generic);
110 assert_eq!(usize::from(reloc.size()), std::mem::size_of::<usize>());
111 assert_eq!(reloc.addend(), 0);
112 let sym = match reloc.target() {
113 object::RelocationTarget::Symbol(id) => id,
114 other => panic!("unknown relocation target {other:?}"),
115 };
116 let sym = obj.symbol_by_index(sym).unwrap().name().unwrap();
117 let libcall = obj::LibCall::from_str(sym)
118 .unwrap_or_else(|| panic!("unknown symbol relocation: {sym}"));
119
120 let offset = usize::try_from(offset).unwrap();
121 relocations.push((offset, libcall));
122 }
123 }
124 UnwindRegistration::SECTION_NAME => unwind = range,
125 obj::ELF_WASM_DATA => wasm_data = range,
126 obj::ELF_WASMTIME_ADDRMAP => address_map_data = range,
127 obj::ELF_WASMTIME_TRAPS => trap_data = range,
128 obj::ELF_NAME_DATA => func_name_data = range,
129 obj::ELF_WASMTIME_INFO => info_data = range,
130 obj::ELF_WASMTIME_DWARF => dwarf = range,
131
132 _ => log::debug!("ignoring section {name}"),
133 }
134 }
135 Ok(Self {
136 mmap: ManuallyDrop::new(mmap),
137 unwind_registration: ManuallyDrop::new(None),
138 published: false,
139 enable_branch_protection: enable_branch_protection
140 .ok_or_else(|| anyhow!("missing `{}` section", obj::ELF_WASM_BTI))?,
141 text,
142 unwind,
143 trap_data,
144 address_map_data,
145 func_name_data,
146 dwarf,
147 info_data,
148 wasm_data,
149 relocations,
150 })
151 }
152
153 /// Returns a reference to the underlying `MmapVec` this memory owns.
154 pub fn mmap(&self) -> &MmapVec {
155 &self.mmap
156 }
157
158 /// Returns the contents of the text section of the ELF executable this
159 /// represents.
160 pub fn text(&self) -> &[u8] {
161 &self.mmap[self.text.clone()]
162 }
163
164 /// Returns the contents of the `ELF_WASMTIME_DWARF` section.
165 pub fn dwarf(&self) -> &[u8] {
166 &self.mmap[self.dwarf.clone()]
167 }
168
169 /// Returns the data in the `ELF_NAME_DATA` section.
170 pub fn func_name_data(&self) -> &[u8] {
171 &self.mmap[self.func_name_data.clone()]
172 }
173
174 /// Returns the concatenated list of all data associated with this wasm
175 /// module.
176 ///
177 /// This is used for initialization of memories and all data ranges stored
178 /// in a `Module` are relative to the slice returned here.
179 pub fn wasm_data(&self) -> &[u8] {
180 &self.mmap[self.wasm_data.clone()]
181 }
182
183 /// Returns the encoded address map section used to pass to
184 /// `wasmtime_environ::lookup_file_pos`.
185 pub fn address_map_data(&self) -> &[u8] {
186 &self.mmap[self.address_map_data.clone()]
187 }
188
189 /// Returns the contents of the `ELF_WASMTIME_INFO` section, or an empty
190 /// slice if it wasn't found.
191 pub fn wasmtime_info(&self) -> &[u8] {
192 &self.mmap[self.info_data.clone()]
193 }
194
195 /// Returns the contents of the `ELF_WASMTIME_TRAPS` section, or an empty
196 /// slice if it wasn't found.
197 pub fn trap_data(&self) -> &[u8] {
198 &self.mmap[self.trap_data.clone()]
199 }
200
201 /// Returns a `VMTrampoline` function pointer for the given function in the
202 /// text section.
203 ///
204 /// # Unsafety
205 ///
206 /// This function is unsafe as there's no guarantee that the returned
207 /// function pointer is valid.
208 pub unsafe fn vmtrampoline(&self, loc: FunctionLoc) -> VMTrampoline {
209 let ptr = self.text()[loc.start as usize..][..loc.length as usize].as_ptr();
210 mem::transmute::<*const u8, VMTrampoline>(ptr)
211 }
212
213 /// Publishes the internal ELF image to be ready for execution.
214 ///
215 /// This method can only be called once and will panic if called twice. This
216 /// will parse the ELF image from the original `MmapVec` and do everything
217 /// necessary to get it ready for execution, including:
218 ///
219 /// * Change page protections from read/write to read/execute.
220 /// * Register unwinding information with the OS
221 ///
222 /// After this function executes all JIT code should be ready to execute.
223 pub fn publish(&mut self) -> Result<()> {
224 assert!(!self.published);
225 self.published = true;
226
227 if self.text().is_empty() {
228 return Ok(());
229 }
230
231 // The unsafety here comes from a few things:
232 //
233 // * We're actually updating some page protections to executable memory.
234 //
235 // * We're registering unwinding information which relies on the
236 // correctness of the information in the first place. This applies to
237 // both the actual unwinding tables as well as the validity of the
238 // pointers we pass in itself.
239 unsafe {
240 // First, if necessary, apply relocations. This can happen for
241 // things like libcalls which happen late in the lowering process
242 // that don't go through the Wasm-based libcalls layer that's
243 // indirected through the `VMContext`. Note that most modules won't
244 // have relocations, so this typically doesn't do anything.
245 self.apply_relocations()?;
246
247 // Next freeze the contents of this image by making all of the
248 // memory readonly. Nothing after this point should ever be modified
249 // so commit everything. For a compiled-in-memory image this will
250 // mean IPIs to evict writable mappings from other cores. For
251 // loaded-from-disk images this shouldn't result in IPIs so long as
252 // there weren't any relocations because nothing should have
253 // otherwise written to the image at any point either.
254 self.mmap.make_readonly(0..self.mmap.len())?;
255
256 let text = self.text();
257
258 // Clear the newly allocated code from cache if the processor requires it
259 //
260 // Do this before marking the memory as R+X, technically we should be able to do it after
261 // but there are some CPU's that have had errata about doing this with read only memory.
262 icache_coherence::clear_cache(text.as_ptr().cast(), text.len())
263 .expect("Failed cache clear");
264
265 // Switch the executable portion from readonly to read/execute.
266 self.mmap
267 .make_executable(self.text.clone(), self.enable_branch_protection)
268 .expect("unable to make memory executable");
269
270 // Flush any in-flight instructions from the pipeline
271 icache_coherence::pipeline_flush_mt().expect("Failed pipeline flush");
272
273 // With all our memory set up use the platform-specific
274 // `UnwindRegistration` implementation to inform the general
275 // runtime that there's unwinding information available for all
276 // our just-published JIT functions.
277 self.register_unwind_info()?;
278 }
279
280 Ok(())
281 }
282
283 unsafe fn apply_relocations(&mut self) -> Result<()> {
284 if self.relocations.is_empty() {
285 return Ok(());
286 }
287
288 for (offset, libcall) in self.relocations.iter() {
289 let offset = self.text.start + offset;
290 let libcall = match libcall {
291 obj::LibCall::FloorF32 => libcalls::relocs::floorf32 as usize,
292 obj::LibCall::FloorF64 => libcalls::relocs::floorf64 as usize,
293 obj::LibCall::NearestF32 => libcalls::relocs::nearestf32 as usize,
294 obj::LibCall::NearestF64 => libcalls::relocs::nearestf64 as usize,
295 obj::LibCall::CeilF32 => libcalls::relocs::ceilf32 as usize,
296 obj::LibCall::CeilF64 => libcalls::relocs::ceilf64 as usize,
297 obj::LibCall::TruncF32 => libcalls::relocs::truncf32 as usize,
298 obj::LibCall::TruncF64 => libcalls::relocs::truncf64 as usize,
299 obj::LibCall::FmaF32 => libcalls::relocs::fmaf32 as usize,
300 obj::LibCall::FmaF64 => libcalls::relocs::fmaf64 as usize,
301 };
302 *self.mmap.as_mut_ptr().add(offset).cast::<usize>() = libcall;
303 }
304 Ok(())
305 }
306
307 unsafe fn register_unwind_info(&mut self) -> Result<()> {
308 if self.unwind.len() == 0 {
309 return Ok(());
310 }
311 let text = self.text();
312 let unwind_info = &self.mmap[self.unwind.clone()];
313 let registration =
314 UnwindRegistration::new(text.as_ptr(), unwind_info.as_ptr(), unwind_info.len())
315 .context("failed to create unwind info registration")?;
316 *self.unwind_registration = Some(registration);
317 Ok(())
318 }
319}