substrate_wasm_builder/lib.rs
1// This file is part of Substrate.
2
3// Copyright (C) Parity Technologies (UK) Ltd.
4// SPDX-License-Identifier: Apache-2.0
5
6// Licensed under the Apache License, Version 2.0 (the "License");
7// you may not use this file except in compliance with the License.
8// You may obtain a copy of the License at
9//
10// http://www.apache.org/licenses/LICENSE-2.0
11//
12// Unless required by applicable law or agreed to in writing, software
13// distributed under the License is distributed on an "AS IS" BASIS,
14// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15// See the License for the specific language governing permissions and
16// limitations under the License.
17
18//! # Wasm builder is a utility for building a project as a Wasm binary
19//!
20//! The Wasm builder is a tool that integrates the process of building the WASM binary of your
21//! project into the main `cargo` build process.
22//!
23//! ## Project setup
24//!
25//! A project that should be compiled as a Wasm binary needs to:
26//!
27//! 1. Add a `build.rs` file.
28//! 2. Add `wasm-builder` as dependency into `build-dependencies`.
29//!
30//! The `build.rs` file needs to contain the following code:
31//!
32//! ```no_run
33//! use substrate_wasm_builder::WasmBuilder;
34//!
35//! fn main() {
36//! // Builds the WASM binary using the recommended defaults.
37//! // If you need more control, you can call `new` or `init_with_defaults`.
38//! WasmBuilder::build_using_defaults();
39//! }
40//! ```
41//!
42//! As the final step, you need to add the following to your project:
43//!
44//! ```ignore
45//! include!(concat!(env!("OUT_DIR"), "/wasm_binary.rs"));
46//! ```
47//!
48//! This will include the generated Wasm binary as two constants `WASM_BINARY` and
49//! `WASM_BINARY_BLOATY`. The former is a compact Wasm binary and the latter is the Wasm binary as
50//! being generated by the compiler. Both variables have `Option<&'static [u8]>` as type.
51//! Additionally it will create the `WASM_BINARY_PATH` which is the path to the WASM blob on the
52//! filesystem.
53//!
54//! ### Feature
55//!
56//! Wasm builder supports to enable cargo features while building the Wasm binary. By default it
57//! will enable all features in the wasm build that are enabled for the native build except the
58//! `default` and `std` features. Besides that, wasm builder supports the special `runtime-wasm`
59//! feature. This `runtime-wasm` feature will be enabled by the wasm builder when it compiles the
60//! Wasm binary. If this feature is not present, it will not be enabled.
61//!
62//! ## Environment variables
63//!
64//! By using environment variables, you can configure which Wasm binaries are built and how:
65//!
66//! - `SUBSTRATE_RUNTIME_TARGET` - Sets the target for building runtime. Supported values are `wasm`
67//! or `riscv` (experimental, do not use it in production!). By default the target is equal to
68//! `wasm`.
69//! - `SKIP_WASM_BUILD` - Skips building any Wasm binary. This is useful when only native should be
70//! recompiled. If this is the first run and there doesn't exist a Wasm binary, this will set both
71//! variables to `None`.
72//! - `WASM_BUILD_TYPE` - Sets the build type for building Wasm binaries. Supported values are
73//! `release` or `debug`. By default the build type is equal to the build type used by the main
74//! build.
75//! - `FORCE_WASM_BUILD` - Can be set to force a Wasm build. On subsequent calls the value of the
76//! variable needs to change. As wasm-builder instructs `cargo` to watch for file changes this
77//! environment variable should only be required in certain circumstances.
78//! - `WASM_BUILD_RUSTFLAGS` - Extend `RUSTFLAGS` given to `cargo build` while building the wasm
79//! binary.
80//! - `WASM_BUILD_NO_COLOR` - Disable color output of the wasm build.
81//! - `WASM_TARGET_DIRECTORY` - Will copy any build Wasm binary to the given directory. The path
82//! needs to be absolute.
83//! - `WASM_BUILD_TOOLCHAIN` - The toolchain that should be used to build the Wasm binaries. The
84//! format needs to be the same as used by cargo, e.g. `nightly-2024-12-26`.
85//! - `WASM_BUILD_WORKSPACE_HINT` - Hint the workspace that is being built. This is normally not
86//! required as we walk up from the target directory until we find a `Cargo.toml`. If the target
87//! directory is changed for the build, this environment variable can be used to point to the
88//! actual workspace.
89//! - `WASM_BUILD_STD` - Sets whether the Rust's standard library crates (`core` and `alloc`) will
90//! also be built. This is necessary to make sure the standard library crates only use the exact
91//! WASM feature set that our executor supports. Enabled by default for RISC-V target and WASM
92//! target (but only if Rust < 1.84). Disabled by default for WASM target and Rust >= 1.84.
93//! - `WASM_BUILD_CARGO_ARGS` - This can take a string as space separated list of `cargo` arguments.
94//! It was added specifically for the use case of enabling JSON diagnostic messages during the
95//! build phase, to be used by IDEs that parse them, but it might be useful for other cases too.
96//! - `CARGO_NET_OFFLINE` - If `true`, `--offline` will be passed to all processes launched to
97//! prevent network access. Useful in offline environments.
98//!
99//! Each project can be skipped individually by using the environment variable
100//! `SKIP_PROJECT_NAME_WASM_BUILD`. Where `PROJECT_NAME` needs to be replaced by the name of the
101//! cargo project, e.g. `kitchensink-runtime` will be `NODE_RUNTIME`.
102//!
103//! ## Prerequisites:
104//!
105//! Wasm builder requires the following prerequisites for building the Wasm binary:
106//! - Rust >= 1.68 and Rust < 1.84:
107//! - `wasm32-unknown-unknown` target
108//! - `rust-src` component
109//! - Rust >= 1.84:
110//! - `wasm32v1-none` target
111//!
112//! If a specific Rust is installed with `rustup`, it is important that the WASM
113//! target is installed as well. For example if installing the Rust from
114//! 26.12.2024 using `rustup install nightly-2024-12-26`, the WASM target
115//! (`wasm32-unknown-unknown` or `wasm32v1-none`) needs to be installed as well
116//! `rustup target add wasm32-unknown-unknown --toolchain nightly-2024-12-26`.
117//! To install the `rust-src` component, use `rustup component add rust-src
118//! --toolchain nightly-2024-12-26`.
119
120use prerequisites::DummyCrate;
121use std::{
122 env, fs,
123 io::BufRead,
124 path::{Path, PathBuf},
125 process::Command,
126};
127use version::Version;
128
129mod builder;
130#[cfg(feature = "metadata-hash")]
131mod metadata_hash;
132mod prerequisites;
133mod version;
134mod wasm_project;
135
136pub use builder::{WasmBuilder, WasmBuilderSelectProject};
137
138/// Environment variable that tells us to skip building the wasm binary.
139const SKIP_BUILD_ENV: &str = "SKIP_WASM_BUILD";
140
141/// Environment variable that tells us whether we should avoid network requests
142const OFFLINE: &str = "CARGO_NET_OFFLINE";
143
144/// Environment variable to force a certain build type when building the wasm binary.
145/// Expects "debug", "release" or "production" as value.
146///
147/// When unset the WASM binary uses the same build type as the main cargo build with
148/// the exception of a debug build: In this case the wasm build defaults to `release` in
149/// order to avoid a slowdown when not explicitly requested.
150const WASM_BUILD_TYPE_ENV: &str = "WASM_BUILD_TYPE";
151
152/// Environment variable to extend the `RUSTFLAGS` variable given to the wasm build.
153const WASM_BUILD_RUSTFLAGS_ENV: &str = "WASM_BUILD_RUSTFLAGS";
154
155/// Environment variable to set the target directory to copy the final wasm binary.
156///
157/// The directory needs to be an absolute path.
158const WASM_TARGET_DIRECTORY: &str = "WASM_TARGET_DIRECTORY";
159
160/// Environment variable to disable color output of the wasm build.
161const WASM_BUILD_NO_COLOR: &str = "WASM_BUILD_NO_COLOR";
162
163/// Environment variable to set the toolchain used to compile the wasm binary.
164const WASM_BUILD_TOOLCHAIN: &str = "WASM_BUILD_TOOLCHAIN";
165
166/// Environment variable that makes sure the WASM build is triggered.
167const FORCE_WASM_BUILD_ENV: &str = "FORCE_WASM_BUILD";
168
169/// Environment variable that hints the workspace we are building.
170const WASM_BUILD_WORKSPACE_HINT: &str = "WASM_BUILD_WORKSPACE_HINT";
171
172/// Environment variable to set whether we'll build `core`/`alloc`.
173const WASM_BUILD_STD: &str = "WASM_BUILD_STD";
174
175/// Environment variable to set additional cargo arguments that might be useful
176/// during the build phase.
177const WASM_BUILD_CARGO_ARGS: &str = "WASM_BUILD_CARGO_ARGS";
178
179/// The target to use for the runtime. Valid values are `wasm` (default) or `riscv`.
180const RUNTIME_TARGET: &str = "SUBSTRATE_RUNTIME_TARGET";
181
182/// Write to the given `file` if the `content` is different.
183fn write_file_if_changed(file: impl AsRef<Path>, content: impl AsRef<str>) {
184 if fs::read_to_string(file.as_ref()).ok().as_deref() != Some(content.as_ref()) {
185 fs::write(file.as_ref(), content.as_ref())
186 .unwrap_or_else(|_| panic!("Writing `{}` can not fail!", file.as_ref().display()));
187 }
188}
189
190/// Copy `src` to `dst` if the `dst` does not exist or is different.
191fn copy_file_if_changed(src: PathBuf, dst: PathBuf) {
192 let src_file = fs::read_to_string(&src).ok();
193 let dst_file = fs::read_to_string(&dst).ok();
194
195 if src_file != dst_file {
196 fs::copy(&src, &dst).unwrap_or_else(|_| {
197 panic!("Copying `{}` to `{}` can not fail; qed", src.display(), dst.display())
198 });
199 }
200}
201
202/// Get a cargo command that should be used to invoke the compilation.
203fn get_cargo_command(target: RuntimeTarget) -> CargoCommand {
204 let env_cargo =
205 CargoCommand::new(&env::var("CARGO").expect("`CARGO` env variable is always set by cargo"));
206 let default_cargo = CargoCommand::new("cargo");
207 let wasm_toolchain = env::var(WASM_BUILD_TOOLCHAIN).ok();
208
209 // First check if the user requested a specific toolchain
210 if let Some(cmd) =
211 wasm_toolchain.map(|t| CargoCommand::new_with_args("rustup", &["run", &t, "cargo"]))
212 {
213 cmd
214 } else if env_cargo.supports_substrate_runtime_env(target) {
215 env_cargo
216 } else if default_cargo.supports_substrate_runtime_env(target) {
217 default_cargo
218 } else {
219 // If no command before provided us with a cargo that supports our Substrate wasm env, we
220 // try to search one with rustup. If that fails as well, we return the default cargo and let
221 // the perquisites check fail.
222 get_rustup_command(target).unwrap_or(default_cargo)
223 }
224}
225
226/// Get the newest rustup command that supports compiling a runtime.
227///
228/// Stable versions are always favored over nightly versions even if the nightly versions are
229/// newer.
230fn get_rustup_command(target: RuntimeTarget) -> Option<CargoCommand> {
231 let output = Command::new("rustup").args(&["toolchain", "list"]).output().ok()?.stdout;
232 let lines = output.as_slice().lines();
233
234 let mut versions = Vec::new();
235 for line in lines.filter_map(|l| l.ok()) {
236 // Split by a space to get rid of e.g. " (default)" at the end.
237 let rustup_version = line.split(" ").next().unwrap();
238 let cmd = CargoCommand::new_with_args("rustup", &["run", &rustup_version, "cargo"]);
239
240 if !cmd.supports_substrate_runtime_env(target) {
241 continue
242 }
243
244 let Some(cargo_version) = cmd.version() else { continue };
245
246 versions.push((cargo_version, rustup_version.to_string()));
247 }
248
249 // Sort by the parsed version to get the latest version (greatest version) at the end of the
250 // vec.
251 versions.sort_by_key(|v| v.0);
252 let version = &versions.last()?.1;
253
254 Some(CargoCommand::new_with_args("rustup", &["run", &version, "cargo"]))
255}
256
257/// Wraps a specific command which represents a cargo invocation.
258#[derive(Debug, Clone)]
259struct CargoCommand {
260 program: String,
261 args: Vec<String>,
262 version: Option<Version>,
263}
264
265impl CargoCommand {
266 fn new(program: &str) -> Self {
267 let version = Self::extract_version(program, &[]);
268
269 CargoCommand { program: program.into(), args: Vec::new(), version }
270 }
271
272 fn new_with_args(program: &str, args: &[&str]) -> Self {
273 let version = Self::extract_version(program, args);
274
275 CargoCommand {
276 program: program.into(),
277 args: args.iter().map(ToString::to_string).collect(),
278 version,
279 }
280 }
281
282 fn command(&self) -> Command {
283 let mut cmd = Command::new(&self.program);
284 cmd.args(&self.args);
285 cmd
286 }
287
288 fn extract_version(program: &str, args: &[&str]) -> Option<Version> {
289 let version = Command::new(program)
290 .args(args)
291 .arg("--version")
292 .output()
293 .ok()
294 .and_then(|o| String::from_utf8(o.stdout).ok())?;
295
296 Version::extract(&version)
297 }
298
299 /// Returns the version of this cargo command or `None` if it failed to extract the version.
300 fn version(&self) -> Option<Version> {
301 self.version
302 }
303
304 /// Returns whether this version of the toolchain supports nightly features.
305 fn supports_nightly_features(&self) -> bool {
306 self.version.map_or(false, |version| version.is_nightly) ||
307 env::var("RUSTC_BOOTSTRAP").is_ok()
308 }
309
310 /// Check if the supplied cargo command supports our runtime environment.
311 fn supports_substrate_runtime_env(&self, target: RuntimeTarget) -> bool {
312 match target {
313 RuntimeTarget::Wasm => self.supports_substrate_runtime_env_wasm(),
314 RuntimeTarget::Riscv => true,
315 }
316 }
317
318 /// Check if the supplied cargo command supports our Substrate wasm environment.
319 ///
320 /// This means that either the cargo version is at minimum 1.68.0 or this is a nightly cargo.
321 ///
322 /// Assumes that cargo version matches the rustc version.
323 fn supports_substrate_runtime_env_wasm(&self) -> bool {
324 // `RUSTC_BOOTSTRAP` tells a stable compiler to behave like a nightly. So, when this env
325 // variable is set, we can assume that whatever rust compiler we have, it is a nightly
326 // compiler. For "more" information, see:
327 // https://github.com/rust-lang/rust/blob/fa0f7d0080d8e7e9eb20aa9cbf8013f96c81287f/src/libsyntax/feature_gate/check.rs#L891
328 if env::var("RUSTC_BOOTSTRAP").is_ok() {
329 return true
330 }
331
332 let Some(version) = self.version() else { return false };
333
334 // Check if major and minor are greater or equal than 1.68 or this is a nightly.
335 version.major > 1 || (version.major == 1 && version.minor >= 68) || version.is_nightly
336 }
337
338 /// Returns whether this version of the toolchain supports the `wasm32v1-none` target.
339 fn supports_wasm32v1_none_target(&self) -> bool {
340 self.version.map_or(false, |version| {
341 // Check if major and minor are greater or equal than 1.84.
342 version.major > 1 || (version.major == 1 && version.minor >= 84)
343 })
344 }
345
346 /// Returns whether the `wasm32v1-none` target is installed in this version of the toolchain.
347 fn is_wasm32v1_none_target_installed(&self) -> bool {
348 let dummy_crate = DummyCrate::new(self, RuntimeTarget::Wasm, true);
349 dummy_crate.is_target_installed("wasm32v1-none").unwrap_or(false)
350 }
351
352 /// Returns whether the `wasm32v1-none` target is available in this version of the toolchain.
353 fn is_wasm32v1_none_target_available(&self) -> bool {
354 // Check if major and minor are greater or equal than 1.84 and that the `wasm32v1-none`
355 // target is installed for this toolchain.
356 self.supports_wasm32v1_none_target() && self.is_wasm32v1_none_target_installed()
357 }
358}
359
360/// Wraps a [`CargoCommand`] and the version of `rustc` the cargo command uses.
361#[derive(Clone)]
362struct CargoCommandVersioned {
363 command: CargoCommand,
364 version: String,
365}
366
367impl CargoCommandVersioned {
368 fn new(command: CargoCommand, version: String) -> Self {
369 Self { command, version }
370 }
371
372 /// Returns the `rustc` version.
373 fn rustc_version(&self) -> &str {
374 &self.version
375 }
376}
377
378impl std::ops::Deref for CargoCommandVersioned {
379 type Target = CargoCommand;
380
381 fn deref(&self) -> &CargoCommand {
382 &self.command
383 }
384}
385
386/// Returns `true` when color output is enabled.
387fn color_output_enabled() -> bool {
388 env::var(crate::WASM_BUILD_NO_COLOR).is_err()
389}
390
391/// Fetches a boolean environment variable. Will exit the process if the value is invalid.
392fn get_bool_environment_variable(name: &str) -> Option<bool> {
393 let value = env::var_os(name)?;
394
395 // We're comparing `OsString`s here so we can't use a `match`.
396 if value == "1" {
397 Some(true)
398 } else if value == "0" {
399 Some(false)
400 } else {
401 build_helper::warning!(
402 "the '{name}' environment variable has an invalid value; it must be either '1' or '0'",
403 );
404 std::process::exit(1);
405 }
406}
407
408#[derive(Copy, Clone, PartialEq, Eq)]
409enum RuntimeTarget {
410 Wasm,
411 Riscv,
412}
413
414impl RuntimeTarget {
415 /// Creates a new instance.
416 fn new() -> Self {
417 let Some(value) = env::var_os(RUNTIME_TARGET) else {
418 return Self::Wasm;
419 };
420
421 if value == "wasm" {
422 Self::Wasm
423 } else if value == "riscv" {
424 Self::Riscv
425 } else {
426 build_helper::warning!(
427 "RUNTIME_TARGET environment variable must be set to either \"wasm\" or \"riscv\""
428 );
429 std::process::exit(1);
430 }
431 }
432
433 /// Figures out the target parameter value for rustc.
434 fn rustc_target(self, cargo_command: &CargoCommand) -> String {
435 match self {
436 RuntimeTarget::Wasm =>
437 if cargo_command.is_wasm32v1_none_target_available() {
438 "wasm32v1-none".into()
439 } else {
440 "wasm32-unknown-unknown".into()
441 },
442 RuntimeTarget::Riscv => {
443 let path = polkavm_linker::target_json_32_path().expect("riscv not found");
444 path.into_os_string().into_string().unwrap()
445 },
446 }
447 }
448
449 /// Figures out the target directory name used by cargo.
450 fn rustc_target_dir(self, cargo_command: &CargoCommand) -> &'static str {
451 match self {
452 RuntimeTarget::Wasm =>
453 if cargo_command.is_wasm32v1_none_target_available() {
454 "wasm32v1-none".into()
455 } else {
456 "wasm32-unknown-unknown".into()
457 },
458 RuntimeTarget::Riscv => "riscv32emac-unknown-none-polkavm",
459 }
460 }
461
462 /// Figures out the build-std argument.
463 fn rustc_target_build_std(self, cargo_command: &CargoCommand) -> Option<&'static str> {
464 if !crate::get_bool_environment_variable(crate::WASM_BUILD_STD).unwrap_or_else(
465 || match self {
466 RuntimeTarget::Wasm => !cargo_command.is_wasm32v1_none_target_available(),
467 RuntimeTarget::Riscv => true,
468 },
469 ) {
470 return None;
471 }
472
473 // This is a nightly-only flag.
474
475 // We only build `core` and `alloc` crates since wasm-builder disables `std` featue for
476 // runtime. Thus the runtime is `#![no_std]` crate.
477
478 Some("build-std=core,alloc")
479 }
480}