frame_benchmarking_cli/pallet/
mod.rs

1// This file is part of Substrate.
2
3// Copyright (C) Parity Technologies (UK) Ltd.
4// SPDX-License-Identifier: Apache-2.0
5
6// Licensed under the Apache License, Version 2.0 (the "License");
7// you may not use this file except in compliance with the License.
8// You may obtain a copy of the License at
9//
10// 	http://www.apache.org/licenses/LICENSE-2.0
11//
12// Unless required by applicable law or agreed to in writing, software
13// distributed under the License is distributed on an "AS IS" BASIS,
14// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15// See the License for the specific language governing permissions and
16// limitations under the License.
17
18mod command;
19mod logging;
20mod types;
21mod writer;
22
23use crate::shared::HostInfoParams;
24use clap::ValueEnum;
25use frame_support::Serialize;
26use sc_cli::{
27	WasmExecutionMethod, WasmtimeInstantiationStrategy, DEFAULT_WASMTIME_INSTANTIATION_STRATEGY,
28	DEFAULT_WASM_EXECUTION_METHOD,
29};
30use std::{fmt::Debug, path::PathBuf};
31
32/// Logging target
33const LOG_TARGET: &'static str = "frame::benchmark::pallet";
34
35// Add a more relaxed parsing for pallet names by allowing pallet directory names with `-` to be
36// used like crate names with `_`
37fn parse_pallet_name(pallet: &str) -> std::result::Result<String, String> {
38	Ok(pallet.replace("-", "_"))
39}
40
41/// List options for available benchmarks.
42#[derive(Debug, Clone, Copy, ValueEnum)]
43pub enum ListOutput {
44	/// List all available pallets and extrinsics.
45	All,
46	/// List all available pallets only.
47	Pallets,
48}
49
50/// Benchmark the extrinsic weight of FRAME Pallets.
51#[derive(Debug, clap::Parser)]
52pub struct PalletCmd {
53	/// Select a FRAME Pallets to benchmark, or `*` for all (in which case `extrinsic` must be
54	/// `*`).
55	#[arg(short, long, alias = "pallet", num_args = 1.., value_delimiter = ',', value_parser = parse_pallet_name, required_unless_present_any = ["list", "json_input", "all"], default_value_if("all", "true", Some("*".into())))]
56	pub pallets: Vec<String>,
57
58	/// Select an extrinsic inside the pallet to benchmark, or `*` or 'all' for all.
59	#[arg(short, long, required_unless_present_any = ["list", "json_input", "all"], default_value_if("all", "true", Some("*".into())))]
60	pub extrinsic: Option<String>,
61
62	/// Comma separated list of pallets that should be excluded from the benchmark.
63	#[arg(long, value_parser, num_args = 1.., value_delimiter = ',')]
64	pub exclude_pallets: Vec<String>,
65
66	/// Comma separated list of `pallet::extrinsic` combinations that should not be run.
67	///
68	/// Example: `frame_system::remark,pallet_balances::transfer_keep_alive`
69	#[arg(long, value_parser, num_args = 1.., value_delimiter = ',')]
70	pub exclude_extrinsics: Vec<String>,
71
72	/// Run benchmarks for all pallets and extrinsics.
73	///
74	/// This is equivalent to running `--pallet * --extrinsic *`.
75	#[arg(long)]
76	pub all: bool,
77
78	/// Select how many samples we should take across the variable components.
79	#[arg(short, long, default_value_t = 50)]
80	pub steps: u32,
81
82	/// Indicates lowest values for each of the component ranges.
83	#[arg(long = "low", value_delimiter = ',')]
84	pub lowest_range_values: Vec<u32>,
85
86	/// Indicates highest values for each of the component ranges.
87	#[arg(long = "high", value_delimiter = ',')]
88	pub highest_range_values: Vec<u32>,
89
90	/// Select how many repetitions of this benchmark should run from within the wasm.
91	#[arg(short, long, default_value_t = 20)]
92	pub repeat: u32,
93
94	/// Select how many repetitions of this benchmark should run from the client.
95	///
96	/// NOTE: Using this alone may give slower results, but will afford you maximum Wasm memory.
97	#[arg(long, default_value_t = 1)]
98	pub external_repeat: u32,
99
100	/// Print the raw results in JSON format.
101	#[arg(long = "json")]
102	pub json_output: bool,
103
104	/// Write the raw results in JSON format into the given file.
105	#[arg(long, conflicts_with = "json_output")]
106	pub json_file: Option<PathBuf>,
107
108	/// Don't print the median-slopes linear regression analysis.
109	#[arg(long)]
110	pub no_median_slopes: bool,
111
112	/// Don't print the min-squares linear regression analysis.
113	#[arg(long)]
114	pub no_min_squares: bool,
115
116	/// Output the benchmarks to a Rust file at the given path.
117	#[arg(long)]
118	pub output: Option<PathBuf>,
119
120	/// Add a header file to your outputted benchmarks.
121	#[arg(long)]
122	pub header: Option<PathBuf>,
123
124	/// Path to Handlebars template file used for outputting benchmark results. (Optional)
125	#[arg(long)]
126	pub template: Option<PathBuf>,
127
128	#[allow(missing_docs)]
129	#[clap(flatten)]
130	pub hostinfo_params: HostInfoParams,
131
132	/// Which analysis function to use when outputting benchmarks:
133	/// * min-squares (default)
134	/// * median-slopes
135	/// * max (max of min squares and median slopes for each value)
136	#[arg(long)]
137	pub output_analysis: Option<String>,
138
139	/// Which analysis function to use when analyzing measured proof sizes.
140	#[arg(long, default_value("median-slopes"))]
141	pub output_pov_analysis: Option<String>,
142
143	/// The PoV estimation mode of a benchmark if no `pov_mode` attribute is present.
144	#[arg(long, default_value("max-encoded-len"), value_enum)]
145	pub default_pov_mode: command::PovEstimationMode,
146
147	/// Ignore the error when PoV modes reference unknown storage items or pallets.
148	#[arg(long)]
149	pub ignore_unknown_pov_mode: bool,
150
151	/// Set the heap pages while running benchmarks. If not set, the default value from the client
152	/// is used.
153	#[arg(long)]
154	pub heap_pages: Option<u64>,
155
156	/// Disable verification logic when running benchmarks.
157	#[arg(long)]
158	pub no_verify: bool,
159
160	/// Display and run extra benchmarks that would otherwise not be needed for weight
161	/// construction.
162	#[arg(long)]
163	pub extra: bool,
164
165	#[allow(missing_docs)]
166	#[clap(flatten)]
167	pub shared_params: sc_cli::SharedParams,
168
169	/// Method for executing Wasm runtime code.
170	#[arg(
171		long = "wasm-execution",
172		value_name = "METHOD",
173		value_enum,
174		ignore_case = true,
175		default_value_t = DEFAULT_WASM_EXECUTION_METHOD,
176	)]
177	pub wasm_method: WasmExecutionMethod,
178
179	/// The WASM instantiation method to use.
180	///
181	/// Only has an effect when `wasm-execution` is set to `compiled`.
182	#[arg(
183		long = "wasm-instantiation-strategy",
184		value_name = "STRATEGY",
185		default_value_t = DEFAULT_WASMTIME_INSTANTIATION_STRATEGY,
186		value_enum,
187	)]
188	pub wasmtime_instantiation_strategy: WasmtimeInstantiationStrategy,
189
190	/// Optional runtime blob to use instead of the one from the genesis config.
191	#[arg(long, conflicts_with = "chain", required_if_eq("genesis_builder", "runtime"))]
192	pub runtime: Option<PathBuf>,
193
194	/// Set the runtime log level.
195	///
196	/// This will overwrite the `RUNTIME_LOG` environment variable. If neither is set, the CLI
197	/// default set by `RUST_LOG` setting is used.
198	#[arg(long)]
199	pub runtime_log: Option<String>,
200
201	/// Do not fail if there are unknown but also unused host functions in the runtime.
202	#[arg(long)]
203	pub allow_missing_host_functions: bool,
204
205	/// How to construct the genesis state.
206	///
207	/// Uses `GenesisBuilderPolicy::Spec` by default.
208	#[arg(long, value_enum, alias = "genesis-builder-policy")]
209	pub genesis_builder: Option<GenesisBuilderPolicy>,
210
211	/// The preset that we expect to find in the GenesisBuilder runtime API.
212	///
213	/// This can be useful when a runtime has a dedicated benchmarking preset instead of using the
214	/// default one.
215	#[arg(long, default_value = sp_genesis_builder::DEV_RUNTIME_PRESET)]
216	pub genesis_builder_preset: String,
217
218	/// DEPRECATED: This argument has no effect.
219	#[arg(long = "execution")]
220	pub execution: Option<String>,
221
222	/// Limit the memory the database cache can use.
223	#[arg(long = "db-cache", value_name = "MiB", default_value_t = 1024)]
224	pub database_cache_size: u32,
225
226	/// List and print available benchmarks in a csv-friendly format.
227	///
228	/// NOTE: `num_args` and `require_equals` are required to allow `--list`
229	#[arg(long, value_enum, ignore_case = true, num_args = 0..=1, require_equals = true, default_missing_value("All"))]
230	pub list: Option<ListOutput>,
231
232	/// Don't include csv header when listing benchmarks.
233	#[arg(long, requires("list"))]
234	pub no_csv_header: bool,
235
236	/// If enabled, the storage info is not displayed in the output next to the analysis.
237	///
238	/// This is independent of the storage info appearing in the *output file*. Use a Handlebar
239	/// template for that purpose.
240	#[arg(long)]
241	pub no_storage_info: bool,
242
243	/// The assumed default maximum size of any `StorageMap`.
244	///
245	/// When the maximum size of a map is not defined by the runtime developer,
246	/// this value is used as a worst case scenario. It will affect the calculated worst case
247	/// PoV size for accessing a value in a map, since the PoV will need to include the trie
248	/// nodes down to the underlying value.
249	#[clap(long = "map-size", default_value = "1000000")]
250	pub worst_case_map_values: u32,
251
252	/// Adjust the PoV estimation by adding additional trie layers to it.
253	///
254	/// This should be set to `log16(n)` where `n` is the number of top-level storage items in the
255	/// runtime, eg. `StorageMap`s and `StorageValue`s. A value of 2 to 3 is usually sufficient.
256	/// Each layer will result in an additional 495 bytes PoV per distinct top-level access.
257	/// Therefore multiple `StorageMap` accesses only suffer from this increase once. The exact
258	/// number of storage items depends on the runtime and the deployed pallets.
259	#[clap(long, default_value = "2")]
260	pub additional_trie_layers: u8,
261
262	/// A path to a `.json` file with existing benchmark results generated with `--json` or
263	/// `--json-file`. When specified the benchmarks are not actually executed, and the data for
264	/// the analysis is read from this file.
265	#[arg(long)]
266	pub json_input: Option<PathBuf>,
267
268	/// Allow overwriting a single file with multiple results.
269	///
270	/// This exists only to restore legacy behaviour. It should never actually be needed.
271	#[arg(long)]
272	pub unsafe_overwrite_results: bool,
273
274	/// Do not print a summary at the end of the run.
275	///
276	/// These summaries can be very long when benchmarking multiple pallets at once. For CI
277	/// use-cases, this option reduces the noise.
278	#[arg(long)]
279	quiet: bool,
280
281	/// Do not enable proof recording during time benchmarking.
282	///
283	/// By default, proof recording is enabled during benchmark execution. This can slightly
284	/// inflate the resulting time weights. For parachains using PoV-reclaim, this is typically the
285	/// correct setting. Chains that ignore the proof size dimension of weight (e.g. relay chain,
286	/// solo-chains) can disable proof recording to get more accurate results.
287	#[arg(long)]
288	disable_proof_recording: bool,
289}
290
291/// How the genesis state for benchmarking should be built.
292#[derive(clap::ValueEnum, Debug, Eq, PartialEq, Clone, Copy, Serialize)]
293#[clap(rename_all = "kebab-case")]
294pub enum GenesisBuilderPolicy {
295	/// Do not provide any genesis state.
296	///
297	/// Benchmarks are advised to function with this, since they should setup their own required
298	/// state. However, to keep backwards compatibility, this is not the default.
299	None,
300	/// Let the runtime build the genesis state through its `BuildGenesisConfig` runtime API.
301	/// This will use the `development` preset by default.
302	Runtime,
303	/// Use the runtime from the Spec file to build the genesis state.
304	SpecRuntime,
305	/// Use the spec file to build the genesis state. This fails when there is no spec.
306	#[value(alias = "spec")]
307	SpecGenesis,
308}
frame_benchmarking_cli/pallet/mod.rs

frame_benchmarking_cli/pallet/
mod.rs
