referrerpolicy=no-referrer-when-downgrade

frame_benchmarking/
utils.rs

1// This file is part of Substrate.
2
3// Copyright (C) Parity Technologies (UK) Ltd.
4// SPDX-License-Identifier: Apache-2.0
5
6// Licensed under the Apache License, Version 2.0 (the "License");
7// you may not use this file except in compliance with the License.
8// You may obtain a copy of the License at
9//
10// 	http://www.apache.org/licenses/LICENSE-2.0
11//
12// Unless required by applicable law or agreed to in writing, software
13// distributed under the License is distributed on an "AS IS" BASIS,
14// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15// See the License for the specific language governing permissions and
16// limitations under the License.
17
18//! Interfaces, types and utils for benchmarking a FRAME runtime.
19use alloc::vec::Vec;
20use codec::{Decode, Encode};
21use frame_support::{dispatch::DispatchErrorWithPostInfo, pallet_prelude::*, traits::StorageInfo};
22use scale_info::TypeInfo;
23#[cfg(feature = "std")]
24use serde::{Deserialize, Serialize};
25use sp_io::hashing::blake2_256;
26use sp_runtime::{
27	traits::TrailingZeroInput, transaction_validity::TransactionValidityError, DispatchError,
28};
29use sp_runtime_interface::pass_by::{
30	AllocateAndReturnByCodec, AllocateAndReturnPointer, PassFatPointerAndDecode,
31	PassFatPointerAndRead,
32};
33use sp_storage::TrackedStorageKey;
34
35/// An alphabet of possible parameters to use for benchmarking.
36#[cfg_attr(feature = "std", derive(Serialize, Deserialize))]
37#[derive(Encode, Decode, Clone, Copy, PartialEq, Debug, TypeInfo)]
38#[allow(missing_docs)]
39#[allow(non_camel_case_types)]
40pub enum BenchmarkParameter {
41	a,
42	b,
43	c,
44	d,
45	e,
46	f,
47	g,
48	h,
49	i,
50	j,
51	k,
52	l,
53	m,
54	n,
55	o,
56	p,
57	q,
58	r,
59	s,
60	t,
61	u,
62	v,
63	w,
64	x,
65	y,
66	z,
67}
68
69#[cfg(feature = "std")]
70impl std::fmt::Display for BenchmarkParameter {
71	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
72		write!(f, "{:?}", self)
73	}
74}
75
76/// The results of a single of benchmark.
77#[cfg_attr(feature = "std", derive(Serialize, Deserialize))]
78#[derive(Encode, Decode, Clone, PartialEq, Debug, TypeInfo)]
79pub struct BenchmarkBatch {
80	/// The pallet containing this benchmark.
81	#[cfg_attr(feature = "std", serde(with = "serde_as_str"))]
82	pub pallet: Vec<u8>,
83	/// The instance of this pallet being benchmarked.
84	#[cfg_attr(feature = "std", serde(with = "serde_as_str"))]
85	pub instance: Vec<u8>,
86	/// The extrinsic (or benchmark name) of this benchmark.
87	#[cfg_attr(feature = "std", serde(with = "serde_as_str"))]
88	pub benchmark: Vec<u8>,
89	/// The results from this benchmark.
90	pub results: Vec<BenchmarkResult>,
91}
92
93// TODO: could probably make API cleaner here.
94/// The results of a single of benchmark, where time and db results are separated.
95#[cfg_attr(feature = "std", derive(Serialize, Deserialize))]
96#[derive(Encode, Decode, Clone, PartialEq, Debug)]
97pub struct BenchmarkBatchSplitResults {
98	/// The pallet containing this benchmark.
99	#[cfg_attr(feature = "std", serde(with = "serde_as_str"))]
100	pub pallet: Vec<u8>,
101	/// The instance of this pallet being benchmarked.
102	#[cfg_attr(feature = "std", serde(with = "serde_as_str"))]
103	pub instance: Vec<u8>,
104	/// The extrinsic (or benchmark name) of this benchmark.
105	#[cfg_attr(feature = "std", serde(with = "serde_as_str"))]
106	pub benchmark: Vec<u8>,
107	/// The extrinsic timing results from this benchmark.
108	pub time_results: Vec<BenchmarkResult>,
109	/// The db tracking results from this benchmark.
110	pub db_results: Vec<BenchmarkResult>,
111}
112
113/// Result from running benchmarks on a FRAME pallet.
114/// Contains duration of the function call in nanoseconds along with the benchmark parameters
115/// used for that benchmark result.
116#[cfg_attr(feature = "std", derive(Serialize, Deserialize))]
117#[derive(Encode, Decode, Default, Clone, PartialEq, Debug, TypeInfo)]
118pub struct BenchmarkResult {
119	pub components: Vec<(BenchmarkParameter, u32)>,
120	pub extrinsic_time: u128,
121	pub storage_root_time: u128,
122	pub reads: u32,
123	pub repeat_reads: u32,
124	pub writes: u32,
125	pub repeat_writes: u32,
126	pub proof_size: u32,
127	#[cfg_attr(feature = "std", serde(skip))]
128	pub keys: Vec<(Vec<u8>, u32, u32, bool)>,
129}
130
131impl BenchmarkResult {
132	pub fn from_weight(w: Weight) -> Self {
133		Self { extrinsic_time: (w.ref_time() / 1_000) as u128, ..Default::default() }
134	}
135}
136
137/// Helper module to make serde serialize `Vec<u8>` as strings.
138#[cfg(feature = "std")]
139mod serde_as_str {
140	pub fn serialize<S>(value: &Vec<u8>, serializer: S) -> Result<S::Ok, S::Error>
141	where
142		S: serde::Serializer,
143	{
144		let s = std::str::from_utf8(value).map_err(serde::ser::Error::custom)?;
145		serializer.collect_str(s)
146	}
147
148	pub fn deserialize<'de, D>(deserializer: D) -> Result<Vec<u8>, D::Error>
149	where
150		D: serde::de::Deserializer<'de>,
151	{
152		let s: &str = serde::de::Deserialize::deserialize(deserializer)?;
153		Ok(s.into())
154	}
155}
156
157/// Possible errors returned from the benchmarking pipeline.
158#[derive(Clone, PartialEq, Debug)]
159pub enum BenchmarkError {
160	/// The benchmarking pipeline should stop and return the inner string.
161	Stop(&'static str),
162	/// The benchmarking pipeline is allowed to fail here, and we should use the
163	/// included weight instead.
164	Override(BenchmarkResult),
165	/// The benchmarking pipeline is allowed to fail here, and we should simply
166	/// skip processing these results.
167	Skip,
168	/// No weight can be determined; set the weight of this call to zero.
169	///
170	/// You can also use `Override` instead, but this is easier to use since `Override` expects the
171	/// correct components to be present.
172	Weightless,
173}
174
175impl From<BenchmarkError> for &'static str {
176	fn from(e: BenchmarkError) -> Self {
177		match e {
178			BenchmarkError::Stop(s) => s,
179			BenchmarkError::Override(_) => "benchmark override",
180			BenchmarkError::Skip => "benchmark skip",
181			BenchmarkError::Weightless => "benchmark weightless",
182		}
183	}
184}
185
186impl From<&'static str> for BenchmarkError {
187	fn from(s: &'static str) -> Self {
188		Self::Stop(s)
189	}
190}
191
192impl From<DispatchErrorWithPostInfo> for BenchmarkError {
193	fn from(e: DispatchErrorWithPostInfo) -> Self {
194		Self::Stop(e.into())
195	}
196}
197
198impl From<DispatchError> for BenchmarkError {
199	fn from(e: DispatchError) -> Self {
200		Self::Stop(e.into())
201	}
202}
203
204impl From<TransactionValidityError> for BenchmarkError {
205	fn from(e: TransactionValidityError) -> Self {
206		Self::Stop(e.into())
207	}
208}
209
210/// Configuration used to setup and run runtime benchmarks.
211#[derive(Encode, Decode, Default, Clone, PartialEq, Debug, TypeInfo)]
212pub struct BenchmarkConfig {
213	/// The encoded name of the pallet to benchmark.
214	pub pallet: Vec<u8>,
215	/// The encoded name of the pallet instance to benchmark.
216	pub instance: Vec<u8>,
217	/// The encoded name of the benchmark/extrinsic to run.
218	pub benchmark: Vec<u8>,
219	/// The selected component values to use when running the benchmark.
220	pub selected_components: Vec<(BenchmarkParameter, u32)>,
221	/// Enable an extra benchmark iteration which runs the verification logic for a benchmark.
222	pub verify: bool,
223	/// Number of times to repeat benchmark within the Wasm environment. (versus in the client)
224	pub internal_repeats: u32,
225}
226
227/// A list of benchmarks available for a particular pallet and instance.
228///
229/// All `Vec<u8>` must be valid utf8 strings.
230#[derive(Encode, Decode, Default, Clone, PartialEq, Debug, TypeInfo)]
231pub struct BenchmarkList {
232	pub pallet: Vec<u8>,
233	pub instance: Vec<u8>,
234	pub benchmarks: Vec<BenchmarkMetadata>,
235}
236
237#[derive(Encode, Decode, Default, Clone, PartialEq, Debug, TypeInfo)]
238pub struct BenchmarkMetadata {
239	pub name: Vec<u8>,
240	pub components: Vec<(BenchmarkParameter, u32, u32)>,
241	pub pov_modes: Vec<(Vec<u8>, Vec<u8>)>,
242}
243
244sp_api::decl_runtime_apis! {
245	/// Runtime api for benchmarking a FRAME runtime.
246	#[api_version(2)]
247	pub trait Benchmark {
248		/// Get the benchmark metadata available for this runtime.
249		///
250		/// Parameters
251		/// - `extra`: Also list benchmarks marked "extra" which would otherwise not be
252		///            needed for weight calculation.
253		fn benchmark_metadata(extra: bool) -> (Vec<BenchmarkList>, Vec<StorageInfo>);
254
255		/// Dispatch the given benchmark.
256		fn dispatch_benchmark(config: BenchmarkConfig) -> Result<Vec<BenchmarkBatch>, alloc::string::String>;
257	}
258}
259
260/// Get the number of nanoseconds passed since the UNIX epoch
261///
262/// WARNING! This is a non-deterministic call. Do not use this within
263/// consensus critical logic.
264pub fn current_time() -> u128 {
265	u128::from_le_bytes(self::benchmarking::current_time())
266}
267
268/// Interface that provides functions for benchmarking the runtime.
269#[sp_runtime_interface::runtime_interface]
270pub trait Benchmarking {
271	/// Get the number of nanoseconds passed since the UNIX epoch, as u128 le-bytes.
272	///
273	/// You may want to use the standalone function [`current_time`].
274	///
275	/// WARNING! This is a non-deterministic call. Do not use this within
276	/// consensus critical logic.
277	fn current_time() -> AllocateAndReturnPointer<[u8; 16], 16> {
278		std::time::SystemTime::now()
279			.duration_since(std::time::SystemTime::UNIX_EPOCH)
280			.expect("Unix time doesn't go backwards; qed")
281			.as_nanos()
282			.to_le_bytes()
283	}
284
285	/// Reset the trie database to the genesis state.
286	fn wipe_db(&mut self) {
287		self.wipe()
288	}
289
290	/// Commit pending storage changes to the trie database and clear the database cache.
291	fn commit_db(&mut self) {
292		self.commit()
293	}
294
295	/// Get the read/write count.
296	fn read_write_count(&self) -> AllocateAndReturnByCodec<(u32, u32, u32, u32)> {
297		self.read_write_count()
298	}
299
300	/// Reset the read/write count.
301	fn reset_read_write_count(&mut self) {
302		self.reset_read_write_count()
303	}
304
305	/// Get the DB whitelist.
306	fn get_whitelist(&self) -> AllocateAndReturnByCodec<Vec<TrackedStorageKey>> {
307		self.get_whitelist()
308	}
309
310	/// Set the DB whitelist.
311	fn set_whitelist(&mut self, new: PassFatPointerAndDecode<Vec<TrackedStorageKey>>) {
312		self.set_whitelist(new)
313	}
314
315	// Add a new item to the DB whitelist.
316	fn add_to_whitelist(&mut self, add: PassFatPointerAndDecode<TrackedStorageKey>) {
317		let mut whitelist = self.get_whitelist();
318		match whitelist.iter_mut().find(|x| x.key == add.key) {
319			// If we already have this key in the whitelist, update to be the most constrained
320			// value.
321			Some(item) => {
322				item.reads += add.reads;
323				item.writes += add.writes;
324				item.whitelisted = item.whitelisted || add.whitelisted;
325			},
326			// If the key does not exist, add it.
327			None => {
328				whitelist.push(add);
329			},
330		}
331		self.set_whitelist(whitelist);
332	}
333
334	// Remove an item from the DB whitelist.
335	fn remove_from_whitelist(&mut self, remove: PassFatPointerAndRead<Vec<u8>>) {
336		let mut whitelist = self.get_whitelist();
337		whitelist.retain(|x| x.key != remove);
338		self.set_whitelist(whitelist);
339	}
340
341	fn get_read_and_written_keys(
342		&self,
343	) -> AllocateAndReturnByCodec<Vec<(Vec<u8>, u32, u32, bool)>> {
344		self.get_read_and_written_keys()
345	}
346
347	/// Get current estimated proof size.
348	fn proof_size(&self) -> AllocateAndReturnByCodec<Option<u32>> {
349		self.proof_size()
350	}
351}
352
353/// The pallet benchmarking trait.
354pub trait Benchmarking {
355	/// Get the benchmarks available for this pallet. Generally there is one benchmark per
356	/// extrinsic, so these are sometimes just called "extrinsics".
357	///
358	/// Parameters
359	/// - `extra`: Also return benchmarks marked "extra" which would otherwise not be needed for
360	///   weight calculation.
361	fn benchmarks(extra: bool) -> Vec<BenchmarkMetadata>;
362
363	/// Run the benchmarks for this pallet.
364	fn run_benchmark(
365		name: &[u8],
366		selected_components: &[(BenchmarkParameter, u32)],
367		whitelist: &[TrackedStorageKey],
368		verify: bool,
369		internal_repeats: u32,
370	) -> Result<Vec<BenchmarkResult>, BenchmarkError>;
371}
372
373/// The recording trait used to mark the start and end of a benchmark.
374pub trait Recording {
375	/// Start the benchmark.
376	fn start(&mut self) {}
377
378	// Stop the benchmark.
379	fn stop(&mut self) {}
380}
381
382/// A no-op recording, used for unit test.
383struct NoopRecording;
384impl Recording for NoopRecording {}
385
386/// A no-op recording, used for tests that should setup some state before running the benchmark.
387struct TestRecording<'a> {
388	on_before_start: Option<&'a dyn Fn()>,
389}
390
391impl<'a> TestRecording<'a> {
392	fn new(on_before_start: &'a dyn Fn()) -> Self {
393		Self { on_before_start: Some(on_before_start) }
394	}
395}
396
397impl<'a> Recording for TestRecording<'a> {
398	fn start(&mut self) {
399		(self.on_before_start.take().expect("start called more than once"))();
400	}
401}
402
403/// Records the time and proof size of a single benchmark iteration.
404pub struct BenchmarkRecording<'a> {
405	on_before_start: Option<&'a dyn Fn()>,
406	start_extrinsic: Option<u128>,
407	finish_extrinsic: Option<u128>,
408	start_pov: Option<u32>,
409	end_pov: Option<u32>,
410}
411
412impl<'a> BenchmarkRecording<'a> {
413	pub fn new(on_before_start: &'a dyn Fn()) -> Self {
414		Self {
415			on_before_start: Some(on_before_start),
416			start_extrinsic: None,
417			finish_extrinsic: None,
418			start_pov: None,
419			end_pov: None,
420		}
421	}
422}
423
424impl<'a> Recording for BenchmarkRecording<'a> {
425	fn start(&mut self) {
426		(self.on_before_start.take().expect("start called more than once"))();
427		self.start_pov = crate::benchmarking::proof_size();
428		self.start_extrinsic = Some(current_time());
429	}
430
431	fn stop(&mut self) {
432		self.finish_extrinsic = Some(current_time());
433		self.end_pov = crate::benchmarking::proof_size();
434	}
435}
436
437impl<'a> BenchmarkRecording<'a> {
438	pub fn start_pov(&self) -> Option<u32> {
439		self.start_pov
440	}
441
442	pub fn end_pov(&self) -> Option<u32> {
443		self.end_pov
444	}
445
446	pub fn diff_pov(&self) -> Option<u32> {
447		self.start_pov.zip(self.end_pov).map(|(start, end)| end.saturating_sub(start))
448	}
449
450	pub fn elapsed_extrinsic(&self) -> Option<u128> {
451		self.start_extrinsic
452			.zip(self.finish_extrinsic)
453			.map(|(start, end)| end.saturating_sub(start))
454	}
455}
456
457/// The required setup for creating a benchmark.
458///
459/// Instance generic parameter is optional and can be used in order to capture unused generics for
460/// instantiable pallets.
461pub trait BenchmarkingSetup<T, I = ()> {
462	/// Return the components and their ranges which should be tested in this benchmark.
463	fn components(&self) -> Vec<(BenchmarkParameter, u32, u32)>;
464
465	/// Set up the storage, and prepare a closure to run the benchmark.
466	fn instance(
467		&self,
468		recording: &mut impl Recording,
469		components: &[(BenchmarkParameter, u32)],
470		verify: bool,
471	) -> Result<(), BenchmarkError>;
472
473	/// Same as `instance` but passing a closure to run before the benchmark starts.
474	fn test_instance(
475		&self,
476		components: &[(BenchmarkParameter, u32)],
477		on_before_start: &dyn Fn(),
478	) -> Result<(), BenchmarkError> {
479		return self.instance(&mut TestRecording::new(on_before_start), components, true);
480	}
481
482	/// Same as `instance` but passing a no-op recording for unit tests.
483	fn unit_test_instance(
484		&self,
485		components: &[(BenchmarkParameter, u32)],
486	) -> Result<(), BenchmarkError> {
487		return self.instance(&mut NoopRecording {}, components, true);
488	}
489}
490
491/// Grab an account, seeded by a name and index.
492pub fn account<AccountId: Decode>(name: &'static str, index: u32, seed: u32) -> AccountId {
493	let entropy = (name, index, seed).using_encoded(blake2_256);
494	Decode::decode(&mut TrailingZeroInput::new(entropy.as_ref()))
495		.expect("infinite length input; no invalid inputs for type; qed")
496}
497
498/// This caller account is automatically whitelisted for DB reads/writes by the benchmarking macro.
499pub fn whitelisted_caller<AccountId: Decode>() -> AccountId {
500	account::<AccountId>("whitelisted_caller", 0, 0)
501}
502
503#[macro_export]
504macro_rules! whitelist_account {
505	($acc:ident) => {
506		frame_benchmarking::benchmarking::add_to_whitelist(
507			frame_system::Account::<T>::hashed_key_for(&$acc).into(),
508		);
509	};
510}