frame_benchmarking_cli/extrinsic/
bench.rs

1// This file is part of Substrate.
2
3// Copyright (C) Parity Technologies (UK) Ltd.
4// SPDX-License-Identifier: Apache-2.0
5
6// Licensed under the Apache License, Version 2.0 (the "License");
7// you may not use this file except in compliance with the License.
8// You may obtain a copy of the License at
9//
10// http://www.apache.org/licenses/LICENSE-2.0
11//
12// Unless required by applicable law or agreed to in writing, software
13// distributed under the License is distributed on an "AS IS" BASIS,
14// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15// See the License for the specific language governing permissions and
16// limitations under the License.
17
18//! Contains the core benchmarking logic.
19
20use sc_block_builder::{BlockBuilderApi, BlockBuilderBuilder, BuiltBlock};
21use sc_cli::{Error, Result};
22use sc_client_api::UsageProvider;
23use sp_api::{ApiExt, CallApiAt, Core, ProvideRuntimeApi};
24use sp_blockchain::{
25	ApplyExtrinsicFailed::Validity,
26	Error::{ApplyExtrinsicFailed, RuntimeApiError},
27};
28use sp_runtime::{
29	traits::Block as BlockT,
30	transaction_validity::{InvalidTransaction, TransactionValidityError},
31	Digest, DigestItem, OpaqueExtrinsic,
32};
33
34use super::ExtrinsicBuilder;
35use crate::shared::{StatSelect, Stats};
36use clap::Args;
37use codec::Encode;
38use log::info;
39use serde::Serialize;
40use sp_trie::proof_size_extension::ProofSizeExt;
41use std::{marker::PhantomData, sync::Arc, time::Instant};
42
43/// Parameters to configure an *overhead* benchmark.
44#[derive(Debug, Default, Serialize, Clone, PartialEq, Args)]
45pub struct BenchmarkParams {
46	/// Rounds of warmups before measuring.
47	#[arg(long, default_value_t = 10)]
48	pub warmup: u32,
49
50	/// How many times the benchmark should be repeated.
51	#[arg(long, default_value_t = 100)]
52	pub repeat: u32,
53
54	/// Maximal number of extrinsics that should be put into a block.
55	///
56	/// Only useful for debugging.
57	#[arg(long)]
58	pub max_ext_per_block: Option<u32>,
59}
60
61/// The results of multiple runs in nano seconds.
62pub(crate) type BenchRecord = Vec<u64>;
63
64/// Holds all objects needed to run the *overhead* benchmarks.
65pub(crate) struct Benchmark<Block, C> {
66	client: Arc<C>,
67	params: BenchmarkParams,
68	inherent_data: sp_inherents::InherentData,
69	digest_items: Vec<DigestItem>,
70	record_proof: bool,
71	_p: PhantomData<Block>,
72}
73
74impl<Block, C> Benchmark<Block, C>
75where
76	Block: BlockT<Extrinsic = OpaqueExtrinsic>,
77	C: ProvideRuntimeApi<Block>
78		+ CallApiAt<Block>
79		+ UsageProvider<Block>
80		+ sp_blockchain::HeaderBackend<Block>,
81	C::Api: ApiExt<Block> + BlockBuilderApi<Block>,
82{
83	/// Create a new [`Self`] from the arguments.
84	pub fn new(
85		client: Arc<C>,
86		params: BenchmarkParams,
87		inherent_data: sp_inherents::InherentData,
88		digest_items: Vec<DigestItem>,
89		record_proof: bool,
90	) -> Self {
91		Self { client, params, inherent_data, digest_items, record_proof, _p: PhantomData }
92	}
93
94	/// Benchmark a block with only inherents.
95	///
96	/// Returns the Ref time stats and the proof size.
97	pub fn bench_block(&self) -> Result<(Stats, u64)> {
98		let (block, _, proof_size) = self.build_block(None)?;
99		let record = self.measure_block(&block)?;
100
101		Ok((Stats::new(&record)?, proof_size))
102	}
103
104	/// Benchmark the time of an extrinsic in a full block.
105	///
106	/// First benchmarks an empty block, analogous to `bench_block` and use it as baseline.
107	/// Then benchmarks a full block built with the given `ext_builder` and subtracts the baseline
108	/// from the result.
109	/// This is necessary to account for the time the inherents use. Returns ref time stats and the
110	/// proof size.
111	pub fn bench_extrinsic(&self, ext_builder: &dyn ExtrinsicBuilder) -> Result<(Stats, u64)> {
112		let (block, _, base_proof_size) = self.build_block(None)?;
113		let base = self.measure_block(&block)?;
114		let base_time = Stats::new(&base)?.select(StatSelect::Average);
115
116		let (block, num_ext, proof_size) = self.build_block(Some(ext_builder))?;
117		let num_ext = num_ext.ok_or_else(|| Error::Input("Block was empty".into()))?;
118		let mut records = self.measure_block(&block)?;
119
120		for r in &mut records {
121			// Subtract the base time.
122			*r = r.saturating_sub(base_time);
123			// Divide by the number of extrinsics in the block.
124			*r = ((*r as f64) / (num_ext as f64)).ceil() as u64;
125		}
126
127		Ok((Stats::new(&records)?, proof_size.saturating_sub(base_proof_size)))
128	}
129
130	/// Builds a block with some optional extrinsics.
131	///
132	/// Returns the block and the number of extrinsics in the block
133	/// that are not inherents together with the proof size.
134	/// Returns a block with only inherents if `ext_builder` is `None`.
135	fn build_block(
136		&self,
137		ext_builder: Option<&dyn ExtrinsicBuilder>,
138	) -> Result<(Block, Option<u64>, u64)> {
139		let chain = self.client.usage_info().chain;
140		let mut builder = BlockBuilderBuilder::new(&*self.client)
141			.on_parent_block(chain.best_hash)
142			.with_parent_block_number(chain.best_number)
143			.with_inherent_digests(Digest { logs: self.digest_items.clone() })
144			.with_proof_recording(self.record_proof)
145			.build()?;
146
147		// Create and insert the inherents.
148		let inherents = builder.create_inherents(self.inherent_data.clone())?;
149		for inherent in inherents {
150			builder.push(inherent)?;
151		}
152
153		let num_ext = match ext_builder {
154			Some(ext_builder) => {
155				// Put as many extrinsics into the block as possible and count them.
156				info!("Building block, this takes some time...");
157				let mut num_ext = 0;
158				for nonce in 0..self.max_ext_per_block() {
159					let ext = ext_builder.build(nonce)?;
160					match builder.push(ext.clone()) {
161						Ok(()) => {},
162						Err(ApplyExtrinsicFailed(Validity(TransactionValidityError::Invalid(
163							InvalidTransaction::ExhaustsResources,
164						)))) => break, // Block is full
165						Err(e) => return Err(Error::Client(e)),
166					}
167					num_ext += 1;
168				}
169				if num_ext == 0 {
170					return Err("A Block must hold at least one extrinsic".into())
171				}
172				info!("Extrinsics per block: {}", num_ext);
173				Some(num_ext)
174			},
175			None => None,
176		};
177
178		let BuiltBlock { block, proof, .. } = builder.build()?;
179
180		Ok((
181			block,
182			num_ext,
183			proof
184				.map(|p| p.encoded_size())
185				.unwrap_or(0)
186				.try_into()
187				.map_err(|_| "Proof size is too large".to_string())?,
188		))
189	}
190
191	/// Measures the time that it take to execute a block or an extrinsic.
192	fn measure_block(&self, block: &Block) -> Result<BenchRecord> {
193		let mut record = BenchRecord::new();
194		let genesis = self.client.info().genesis_hash;
195
196		let measure_block = || -> Result<u128> {
197			let block = block.clone();
198			let mut runtime_api = self.client.runtime_api();
199			if self.record_proof {
200				runtime_api.record_proof();
201				let recorder = runtime_api
202					.proof_recorder()
203					.expect("Proof recording is enabled in the line above; qed.");
204				runtime_api.register_extension(ProofSizeExt::new(recorder));
205			}
206			let start = Instant::now();
207
208			runtime_api
209				.execute_block(genesis, block)
210				.map_err(|e| Error::Client(RuntimeApiError(e)))?;
211
212			Ok(start.elapsed().as_nanos())
213		};
214
215		info!("Running {} warmups...", self.params.warmup);
216		for _ in 0..self.params.warmup {
217			measure_block()?;
218		}
219
220		info!("Executing block {} times", self.params.repeat);
221		// Interesting part here:
222		// Execute a block multiple times and record each execution time.
223		for _ in 0..self.params.repeat {
224			let elapsed = measure_block()?;
225			record.push(elapsed as u64);
226		}
227
228		Ok(record)
229	}
230
231	fn max_ext_per_block(&self) -> u32 {
232		self.params.max_ext_per_block.unwrap_or(u32::MAX)
233	}
234}
frame_benchmarking_cli/extrinsic/bench.rs

frame_benchmarking_cli/extrinsic/
bench.rs
