frame_benchmarking_cli/shared/
stats.rs

1// This file is part of Substrate.
2
3// Copyright (C) Parity Technologies (UK) Ltd.
4// SPDX-License-Identifier: Apache-2.0
5
6// Licensed under the Apache License, Version 2.0 (the "License");
7// you may not use this file except in compliance with the License.
8// You may obtain a copy of the License at
9//
10// http://www.apache.org/licenses/LICENSE-2.0
11//
12// Unless required by applicable law or agreed to in writing, software
13// distributed under the License is distributed on an "AS IS" BASIS,
14// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15// See the License for the specific language governing permissions and
16// limitations under the License.
17
18//! Handles statistics that were generated from benchmarking results and
19//! that can be used to fill out weight templates.
20
21use sc_cli::Result;
22
23use serde::Serialize;
24use std::{fmt, result, str::FromStr};
25
26/// Various statistics that help to gauge the quality of the produced weights.
27/// Will be written to the weight file and printed to console.
28#[derive(Serialize, Default, Clone)]
29pub struct Stats {
30	/// Sum of all values.
31	pub sum: u64,
32	/// Minimal observed value.
33	pub min: u64,
34	/// Maximal observed value.
35	pub max: u64,
36
37	/// Average of all values.
38	pub avg: u64,
39	/// Median of all values.
40	pub median: u64,
41	/// Standard derivation of all values.
42	pub stddev: f64,
43
44	/// 99th percentile. At least 99% of all values are below this threshold.
45	pub p99: u64,
46	/// 95th percentile. At least 95% of all values are below this threshold.
47	pub p95: u64,
48	/// 75th percentile. At least 75% of all values are below this threshold.
49	pub p75: u64,
50}
51
52/// Selects a specific field from a [`Stats`] object.
53/// Not all fields are available.
54#[derive(Debug, Clone, Copy, Serialize, PartialEq)]
55pub enum StatSelect {
56	/// Select the maximum.
57	Maximum,
58	/// Select the average.
59	Average,
60	/// Select the median.
61	Median,
62	/// Select the 99th percentile.
63	P99Percentile,
64	/// Select the 95th percentile.
65	P95Percentile,
66	/// Select the 75th percentile.
67	P75Percentile,
68}
69
70impl Stats {
71	/// Calculates statistics and returns them.
72	pub fn new(xs: &Vec<u64>) -> Result<Self> {
73		if xs.is_empty() {
74			return Err("Empty input is invalid".into())
75		}
76		let (avg, stddev) = Self::avg_and_stddev(xs);
77
78		Ok(Self {
79			sum: xs.iter().sum(),
80			min: *xs.iter().min().expect("Checked for non-empty above"),
81			max: *xs.iter().max().expect("Checked for non-empty above"),
82
83			avg: avg as u64,
84			median: Self::percentile(xs.clone(), 0.50),
85			stddev: (stddev * 100.0).round() / 100.0, // round to 1/100
86
87			p99: Self::percentile(xs.clone(), 0.99),
88			p95: Self::percentile(xs.clone(), 0.95),
89			p75: Self::percentile(xs.clone(), 0.75),
90		})
91	}
92
93	/// Returns the selected stat.
94	pub fn select(&self, s: StatSelect) -> u64 {
95		match s {
96			StatSelect::Maximum => self.max,
97			StatSelect::Average => self.avg,
98			StatSelect::Median => self.median,
99			StatSelect::P99Percentile => self.p99,
100			StatSelect::P95Percentile => self.p95,
101			StatSelect::P75Percentile => self.p75,
102		}
103	}
104
105	/// Returns the *average* and the *standard derivation*.
106	fn avg_and_stddev(xs: &Vec<u64>) -> (f64, f64) {
107		let avg = xs.iter().map(|x| *x as f64).sum::<f64>() / xs.len() as f64;
108		let variance = xs.iter().map(|x| (*x as f64 - avg).powi(2)).sum::<f64>() / xs.len() as f64;
109		(avg, variance.sqrt())
110	}
111
112	/// Returns the specified percentile for the given data.
113	/// This is best effort since it ignores the interpolation case.
114	fn percentile(mut xs: Vec<u64>, p: f64) -> u64 {
115		xs.sort();
116		let index = (xs.len() as f64 * p).ceil() as usize - 1;
117		xs[index.clamp(0, xs.len() - 1)]
118	}
119}
120
121impl fmt::Debug for Stats {
122	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
123		writeln!(f, "Total: {}", self.sum)?;
124		writeln!(f, "Min: {}, Max: {}", self.min, self.max)?;
125		writeln!(f, "Average: {}, Median: {}, Stddev: {}", self.avg, self.median, self.stddev)?;
126		write!(f, "Percentiles 99th, 95th, 75th: {}, {}, {}", self.p99, self.p95, self.p75)
127	}
128}
129
130impl Default for StatSelect {
131	/// Returns the `Average` selector.
132	fn default() -> Self {
133		Self::Average
134	}
135}
136
137impl FromStr for StatSelect {
138	type Err = &'static str;
139
140	fn from_str(day: &str) -> result::Result<Self, Self::Err> {
141		match day.to_lowercase().as_str() {
142			"max" => Ok(Self::Maximum),
143			"average" => Ok(Self::Average),
144			"median" => Ok(Self::Median),
145			"p99" => Ok(Self::P99Percentile),
146			"p95" => Ok(Self::P95Percentile),
147			"p75" => Ok(Self::P75Percentile),
148			_ => Err("String was not a StatSelect"),
149		}
150	}
151}
152
153#[cfg(test)]
154mod test_stats {
155	use super::Stats;
156	use rand::{seq::SliceRandom, thread_rng};
157
158	#[test]
159	fn stats_correct() {
160		let mut data: Vec<u64> = (1..=100).collect();
161		data.shuffle(&mut thread_rng());
162		let stats = Stats::new(&data).unwrap();
163
164		assert_eq!(stats.sum, 5050);
165		assert_eq!(stats.min, 1);
166		assert_eq!(stats.max, 100);
167
168		assert_eq!(stats.avg, 50);
169		assert_eq!(stats.median, 50); // 50.5 to be exact.
170		assert_eq!(stats.stddev, 28.87); // Rounded with 1/100 precision.
171
172		assert_eq!(stats.p99, 99);
173		assert_eq!(stats.p95, 95);
174		assert_eq!(stats.p75, 75);
175	}
176
177	#[test]
178	fn no_panic_short_lengths() {
179		// Empty input does error.
180		assert!(Stats::new(&vec![]).is_err());
181
182		// Different small input lengths are fine.
183		for l in 1..10 {
184			let data = (0..=l).collect();
185			assert!(Stats::new(&data).is_ok());
186		}
187	}
188}
frame_benchmarking_cli/shared/stats.rs

frame_benchmarking_cli/shared/
stats.rs
