polkadot_subsystem_bench/
usage.rs

1// Copyright (C) Parity Technologies (UK) Ltd.
2// This file is part of Polkadot.
3
4// Polkadot is free software: you can redistribute it and/or modify
5// it under the terms of the GNU General Public License as published by
6// the Free Software Foundation, either version 3 of the License, or
7// (at your option) any later version.
8
9// Polkadot is distributed in the hope that it will be useful,
10// but WITHOUT ANY WARRANTY; without even the implied warranty of
11// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12// GNU General Public License for more details.
13
14// You should have received a copy of the GNU General Public License
15// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
16
17//! Test usage implementation
18
19use colored::Colorize;
20use itertools::Itertools;
21use serde::{Deserialize, Serialize};
22use std::collections::HashMap;
23
24#[derive(Debug, Serialize, Deserialize, Clone)]
25pub struct BenchmarkUsage {
26	pub network_usage: Vec<ResourceUsage>,
27	pub cpu_usage: Vec<ResourceUsage>,
28}
29
30impl std::fmt::Display for BenchmarkUsage {
31	fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
32		write!(
33			f,
34			"\n{}\n{}\n\n{}\n{}\n",
35			format!("{:<64}{:>12}{:>12}", "Network usage, KiB", "total", "per block").blue(),
36			self.network_usage
37				.iter()
38				.map(|v| v.to_string())
39				.sorted()
40				.collect::<Vec<String>>()
41				.join("\n"),
42			format!("{:<64}{:>12}{:>12}", "CPU usage, seconds", "total", "per block").blue(),
43			self.cpu_usage
44				.iter()
45				.map(|v| v.to_string())
46				.sorted()
47				.collect::<Vec<String>>()
48				.join("\n")
49		)
50	}
51}
52
53impl BenchmarkUsage {
54	pub fn average(usages: &[Self]) -> Self {
55		let all_network_usages: Vec<&ResourceUsage> =
56			usages.iter().flat_map(|v| &v.network_usage).collect();
57		let all_cpu_usage: Vec<&ResourceUsage> = usages.iter().flat_map(|v| &v.cpu_usage).collect();
58
59		Self {
60			network_usage: ResourceUsage::average_by_resource_name(&all_network_usages),
61			cpu_usage: ResourceUsage::average_by_resource_name(&all_cpu_usage),
62		}
63	}
64
65	pub fn check_network_usage(&self, checks: &[ResourceUsageCheck]) -> Vec<String> {
66		check_usage(&self.network_usage, checks)
67	}
68
69	pub fn check_cpu_usage(&self, checks: &[ResourceUsageCheck]) -> Vec<String> {
70		check_usage(&self.cpu_usage, checks)
71	}
72
73	pub fn cpu_usage_diff(&self, other: &Self, resource_name: &str) -> Option<f64> {
74		let self_res = self.cpu_usage.iter().find(|v| v.resource_name == resource_name);
75		let other_res = other.cpu_usage.iter().find(|v| v.resource_name == resource_name);
76
77		match (self_res, other_res) {
78			(Some(self_res), Some(other_res)) => Some(self_res.diff(other_res)),
79			_ => None,
80		}
81	}
82
83	// Prepares a json string for a graph representation
84	// See: https://github.com/benchmark-action/github-action-benchmark?tab=readme-ov-file#examples
85	pub fn to_chart_json(&self) -> color_eyre::eyre::Result<String> {
86		let chart = self
87			.network_usage
88			.iter()
89			.map(|v| ChartItem {
90				name: v.resource_name.clone(),
91				unit: "KiB".to_string(),
92				value: v.per_block,
93			})
94			.chain(self.cpu_usage.iter().map(|v| ChartItem {
95				name: v.resource_name.clone(),
96				unit: "seconds".to_string(),
97				value: v.per_block,
98			}))
99			.collect::<Vec<_>>();
100
101		Ok(serde_json::to_string(&chart)?)
102	}
103}
104
105fn check_usage(usage: &[ResourceUsage], checks: &[ResourceUsageCheck]) -> Vec<String> {
106	checks.iter().filter_map(|check| check_resource_usage(usage, check)).collect()
107}
108
109fn check_resource_usage(
110	usage: &[ResourceUsage],
111	(resource_name, base, precision): &ResourceUsageCheck,
112) -> Option<String> {
113	if let Some(usage) = usage.iter().find(|v| v.resource_name == *resource_name) {
114		let diff = (base - usage.per_block).abs() / base;
115		if diff < *precision {
116			None
117		} else {
118			Some(format!(
119				"The resource `{}` is expected to be equal to {} with a precision {}, but the current value is {} ({})",
120				resource_name, base, precision, usage.per_block, diff
121			))
122		}
123	} else {
124		Some(format!("The resource `{resource_name}` is not found"))
125	}
126}
127
128#[derive(Debug, Serialize, Deserialize, Clone)]
129pub struct ResourceUsage {
130	pub resource_name: String,
131	pub total: f64,
132	pub per_block: f64,
133}
134
135impl std::fmt::Display for ResourceUsage {
136	fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
137		write!(f, "{:<64}{:>12.4}{:>12.4}", self.resource_name.cyan(), self.total, self.per_block)
138	}
139}
140
141impl ResourceUsage {
142	fn average_by_resource_name(usages: &[&Self]) -> Vec<Self> {
143		let mut by_name: HashMap<String, Vec<&Self>> = Default::default();
144		for usage in usages {
145			by_name.entry(usage.resource_name.clone()).or_default().push(usage);
146		}
147		let mut average = vec![];
148		for (resource_name, values) in by_name {
149			let total = values.iter().map(|v| v.total).sum::<f64>() / values.len() as f64;
150			let per_block = values.iter().map(|v| v.per_block).sum::<f64>() / values.len() as f64;
151			let per_block_sd =
152				standard_deviation(&values.iter().map(|v| v.per_block).collect::<Vec<f64>>());
153			println!(
154				"[{}] standart_deviation {:.2}%",
155				resource_name,
156				per_block_sd / per_block * 100.0
157			);
158			average.push(Self { resource_name, total, per_block });
159		}
160		average
161	}
162
163	fn diff(&self, other: &Self) -> f64 {
164		(self.per_block - other.per_block).abs() / self.per_block
165	}
166}
167
168type ResourceUsageCheck<'a> = (&'a str, f64, f64);
169
170#[derive(Debug, Serialize)]
171pub struct ChartItem {
172	pub name: String,
173	pub unit: String,
174	pub value: f64,
175}
176
177fn standard_deviation(values: &[f64]) -> f64 {
178	let n = values.len() as f64;
179	let mean = values.iter().sum::<f64>() / n;
180	let variance = values.iter().map(|v| (v - mean).powi(2)).sum::<f64>() / (n - 1.0);
181
182	variance.sqrt()
183}
polkadot_subsystem_bench/usage.rs

polkadot_subsystem_bench/
usage.rs
