foundry_bench/
results.rs

1use crate::RepoConfig;
2use eyre::Result;
3use serde::{Deserialize, Serialize};
4use std::{collections::HashMap, process::Command};
5
6/// Hyperfine benchmark result
7#[derive(Debug, Deserialize, Serialize)]
8pub struct HyperfineResult {
9    pub command: String,
10    pub mean: f64,
11    pub stddev: Option<f64>,
12    pub median: f64,
13    pub user: f64,
14    pub system: f64,
15    pub min: f64,
16    pub max: f64,
17    pub times: Vec<f64>,
18    #[serde(skip_serializing_if = "Option::is_none")]
19    pub exit_codes: Option<Vec<i32>>,
20    #[serde(skip_serializing_if = "Option::is_none")]
21    pub parameters: Option<HashMap<String, serde_json::Value>>,
22}
23
24/// Hyperfine JSON output format
25#[derive(Debug, Deserialize, Serialize)]
26pub struct HyperfineOutput {
27    pub results: Vec<HyperfineResult>,
28}
29
30/// Aggregated benchmark results
31#[derive(Debug, Default)]
32pub struct BenchmarkResults {
33    /// Map of benchmark_name -> version -> repo -> result
34    pub data: HashMap<String, HashMap<String, HashMap<String, HyperfineResult>>>,
35    /// Track the baseline version for comparison
36    pub baseline_version: Option<String>,
37    /// Map of version name -> full version details
38    pub version_details: HashMap<String, String>,
39}
40
41impl BenchmarkResults {
42    pub fn new() -> Self {
43        Self::default()
44    }
45
46    pub fn set_baseline_version(&mut self, version: String) {
47        self.baseline_version = Some(version);
48    }
49
50    pub fn add_result(
51        &mut self,
52        benchmark: &str,
53        version: &str,
54        repo: &str,
55        result: HyperfineResult,
56    ) {
57        self.data
58            .entry(benchmark.to_string())
59            .or_default()
60            .entry(version.to_string())
61            .or_default()
62            .insert(repo.to_string(), result);
63    }
64
65    pub fn add_version_details(&mut self, version: &str, details: String) {
66        self.version_details.insert(version.to_string(), details);
67    }
68
69    pub fn generate_markdown(&self, versions: &[String], repos: &[RepoConfig]) -> String {
70        let mut output = String::new();
71
72        // Header
73        output.push_str("# Foundry Benchmark Results\n\n");
74        output.push_str(&format!(
75            "**Date**: {}\n\n",
76            chrono::Local::now().format("%Y-%m-%d %H:%M:%S")
77        ));
78
79        // Summary
80        output.push_str("## Summary\n\n");
81        // Count actual repos that have results
82        let mut repos_with_results = std::collections::HashSet::new();
83        for version_data in self.data.values() {
84            for repo_data in version_data.values() {
85                for repo_name in repo_data.keys() {
86                    repos_with_results.insert(repo_name.clone());
87                }
88            }
89        }
90        output.push_str(&format!(
91            "Benchmarked {} Foundry versions across {} repositories.\n\n",
92            versions.len(),
93            repos_with_results.len()
94        ));
95
96        // Repositories tested
97        output.push_str("### Repositories Tested\n\n");
98        for (i, repo) in repos.iter().enumerate() {
99            output.push_str(&format!(
100                "{}. [{}/{}](https://github.com/{}/{})\n",
101                i + 1,
102                repo.org,
103                repo.repo,
104                repo.org,
105                repo.repo
106            ));
107        }
108        output.push('\n');
109
110        // Versions tested
111        output.push_str("### Foundry Versions\n\n");
112        for version in versions {
113            if let Some(details) = self.version_details.get(version) {
114                output.push_str(&format!("- **{version}**: {}\n", details.trim()));
115            } else {
116                output.push_str(&format!("- {version}\n"));
117            }
118        }
119        output.push('\n');
120
121        // Results for each benchmark type
122        for (benchmark_name, version_data) in &self.data {
123            output.push_str(&self.generate_benchmark_table(
124                benchmark_name,
125                version_data,
126                versions,
127                repos,
128            ));
129        }
130
131        // System info
132        output.push_str("## System Information\n\n");
133        output.push_str(&format!("- **OS**: {}\n", std::env::consts::OS));
134        output.push_str(&format!("- **CPU**: {}\n", num_cpus::get()));
135        output.push_str(&format!(
136            "- **Rustc**: {}\n",
137            get_rustc_version().unwrap_or_else(|_| "unknown".to_string())
138        ));
139
140        output
141    }
142
143    /// Generate a complete markdown table for a single benchmark type
144    ///
145    /// This includes the section header, table header, separator, and all rows
146    fn generate_benchmark_table(
147        &self,
148        benchmark_name: &str,
149        version_data: &HashMap<String, HashMap<String, HyperfineResult>>,
150        versions: &[String],
151        repos: &[RepoConfig],
152    ) -> String {
153        let mut output = String::new();
154
155        // Section header
156        output.push_str(&format!("## {}\n\n", format_benchmark_name(benchmark_name)));
157
158        // Create table header
159        output.push_str("| Repository |");
160        for version in versions {
161            output.push_str(&format!(" {version} |"));
162        }
163        output.push('\n');
164
165        // Table separator
166        output.push_str("|------------|");
167        for _ in versions {
168            output.push_str("----------|");
169        }
170        output.push('\n');
171
172        // Table rows
173        output.push_str(&generate_table_rows(version_data, versions, repos));
174        output.push('\n');
175
176        output
177    }
178}
179
180/// Generate table rows for benchmark results
181///
182/// This function creates the markdown table rows for each repository,
183/// showing the benchmark results for each version.
184fn generate_table_rows(
185    version_data: &HashMap<String, HashMap<String, HyperfineResult>>,
186    versions: &[String],
187    repos: &[RepoConfig],
188) -> String {
189    let mut output = String::new();
190
191    for repo in repos {
192        output.push_str(&format!("| {} |", repo.name));
193
194        for version in versions {
195            let cell_content = get_benchmark_cell_content(version_data, version, &repo.name);
196            output.push_str(&format!(" {cell_content} |"));
197        }
198
199        output.push('\n');
200    }
201
202    output
203}
204
205/// Get the content for a single benchmark table cell
206///
207/// Returns the formatted duration or "N/A" if no data is available.
208/// The nested if-let statements handle the following cases:
209/// 1. Check if version data exists
210/// 2. Check if repository data exists for this version
211fn get_benchmark_cell_content(
212    version_data: &HashMap<String, HashMap<String, HyperfineResult>>,
213    version: &str,
214    repo_name: &str,
215) -> String {
216    // Check if we have data for this version
217    if let Some(repo_data) = version_data.get(version) &&
218    // Check if we have data for this repository
219        let Some(result) = repo_data.get(repo_name)
220    {
221        return format_duration_seconds(result.mean);
222    }
223
224    "N/A".to_string()
225}
226
227pub fn format_benchmark_name(name: &str) -> String {
228    match name {
229        "forge_test" => "Forge Test",
230        "forge_build_no_cache" => "Forge Build (No Cache)",
231        "forge_build_with_cache" => "Forge Build (With Cache)",
232        "forge_fuzz_test" => "Forge Fuzz Test",
233        "forge_coverage" => "Forge Coverage",
234        "forge_isolate_test" => "Forge Test (Isolated)",
235        _ => name,
236    }
237    .to_string()
238}
239
240pub fn format_duration_seconds(seconds: f64) -> String {
241    if seconds < 0.001 {
242        format!("{:.2} ms", seconds * 1000.0)
243    } else if seconds < 1.0 {
244        format!("{seconds:.3} s")
245    } else if seconds < 60.0 {
246        format!("{seconds:.2} s")
247    } else {
248        let minutes = (seconds / 60.0).floor();
249        let remaining_seconds = seconds % 60.0;
250        format!("{minutes:.0}m {remaining_seconds:.1}s")
251    }
252}
253
254pub fn get_rustc_version() -> Result<String> {
255    let output = Command::new("rustc").arg("--version").output()?;
256
257    Ok(String::from_utf8_lossy(&output.stdout).trim().to_string())
258}
foundry_bench/results.rs

foundry_bench/
results.rs
