diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 0b63d294a4..ca05694418 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -282,8 +282,8 @@ jobs: - name: Run Evaluation run: | - cd crates/eval - RUSTFLAGS='-C target-cpu=native' cargo run --release -- \ + cd crates/perf + RUSTFLAGS='-C target-cpu=native' cargo run --bin evaluate --release -- \ --programs fibonacci,ssz-withdrawals,tendermint \ --post-to-slack ${{ github.ref == 'refs/heads/dev' }} \ --slack-channel-id "${{ secrets.SLACK_CHANNEL_ID }}" \ diff --git a/crates/perf/Cargo.toml b/crates/perf/Cargo.toml index 38300783fe..a9f418dcec 100644 --- a/crates/perf/Cargo.toml +++ b/crates/perf/Cargo.toml @@ -18,9 +18,15 @@ sp1-stark = { workspace = true } sp1-cuda = { workspace = true } test-artifacts = { workspace = true } -clap = { version = "4.5.9", features = ["derive"] } +anyhow = "1.0" bincode = "1.3.3" +clap = { version = "4.5.9", features = ["derive"] } +reqwest = { version = "0.11", features = ["json"] } +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +slack-rust = "0.1" time = "0.3.26" +tokio = { version = "1.0", features = ["full"] } [features] native-gnark = ["sp1-sdk/native-gnark"] diff --git a/crates/perf/programs/fibonacci/elf b/crates/perf/programs/fibonacci/elf new file mode 100644 index 0000000000..10b446cae3 Binary files /dev/null and b/crates/perf/programs/fibonacci/elf differ diff --git a/crates/perf/programs/fibonacci/input.bin b/crates/perf/programs/fibonacci/input.bin new file mode 100644 index 0000000000..a8e468e776 Binary files /dev/null and b/crates/perf/programs/fibonacci/input.bin differ diff --git a/crates/perf/programs/ssz-withdrawals/elf b/crates/perf/programs/ssz-withdrawals/elf new file mode 100644 index 0000000000..ef27bf0f37 Binary files /dev/null and b/crates/perf/programs/ssz-withdrawals/elf differ diff --git a/crates/perf/programs/ssz-withdrawals/input.bin b/crates/perf/programs/ssz-withdrawals/input.bin new file mode 100644 index 0000000000..4ac5fc6cf8 Binary files /dev/null and b/crates/perf/programs/ssz-withdrawals/input.bin differ diff --git a/crates/perf/programs/tendermint/elf b/crates/perf/programs/tendermint/elf new file mode 100644 index 0000000000..6ecc25003e Binary files /dev/null and b/crates/perf/programs/tendermint/elf differ diff --git a/crates/perf/programs/tendermint/input.bin b/crates/perf/programs/tendermint/input.bin new file mode 100644 index 0000000000..e0d8f73d36 Binary files /dev/null and b/crates/perf/programs/tendermint/input.bin differ diff --git a/crates/perf/src/bin/evaluate.rs b/crates/perf/src/bin/evaluate.rs new file mode 100644 index 0000000000..76ba084040 --- /dev/null +++ b/crates/perf/src/bin/evaluate.rs @@ -0,0 +1,10 @@ +use anyhow::Result; +use sp1_perf::evaluate_performance; +use sp1_prover::components::DefaultProverComponents; +use sp1_stark::SP1ProverOpts; + +#[tokio::main] +async fn main() -> Result<(), Box> { + let opts = SP1ProverOpts::default(); + evaluate_performance::(opts).await +} diff --git a/crates/perf/src/lib.rs b/crates/perf/src/lib.rs new file mode 100644 index 0000000000..06f85a0a68 --- /dev/null +++ b/crates/perf/src/lib.rs @@ -0,0 +1,420 @@ +use anyhow::Result; +use clap::{command, Parser}; +use reqwest::Client; +use serde::Serialize; +use serde_json::json; +use slack_rust::chat::post_message::{post_message, PostMessageRequest}; +use slack_rust::http_client::default_client; +use sp1_prover::{components::SP1ProverComponents, utils::get_cycles, SP1Prover}; +use sp1_sdk::{SP1Context, SP1Stdin}; +use sp1_stark::SP1ProverOpts; +use std::time::{Duration, Instant}; + +use program::load_program; + +use crate::program::{TesterProgram, PROGRAMS}; + +mod program; + +#[derive(Parser, Clone)] +#[command(about = "Evaluate the performance of SP1 on programs.")] +struct EvalArgs { + /// The programs to evaluate, specified by name. If not specified, all programs will be evaluated. + #[arg(long, use_value_delimiter = true, value_delimiter = ',')] + pub programs: Vec, + + /// The shard size to use for the prover. + #[arg(long)] + pub shard_size: Option, + + /// Whether to post results to Slack. + #[arg(long, default_missing_value="true", num_args=0..=1)] + pub post_to_slack: Option, + + /// The Slack channel ID to post results to, only used if post_to_slack is true. + #[arg(long)] + pub slack_channel_id: Option, + + /// The Slack bot token to post results to, only used if post_to_slack is true. + #[arg(long)] + pub slack_token: Option, + + /// Whether to post results to GitHub PR. + #[arg(long, default_missing_value="true", num_args=0..=1)] + pub post_to_github: Option, + + /// The GitHub token for authentication, only used if post_to_github is true. + #[arg(long)] + pub github_token: Option, + + /// The GitHub repository owner. + #[arg(long)] + pub repo_owner: Option, + + /// The GitHub repository name. + #[arg(long)] + pub repo_name: Option, + + /// The GitHub PR number. + #[arg(long)] + pub pr_number: Option, + + /// The name of the branch. + #[arg(long)] + pub branch_name: Option, + + /// The commit hash. + #[arg(long)] + pub commit_hash: Option, + + /// The author of the commit. + #[arg(long)] + pub author: Option, +} + +pub async fn evaluate_performance( + opts: SP1ProverOpts, +) -> Result<(), Box> { + println!("opts: {:?}", opts); + + let args = EvalArgs::parse(); + + // Set environment variables to configure the prover. + if let Some(shard_size) = args.shard_size { + std::env::set_var("SHARD_SIZE", format!("{}", 1 << shard_size)); + } + + // Choose which programs to evaluate. + let programs: Vec<&TesterProgram> = if args.programs.is_empty() { + PROGRAMS.iter().collect() + } else { + PROGRAMS + .iter() + .filter(|p| args.programs.iter().any(|arg| arg.eq_ignore_ascii_case(p.name))) + .collect() + }; + + sp1_sdk::utils::setup_logger(); + + // Run the evaluations on each program. + let mut reports = Vec::new(); + for program in &programs { + println!("Evaluating program: {}", program.name); + let (elf, stdin) = load_program(program.elf, program.input); + let report = run_evaluation::(program.name, &elf, &stdin, opts); + reports.push(report); + println!("Finished Program: {}", program.name); + } + + // Prepare and format the results. + let reports_len = reports.len(); + let success_count = reports.iter().filter(|r| r.success).count(); + let results_text = format_results(&args, &reports); + + // Print results + println!("{}", results_text.join("\n")); + + // Post to Slack if applicable + if args.post_to_slack.unwrap_or(false) { + match (&args.slack_token, &args.slack_channel_id) { + (Some(token), Some(channel)) => { + for message in &results_text { + post_to_slack(token, channel, message).await?; + } + } + _ => println!("Warning: post_to_slack is true, required Slack arguments are missing."), + } + } + + // Post to GitHub PR if applicable + if args.post_to_github.unwrap_or(false) { + match (&args.repo_owner, &args.repo_name, &args.pr_number, &args.github_token) { + (Some(owner), Some(repo), Some(pr_number), Some(token)) => { + let message = format_github_message(&results_text); + post_to_github_pr(owner, repo, pr_number, token, &message).await?; + } + _ => { + println!("Warning: post_to_github is true, required GitHub arguments are missing.") + } + } + } + + // Exit with an error if any programs failed. + let all_successful = success_count == reports_len; + if !all_successful { + println!("Some programs failed. Please check the results above."); + std::process::exit(1); + } + + Ok(()) +} + +#[derive(Debug, Serialize)] +pub struct PerformanceReport { + program: String, + cycles: u64, + exec_khz: f64, + core_khz: f64, + compressed_khz: f64, + time: f64, + success: bool, +} + +fn run_evaluation( + program_name: &str, + elf: &[u8], + stdin: &SP1Stdin, + opts: SP1ProverOpts, +) -> PerformanceReport { + let cycles = get_cycles(elf, stdin); + + let prover = SP1Prover::::new(); + let (pk, vk) = prover.setup(elf); + + let context = SP1Context::default(); + + let (_, exec_duration) = time_operation(|| prover.execute(elf, stdin, context.clone())); + + let (core_proof, core_duration) = + time_operation(|| prover.prove_core(&pk, stdin, opts, context).unwrap()); + + let (_, compress_duration) = + time_operation(|| prover.compress(&vk, core_proof, vec![], opts).unwrap()); + + let total_duration = exec_duration + core_duration + compress_duration; + + PerformanceReport { + program: program_name.to_string(), + cycles, + exec_khz: calculate_khz(cycles, exec_duration), + core_khz: calculate_khz(cycles, core_duration), + compressed_khz: calculate_khz(cycles, compress_duration + core_duration), + time: total_duration.as_secs_f64(), + success: true, + } +} + +fn format_results(args: &EvalArgs, results: &[PerformanceReport]) -> Vec { + let mut detail_text = String::new(); + if let Some(branch_name) = &args.branch_name { + detail_text.push_str(&format!("*Branch*: {}\n", branch_name)); + } + if let Some(commit_hash) = &args.commit_hash { + detail_text.push_str(&format!("*Commit*: {}\n", &commit_hash[..8])); + } + if let Some(author) = &args.author { + detail_text.push_str(&format!("*Author*: {}\n", author)); + } + + let mut table_text = String::new(); + table_text.push_str("```\n"); + table_text.push_str("| program | cycles | execute (mHz) | core (kHZ) | compress (KHz) | time | success |\n"); + table_text.push_str("|-------------------|-------------|----------------|----------------|----------------|--------|----------|"); + + for result in results.iter() { + table_text.push_str(&format!( + "\n| {:<17} | {:>11} | {:>14.2} | {:>14.2} | {:>14.2} | {:>6} | {:<7} |", + result.program, + result.cycles, + result.exec_khz / 1000.0, + result.core_khz, + result.compressed_khz, + format_duration(result.time), + if result.success { "✅" } else { "❌" } + )); + } + table_text.push_str("\n```"); + + vec!["*SP1 Performance Test Results*\n".to_string(), detail_text, table_text] +} + +pub fn time_operation T>(operation: F) -> (T, Duration) { + let start = Instant::now(); + let result = operation(); + let duration = start.elapsed(); + (result, duration) +} + +fn calculate_khz(cycles: u64, duration: Duration) -> f64 { + let duration_secs = duration.as_secs_f64(); + if duration_secs > 0.0 { + (cycles as f64 / duration_secs) / 1_000.0 + } else { + 0.0 + } +} + +fn format_duration(duration: f64) -> String { + let secs = duration.round() as u64; + let minutes = secs / 60; + let seconds = secs % 60; + + if minutes > 0 { + format!("{}m{}s", minutes, seconds) + } else if seconds > 0 { + format!("{}s", seconds) + } else { + format!("{}ms", (duration * 1000.0).round() as u64) + } +} + +async fn post_to_slack(slack_token: &str, slack_channel_id: &str, message: &str) -> Result<()> { + let slack_api_client = default_client(); + let request = PostMessageRequest { + channel: slack_channel_id.to_string(), + text: Some(message.to_string()), + ..Default::default() + }; + + post_message(&slack_api_client, &request, slack_token).await.expect("slack api call error"); + + Ok(()) +} + +fn format_github_message(results_text: &[String]) -> String { + let mut formatted_message = String::new(); + + if let Some(title) = results_text.first() { + // Add an extra asterisk for GitHub bold formatting + formatted_message.push_str(&title.replace('*', "**")); + formatted_message.push('\n'); + } + + if let Some(details) = results_text.get(1) { + // Add an extra asterisk for GitHub bold formatting + formatted_message.push_str(&details.replace('*', "**")); + formatted_message.push('\n'); + } + + if let Some(table) = results_text.get(2) { + // Remove the triple backticks as GitHub doesn't require them for table formatting + let cleaned_table = table.trim_start_matches("```").trim_end_matches("```"); + formatted_message.push_str(cleaned_table); + } + + formatted_message +} + +async fn post_to_github_pr( + owner: &str, + repo: &str, + pr_number: &str, + token: &str, + message: &str, +) -> Result<(), Box> { + let client = Client::new(); + let base_url = format!("https://api.github.com/repos/{}/{}", owner, repo); + + // Get all comments on the PR + let comments_url = format!("{}/issues/{}/comments", base_url, pr_number); + let comments_response = client + .get(&comments_url) + .header("Authorization", format!("token {}", token)) + .header("User-Agent", "sp1-perf-bot") + .send() + .await?; + + let comments: Vec = comments_response.json().await?; + + // Look for an existing comment from our bot + let bot_comment = comments.iter().find(|comment| { + comment["user"]["login"] + .as_str() + .map(|login| login == "github-actions[bot]") + .unwrap_or(false) + }); + + if let Some(existing_comment) = bot_comment { + // Update the existing comment + let comment_url = existing_comment["url"].as_str().unwrap(); + let response = client + .patch(comment_url) + .header("Authorization", format!("token {}", token)) + .header("User-Agent", "sp1-perf-bot") + .json(&json!({ + "body": message + })) + .send() + .await?; + + if !response.status().is_success() { + return Err(format!("Failed to update comment: {:?}", response.text().await?).into()); + } + } else { + // Create a new comment + let response = client + .post(&comments_url) + .header("Authorization", format!("token {}", token)) + .header("User-Agent", "sp1-perf-bot") + .json(&json!({ + "body": message + })) + .send() + .await?; + + if !response.status().is_success() { + return Err(format!("Failed to post comment: {:?}", response.text().await?).into()); + } + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_format_results() { + let dummy_reports = vec![ + PerformanceReport { + program: "fibonacci".to_string(), + cycles: 11291, + exec_khz: 29290.0, + core_khz: 30.0, + compressed_khz: 0.1, + time: 622.385, + success: true, + }, + PerformanceReport { + program: "super-program".to_string(), + cycles: 275735600, + exec_khz: 70190.0, + core_khz: 310.0, + compressed_khz: 120.0, + time: 812.285, + success: true, + }, + ]; + + let args = EvalArgs { + programs: vec!["fibonacci".to_string(), "super-program".to_string()], + shard_size: None, + post_to_slack: Some(false), + slack_channel_id: None, + slack_token: None, + post_to_github: Some(true), + github_token: Some("abcdef1234567890".to_string()), + repo_owner: Some("succinctlabs".to_string()), + repo_name: Some("sp1".to_string()), + pr_number: Some("123456".to_string()), + branch_name: Some("feature-branch".to_string()), + commit_hash: Some("abcdef1234567890".to_string()), + author: Some("John Doe".to_string()), + }; + + let formatted_results = format_results(&args, &dummy_reports); + + for line in &formatted_results { + println!("{}", line); + } + + assert_eq!(formatted_results.len(), 3); + assert!(formatted_results[0].contains("SP1 Performance Test Results")); + assert!(formatted_results[1].contains("*Branch*: feature-branch")); + assert!(formatted_results[1].contains("*Commit*: abcdef12")); + assert!(formatted_results[1].contains("*Author*: John Doe")); + assert!(formatted_results[2].contains("fibonacci")); + assert!(formatted_results[2].contains("super-program")); + } +} diff --git a/crates/perf/src/program.rs b/crates/perf/src/program.rs new file mode 100644 index 0000000000..ec9e5aff31 --- /dev/null +++ b/crates/perf/src/program.rs @@ -0,0 +1,37 @@ +use sp1_sdk::SP1Stdin; + +#[derive(Clone)] +pub struct TesterProgram { + pub name: &'static str, + pub elf: &'static [u8], + pub input: &'static [u8], +} + +impl TesterProgram { + const fn new(name: &'static str, elf: &'static [u8], input: &'static [u8]) -> Self { + Self { name, elf, input } + } +} + +pub const PROGRAMS: &[TesterProgram] = &[ + TesterProgram::new( + "fibonacci", + include_bytes!("../programs/fibonacci/elf"), + include_bytes!("../programs/fibonacci/input.bin"), + ), + TesterProgram::new( + "ssz-withdrawals", + include_bytes!("../programs/ssz-withdrawals/elf"), + include_bytes!("../programs/ssz-withdrawals/input.bin"), + ), + TesterProgram::new( + "tendermint", + include_bytes!("../programs/tendermint/elf"), + include_bytes!("../programs/tendermint/input.bin"), + ), +]; + +pub fn load_program(elf: &[u8], input: &[u8]) -> (Vec, SP1Stdin) { + let stdin: SP1Stdin = bincode::deserialize(input).expect("failed to deserialize input"); + (elf.to_vec(), stdin) +}