From 2ec7eb4b650a92d5c3b1a315c3d04a70d547c87c Mon Sep 17 00:00:00 2001 From: mattstam Date: Thu, 29 Aug 2024 14:24:34 -0700 Subject: [PATCH 1/4] chore: eval --- crates/eval/src/lib.rs | 10 ++++----- crates/eval/src/program.rs | 42 +++++++++++++++++++------------------- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/crates/eval/src/lib.rs b/crates/eval/src/lib.rs index d30027bff..be1c2878a 100644 --- a/crates/eval/src/lib.rs +++ b/crates/eval/src/lib.rs @@ -6,7 +6,7 @@ use serde_json::json; use slack_rust::chat::post_message::{post_message, PostMessageRequest}; use slack_rust::http_client::default_client; use sp1_prover::{components::SP1ProverComponents, utils::get_cycles, SP1Prover}; -use sp1_sdk::SP1Context; +use sp1_sdk::{SP1Context, SP1Stdin}; use sp1_stark::SP1ProverOpts; use std::time::{Duration, Instant}; @@ -99,7 +99,8 @@ pub async fn evaluate_performance() -> Result<(), Box(program.name, program.elf, program.input); + let (elf, stdin) = load_program(program.elf, program.input); + let report = run_evaluation::(program.name, &elf, &stdin); reports.push(report); println!("Program: {} completed", program.name); } @@ -160,10 +161,9 @@ pub struct PerformanceReport { fn run_evaluation( program_name: &str, - elf_path: &str, - input_path: &str, + elf: &[u8], + stdin: &SP1Stdin, ) -> PerformanceReport { - let (elf, stdin) = load_program(elf_path, input_path); let cycles = get_cycles(&elf, &stdin); let prover = SP1Prover::::new(); diff --git a/crates/eval/src/program.rs b/crates/eval/src/program.rs index 49d576a9e..ec9e5aff3 100644 --- a/crates/eval/src/program.rs +++ b/crates/eval/src/program.rs @@ -1,37 +1,37 @@ use sp1_sdk::SP1Stdin; -use std::fs::File; -use std::io::Read; #[derive(Clone)] pub struct TesterProgram { pub name: &'static str, - pub elf: &'static str, - pub input: &'static str, + pub elf: &'static [u8], + pub input: &'static [u8], } impl TesterProgram { - const fn new(name: &'static str, elf: &'static str, input: &'static str) -> Self { + const fn new(name: &'static str, elf: &'static [u8], input: &'static [u8]) -> Self { Self { name, elf, input } } } pub const PROGRAMS: &[TesterProgram] = &[ - TesterProgram::new("fibonacci", "fibonacci/elf", "fibonacci/input.bin"), - TesterProgram::new("ssz-withdrawals", "ssz-withdrawals/elf", "ssz-withdrawals/input.bin"), - TesterProgram::new("tendermint", "tendermint/elf", "tendermint/input.bin"), + TesterProgram::new( + "fibonacci", + include_bytes!("../programs/fibonacci/elf"), + include_bytes!("../programs/fibonacci/input.bin"), + ), + TesterProgram::new( + "ssz-withdrawals", + include_bytes!("../programs/ssz-withdrawals/elf"), + include_bytes!("../programs/ssz-withdrawals/input.bin"), + ), + TesterProgram::new( + "tendermint", + include_bytes!("../programs/tendermint/elf"), + include_bytes!("../programs/tendermint/input.bin"), + ), ]; -pub fn load_program(elf_path: &str, input_path: &str) -> (Vec, SP1Stdin) { - let elf_path = format!("./programs/{}", elf_path); - let input_path = format!("./programs/{}", input_path); - - let mut elf_file = File::open(elf_path).expect("failed to open elf"); - let mut elf = Vec::new(); - elf_file.read_to_end(&mut elf).expect("failed to read elf"); - - let input_file = File::open(input_path).expect("failed to open input"); - let stdin: SP1Stdin = - bincode::deserialize_from(input_file).expect("failed to deserialize input"); - - (elf, stdin) +pub fn load_program(elf: &[u8], input: &[u8]) -> (Vec, SP1Stdin) { + let stdin: SP1Stdin = bincode::deserialize(input).expect("failed to deserialize input"); + (elf.to_vec(), stdin) } From d46e862e46fb5e9eb5c0bf9f02cbd6b2f15fe01c Mon Sep 17 00:00:00 2001 From: mattstam Date: Thu, 29 Aug 2024 15:00:12 -0700 Subject: [PATCH 2/4] x --- .github/workflows/pr.yml | 3 +-- crates/eval/src/lib.rs | 26 +++++++++----------------- crates/eval/src/main.rs | 4 +++- 3 files changed, 13 insertions(+), 20 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 498ab0960..73cad72ab 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -247,7 +247,7 @@ jobs: ~/.sp1/bin/sp1up ~/.sp1/bin/cargo-prove prove --version - - name: Build program and run script + - name: Run Evaluation run: | cd crates/eval RUSTFLAGS='-C target-cpu=native' cargo run --release -- \ @@ -260,7 +260,6 @@ jobs: --repo-owner "${{ github.repository_owner }}" \ --repo-name "${{ github.event.repository.name }}" \ --pr-number "${{ github.event.pull_request.number }}" \ - --pr-name "${{ github.event.pull_request.title }}" \ --branch-name "${{ github.head_ref || github.ref_name }}" \ --commit-hash "${{ github.sha }}" \ --author "${{ github.event.pull_request.user.login || github.actor }}" diff --git a/crates/eval/src/lib.rs b/crates/eval/src/lib.rs index be1c2878a..3b3a14acc 100644 --- a/crates/eval/src/lib.rs +++ b/crates/eval/src/lib.rs @@ -59,10 +59,6 @@ struct EvalArgs { #[arg(long)] pub pr_number: Option, - /// The name of the pull request. - #[arg(long)] - pub pr_name: Option, - /// The name of the branch. #[arg(long)] pub branch_name: Option, @@ -76,8 +72,9 @@ struct EvalArgs { pub author: Option, } -pub async fn evaluate_performance() -> Result<(), Box> -{ +pub async fn evaluate_performance( + opts: SP1ProverOpts, +) -> Result<(), Box> { let args = EvalArgs::parse(); // Set environment variables to configure the prover. @@ -100,7 +97,7 @@ pub async fn evaluate_performance() -> Result<(), Box(program.name, &elf, &stdin); + let report = run_evaluation::(program.name, &elf, &stdin, opts); reports.push(report); println!("Program: {} completed", program.name); } @@ -163,19 +160,19 @@ fn run_evaluation( program_name: &str, elf: &[u8], stdin: &SP1Stdin, + opts: SP1ProverOpts, ) -> PerformanceReport { - let cycles = get_cycles(&elf, &stdin); + let cycles = get_cycles(elf, stdin); let prover = SP1Prover::::new(); - let (pk, vk) = prover.setup(&elf); + let (pk, vk) = prover.setup(elf); - let opts = SP1ProverOpts::default(); let context = SP1Context::default(); - let (_, exec_duration) = time_operation(|| prover.execute(&elf, &stdin, context.clone())); + let (_, exec_duration) = time_operation(|| prover.execute(elf, stdin, context.clone())); let (core_proof, core_duration) = - time_operation(|| prover.prove_core(&pk, &stdin, opts, context).unwrap()); + time_operation(|| prover.prove_core(&pk, stdin, opts, context).unwrap()); let (_, compress_duration) = time_operation(|| prover.compress(&vk, core_proof, vec![], opts).unwrap()); @@ -195,9 +192,6 @@ fn run_evaluation( fn format_results(args: &EvalArgs, results: &[PerformanceReport]) -> Vec { let mut detail_text = String::new(); - if let Some(pr_name) = &args.pr_name { - detail_text.push_str(&format!("*PR*: {}\n", pr_name)); - } if let Some(branch_name) = &args.branch_name { detail_text.push_str(&format!("*Branch*: {}\n", branch_name)); } @@ -400,7 +394,6 @@ mod tests { repo_owner: Some("succinctlabs".to_string()), repo_name: Some("sp1".to_string()), pr_number: Some("123456".to_string()), - pr_name: Some("Test PR".to_string()), branch_name: Some("feature-branch".to_string()), commit_hash: Some("abcdef1234567890".to_string()), author: Some("John Doe".to_string()), @@ -414,7 +407,6 @@ mod tests { assert_eq!(formatted_results.len(), 3); assert!(formatted_results[0].contains("SP1 Performance Test Results")); - assert!(formatted_results[1].contains("*PR*: Test PR")); assert!(formatted_results[1].contains("*Branch*: feature-branch")); assert!(formatted_results[1].contains("*Commit*: abcdef12")); assert!(formatted_results[1].contains("*Author*: John Doe")); diff --git a/crates/eval/src/main.rs b/crates/eval/src/main.rs index e0e5b7888..ca0e75a37 100644 --- a/crates/eval/src/main.rs +++ b/crates/eval/src/main.rs @@ -1,8 +1,10 @@ use anyhow::Result; use sp1_eval::evaluate_performance; use sp1_prover::components::DefaultProverComponents; +use sp1_stark::SP1ProverOpts; #[tokio::main] async fn main() -> Result<(), Box> { - evaluate_performance::().await + let opts = SP1ProverOpts::default(); + evaluate_performance::(opts).await } From 019a73a617a83d8e0e10413c8eeae8d9c013bc10 Mon Sep 17 00:00:00 2001 From: mattstam Date: Thu, 29 Aug 2024 15:17:55 -0700 Subject: [PATCH 3/4] setup logger --- crates/eval/src/lib.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/crates/eval/src/lib.rs b/crates/eval/src/lib.rs index 3b3a14acc..88c6faf57 100644 --- a/crates/eval/src/lib.rs +++ b/crates/eval/src/lib.rs @@ -75,6 +75,8 @@ struct EvalArgs { pub async fn evaluate_performance( opts: SP1ProverOpts, ) -> Result<(), Box> { + println!("opts: {:?}", opts); + let args = EvalArgs::parse(); // Set environment variables to configure the prover. @@ -92,6 +94,8 @@ pub async fn evaluate_performance( .collect() }; + sp1_sdk::utils::setup_logger(); + // Run the evaluations on each program. let mut reports = Vec::new(); for program in &programs { @@ -99,7 +103,7 @@ pub async fn evaluate_performance( let (elf, stdin) = load_program(program.elf, program.input); let report = run_evaluation::(program.name, &elf, &stdin, opts); reports.push(report); - println!("Program: {} completed", program.name); + println!("Finished Program: {}", program.name); } // Prepare and format the results. From 16f3acf64e9446fa1edd8f2ed0a384caf1fb7f6c Mon Sep 17 00:00:00 2001 From: mattstam Date: Thu, 29 Aug 2024 16:25:24 -0700 Subject: [PATCH 4/4] cratesio --- Cargo.lock | 9 +++++---- crates/eval/Cargo.toml | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index efb79c4ff..22b515020 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5839,9 +5839,10 @@ dependencies = [ ] [[package]] -name = "slack-rust" -version = "0.0.1-alpha" -source = "git+https://github.com/dselans/slack-rust.git?branch=main#8212d4e8de30ebb3f536da35eab27e7e0820461b" +name = "slack-rust-rs" +version = "0.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b12b682cb9dfa4f3b4a57315c241e676103a79a01de6cb72802489fb2fc9bd8" dependencies = [ "async-std", "async-tls", @@ -6120,7 +6121,7 @@ dependencies = [ "reqwest 0.12.5", "serde", "serde_json", - "slack-rust", + "slack-rust-rs", "sp1-prover", "sp1-sdk", "sp1-stark", diff --git a/crates/eval/Cargo.toml b/crates/eval/Cargo.toml index 2f6163b98..6cb700a46 100644 --- a/crates/eval/Cargo.toml +++ b/crates/eval/Cargo.toml @@ -21,7 +21,7 @@ csv = "1.3.0" serde = "1.0.204" bincode = "1.3.3" time = "0.3.26" -slack-rust = { git = "https://github.com/dselans/slack-rust.git", branch = "main" } +slack-rust = { package = "slack-rust-rs", version = "0.0.1" } tokio = { version = "1.39.0", features = ["full"] } reqwest = { version = "0.12.4", features = ["json"] } serde_json = "1.0.104"