feat(nargo): Consume CommonReferenceString functions & manage caching (…

…#1348)

Cargo.toml

@@ -52,5 +52,5 @@ wasm-bindgen = { version = "0.2.83", features = ["serde-serialize"] }
 wasm-bindgen-test = "0.3.33"
 
 [patch.crates-io]
-acvm = { package = "acvm", git = "https://github.com/noir-lang/acvm", rev = "bbd9ab7ca5be3fb31f3e141fee2522704852f5de" }
-acvm-backend-barretenberg = { git = "https://github.com/noir-lang/acvm-backend-barretenberg", rev = "4f622b58144330ebc53d8458bd9e456a8efb147c" }
+acvm = { package = "acvm", git = "https://github.com/noir-lang/acvm", rev = "eeddcf179880f246383f7f67a11e589269c4e3ff" }
+acvm-backend-barretenberg = { git = "https://github.com/noir-lang/acvm-backend-barretenberg", rev = "04c9d0f6b5e0f3625fd7d1e2bb5bacb020ae279d" }

crates/nargo/Cargo.toml

@@ -14,7 +14,6 @@ rustc_version = "0.4.0"
 acvm.workspace = true
 noirc_abi.workspace = true
 noirc_driver.workspace = true
-iter-extended.workspace = true
 toml.workspace = true
 serde.workspace = true
 thiserror.workspace = true

crates/nargo/src/ops/codegen_verifier.rs

@@ -2,7 +2,8 @@ use acvm::SmartContract;
 
 pub fn codegen_verifier<B: SmartContract>(
     backend: &B,
+    common_reference_string: &[u8],
     verification_key: &[u8],
 ) -> Result<String, B::Error> {
-    backend.eth_contract_from_vk(verification_key)
+    backend.eth_contract_from_vk(common_reference_string, verification_key)
 }

crates/nargo/src/ops/mod.rs

@@ -1,6 +1,6 @@
 pub use self::codegen_verifier::codegen_verifier;
 pub use self::execute::execute_circuit;
-pub use self::preprocess::{preprocess_contract, preprocess_program};
+pub use self::preprocess::{preprocess_contract_function, preprocess_program};
 pub use self::prove::prove_execution;
 pub use self::verify::verify_proof;
 

crates/nargo/src/ops/preprocess.rs

@@ -1,23 +1,21 @@
 use acvm::ProofSystemCompiler;
-use iter_extended::try_vecmap;
-use noirc_driver::{CompiledContract, CompiledProgram};
+use noirc_driver::{CompiledProgram, ContractFunction};
 
-use crate::artifacts::{
-    contract::{PreprocessedContract, PreprocessedContractFunction},
-    program::PreprocessedProgram,
-};
+use crate::artifacts::{contract::PreprocessedContractFunction, program::PreprocessedProgram};
 
 // TODO: pull this from backend.
 const BACKEND_IDENTIFIER: &str = "acvm-backend-barretenberg";
 
 pub fn preprocess_program<B: ProofSystemCompiler>(
     backend: &B,
+    common_reference_string: &[u8],
     compiled_program: CompiledProgram,
 ) -> Result<PreprocessedProgram, B::Error> {
     // TODO: currently `compiled_program`'s bytecode is already optimized for the backend.
     // In future we'll need to apply those optimizations here.
     let optimized_bytecode = compiled_program.circuit;
-    let (proving_key, verification_key) = backend.preprocess(&optimized_bytecode)?;
+    let (proving_key, verification_key) =
+        backend.preprocess(common_reference_string, &optimized_bytecode)?;
 
     Ok(PreprocessedProgram {
         backend: String::from(BACKEND_IDENTIFIER),
@@ -28,30 +26,24 @@ pub fn preprocess_program<B: ProofSystemCompiler>(
     })
 }
 
-pub fn preprocess_contract<B: ProofSystemCompiler>(
+pub fn preprocess_contract_function<B: ProofSystemCompiler>(
     backend: &B,
-    compiled_contract: CompiledContract,
-) -> Result<PreprocessedContract, B::Error> {
-    let preprocessed_contract_functions = try_vecmap(compiled_contract.functions, |func| {
-        // TODO: currently `func`'s bytecode is already optimized for the backend.
-        // In future we'll need to apply those optimizations here.
-        let optimized_bytecode = func.bytecode;
-        let (proving_key, verification_key) = backend.preprocess(&optimized_bytecode)?;
-
-        Ok(PreprocessedContractFunction {
-            name: func.name,
-            function_type: func.function_type,
-            abi: func.abi,
+    common_reference_string: &[u8],
+    func: ContractFunction,
+) -> Result<PreprocessedContractFunction, B::Error> {
+    // TODO: currently `func`'s bytecode is already optimized for the backend.
+    // In future we'll need to apply those optimizations here.
+    let optimized_bytecode = func.bytecode;
+    let (proving_key, verification_key) =
+        backend.preprocess(common_reference_string, &optimized_bytecode)?;
 
-            bytecode: optimized_bytecode,
-            proving_key,
-            verification_key,
-        })
-    })?;
+    Ok(PreprocessedContractFunction {
+        name: func.name,
+        function_type: func.function_type,
+        abi: func.abi,
 
-    Ok(PreprocessedContract {
-        name: compiled_contract.name,
-        backend: String::from(BACKEND_IDENTIFIER),
-        functions: preprocessed_contract_functions,
+        bytecode: optimized_bytecode,
+        proving_key,
+        verification_key,
     })
 }

crates/nargo/src/ops/prove.rs

@@ -3,9 +3,10 @@ use acvm::ProofSystemCompiler;
 
 pub fn prove_execution<B: ProofSystemCompiler>(
     backend: &B,
+    common_reference_string: &[u8],
     circuit: &Circuit,
     solved_witness: WitnessMap,
     proving_key: &[u8],
 ) -> Result<Vec<u8>, B::Error> {
-    backend.prove_with_pk(circuit, solved_witness, proving_key)
+    backend.prove_with_pk(common_reference_string, circuit, solved_witness, proving_key)
 }

crates/nargo/src/ops/verify.rs

@@ -3,10 +3,11 @@ use acvm::ProofSystemCompiler;
 
 pub fn verify_proof<B: ProofSystemCompiler>(
     backend: &B,
+    common_reference_string: &[u8],
     circuit: &Circuit,
     proof: &[u8],
     public_inputs: WitnessMap,
     verification_key: &[u8],
 ) -> Result<bool, B::Error> {
-    backend.verify_with_vk(proof, public_inputs, circuit, verification_key)
+    backend.verify_with_vk(common_reference_string, proof, public_inputs, circuit, verification_key)
 }

crates/nargo_cli/Cargo.toml

@@ -35,6 +35,7 @@ hex = "0.4.2"
 serde_json = "1.0"
 termcolor = "1.1.2"
 color-eyre = "0.6.2"
+tokio = "1.0"
 
 # Backends
 acvm-backend-barretenberg = { version = "0.1.2", default-features = false }

crates/nargo_cli/src/cli/codegen_verifier_cmd.rs

@@ -1,4 +1,12 @@
-use super::fs::{create_named_dir, program::read_program_from_file, write_to_file};
+use super::fs::{
+    common_reference_string::{
+        read_cached_common_reference_string, update_common_reference_string,
+        write_cached_common_reference_string,
+    },
+    create_named_dir,
+    program::read_program_from_file,
+    write_to_file,
+};
 use super::NargoConfig;
 use crate::{
     cli::compile_cmd::compile_circuit, constants::CONTRACT_DIR, constants::TARGET_DIR,
@@ -29,18 +37,36 @@ pub(crate) fn run<B: Backend>(
         .circuit_name
         .map(|circuit_name| config.program_dir.join(TARGET_DIR).join(circuit_name));
 
-    let preprocessed_program = match circuit_build_path {
-        Some(circuit_build_path) => read_program_from_file(circuit_build_path)?,
+    let common_reference_string = read_cached_common_reference_string();
+
+    let (common_reference_string, preprocessed_program) = match circuit_build_path {
+        Some(circuit_build_path) => {
+            let program = read_program_from_file(circuit_build_path)?;
+            let common_reference_string = update_common_reference_string(
+                backend,
+                &common_reference_string,
+                &program.bytecode,
+            )
+            .map_err(CliError::CommonReferenceStringError)?;
+            (common_reference_string, program)
+        }
         None => {
-            let compiled_program =
+            let program =
                 compile_circuit(backend, config.program_dir.as_ref(), &args.compile_options)?;
-            preprocess_program(backend, compiled_program)
-                .map_err(CliError::ProofSystemCompilerError)?
+            let common_reference_string =
+                update_common_reference_string(backend, &common_reference_string, &program.circuit)
+                    .map_err(CliError::CommonReferenceStringError)?;
+            let program = preprocess_program(backend, &common_reference_string, program)
+                .map_err(CliError::ProofSystemCompilerError)?;
+            (common_reference_string, program)
         }
     };
 
-    let smart_contract_string = codegen_verifier(backend, &preprocessed_program.verification_key)
-        .map_err(CliError::SmartContractError)?;
+    let smart_contract_string =
+        codegen_verifier(backend, &common_reference_string, &preprocessed_program.verification_key)
+            .map_err(CliError::SmartContractError)?;
+
+    write_cached_common_reference_string(&common_reference_string);
 
     let contract_dir = config.program_dir.join(CONTRACT_DIR);
     create_named_dir(&contract_dir, "contract");

crates/nargo_cli/src/cli/compile_cmd.rs

@@ -1,18 +1,28 @@
 use acvm::Backend;
 use iter_extended::try_vecmap;
+use nargo::artifacts::contract::PreprocessedContract;
 use noirc_driver::{CompileOptions, CompiledProgram, Driver};
 use std::path::Path;
 
 use clap::Args;
 
-use nargo::ops::{preprocess_contract, preprocess_program};
+use nargo::ops::{preprocess_contract_function, preprocess_program};
 
 use crate::resolver::DependencyResolutionError;
 use crate::{constants::TARGET_DIR, errors::CliError, resolver::Resolver};
 
-use super::fs::program::{save_contract_to_file, save_program_to_file};
+use super::fs::{
+    common_reference_string::{
+        read_cached_common_reference_string, update_common_reference_string,
+        write_cached_common_reference_string,
+    },
+    program::{save_contract_to_file, save_program_to_file},
+};
 use super::NargoConfig;
 
+// TODO: pull this from backend.
+const BACKEND_IDENTIFIER: &str = "acvm-backend-barretenberg";
+
 /// Compile the program and its secret execution trace into ACIR format
 #[derive(Debug, Clone, Args)]
 pub(crate) struct CompileCommand {
@@ -34,16 +44,40 @@ pub(crate) fn run<B: Backend>(
 ) -> Result<(), CliError<B>> {
     let circuit_dir = config.program_dir.join(TARGET_DIR);
 
+    let mut common_reference_string = read_cached_common_reference_string();
+
     // If contracts is set we're compiling every function in a 'contract' rather than just 'main'.
     if args.contracts {
         let mut driver = setup_driver(backend, &config.program_dir)?;
         let compiled_contracts = driver
             .compile_contracts(&args.compile_options)
             .map_err(|_| CliError::CompilationError)?;
-        let preprocessed_contracts = try_vecmap(compiled_contracts, |contract| {
-            preprocess_contract(backend, contract).map_err(CliError::ProofSystemCompilerError)
-        })?;
-        for contract in preprocessed_contracts {
+
+        // TODO: I wonder if it is incorrect for nargo-core to know anything about contracts.
+        // As can be seen here, It seems like a leaky abstraction where ContractFunctions (essentially CompiledPrograms)
+        // are compiled via nargo-core and then the PreprocessedContract is constructed here.
+        // This is due to EACH function needing it's own CRS, PKey, and VKey from the backend.
+        let preprocessed_contracts: Result<Vec<PreprocessedContract>, CliError<B>> =
+            try_vecmap(compiled_contracts, |contract| {
+                let preprocessed_contract_functions = try_vecmap(contract.functions, |func| {
+                    common_reference_string = update_common_reference_string(
+                        backend,
+                        &common_reference_string,
+                        &func.bytecode,
+                    )
+                    .map_err(CliError::CommonReferenceStringError)?;
+
+                    preprocess_contract_function(backend, &common_reference_string, func)
+                        .map_err(CliError::ProofSystemCompilerError)
+                })?;
+
+                Ok(PreprocessedContract {
+                    name: contract.name,
+                    backend: String::from(BACKEND_IDENTIFIER),
+                    functions: preprocessed_contract_functions,
+                })
+            });
+        for contract in preprocessed_contracts? {
             save_contract_to_file(
                 &contract,
                 &format!("{}-{}", &args.circuit_name, contract.name),
@@ -52,10 +86,17 @@ pub(crate) fn run<B: Backend>(
         }
     } else {
         let program = compile_circuit(backend, &config.program_dir, &args.compile_options)?;
-        let preprocessed_program =
-            preprocess_program(backend, program).map_err(CliError::ProofSystemCompilerError)?;
+        common_reference_string =
+            update_common_reference_string(backend, &common_reference_string, &program.circuit)
+                .map_err(CliError::CommonReferenceStringError)?;
+
+        let preprocessed_program = preprocess_program(backend, &common_reference_string, program)
+            .map_err(CliError::ProofSystemCompilerError)?;
         save_program_to_file(&preprocessed_program, &args.circuit_name, circuit_dir);
     }
+
+    write_cached_common_reference_string(&common_reference_string);
+
     Ok(())
 }
 

crates/nargo_cli/src/cli/fs/common_reference_string.rs

@@ -0,0 +1,55 @@
+use std::{env, path::PathBuf};
+
+use acvm::{acir::circuit::Circuit, CommonReferenceString};
+
+use super::{create_named_dir, write_to_file};
+
+// TODO: pull this from backend.
+const BACKEND_IDENTIFIER: &str = "acvm-backend-barretenberg";
+const TRANSCRIPT_NAME: &str = "common-reference-string.bin";
+
+fn common_reference_string_location() -> PathBuf {
+    let cache_dir = match env::var("NARGO_BACKEND_CACHE_DIR") {
+        Ok(cache_dir) => PathBuf::from(cache_dir),
+        Err(_) => dirs::home_dir().unwrap().join(".nargo").join("backends"),
+    };
+    cache_dir.join(BACKEND_IDENTIFIER).join(TRANSCRIPT_NAME)
+}
+
+pub(crate) fn read_cached_common_reference_string() -> Vec<u8> {
+    let crs_path = common_reference_string_location();
+
+    // TODO: Implement checksum
+    match std::fs::read(crs_path) {
+        Ok(common_reference_string) => common_reference_string,
+        Err(_) => vec![],
+    }
+}
+
+pub(crate) fn update_common_reference_string<B: CommonReferenceString>(
+    backend: &B,
+    common_reference_string: &[u8],
+    circuit: &Circuit,
+) -> Result<Vec<u8>, B::Error> {
+    use tokio::runtime::Builder;
+
+    let runtime = Builder::new_current_thread().enable_all().build().unwrap();
+
+    // TODO: Implement retries
+    // If the read data is empty, we don't have a CRS and need to generate one
+    let fut = if common_reference_string.is_empty() {
+        backend.generate_common_reference_string(circuit)
+    } else {
+        backend.update_common_reference_string(common_reference_string.to_vec(), circuit)
+    };
+
+    runtime.block_on(fut)
+}
+
+pub(crate) fn write_cached_common_reference_string(common_reference_string: &[u8]) {
+    let crs_path = common_reference_string_location();
+
+    create_named_dir(crs_path.parent().unwrap(), "crs");
+
+    write_to_file(common_reference_string, &crs_path);
+}

crates/nargo_cli/src/cli/fs/mod.rs

@@ -6,6 +6,7 @@ use std::{
 
 use crate::errors::FilesystemError;
 
+pub(super) mod common_reference_string;
 pub(super) mod inputs;
 pub(super) mod program;
 pub(super) mod proof;