From 662c8db3bd8b5935d9e3f3a159de079adc819085 Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Sat, 21 Sep 2024 09:13:48 -0700 Subject: [PATCH 01/91] refactoring --- .../src/fixed_recursive_verifier.rs | 95 +++------ evm_arithmetization/src/prover.rs | 181 +++--------------- evm_arithmetization/src/verifier.rs | 103 +++------- 3 files changed, 79 insertions(+), 300 deletions(-) diff --git a/evm_arithmetization/src/fixed_recursive_verifier.rs b/evm_arithmetization/src/fixed_recursive_verifier.rs index 7f3842826..cb3cec8a8 100644 --- a/evm_arithmetization/src/fixed_recursive_verifier.rs +++ b/evm_arithmetization/src/fixed_recursive_verifier.rs @@ -697,77 +697,30 @@ where // Sanity check on the provided config assert_eq!(DEFAULT_CAP_LEN, 1 << stark_config.fri_config.cap_height); - let arithmetic = RecursiveCircuitsForTable::new( - Table::Arithmetic, - &all_stark.arithmetic_stark, - degree_bits_ranges[*Table::Arithmetic].clone(), - &all_stark.cross_table_lookups, - stark_config, - ); - let byte_packing = RecursiveCircuitsForTable::new( - Table::BytePacking, - &all_stark.byte_packing_stark, - degree_bits_ranges[*Table::BytePacking].clone(), - &all_stark.cross_table_lookups, - stark_config, - ); - let cpu = RecursiveCircuitsForTable::new( - Table::Cpu, - &all_stark.cpu_stark, - degree_bits_ranges[*Table::Cpu].clone(), - &all_stark.cross_table_lookups, - stark_config, - ); - let keccak = RecursiveCircuitsForTable::new( - Table::Keccak, - &all_stark.keccak_stark, - degree_bits_ranges[*Table::Keccak].clone(), - &all_stark.cross_table_lookups, - stark_config, - ); - let keccak_sponge = RecursiveCircuitsForTable::new( - Table::KeccakSponge, - &all_stark.keccak_sponge_stark, - degree_bits_ranges[*Table::KeccakSponge].clone(), - &all_stark.cross_table_lookups, - stark_config, - ); - let logic = RecursiveCircuitsForTable::new( - Table::Logic, - &all_stark.logic_stark, - degree_bits_ranges[*Table::Logic].clone(), - &all_stark.cross_table_lookups, - stark_config, - ); - let memory = RecursiveCircuitsForTable::new( - Table::Memory, - &all_stark.memory_stark, - degree_bits_ranges[*Table::Memory].clone(), - &all_stark.cross_table_lookups, - stark_config, - ); - let mem_before = RecursiveCircuitsForTable::new( - Table::MemBefore, - &all_stark.mem_before_stark, - degree_bits_ranges[*Table::MemBefore].clone(), - &all_stark.cross_table_lookups, - stark_config, - ); - let mem_after = RecursiveCircuitsForTable::new( - Table::MemAfter, - &all_stark.mem_after_stark, - degree_bits_ranges[*Table::MemAfter].clone(), - &all_stark.cross_table_lookups, - stark_config, - ); + macro_rules! create_recursive_circuit { + ($table_enum:expr, $stark_field:ident) => { + RecursiveCircuitsForTable::new( + $table_enum, + &all_stark.$stark_field, + degree_bits_ranges[*$table_enum].clone(), + &all_stark.cross_table_lookups, + stark_config, + ) + }; + } + + let arithmetic = create_recursive_circuit!(Table::Arithmetic, arithmetic_stark); + let byte_packing = create_recursive_circuit!(Table::BytePacking, byte_packing_stark); + let cpu = create_recursive_circuit!(Table::Cpu, cpu_stark); + let keccak = create_recursive_circuit!(Table::Keccak, keccak_stark); + let keccak_sponge = create_recursive_circuit!(Table::KeccakSponge, keccak_sponge_stark); + let logic = create_recursive_circuit!(Table::Logic, logic_stark); + let memory = create_recursive_circuit!(Table::Memory, memory_stark); + let mem_before = create_recursive_circuit!(Table::MemBefore, mem_before_stark); + let mem_after = create_recursive_circuit!(Table::MemAfter, mem_after_stark); + #[cfg(feature = "cdk_erigon")] - let poseidon = RecursiveCircuitsForTable::new( - Table::Poseidon, - &all_stark.poseidon_stark, - degree_bits_ranges[*Table::Poseidon].clone(), - &all_stark.cross_table_lookups, - stark_config, - ); + let poseidon = create_recursive_circuit!(Table::Poseidon, poseidon_stark); let by_table = [ arithmetic, @@ -782,6 +735,7 @@ where #[cfg(feature = "cdk_erigon")] poseidon, ]; + let root = Self::create_segment_circuit(&by_table, stark_config); let segment_aggregation = Self::create_segment_aggregation_circuit(&root); let txn_aggregation = @@ -789,6 +743,7 @@ where let block = Self::create_block_circuit(&txn_aggregation); let block_wrapper = Self::create_block_wrapper_circuit(&block); let two_to_one_block = Self::create_two_to_one_block_circuit(&block_wrapper); + Self { root, segment_aggregation, diff --git a/evm_arithmetization/src/prover.rs b/evm_arithmetization/src/prover.rs index 0cd825e87..079fe3469 100644 --- a/evm_arithmetization/src/prover.rs +++ b/evm_arithmetization/src/prover.rs @@ -239,157 +239,38 @@ where F: RichField + Extendable, C: GenericConfig, { - let (arithmetic_proof, _) = timed!( - timing, - "prove Arithmetic STARK", - prove_single_table( - &all_stark.arithmetic_stark, - config, - &trace_poly_values[*Table::Arithmetic], - &trace_commitments[*Table::Arithmetic], - &ctl_data_per_table[*Table::Arithmetic], - ctl_challenges, - challenger, - timing, - abort_signal.clone(), - )? - ); - let (byte_packing_proof, _) = timed!( - timing, - "prove byte packing STARK", - prove_single_table( - &all_stark.byte_packing_stark, - config, - &trace_poly_values[*Table::BytePacking], - &trace_commitments[*Table::BytePacking], - &ctl_data_per_table[*Table::BytePacking], - ctl_challenges, - challenger, - timing, - abort_signal.clone(), - )? - ); - let (cpu_proof, _) = timed!( - timing, - "prove CPU STARK", - prove_single_table( - &all_stark.cpu_stark, - config, - &trace_poly_values[*Table::Cpu], - &trace_commitments[*Table::Cpu], - &ctl_data_per_table[*Table::Cpu], - ctl_challenges, - challenger, - timing, - abort_signal.clone(), - )? - ); - let (keccak_proof, _) = timed!( - timing, - "prove Keccak STARK", - prove_single_table( - &all_stark.keccak_stark, - config, - &trace_poly_values[*Table::Keccak], - &trace_commitments[*Table::Keccak], - &ctl_data_per_table[*Table::Keccak], - ctl_challenges, - challenger, - timing, - abort_signal.clone(), - )? - ); - let (keccak_sponge_proof, _) = timed!( - timing, - "prove Keccak sponge STARK", - prove_single_table( - &all_stark.keccak_sponge_stark, - config, - &trace_poly_values[*Table::KeccakSponge], - &trace_commitments[*Table::KeccakSponge], - &ctl_data_per_table[*Table::KeccakSponge], - ctl_challenges, - challenger, - timing, - abort_signal.clone(), - )? - ); - let (logic_proof, _) = timed!( - timing, - "prove logic STARK", - prove_single_table( - &all_stark.logic_stark, - config, - &trace_poly_values[*Table::Logic], - &trace_commitments[*Table::Logic], - &ctl_data_per_table[*Table::Logic], - ctl_challenges, - challenger, - timing, - abort_signal.clone(), - )? - ); - let (memory_proof, _) = timed!( - timing, - "prove memory STARK", - prove_single_table( - &all_stark.memory_stark, - config, - &trace_poly_values[*Table::Memory], - &trace_commitments[*Table::Memory], - &ctl_data_per_table[*Table::Memory], - ctl_challenges, - challenger, - timing, - abort_signal.clone(), - )? - ); - let (mem_before_proof, mem_before_cap) = timed!( - timing, - "prove mem_before STARK", - prove_single_table( - &all_stark.mem_before_stark, - config, - &trace_poly_values[*Table::MemBefore], - &trace_commitments[*Table::MemBefore], - &ctl_data_per_table[*Table::MemBefore], - ctl_challenges, - challenger, - timing, - abort_signal.clone(), - )? - ); - let (mem_after_proof, mem_after_cap) = timed!( - timing, - "prove mem_after STARK", - prove_single_table( - &all_stark.mem_after_stark, - config, - &trace_poly_values[*Table::MemAfter], - &trace_commitments[*Table::MemAfter], - &ctl_data_per_table[*Table::MemAfter], - ctl_challenges, - challenger, - timing, - abort_signal.clone(), - )? - ); + macro_rules! prove_table { + ($stark:ident, $table:expr) => { + timed!( + timing, + &format!("prove {} STARK", stringify!($stark)), + prove_single_table( + &all_stark.$stark, + config, + &trace_poly_values[*$table], + &trace_commitments[*$table], + &ctl_data_per_table[*$table], + ctl_challenges, + challenger, + timing, + abort_signal.clone(), + )? + ) + }; + } + + let (arithmetic_proof, _) = prove_table!(arithmetic_stark, Table::Arithmetic); + let (byte_packing_proof, _) = prove_table!(byte_packing_stark, Table::BytePacking); + let (cpu_proof, _) = prove_table!(cpu_stark, Table::Cpu); + let (keccak_proof, _) = prove_table!(keccak_stark, Table::Keccak); + let (keccak_sponge_proof, _) = prove_table!(keccak_sponge_stark, Table::KeccakSponge); + let (logic_proof, _) = prove_table!(logic_stark, Table::Logic); + let (memory_proof, _) = prove_table!(memory_stark, Table::Memory); + let (mem_before_proof, mem_before_cap) = prove_table!(mem_before_stark, Table::MemBefore); + let (mem_after_proof, mem_after_cap) = prove_table!(mem_after_stark, Table::MemAfter); + #[cfg(feature = "cdk_erigon")] - let (poseidon_proof, _) = timed!( - timing, - "prove poseidon STARK", - prove_single_table( - &all_stark.poseidon_stark, - config, - &trace_poly_values[*Table::Poseidon], - &trace_commitments[*Table::Poseidon], - &ctl_data_per_table[*Table::Poseidon], - ctl_challenges, - challenger, - timing, - abort_signal, - )? - ); + let (poseidon_proof, _) = prove_table!(poseidon_stark, Table::Poseidon); Ok(( [ diff --git a/evm_arithmetization/src/verifier.rs b/evm_arithmetization/src/verifier.rs index 9ee5750f8..caa01859e 100644 --- a/evm_arithmetization/src/verifier.rs +++ b/evm_arithmetization/src/verifier.rs @@ -156,88 +156,31 @@ fn verify_proof, C: GenericConfig, const let stark_proofs = &all_proof.multi_proof.stark_proofs; - verify_stark_proof_with_challenges( - arithmetic_stark, - &stark_proofs[*Table::Arithmetic].proof, - &stark_challenges[*Table::Arithmetic], - Some(&ctl_vars_per_table[*Table::Arithmetic]), - &[], - config, - )?; + macro_rules! verify_table { + ($stark:ident, $table:expr) => { + verify_stark_proof_with_challenges( + $stark, + &stark_proofs[*$table].proof, + &stark_challenges[*$table], + Some(&ctl_vars_per_table[*$table]), + &[], + config, + )?; + }; + } + + verify_table!(arithmetic_stark, Table::Arithmetic); + verify_table!(byte_packing_stark, Table::BytePacking); + verify_table!(cpu_stark, Table::Cpu); + verify_table!(keccak_stark, Table::Keccak); + verify_table!(keccak_sponge_stark, Table::KeccakSponge); + verify_table!(logic_stark, Table::Logic); + verify_table!(memory_stark, Table::Memory); + verify_table!(mem_before_stark, Table::MemBefore); + verify_table!(mem_after_stark, Table::MemAfter); - verify_stark_proof_with_challenges( - byte_packing_stark, - &stark_proofs[*Table::BytePacking].proof, - &stark_challenges[*Table::BytePacking], - Some(&ctl_vars_per_table[*Table::BytePacking]), - &[], - config, - )?; - verify_stark_proof_with_challenges( - cpu_stark, - &stark_proofs[*Table::Cpu].proof, - &stark_challenges[*Table::Cpu], - Some(&ctl_vars_per_table[*Table::Cpu]), - &[], - config, - )?; - verify_stark_proof_with_challenges( - keccak_stark, - &stark_proofs[*Table::Keccak].proof, - &stark_challenges[*Table::Keccak], - Some(&ctl_vars_per_table[*Table::Keccak]), - &[], - config, - )?; - verify_stark_proof_with_challenges( - keccak_sponge_stark, - &stark_proofs[*Table::KeccakSponge].proof, - &stark_challenges[*Table::KeccakSponge], - Some(&ctl_vars_per_table[*Table::KeccakSponge]), - &[], - config, - )?; - verify_stark_proof_with_challenges( - logic_stark, - &stark_proofs[*Table::Logic].proof, - &stark_challenges[*Table::Logic], - Some(&ctl_vars_per_table[*Table::Logic]), - &[], - config, - )?; - verify_stark_proof_with_challenges( - memory_stark, - &stark_proofs[*Table::Memory].proof, - &stark_challenges[*Table::Memory], - Some(&ctl_vars_per_table[*Table::Memory]), - &[], - config, - )?; - verify_stark_proof_with_challenges( - mem_before_stark, - &stark_proofs[*Table::MemBefore].proof, - &stark_challenges[*Table::MemBefore], - Some(&ctl_vars_per_table[*Table::MemBefore]), - &[], - config, - )?; - verify_stark_proof_with_challenges( - mem_after_stark, - &stark_proofs[*Table::MemAfter].proof, - &stark_challenges[*Table::MemAfter], - Some(&ctl_vars_per_table[*Table::MemAfter]), - &[], - config, - )?; #[cfg(feature = "cdk_erigon")] - verify_stark_proof_with_challenges( - poseidon_stark, - &stark_proofs[*Table::Poseidon].proof, - &stark_challenges[*Table::Poseidon], - Some(&ctl_vars_per_table[*Table::Poseidon]), - &[], - config, - )?; + verify_table!(poseidon_stark, Table::Poseidon); let public_values = all_proof.public_values; From d82b1a88d3a6824e4b84525fdc4d866553b4e8dc Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Sat, 21 Sep 2024 09:39:48 -0700 Subject: [PATCH 02/91] fix clippy --- evm_arithmetization/src/arithmetic/arithmetic_stark.rs | 3 ++- evm_arithmetization/src/byte_packing/byte_packing_stark.rs | 3 ++- evm_arithmetization/src/cpu/cpu_stark.rs | 3 ++- evm_arithmetization/src/keccak/keccak_stark.rs | 3 ++- evm_arithmetization/src/keccak_sponge/keccak_sponge_stark.rs | 3 ++- evm_arithmetization/src/logic.rs | 3 ++- evm_arithmetization/src/memory/memory_stark.rs | 3 ++- .../src/memory_continuation/memory_continuation_stark.rs | 3 ++- evm_arithmetization/src/poseidon/poseidon_stark.rs | 3 ++- 9 files changed, 18 insertions(+), 9 deletions(-) diff --git a/evm_arithmetization/src/arithmetic/arithmetic_stark.rs b/evm_arithmetization/src/arithmetic/arithmetic_stark.rs index d0712a3bc..2b7eb2507 100644 --- a/evm_arithmetization/src/arithmetic/arithmetic_stark.rs +++ b/evm_arithmetization/src/arithmetic/arithmetic_stark.rs @@ -191,7 +191,8 @@ impl ArithmeticStark { } impl, const D: usize> Stark for ArithmeticStark { - type EvaluationFrame = EvmStarkFrame + type EvaluationFrame + = EvmStarkFrame where FE: FieldExtension, P: PackedField; diff --git a/evm_arithmetization/src/byte_packing/byte_packing_stark.rs b/evm_arithmetization/src/byte_packing/byte_packing_stark.rs index 018c23e14..af62a5079 100644 --- a/evm_arithmetization/src/byte_packing/byte_packing_stark.rs +++ b/evm_arithmetization/src/byte_packing/byte_packing_stark.rs @@ -285,7 +285,8 @@ impl, const D: usize> BytePackingStark { } impl, const D: usize> Stark for BytePackingStark { - type EvaluationFrame = EvmStarkFrame + type EvaluationFrame + = EvmStarkFrame where FE: FieldExtension, P: PackedField; diff --git a/evm_arithmetization/src/cpu/cpu_stark.rs b/evm_arithmetization/src/cpu/cpu_stark.rs index 386f89519..a0dd45e1e 100644 --- a/evm_arithmetization/src/cpu/cpu_stark.rs +++ b/evm_arithmetization/src/cpu/cpu_stark.rs @@ -581,7 +581,8 @@ pub(crate) struct CpuStark { } impl, const D: usize> Stark for CpuStark { - type EvaluationFrame = EvmStarkFrame + type EvaluationFrame + = EvmStarkFrame where FE: FieldExtension, P: PackedField; diff --git a/evm_arithmetization/src/keccak/keccak_stark.rs b/evm_arithmetization/src/keccak/keccak_stark.rs index 5f505cf06..0a086d03d 100644 --- a/evm_arithmetization/src/keccak/keccak_stark.rs +++ b/evm_arithmetization/src/keccak/keccak_stark.rs @@ -255,7 +255,8 @@ impl, const D: usize> KeccakStark { } impl, const D: usize> Stark for KeccakStark { - type EvaluationFrame = EvmStarkFrame + type EvaluationFrame + = EvmStarkFrame where FE: FieldExtension, P: PackedField; diff --git a/evm_arithmetization/src/keccak_sponge/keccak_sponge_stark.rs b/evm_arithmetization/src/keccak_sponge/keccak_sponge_stark.rs index b517117a3..26cc76431 100644 --- a/evm_arithmetization/src/keccak_sponge/keccak_sponge_stark.rs +++ b/evm_arithmetization/src/keccak_sponge/keccak_sponge_stark.rs @@ -534,7 +534,8 @@ impl, const D: usize> KeccakSpongeStark { } impl, const D: usize> Stark for KeccakSpongeStark { - type EvaluationFrame = EvmStarkFrame + type EvaluationFrame + = EvmStarkFrame where FE: FieldExtension, P: PackedField; diff --git a/evm_arithmetization/src/logic.rs b/evm_arithmetization/src/logic.rs index eae7c972b..0ba378b52 100644 --- a/evm_arithmetization/src/logic.rs +++ b/evm_arithmetization/src/logic.rs @@ -237,7 +237,8 @@ impl LogicStark { } impl, const D: usize> Stark for LogicStark { - type EvaluationFrame = EvmStarkFrame + type EvaluationFrame + = EvmStarkFrame where FE: FieldExtension, P: PackedField; diff --git a/evm_arithmetization/src/memory/memory_stark.rs b/evm_arithmetization/src/memory/memory_stark.rs index e58a85a26..357e72095 100644 --- a/evm_arithmetization/src/memory/memory_stark.rs +++ b/evm_arithmetization/src/memory/memory_stark.rs @@ -463,7 +463,8 @@ impl, const D: usize> MemoryStark { } impl, const D: usize> Stark for MemoryStark { - type EvaluationFrame = EvmStarkFrame + type EvaluationFrame + = EvmStarkFrame where FE: FieldExtension, P: PackedField; diff --git a/evm_arithmetization/src/memory_continuation/memory_continuation_stark.rs b/evm_arithmetization/src/memory_continuation/memory_continuation_stark.rs index 72c869b61..107c8dd5d 100644 --- a/evm_arithmetization/src/memory_continuation/memory_continuation_stark.rs +++ b/evm_arithmetization/src/memory_continuation/memory_continuation_stark.rs @@ -99,7 +99,8 @@ impl, const D: usize> MemoryContinuationStark } impl, const D: usize> Stark for MemoryContinuationStark { - type EvaluationFrame = EvmStarkFrame + type EvaluationFrame + = EvmStarkFrame where FE: FieldExtension, P: PackedField; diff --git a/evm_arithmetization/src/poseidon/poseidon_stark.rs b/evm_arithmetization/src/poseidon/poseidon_stark.rs index 0aacbaa50..ff4b5aed7 100644 --- a/evm_arithmetization/src/poseidon/poseidon_stark.rs +++ b/evm_arithmetization/src/poseidon/poseidon_stark.rs @@ -427,7 +427,8 @@ impl, const D: usize> PoseidonStark { } impl, const D: usize> Stark for PoseidonStark { - type EvaluationFrame = EvmStarkFrame + type EvaluationFrame + = EvmStarkFrame where FE: FieldExtension, P: PackedField; From 449887052f52b9ff58fa628757caf745603c075d Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Sat, 21 Sep 2024 10:00:17 -0700 Subject: [PATCH 03/91] fix clippy --- evm_arithmetization/src/poseidon/poseidon_stark.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/evm_arithmetization/src/poseidon/poseidon_stark.rs b/evm_arithmetization/src/poseidon/poseidon_stark.rs index ff4b5aed7..ae6f485fb 100644 --- a/evm_arithmetization/src/poseidon/poseidon_stark.rs +++ b/evm_arithmetization/src/poseidon/poseidon_stark.rs @@ -429,9 +429,9 @@ impl, const D: usize> PoseidonStark { impl, const D: usize> Stark for PoseidonStark { type EvaluationFrame = EvmStarkFrame - where - FE: FieldExtension, - P: PackedField; + where + FE: FieldExtension, + P: PackedField; type EvaluationFrameTarget = EvmStarkFrame, ExtensionTarget, NUM_COLUMNS>; From b14417fa8df83786c8354f3dbdf93767ed7fd77d Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Sun, 22 Sep 2024 14:36:52 -0700 Subject: [PATCH 04/91] wip --- .../src/fixed_recursive_verifier.rs | 40 +++++++++ evm_arithmetization/src/testing_utils.rs | 84 ++++++++++++++++++- 2 files changed, 122 insertions(+), 2 deletions(-) diff --git a/evm_arithmetization/src/fixed_recursive_verifier.rs b/evm_arithmetization/src/fixed_recursive_verifier.rs index cb3cec8a8..17b504b5c 100644 --- a/evm_arithmetization/src/fixed_recursive_verifier.rs +++ b/evm_arithmetization/src/fixed_recursive_verifier.rs @@ -36,6 +36,7 @@ use starky::proof::StarkProofWithMetadata; use starky::stark::Stark; use crate::all_stark::{all_cross_table_lookups, AllStark, Table, NUM_TABLES}; +use crate::all_stark::Table::Keccak; use crate::cpu::kernel::aggregator::KERNEL; use crate::generation::segments::{GenerationSegmentData, SegmentDataIterator, SegmentError}; use crate::generation::{GenerationInputs, TrimmedGenerationInputs}; @@ -1816,6 +1817,8 @@ where timing, abort_signal.clone(), )?; + dbg!(&all_proof.multi_proof.stark_proofs[*Table::Keccak].proof.trace_cap); + dbg!(&all_proof.multi_proof.stark_proofs[*Table::KeccakSponge].proof.trace_cap); let mut root_inputs = PartialWitness::new(); for table in 0..NUM_TABLES { @@ -2944,3 +2947,40 @@ where circuit.verifier_only.circuit_digest.elements.len() + (1 << circuit.common.config.fri_config.cap_height) * NUM_HASH_OUT_ELTS } + +#[cfg(test)] +mod tests { + use plonky2::field::goldilocks_field::GoldilocksField; + use plonky2::plonk::config::PoseidonGoldilocksConfig; + + use super::*; + use crate::testing_utils::{dummy_payload, init_logger}; + + type F = GoldilocksField; + const D: usize = 2; + type C = PoseidonGoldilocksConfig; + + #[test] + #[ignore] + fn test_root_proof_generation() -> anyhow::Result<()> { + init_logger(); + + let all_stark = AllStark::::default(); + let config = StarkConfig::standard_fast_config(); + + let all_circuits = AllRecursiveCircuits::::new( + &all_stark, + &[16..17, 8..9, 9..10, 4..9, 8..9, 4..7, 17..18, 17..18, 7..18], + &config, + ); + let dummy = dummy_payload(100, true)?; + + let timing = &mut TimingTree::new(&format!("Blockproof"), log::Level::Info); + let dummy_proof = + all_circuits.prove_all_segments(&all_stark, &config, dummy, 9, timing, None)?; + all_circuits.verify_root(dummy_proof[0].proof_with_pis.clone())?; + timing.print(); + + Ok(()) + } +} diff --git a/evm_arithmetization/src/testing_utils.rs b/evm_arithmetization/src/testing_utils.rs index 7018e06be..528cae9ed 100644 --- a/evm_arithmetization/src/testing_utils.rs +++ b/evm_arithmetization/src/testing_utils.rs @@ -1,18 +1,22 @@ //! A set of utility functions and constants to be used by `evm_arithmetization` //! unit and integration tests. +use anyhow::Result; use env_logger::{try_init_from_env, Env, DEFAULT_FILTER_ENV}; -use ethereum_types::{BigEndianHash, H256, U256}; +use ethereum_types::{Address, BigEndianHash, H256, U256}; use hex_literal::hex; use keccak_hash::keccak; use mpt_trie::{ nibbles::Nibbles, partial_trie::{HashedPartialTrie, Node, PartialTrie}, }; +use plonky2::field::goldilocks_field::GoldilocksField; pub use crate::cpu::kernel::cancun_constants::*; pub use crate::cpu::kernel::constants::global_exit_root::*; -use crate::{generation::mpt::AccountRlp, proof::BlockMetadata, util::h2u}; +use crate::generation::TrieInputs; +use crate::proof::TrieRoots; +use crate::{generation::mpt::AccountRlp, proof::BlockMetadata, util::h2u, GenerationInputs}; pub const EMPTY_NODE_HASH: H256 = H256(hex!( "56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421" @@ -161,3 +165,79 @@ pub fn scalable_contract_from_storage(storage_trie: &HashedPartialTrie) -> Accou ..Default::default() } } + +/// Get `GenerationInputs` for a dummy payload, where the block has the given +/// timestamp. +pub fn dummy_payload( + timestamp: u64, + is_first_payload: bool, +) -> Result> { + let beneficiary = hex!("deadbeefdeadbeefdeadbeefdeadbeefdeadbeef"); + + let block_metadata = BlockMetadata { + block_beneficiary: Address::from(beneficiary), + block_timestamp: timestamp.into(), + block_number: 1.into(), + block_difficulty: 0x020000.into(), + block_random: H256::from_uint(&0x020000.into()), + block_gaslimit: 0xff112233u32.into(), + block_chain_id: 1.into(), + block_base_fee: 0xa.into(), + ..Default::default() + }; + + let (mut state_trie_before, mut storage_tries) = preinitialized_state_and_storage_tries()?; + let checkpoint_state_trie_root = state_trie_before.hash(); + let mut beacon_roots_account_storage = storage_tries[0].1.clone(); + + update_beacon_roots_account_storage( + &mut beacon_roots_account_storage, + block_metadata.block_timestamp, + block_metadata.parent_beacon_block_root, + )?; + let updated_beacon_roots_account = + beacon_roots_contract_from_storage(&beacon_roots_account_storage); + + if !is_first_payload { + // This isn't the first dummy payload being processed. We need to update the + // initial state trie to account for the update on the beacon roots contract. + state_trie_before.insert( + beacon_roots_account_nibbles(), + rlp::encode(&updated_beacon_roots_account).to_vec(), + )?; + storage_tries[0].1 = beacon_roots_account_storage; + } + + let tries_before = TrieInputs { + state_trie: state_trie_before, + storage_tries, + ..Default::default() + }; + + let expected_state_trie_after: HashedPartialTrie = { + let mut state_trie_after = HashedPartialTrie::from(crate::Node::Empty); + state_trie_after.insert( + beacon_roots_account_nibbles(), + rlp::encode(&updated_beacon_roots_account).to_vec(), + )?; + + state_trie_after + }; + + let trie_roots_after = TrieRoots { + state_root: expected_state_trie_after.hash(), + transactions_root: tries_before.transactions_trie.hash(), + receipts_root: tries_before.receipts_trie.hash(), + }; + + let inputs = GenerationInputs { + tries: tries_before.clone(), + burn_addr: None, + trie_roots_after, + checkpoint_state_trie_root, + block_metadata, + ..Default::default() + }; + + Ok(inputs) +} From eabcf5e51395e8b0d788ca6bed9bb0c3523c5788 Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Sun, 22 Sep 2024 15:58:50 -0700 Subject: [PATCH 05/91] wip --- evm_arithmetization/src/all_stark.rs | 4 ++ .../src/fixed_recursive_verifier.rs | 39 +++++++++++++++---- 2 files changed, 35 insertions(+), 8 deletions(-) diff --git a/evm_arithmetization/src/all_stark.rs b/evm_arithmetization/src/all_stark.rs index f37021897..d899d9e50 100644 --- a/evm_arithmetization/src/all_stark.rs +++ b/evm_arithmetization/src/all_stark.rs @@ -126,6 +126,10 @@ pub const NUM_TABLES: usize = if cfg!(feature = "cdk_erigon") { Table::MemAfter as usize + 1 }; +/// Indices of Keccak Tables +pub const KECCAK_TABLES_INDICES: [usize; 2] = + [Table::Keccak as usize, Table::KeccakSponge as usize]; + impl Table { /// Returns all STARK table indices. pub(crate) const fn all() -> [Self; NUM_TABLES] { diff --git a/evm_arithmetization/src/fixed_recursive_verifier.rs b/evm_arithmetization/src/fixed_recursive_verifier.rs index 17b504b5c..927712b52 100644 --- a/evm_arithmetization/src/fixed_recursive_verifier.rs +++ b/evm_arithmetization/src/fixed_recursive_verifier.rs @@ -35,8 +35,10 @@ use starky::lookup::{get_grand_product_challenge_set_target, GrandProductChallen use starky::proof::StarkProofWithMetadata; use starky::stark::Stark; -use crate::all_stark::{all_cross_table_lookups, AllStark, Table, NUM_TABLES}; use crate::all_stark::Table::Keccak; +use crate::all_stark::{ + all_cross_table_lookups, AllStark, Table, KECCAK_TABLES_INDICES, NUM_TABLES, +}; use crate::cpu::kernel::aggregator::KERNEL; use crate::generation::segments::{GenerationSegmentData, SegmentDataIterator, SegmentError}; use crate::generation::{GenerationInputs, TrimmedGenerationInputs}; @@ -129,6 +131,8 @@ where /// for EVM root proofs; the circuit has them just to match the /// structure of aggregation proofs. cyclic_vk: VerifierCircuitTarget, + /// We can skip verifying Keccak tables when they are not in use. + enable_keccak_tables: BoolTarget, } impl RootCircuitData @@ -785,6 +789,7 @@ where let mut builder = CircuitBuilder::new(CircuitConfig::standard_recursion_config()); + let enable_keccak_tables = builder.add_virtual_bool_target_safe(); let public_values = add_virtual_public_values_public_input(&mut builder); let recursive_proofs = @@ -884,11 +889,20 @@ where let inner_verifier_data = builder.random_access_verifier_data(index_verifier_data[i], possible_vks); - builder.verify_proof::( - &recursive_proofs[i], - &inner_verifier_data, - inner_common_data[i], - ); + if KECCAK_TABLES_INDICES.contains(&i) { + builder.conditionally_verify_proof_or_dummy::( + enable_keccak_tables, + &recursive_proofs[i], + &inner_verifier_data, + inner_common_data[i], + )?; + } else { + builder.verify_proof::( + &recursive_proofs[i], + &inner_verifier_data, + inner_common_data[i], + ); + } } let merkle_before = @@ -919,6 +933,7 @@ where index_verifier_data, public_values, cyclic_vk, + enable_keccak_tables, } } @@ -1817,8 +1832,16 @@ where timing, abort_signal.clone(), )?; - dbg!(&all_proof.multi_proof.stark_proofs[*Table::Keccak].proof.trace_cap); - dbg!(&all_proof.multi_proof.stark_proofs[*Table::KeccakSponge].proof.trace_cap); + dbg!( + &all_proof.multi_proof.stark_proofs[*Table::Keccak] + .proof + .trace_cap + ); + dbg!( + &all_proof.multi_proof.stark_proofs[*Table::KeccakSponge] + .proof + .trace_cap + ); let mut root_inputs = PartialWitness::new(); for table in 0..NUM_TABLES { From 2b2e737ac9217608ac0a5376bf9e411140cb0dcb Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Mon, 23 Sep 2024 09:27:55 -0700 Subject: [PATCH 06/91] wip --- .../src/fixed_recursive_verifier.rs | 78 ++++++++++++++----- 1 file changed, 59 insertions(+), 19 deletions(-) diff --git a/evm_arithmetization/src/fixed_recursive_verifier.rs b/evm_arithmetization/src/fixed_recursive_verifier.rs index 927712b52..8c3851d36 100644 --- a/evm_arithmetization/src/fixed_recursive_verifier.rs +++ b/evm_arithmetization/src/fixed_recursive_verifier.rs @@ -7,6 +7,7 @@ use std::sync::Arc; use anyhow::anyhow; use hashbrown::HashMap; use itertools::{zip_eq, Itertools}; +use log::info; use mpt_trie::partial_trie::{HashedPartialTrie, Node, PartialTrie}; use plonky2::field::extension::Extendable; use plonky2::fri::FriParams; @@ -131,8 +132,8 @@ where /// for EVM root proofs; the circuit has them just to match the /// structure of aggregation proofs. cyclic_vk: VerifierCircuitTarget, - /// We can skip verifying Keccak tables when they are not in use. - enable_keccak_tables: BoolTarget, + // We can skip verifying Keccak tables when they are not in use. + // enable_keccak_tables: BoolTarget, } impl RootCircuitData @@ -790,6 +791,7 @@ where let mut builder = CircuitBuilder::new(CircuitConfig::standard_recursion_config()); let enable_keccak_tables = builder.add_virtual_bool_target_safe(); + let disable_keccak_tables = builder.not(enable_keccak_tables); let public_values = add_virtual_public_values_public_input(&mut builder); let recursive_proofs = @@ -860,6 +862,14 @@ where }) .collect_vec(); + // When Keccak Tables are disabled, Keccak Tables' ctl_zs_first should be 0s. + for &tbl in KECCAK_TABLES_INDICES.iter() { + for &t in pis[tbl].ctl_zs_first.iter() { + let keccak_ctl_check = builder.mul(disable_keccak_tables.target, t); + builder.assert_zero(keccak_ctl_check); + } + } + // Verify the CTL checks. verify_cross_table_lookups_circuit::( &mut builder, @@ -889,20 +899,20 @@ where let inner_verifier_data = builder.random_access_verifier_data(index_verifier_data[i], possible_vks); - if KECCAK_TABLES_INDICES.contains(&i) { - builder.conditionally_verify_proof_or_dummy::( - enable_keccak_tables, - &recursive_proofs[i], - &inner_verifier_data, - inner_common_data[i], - )?; - } else { - builder.verify_proof::( - &recursive_proofs[i], - &inner_verifier_data, - inner_common_data[i], - ); - } + // if KECCAK_TABLES_INDICES.contains(&i) { + // builder.conditionally_verify_proof_or_dummy::( + // enable_keccak_tables, + // &recursive_proofs[i], + // &inner_verifier_data, + // inner_common_data[i], + // )?; + // } else { + builder.verify_proof::( + &recursive_proofs[i], + &inner_verifier_data, + inner_common_data[i], + ); + // } } let merkle_before = @@ -933,7 +943,7 @@ where index_verifier_data, public_values, cyclic_vk, - enable_keccak_tables, + // enable_keccak_tables, } } @@ -1832,16 +1842,46 @@ where timing, abort_signal.clone(), )?; - dbg!( + info!( + "Debugging trace_cap.0[0]: {:?}", &all_proof.multi_proof.stark_proofs[*Table::Keccak] .proof .trace_cap + .0[0] + ); + + info!( + "Debugging openings.ctl_zs_first: {:?}", + &all_proof.multi_proof.stark_proofs[*Table::Keccak] + .proof + .openings + .ctl_zs_first ); - dbg!( + + info!( + "Debugging trace_cap.0[0] for KeccakSponge: {:?}", &all_proof.multi_proof.stark_proofs[*Table::KeccakSponge] .proof .trace_cap + .0[0] + ); + + info!( + "Debugging openings.ctl_zs_first for KeccakSponge: {:?}", + &all_proof.multi_proof.stark_proofs[*Table::KeccakSponge] + .proof + .openings + .ctl_zs_first ); + + info!( + "Debugging openings.ctl_zs_first for Logic: {:?}", + &all_proof.multi_proof.stark_proofs[*Table::Logic] + .proof + .openings + .ctl_zs_first + ); + let mut root_inputs = PartialWitness::new(); for table in 0..NUM_TABLES { From f63363b6ac07fd8d4c875223ce3997d6835e3264 Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Mon, 23 Sep 2024 09:31:48 -0700 Subject: [PATCH 07/91] cleanup --- evm_arithmetization/src/all_stark.rs | 4 -- .../src/fixed_recursive_verifier.rs | 68 +------------------ 2 files changed, 1 insertion(+), 71 deletions(-) diff --git a/evm_arithmetization/src/all_stark.rs b/evm_arithmetization/src/all_stark.rs index d899d9e50..f37021897 100644 --- a/evm_arithmetization/src/all_stark.rs +++ b/evm_arithmetization/src/all_stark.rs @@ -126,10 +126,6 @@ pub const NUM_TABLES: usize = if cfg!(feature = "cdk_erigon") { Table::MemAfter as usize + 1 }; -/// Indices of Keccak Tables -pub const KECCAK_TABLES_INDICES: [usize; 2] = - [Table::Keccak as usize, Table::KeccakSponge as usize]; - impl Table { /// Returns all STARK table indices. pub(crate) const fn all() -> [Self; NUM_TABLES] { diff --git a/evm_arithmetization/src/fixed_recursive_verifier.rs b/evm_arithmetization/src/fixed_recursive_verifier.rs index 8c3851d36..436a46bd2 100644 --- a/evm_arithmetization/src/fixed_recursive_verifier.rs +++ b/evm_arithmetization/src/fixed_recursive_verifier.rs @@ -7,7 +7,6 @@ use std::sync::Arc; use anyhow::anyhow; use hashbrown::HashMap; use itertools::{zip_eq, Itertools}; -use log::info; use mpt_trie::partial_trie::{HashedPartialTrie, Node, PartialTrie}; use plonky2::field::extension::Extendable; use plonky2::fri::FriParams; @@ -36,10 +35,7 @@ use starky::lookup::{get_grand_product_challenge_set_target, GrandProductChallen use starky::proof::StarkProofWithMetadata; use starky::stark::Stark; -use crate::all_stark::Table::Keccak; -use crate::all_stark::{ - all_cross_table_lookups, AllStark, Table, KECCAK_TABLES_INDICES, NUM_TABLES, -}; +use crate::all_stark::{all_cross_table_lookups, AllStark, Table, NUM_TABLES}; use crate::cpu::kernel::aggregator::KERNEL; use crate::generation::segments::{GenerationSegmentData, SegmentDataIterator, SegmentError}; use crate::generation::{GenerationInputs, TrimmedGenerationInputs}; @@ -132,8 +128,6 @@ where /// for EVM root proofs; the circuit has them just to match the /// structure of aggregation proofs. cyclic_vk: VerifierCircuitTarget, - // We can skip verifying Keccak tables when they are not in use. - // enable_keccak_tables: BoolTarget, } impl RootCircuitData @@ -790,8 +784,6 @@ where let mut builder = CircuitBuilder::new(CircuitConfig::standard_recursion_config()); - let enable_keccak_tables = builder.add_virtual_bool_target_safe(); - let disable_keccak_tables = builder.not(enable_keccak_tables); let public_values = add_virtual_public_values_public_input(&mut builder); let recursive_proofs = @@ -862,14 +854,6 @@ where }) .collect_vec(); - // When Keccak Tables are disabled, Keccak Tables' ctl_zs_first should be 0s. - for &tbl in KECCAK_TABLES_INDICES.iter() { - for &t in pis[tbl].ctl_zs_first.iter() { - let keccak_ctl_check = builder.mul(disable_keccak_tables.target, t); - builder.assert_zero(keccak_ctl_check); - } - } - // Verify the CTL checks. verify_cross_table_lookups_circuit::( &mut builder, @@ -899,20 +883,11 @@ where let inner_verifier_data = builder.random_access_verifier_data(index_verifier_data[i], possible_vks); - // if KECCAK_TABLES_INDICES.contains(&i) { - // builder.conditionally_verify_proof_or_dummy::( - // enable_keccak_tables, - // &recursive_proofs[i], - // &inner_verifier_data, - // inner_common_data[i], - // )?; - // } else { builder.verify_proof::( &recursive_proofs[i], &inner_verifier_data, inner_common_data[i], ); - // } } let merkle_before = @@ -943,7 +918,6 @@ where index_verifier_data, public_values, cyclic_vk, - // enable_keccak_tables, } } @@ -1842,46 +1816,6 @@ where timing, abort_signal.clone(), )?; - info!( - "Debugging trace_cap.0[0]: {:?}", - &all_proof.multi_proof.stark_proofs[*Table::Keccak] - .proof - .trace_cap - .0[0] - ); - - info!( - "Debugging openings.ctl_zs_first: {:?}", - &all_proof.multi_proof.stark_proofs[*Table::Keccak] - .proof - .openings - .ctl_zs_first - ); - - info!( - "Debugging trace_cap.0[0] for KeccakSponge: {:?}", - &all_proof.multi_proof.stark_proofs[*Table::KeccakSponge] - .proof - .trace_cap - .0[0] - ); - - info!( - "Debugging openings.ctl_zs_first for KeccakSponge: {:?}", - &all_proof.multi_proof.stark_proofs[*Table::KeccakSponge] - .proof - .openings - .ctl_zs_first - ); - - info!( - "Debugging openings.ctl_zs_first for Logic: {:?}", - &all_proof.multi_proof.stark_proofs[*Table::Logic] - .proof - .openings - .ctl_zs_first - ); - let mut root_inputs = PartialWitness::new(); for table in 0..NUM_TABLES { From cc9d41be6d29aad230a21e8e930d7a1821f757ec Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Mon, 23 Sep 2024 10:01:03 -0700 Subject: [PATCH 08/91] prove one segment --- .../src/fixed_recursive_verifier.rs | 74 ++++++++++++++++--- 1 file changed, 63 insertions(+), 11 deletions(-) diff --git a/evm_arithmetization/src/fixed_recursive_verifier.rs b/evm_arithmetization/src/fixed_recursive_verifier.rs index 436a46bd2..8d6dd8e05 100644 --- a/evm_arithmetization/src/fixed_recursive_verifier.rs +++ b/evm_arithmetization/src/fixed_recursive_verifier.rs @@ -2949,6 +2949,7 @@ where mod tests { use plonky2::field::goldilocks_field::GoldilocksField; use plonky2::plonk::config::PoseidonGoldilocksConfig; + use plonky2::timed; use super::*; use crate::testing_utils::{dummy_payload, init_logger}; @@ -2958,24 +2959,75 @@ mod tests { type C = PoseidonGoldilocksConfig; #[test] - #[ignore] - fn test_root_proof_generation() -> anyhow::Result<()> { + fn test_root_proof_generation_without_keccak() -> anyhow::Result<()> { + let timing = &mut TimingTree::new(&format!("segment proof"), log::Level::Info); init_logger(); let all_stark = AllStark::::default(); let config = StarkConfig::standard_fast_config(); - let all_circuits = AllRecursiveCircuits::::new( - &all_stark, - &[16..17, 8..9, 9..10, 4..9, 8..9, 4..7, 17..18, 17..18, 7..18], - &config, + let all_circuits = timed!( + timing, + log::Level::Info, + "create all circuits.", + AllRecursiveCircuits::::new( + &all_stark, + &[ + 16..17, + 8..9, + 9..10, + 4..5, + 8..9, + 4..5, + 17..18, + 17..18, + 17..18 + ], + &config, + ) + ); + let dummy = timed!( + timing, + log::Level::Info, + "generate dummy payload.", + dummy_payload(100, true)? ); - let dummy = dummy_payload(100, true)?; - let timing = &mut TimingTree::new(&format!("Blockproof"), log::Level::Info); - let dummy_proof = - all_circuits.prove_all_segments(&all_stark, &config, dummy, 9, timing, None)?; - all_circuits.verify_root(dummy_proof[0].proof_with_pis.clone())?; + let max_cpu_len_log = 9; + let segment_iterator = SegmentDataIterator::::new(&dummy, Some(max_cpu_len_log)); + + let mut dummy_proof = vec![]; + + let proof_without_keccak_index = 3; + for (i, segment_run) in segment_iterator.enumerate() { + if i < proof_without_keccak_index { + continue; + } + let (_, mut next_data) = + segment_run.map_err(|e: SegmentError| anyhow::format_err!(e))?; + let proof = timed!( + timing, + log::Level::Info, + "prove the segment.", + all_circuits.prove_segment( + &all_stark, + &config, + dummy.trim(), + &mut next_data, + timing, + None, + )? + ); + dummy_proof.push(proof); + break; + } + + timed!( + timing, + log::Level::Info, + "verify segment proofs.", + all_circuits.verify_root(dummy_proof[0].proof_with_pis.clone())? + ); timing.print(); Ok(()) From fe88819ad9d1c59e94d1b1440d93304071e4d5ec Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Mon, 23 Sep 2024 10:06:08 -0700 Subject: [PATCH 09/91] polish the code --- .../src/fixed_recursive_verifier.rs | 44 +++++++++++-------- 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/evm_arithmetization/src/fixed_recursive_verifier.rs b/evm_arithmetization/src/fixed_recursive_verifier.rs index 8d6dd8e05..7ee352c91 100644 --- a/evm_arithmetization/src/fixed_recursive_verifier.rs +++ b/evm_arithmetization/src/fixed_recursive_verifier.rs @@ -2959,8 +2959,8 @@ mod tests { type C = PoseidonGoldilocksConfig; #[test] - fn test_root_proof_generation_without_keccak() -> anyhow::Result<()> { - let timing = &mut TimingTree::new(&format!("segment proof"), log::Level::Info); + fn test_segment_proof_generation_without_keccak() -> anyhow::Result<()> { + let timing = &mut TimingTree::new("Segment Proof Generation", log::Level::Info); init_logger(); let all_stark = AllStark::::default(); @@ -2969,7 +2969,7 @@ mod tests { let all_circuits = timed!( timing, log::Level::Info, - "create all circuits.", + "Create all recursive circuits", AllRecursiveCircuits::::new( &all_stark, &[ @@ -2986,48 +2986,56 @@ mod tests { &config, ) ); - let dummy = timed!( + + // Generate a dummy payload for testing + let dummy_payload = timed!( timing, log::Level::Info, - "generate dummy payload.", + "Generate dummy payload", dummy_payload(100, true)? ); let max_cpu_len_log = 9; - let segment_iterator = SegmentDataIterator::::new(&dummy, Some(max_cpu_len_log)); + let segment_iterator = SegmentDataIterator::::new(&dummy_payload, Some(max_cpu_len_log)); - let mut dummy_proof = vec![]; + let mut proofs_without_keccak = vec![]; - let proof_without_keccak_index = 3; + let skip_proofs_before_index = 3; for (i, segment_run) in segment_iterator.enumerate() { - if i < proof_without_keccak_index { + if i < skip_proofs_before_index { continue; } - let (_, mut next_data) = + + // Process and prove segment + let (_, mut segment_data) = segment_run.map_err(|e: SegmentError| anyhow::format_err!(e))?; - let proof = timed!( + let segment_proof = timed!( timing, log::Level::Info, - "prove the segment.", + "Prove segment", all_circuits.prove_segment( &all_stark, &config, - dummy.trim(), - &mut next_data, + dummy_payload.trim(), + &mut segment_data, timing, None, )? ); - dummy_proof.push(proof); - break; + + proofs_without_keccak.push(segment_proof); + break; // Process only one proof } + // Verify the generated segment proof timed!( timing, log::Level::Info, - "verify segment proofs.", - all_circuits.verify_root(dummy_proof[0].proof_with_pis.clone())? + "Verify segment proof", + all_circuits.verify_root(proofs_without_keccak[0].proof_with_pis.clone())? ); + + // Print timing details timing.print(); Ok(()) From 7ff2fb11ab58eb462d98fdd62359f1287e313669 Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Mon, 23 Sep 2024 10:14:11 -0700 Subject: [PATCH 10/91] cleanup --- evm_arithmetization/tests/two_to_one_block.rs | 90 +------------------ 1 file changed, 3 insertions(+), 87 deletions(-) diff --git a/evm_arithmetization/tests/two_to_one_block.rs b/evm_arithmetization/tests/two_to_one_block.rs index a70c43c8c..9750f1dc7 100644 --- a/evm_arithmetization/tests/two_to_one_block.rs +++ b/evm_arithmetization/tests/two_to_one_block.rs @@ -1,22 +1,12 @@ #![cfg(feature = "eth_mainnet")] -use ethereum_types::{Address, BigEndianHash, H256}; use evm_arithmetization::fixed_recursive_verifier::{ extract_block_final_public_values, extract_two_to_one_block_hash, }; -use evm_arithmetization::generation::{GenerationInputs, TrieInputs}; -use evm_arithmetization::proof::{ - BlockMetadata, FinalPublicValues, PublicValues, TrieRoots, EMPTY_CONSOLIDATED_BLOCKHASH, -}; -use evm_arithmetization::testing_utils::{ - beacon_roots_account_nibbles, beacon_roots_contract_from_storage, init_logger, - preinitialized_state_and_storage_tries, update_beacon_roots_account_storage, -}; -use evm_arithmetization::{AllRecursiveCircuits, AllStark, Node, StarkConfig}; -use hex_literal::hex; -use mpt_trie::partial_trie::{HashedPartialTrie, PartialTrie}; +use evm_arithmetization::proof::{FinalPublicValues, PublicValues}; +use evm_arithmetization::testing_utils::{dummy_payload, init_logger}; +use evm_arithmetization::{AllRecursiveCircuits, AllStark, StarkConfig}; use plonky2::field::goldilocks_field::GoldilocksField; -use plonky2::field::types::Field; use plonky2::hash::poseidon::PoseidonHash; use plonky2::plonk::config::{Hasher, PoseidonGoldilocksConfig}; use plonky2::plonk::proof::ProofWithPublicInputs; @@ -26,80 +16,6 @@ type F = GoldilocksField; const D: usize = 2; type C = PoseidonGoldilocksConfig; -/// Get `GenerationInputs` for a dummy payload, where the block has the given -/// timestamp. -fn dummy_payload(timestamp: u64, is_first_payload: bool) -> anyhow::Result> { - let beneficiary = hex!("deadbeefdeadbeefdeadbeefdeadbeefdeadbeef"); - - let block_metadata = BlockMetadata { - block_beneficiary: Address::from(beneficiary), - block_timestamp: timestamp.into(), - block_number: 1.into(), - block_difficulty: 0x020000.into(), - block_random: H256::from_uint(&0x020000.into()), - block_gaslimit: 0xff112233u32.into(), - block_chain_id: 1.into(), - block_base_fee: 0xa.into(), - ..Default::default() - }; - - let (mut state_trie_before, mut storage_tries) = preinitialized_state_and_storage_tries()?; - let checkpoint_state_trie_root = state_trie_before.hash(); - let mut beacon_roots_account_storage = storage_tries[0].1.clone(); - - update_beacon_roots_account_storage( - &mut beacon_roots_account_storage, - block_metadata.block_timestamp, - block_metadata.parent_beacon_block_root, - )?; - let updated_beacon_roots_account = - beacon_roots_contract_from_storage(&beacon_roots_account_storage); - - if !is_first_payload { - // This isn't the first dummy payload being processed. We need to update the - // initial state trie to account for the update on the beacon roots contract. - state_trie_before.insert( - beacon_roots_account_nibbles(), - rlp::encode(&updated_beacon_roots_account).to_vec(), - )?; - storage_tries[0].1 = beacon_roots_account_storage; - } - - let tries_before = TrieInputs { - state_trie: state_trie_before, - storage_tries, - ..Default::default() - }; - - let expected_state_trie_after: HashedPartialTrie = { - let mut state_trie_after = HashedPartialTrie::from(Node::Empty); - state_trie_after.insert( - beacon_roots_account_nibbles(), - rlp::encode(&updated_beacon_roots_account).to_vec(), - )?; - - state_trie_after - }; - - let trie_roots_after = TrieRoots { - state_root: expected_state_trie_after.hash(), - transactions_root: tries_before.transactions_trie.hash(), - receipts_root: tries_before.receipts_trie.hash(), - }; - - let inputs = GenerationInputs { - tries: tries_before.clone(), - burn_addr: None, - trie_roots_after, - checkpoint_state_trie_root, - checkpoint_consolidated_hash: EMPTY_CONSOLIDATED_BLOCKHASH.map(F::from_canonical_u64), - block_metadata, - ..Default::default() - }; - - Ok(inputs) -} - fn get_test_block_proof( timestamp: u64, all_circuits: &AllRecursiveCircuits, From 6ce3cbe374fb6c6d4644d756f7495ef037cc5f07 Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Mon, 23 Sep 2024 10:29:20 -0700 Subject: [PATCH 11/91] Revert "cleanup" This reverts commit f63363b6ac07fd8d4c875223ce3997d6835e3264. --- evm_arithmetization/src/all_stark.rs | 4 ++ .../src/fixed_recursive_verifier.rs | 68 ++++++++++++++++++- 2 files changed, 71 insertions(+), 1 deletion(-) diff --git a/evm_arithmetization/src/all_stark.rs b/evm_arithmetization/src/all_stark.rs index f37021897..d899d9e50 100644 --- a/evm_arithmetization/src/all_stark.rs +++ b/evm_arithmetization/src/all_stark.rs @@ -126,6 +126,10 @@ pub const NUM_TABLES: usize = if cfg!(feature = "cdk_erigon") { Table::MemAfter as usize + 1 }; +/// Indices of Keccak Tables +pub const KECCAK_TABLES_INDICES: [usize; 2] = + [Table::Keccak as usize, Table::KeccakSponge as usize]; + impl Table { /// Returns all STARK table indices. pub(crate) const fn all() -> [Self; NUM_TABLES] { diff --git a/evm_arithmetization/src/fixed_recursive_verifier.rs b/evm_arithmetization/src/fixed_recursive_verifier.rs index 7ee352c91..c09051ec9 100644 --- a/evm_arithmetization/src/fixed_recursive_verifier.rs +++ b/evm_arithmetization/src/fixed_recursive_verifier.rs @@ -7,6 +7,7 @@ use std::sync::Arc; use anyhow::anyhow; use hashbrown::HashMap; use itertools::{zip_eq, Itertools}; +use log::info; use mpt_trie::partial_trie::{HashedPartialTrie, Node, PartialTrie}; use plonky2::field::extension::Extendable; use plonky2::fri::FriParams; @@ -35,7 +36,10 @@ use starky::lookup::{get_grand_product_challenge_set_target, GrandProductChallen use starky::proof::StarkProofWithMetadata; use starky::stark::Stark; -use crate::all_stark::{all_cross_table_lookups, AllStark, Table, NUM_TABLES}; +use crate::all_stark::Table::Keccak; +use crate::all_stark::{ + all_cross_table_lookups, AllStark, Table, KECCAK_TABLES_INDICES, NUM_TABLES, +}; use crate::cpu::kernel::aggregator::KERNEL; use crate::generation::segments::{GenerationSegmentData, SegmentDataIterator, SegmentError}; use crate::generation::{GenerationInputs, TrimmedGenerationInputs}; @@ -128,6 +132,8 @@ where /// for EVM root proofs; the circuit has them just to match the /// structure of aggregation proofs. cyclic_vk: VerifierCircuitTarget, + // We can skip verifying Keccak tables when they are not in use. + // enable_keccak_tables: BoolTarget, } impl RootCircuitData @@ -784,6 +790,8 @@ where let mut builder = CircuitBuilder::new(CircuitConfig::standard_recursion_config()); + let enable_keccak_tables = builder.add_virtual_bool_target_safe(); + let disable_keccak_tables = builder.not(enable_keccak_tables); let public_values = add_virtual_public_values_public_input(&mut builder); let recursive_proofs = @@ -854,6 +862,14 @@ where }) .collect_vec(); + // When Keccak Tables are disabled, Keccak Tables' ctl_zs_first should be 0s. + for &tbl in KECCAK_TABLES_INDICES.iter() { + for &t in pis[tbl].ctl_zs_first.iter() { + let keccak_ctl_check = builder.mul(disable_keccak_tables.target, t); + builder.assert_zero(keccak_ctl_check); + } + } + // Verify the CTL checks. verify_cross_table_lookups_circuit::( &mut builder, @@ -883,11 +899,20 @@ where let inner_verifier_data = builder.random_access_verifier_data(index_verifier_data[i], possible_vks); + // if KECCAK_TABLES_INDICES.contains(&i) { + // builder.conditionally_verify_proof_or_dummy::( + // enable_keccak_tables, + // &recursive_proofs[i], + // &inner_verifier_data, + // inner_common_data[i], + // )?; + // } else { builder.verify_proof::( &recursive_proofs[i], &inner_verifier_data, inner_common_data[i], ); + // } } let merkle_before = @@ -918,6 +943,7 @@ where index_verifier_data, public_values, cyclic_vk, + // enable_keccak_tables, } } @@ -1816,6 +1842,46 @@ where timing, abort_signal.clone(), )?; + info!( + "Debugging trace_cap.0[0]: {:?}", + &all_proof.multi_proof.stark_proofs[*Table::Keccak] + .proof + .trace_cap + .0[0] + ); + + info!( + "Debugging openings.ctl_zs_first: {:?}", + &all_proof.multi_proof.stark_proofs[*Table::Keccak] + .proof + .openings + .ctl_zs_first + ); + + info!( + "Debugging trace_cap.0[0] for KeccakSponge: {:?}", + &all_proof.multi_proof.stark_proofs[*Table::KeccakSponge] + .proof + .trace_cap + .0[0] + ); + + info!( + "Debugging openings.ctl_zs_first for KeccakSponge: {:?}", + &all_proof.multi_proof.stark_proofs[*Table::KeccakSponge] + .proof + .openings + .ctl_zs_first + ); + + info!( + "Debugging openings.ctl_zs_first for Logic: {:?}", + &all_proof.multi_proof.stark_proofs[*Table::Logic] + .proof + .openings + .ctl_zs_first + ); + let mut root_inputs = PartialWitness::new(); for table in 0..NUM_TABLES { From 8c940b04414cde430ef4c7a2e6d41c017289878a Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Mon, 23 Sep 2024 10:47:49 -0700 Subject: [PATCH 12/91] add [ignore] --- evm_arithmetization/src/fixed_recursive_verifier.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/evm_arithmetization/src/fixed_recursive_verifier.rs b/evm_arithmetization/src/fixed_recursive_verifier.rs index 7ee352c91..80f7f1b66 100644 --- a/evm_arithmetization/src/fixed_recursive_verifier.rs +++ b/evm_arithmetization/src/fixed_recursive_verifier.rs @@ -2959,6 +2959,7 @@ mod tests { type C = PoseidonGoldilocksConfig; #[test] + #[ignore] fn test_segment_proof_generation_without_keccak() -> anyhow::Result<()> { let timing = &mut TimingTree::new("Segment Proof Generation", log::Level::Info); init_logger(); From d6af500d3c1e485bf8c33e64d5ab5e3971f7ae5f Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Mon, 23 Sep 2024 11:12:18 -0700 Subject: [PATCH 13/91] add challenger state check --- .../src/fixed_recursive_verifier.rs | 47 ++++++++++++------- 1 file changed, 30 insertions(+), 17 deletions(-) diff --git a/evm_arithmetization/src/fixed_recursive_verifier.rs b/evm_arithmetization/src/fixed_recursive_verifier.rs index c09051ec9..befb03bb8 100644 --- a/evm_arithmetization/src/fixed_recursive_verifier.rs +++ b/evm_arithmetization/src/fixed_recursive_verifier.rs @@ -132,8 +132,8 @@ where /// for EVM root proofs; the circuit has them just to match the /// structure of aggregation proofs. cyclic_vk: VerifierCircuitTarget, - // We can skip verifying Keccak tables when they are not in use. - // enable_keccak_tables: BoolTarget, + /// We can skip verifying Keccak tables when they are not in use. + enable_keccak_tables: BoolTarget, } impl RootCircuitData @@ -844,6 +844,19 @@ where ) { builder.connect(before, after); } + if KECCAK_TABLES_INDICES.contains(&i) { + for (&before, &after) in zip_eq( + pis[i].challenger_state_before.as_ref(), + pis[i].challenger_state_after.as_ref(), + ) { + // Ensure that the challenger state remains consistent before and after Keccak + // tables. + let state_difference = builder.sub(before, after); + let keccak_consistency_check = + builder.mul(disable_keccak_tables.target, state_difference); + builder.assert_zero(keccak_consistency_check); + } + } } // Extra sums to add to the looked last value. @@ -899,20 +912,20 @@ where let inner_verifier_data = builder.random_access_verifier_data(index_verifier_data[i], possible_vks); - // if KECCAK_TABLES_INDICES.contains(&i) { - // builder.conditionally_verify_proof_or_dummy::( - // enable_keccak_tables, - // &recursive_proofs[i], - // &inner_verifier_data, - // inner_common_data[i], - // )?; - // } else { - builder.verify_proof::( - &recursive_proofs[i], - &inner_verifier_data, - inner_common_data[i], - ); - // } + if KECCAK_TABLES_INDICES.contains(&i) { + builder.conditionally_verify_proof_or_dummy::( + enable_keccak_tables, + &recursive_proofs[i], + &inner_verifier_data, + inner_common_data[i], + )?; + } else { + builder.verify_proof::( + &recursive_proofs[i], + &inner_verifier_data, + inner_common_data[i], + ); + } } let merkle_before = @@ -943,7 +956,7 @@ where index_verifier_data, public_values, cyclic_vk, - // enable_keccak_tables, + enable_keccak_tables, } } From f5deedbd1e75d64ad0572664bc44f7a8b28cd81e Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Mon, 23 Sep 2024 15:52:57 -0700 Subject: [PATCH 14/91] add more checks --- .../src/fixed_recursive_verifier.rs | 99 ++++++++++++------- evm_arithmetization/src/recursive_verifier.rs | 4 + 2 files changed, 68 insertions(+), 35 deletions(-) diff --git a/evm_arithmetization/src/fixed_recursive_verifier.rs b/evm_arithmetization/src/fixed_recursive_verifier.rs index befb03bb8..0dc837407 100644 --- a/evm_arithmetization/src/fixed_recursive_verifier.rs +++ b/evm_arithmetization/src/fixed_recursive_verifier.rs @@ -36,7 +36,6 @@ use starky::lookup::{get_grand_product_challenge_set_target, GrandProductChallen use starky::proof::StarkProofWithMetadata; use starky::stark::Stark; -use crate::all_stark::Table::Keccak; use crate::all_stark::{ all_cross_table_lookups, AllStark, Table, KECCAK_TABLES_INDICES, NUM_TABLES, }; @@ -156,6 +155,7 @@ where } self.public_values.to_buffer(buffer)?; buffer.write_target_verifier_circuit(&self.cyclic_vk)?; + buffer.write_target_bool(self.enable_keccak_tables)?; Ok(()) } @@ -175,6 +175,7 @@ where } let public_values = PublicValuesTarget::from_buffer(buffer)?; let cyclic_vk = buffer.read_target_verifier_circuit()?; + let enable_keccak_tables = buffer.read_target_bool()?; Ok(Self { circuit, @@ -182,6 +183,7 @@ where index_verifier_data: index_verifier_data.try_into().unwrap(), public_values, cyclic_vk, + enable_keccak_tables, }) } } @@ -811,6 +813,15 @@ where } } + for i in KECCAK_TABLES_INDICES { + for h in &pis[i].trace_cap { + for t in h { + let trace_cap_check = builder.mul(disable_keccak_tables.target, *t); + builder.assert_zero(trace_cap_check); + } + } + } + observe_public_values_target::(&mut challenger, &public_values); let ctl_challenges = get_grand_product_challenge_set_target( @@ -819,16 +830,35 @@ where stark_config.num_challenges, ); // Check that the correct CTL challenges are used in every proof. - for pi in &pis { - for i in 0..stark_config.num_challenges { - builder.connect( - ctl_challenges.challenges[i].beta, - pi.ctl_challenges.challenges[i].beta, - ); - builder.connect( - ctl_challenges.challenges[i].gamma, - pi.ctl_challenges.challenges[i].gamma, - ); + for (i, pi) in pis.iter().enumerate() { + if KECCAK_TABLES_INDICES.contains(&i) { + // Ensures that the correct CTL challenges are used in Keccak tables when + // `enable_keccak_tables` is true. + for i in 0..stark_config.num_challenges { + let beta_diff = builder.sub( + ctl_challenges.challenges[i].beta, + pi.ctl_challenges.challenges[i].beta, + ); + let gamma_diff = builder.sub( + ctl_challenges.challenges[i].gamma, + pi.ctl_challenges.challenges[i].gamma, + ); + let beta_check = builder.mul(enable_keccak_tables.target, beta_diff); + let gamma_check = builder.mul(enable_keccak_tables.target, gamma_diff); + builder.assert_zero(beta_check); + builder.assert_zero(gamma_check); + } + } else { + for i in 0..stark_config.num_challenges { + builder.connect( + ctl_challenges.challenges[i].beta, + pi.ctl_challenges.challenges[i].beta, + ); + builder.connect( + ctl_challenges.challenges[i].gamma, + pi.ctl_challenges.challenges[i].gamma, + ); + } } } @@ -845,16 +875,16 @@ where builder.connect(before, after); } if KECCAK_TABLES_INDICES.contains(&i) { + // Ensure that the challenger state remains consistent before and after Keccak + // tables. for (&before, &after) in zip_eq( pis[i].challenger_state_before.as_ref(), pis[i].challenger_state_after.as_ref(), ) { - // Ensure that the challenger state remains consistent before and after Keccak - // tables. let state_difference = builder.sub(before, after); - let keccak_consistency_check = + let challenger_state_check = builder.mul(disable_keccak_tables.target, state_difference); - builder.assert_zero(keccak_consistency_check); + builder.assert_zero(challenger_state_check); } } } @@ -878,8 +908,8 @@ where // When Keccak Tables are disabled, Keccak Tables' ctl_zs_first should be 0s. for &tbl in KECCAK_TABLES_INDICES.iter() { for &t in pis[tbl].ctl_zs_first.iter() { - let keccak_ctl_check = builder.mul(disable_keccak_tables.target, t); - builder.assert_zero(keccak_ctl_check); + let ctl_check = builder.mul(disable_keccak_tables.target, t); + builder.assert_zero(ctl_check); } } @@ -913,12 +943,14 @@ where builder.random_access_verifier_data(index_verifier_data[i], possible_vks); if KECCAK_TABLES_INDICES.contains(&i) { - builder.conditionally_verify_proof_or_dummy::( - enable_keccak_tables, - &recursive_proofs[i], - &inner_verifier_data, - inner_common_data[i], - )?; + builder + .conditionally_verify_proof_or_dummy::( + enable_keccak_tables, + &recursive_proofs[i], + &inner_verifier_data, + inner_common_data[i], + ) + .expect("Unable conditionally verify Keccak proofs in the root circuit"); } else { builder.verify_proof::( &recursive_proofs[i], @@ -1940,6 +1972,9 @@ where anyhow::Error::msg("Invalid conversion when setting public values targets.") })?; + // TODO(sdeng): Set to false when this segment contains no Keccak operations. + root_inputs.set_bool_target(self.root.enable_keccak_tables, true); + let root_proof = self.root.circuit.prove(root_inputs)?; Ok(ProverOutputData { @@ -2080,6 +2115,9 @@ where anyhow::Error::msg("Invalid conversion when setting public values targets.") })?; + // TODO(sdeng): Set to false when this segment contains no Keccak operations. + root_inputs.set_bool_target(self.root.enable_keccak_tables, true); + let root_proof = self.root.circuit.prove(root_inputs)?; Ok((root_proof, all_proof.public_values)) @@ -3038,6 +3076,7 @@ mod tests { type C = PoseidonGoldilocksConfig; #[test] + #[ignore] fn test_segment_proof_generation_without_keccak() -> anyhow::Result<()> { let timing = &mut TimingTree::new("Segment Proof Generation", log::Level::Info); init_logger(); @@ -3051,17 +3090,7 @@ mod tests { "Create all recursive circuits", AllRecursiveCircuits::::new( &all_stark, - &[ - 16..17, - 8..9, - 9..10, - 4..5, - 8..9, - 4..5, - 17..18, - 17..18, - 17..18 - ], + &[16..17, 8..9, 9..10, 4..9, 8..9, 4..7, 17..18, 17..18, 7..18], &config, ) ); @@ -3103,7 +3132,7 @@ mod tests { ); proofs_without_keccak.push(segment_proof); - break; // Process only one proof + // break; // Process only one proof } // Verify the generated segment proof diff --git a/evm_arithmetization/src/recursive_verifier.rs b/evm_arithmetization/src/recursive_verifier.rs index 417bf468d..14ed5957e 100644 --- a/evm_arithmetization/src/recursive_verifier.rs +++ b/evm_arithmetization/src/recursive_verifier.rs @@ -4,6 +4,7 @@ use core::fmt::Debug; use anyhow::Result; use ethereum_types::{BigEndianHash, U256}; use plonky2::field::extension::Extendable; +use plonky2::gates::constant::ConstantGate; use plonky2::gates::exponentiation::ExponentiationGate; use plonky2::gates::gate::GateRef; use plonky2::gates::noop::NoopGate; @@ -326,6 +327,9 @@ pub(crate) fn add_common_recursion_gates, const D: builder.add_gate_to_gate_set(GateRef::new(ExponentiationGate::new_from_config( &builder.config, ))); + builder.add_gate_to_gate_set(GateRef::new(ConstantGate::new( + builder.config.num_constants, + ))); } /// Recursive version of `get_memory_extra_looking_sum`. From e39894ad6dfc2f2217fb8b9f5550c2ab77ad22d9 Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Mon, 23 Sep 2024 16:06:33 -0700 Subject: [PATCH 15/91] cleanup --- .../src/fixed_recursive_verifier.rs | 34 +++++++++---------- 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/evm_arithmetization/src/fixed_recursive_verifier.rs b/evm_arithmetization/src/fixed_recursive_verifier.rs index 0dc837407..4beda37b6 100644 --- a/evm_arithmetization/src/fixed_recursive_verifier.rs +++ b/evm_arithmetization/src/fixed_recursive_verifier.rs @@ -831,32 +831,30 @@ where ); // Check that the correct CTL challenges are used in every proof. for (i, pi) in pis.iter().enumerate() { - if KECCAK_TABLES_INDICES.contains(&i) { - // Ensures that the correct CTL challenges are used in Keccak tables when - // `enable_keccak_tables` is true. - for i in 0..stark_config.num_challenges { + for j in 0..stark_config.num_challenges { + if KECCAK_TABLES_INDICES.contains(&i) { + // Ensures that the correct CTL challenges are used in Keccak tables when + // `enable_keccak_tables` is true. let beta_diff = builder.sub( - ctl_challenges.challenges[i].beta, - pi.ctl_challenges.challenges[i].beta, + ctl_challenges.challenges[j].beta, + pi.ctl_challenges.challenges[j].beta, ); let gamma_diff = builder.sub( - ctl_challenges.challenges[i].gamma, - pi.ctl_challenges.challenges[i].gamma, + ctl_challenges.challenges[j].gamma, + pi.ctl_challenges.challenges[j].gamma, ); let beta_check = builder.mul(enable_keccak_tables.target, beta_diff); let gamma_check = builder.mul(enable_keccak_tables.target, gamma_diff); builder.assert_zero(beta_check); builder.assert_zero(gamma_check); - } - } else { - for i in 0..stark_config.num_challenges { + } else { builder.connect( - ctl_challenges.challenges[i].beta, - pi.ctl_challenges.challenges[i].beta, + ctl_challenges.challenges[j].beta, + pi.ctl_challenges.challenges[j].beta, ); builder.connect( - ctl_challenges.challenges[i].gamma, - pi.ctl_challenges.challenges[i].gamma, + ctl_challenges.challenges[j].gamma, + pi.ctl_challenges.challenges[j].gamma, ); } } @@ -906,8 +904,8 @@ where .collect_vec(); // When Keccak Tables are disabled, Keccak Tables' ctl_zs_first should be 0s. - for &tbl in KECCAK_TABLES_INDICES.iter() { - for &t in pis[tbl].ctl_zs_first.iter() { + for &i in KECCAK_TABLES_INDICES.iter() { + for &t in pis[i].ctl_zs_first.iter() { let ctl_check = builder.mul(disable_keccak_tables.target, t); builder.assert_zero(ctl_check); } @@ -3132,7 +3130,7 @@ mod tests { ); proofs_without_keccak.push(segment_proof); - // break; // Process only one proof + break; // Process only one proof } // Verify the generated segment proof From 948d74235b4b36ae0798f6a5c4917c25bda60a21 Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Tue, 24 Sep 2024 11:17:41 -0700 Subject: [PATCH 16/91] add empty keccak tables bool --- evm_arithmetization/src/generation/mod.rs | 16 ++++++++++++++-- evm_arithmetization/src/proof.rs | 4 ++++ evm_arithmetization/src/prover.rs | 21 +++++++++++++++------ 3 files changed, 33 insertions(+), 8 deletions(-) diff --git a/evm_arithmetization/src/generation/mod.rs b/evm_arithmetization/src/generation/mod.rs index 9c7625d2b..3b24e46b1 100644 --- a/evm_arithmetization/src/generation/mod.rs +++ b/evm_arithmetization/src/generation/mod.rs @@ -453,7 +453,11 @@ fn get_all_memory_address_and_values(memory_before: &MemoryState) -> Vec<(Memory res } -type TablesWithPVsAndFinalMem = ([Vec>; NUM_TABLES], PublicValues); +pub(crate) struct TablesWithPVsAndFinalMem { + pub tables: [Vec>; NUM_TABLES], + pub empty_keccak_tables: bool, + pub public_values: PublicValues, +} pub fn generate_traces, const D: usize>( all_stark: &AllStark, @@ -550,6 +554,9 @@ pub fn generate_traces, const D: usize>( mem_after: MemCap::default(), }; + let empty_keccak_tables = + state.traces.keccak_inputs.is_empty() && state.traces.keccak_sponge_ops.is_empty(); + let tables = timed!( timing, "convert trace data to tables", @@ -562,7 +569,12 @@ pub fn generate_traces, const D: usize>( timing ) ); - Ok((tables, public_values)) + + Ok(TablesWithPVsAndFinalMem { + tables, + empty_keccak_tables, + public_values, + }) } fn simulate_cpu( diff --git a/evm_arithmetization/src/proof.rs b/evm_arithmetization/src/proof.rs index d4a04deff..76eef9f54 100644 --- a/evm_arithmetization/src/proof.rs +++ b/evm_arithmetization/src/proof.rs @@ -31,6 +31,10 @@ pub struct AllProof, C: GenericConfig, co pub multi_proof: MultiProof, /// Public memory values used for the recursive proofs. pub public_values: PublicValues, + /// A flag indicating whether the Keccak and KeccakSponge tables contain + /// only padding values (i.e., no meaningful data). This is set when no + /// actual Keccak operations were performed. + pub empty_keccak_tables: bool, } impl, C: GenericConfig, const D: usize> AllProof { diff --git a/evm_arithmetization/src/prover.rs b/evm_arithmetization/src/prover.rs index 079fe3469..137bed656 100644 --- a/evm_arithmetization/src/prover.rs +++ b/evm_arithmetization/src/prover.rs @@ -20,8 +20,9 @@ use starky::proof::{MultiProof, StarkProofWithMetadata}; use starky::prover::prove_with_commitment; use starky::stark::Stark; -use crate::all_stark::{AllStark, Table, NUM_TABLES}; +use crate::all_stark::{AllStark, Table, KECCAK_TABLES_INDICES, NUM_TABLES}; use crate::cpu::kernel::aggregator::KERNEL; +use crate::cpu::kernel::constants::trie_type::PartialTrieType::Hash; use crate::generation::segments::GenerationSegmentData; use crate::generation::{generate_traces, GenerationInputs, TrimmedGenerationInputs}; use crate::get_challenges::observe_public_values; @@ -47,7 +48,7 @@ where timed!(timing, "build kernel", Lazy::force(&KERNEL)); - let (traces, mut public_values) = timed!( + let mut tables_with_pvs = timed!( timing, "generate all traces", generate_traces(all_stark, &inputs, config, segment_data, timing)? @@ -58,8 +59,9 @@ where let proof = prove_with_traces( all_stark, config, - traces, - &mut public_values, + tables_with_pvs.tables, + tables_with_pvs.empty_keccak_tables, + &mut tables_with_pvs.public_values, timing, abort_signal, )?; @@ -72,6 +74,7 @@ pub(crate) fn prove_with_traces( all_stark: &AllStark, config: &StarkConfig, trace_poly_values: [Vec>; NUM_TABLES], + empty_keccak_tables: bool, public_values: &mut PublicValues, timing: &mut TimingTree, abort_signal: Option>, @@ -114,8 +117,13 @@ where .map(|c| c.merkle_tree.cap.clone()) .collect::>(); let mut challenger = Challenger::::new(); - for cap in &trace_caps { - challenger.observe_cap(cap); + for (i, cap) in trace_caps.iter().enumerate() { + if KECCAK_TABLES_INDICES.contains(&i) { + let zero_merkle_cap = cap.flatten().iter().map(|_| F::ZERO).collect::>(); + challenger.observe_elements(&zero_merkle_cap); + } else { + challenger.observe_cap(cap); + } } observe_public_values::(&mut challenger, public_values) @@ -206,6 +214,7 @@ where ctl_challenges, }, public_values: public_values.clone(), + empty_keccak_tables, }) } From 49bb96b9238f9ea6dbc4841d8e29378027e2567e Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Tue, 24 Sep 2024 14:05:12 -0700 Subject: [PATCH 17/91] enable no keccak recursion --- .../src/fixed_recursive_verifier.rs | 33 ++++++++++++++++--- evm_arithmetization/src/generation/mod.rs | 2 +- evm_arithmetization/src/prover.rs | 1 - 3 files changed, 29 insertions(+), 7 deletions(-) diff --git a/evm_arithmetization/src/fixed_recursive_verifier.rs b/evm_arithmetization/src/fixed_recursive_verifier.rs index 4beda37b6..4d01a09cf 100644 --- a/evm_arithmetization/src/fixed_recursive_verifier.rs +++ b/evm_arithmetization/src/fixed_recursive_verifier.rs @@ -1928,9 +1928,21 @@ where let mut root_inputs = PartialWitness::new(); for table in 0..NUM_TABLES { + let table_circuits = &self.by_table[table]; + if KECCAK_TABLES_INDICES.contains(&table) && all_proof.empty_keccak_tables { + let index_verifier_data = table_circuits + .by_stark_size + .keys() + .min() + .expect("No valid size found in the shrinking circuits"); + root_inputs.set_target( + self.root.index_verifier_data[table], + F::from_canonical_usize(*index_verifier_data), + ); + continue; + } let stark_proof = &all_proof.multi_proof.stark_proofs[table]; let original_degree_bits = stark_proof.proof.recover_degree_bits(config); - let table_circuits = &self.by_table[table]; let shrunk_proof = table_circuits .by_stark_size .get(&original_degree_bits) @@ -1970,8 +1982,10 @@ where anyhow::Error::msg("Invalid conversion when setting public values targets.") })?; - // TODO(sdeng): Set to false when this segment contains no Keccak operations. - root_inputs.set_bool_target(self.root.enable_keccak_tables, true); + root_inputs.set_bool_target( + self.root.enable_keccak_tables, + all_proof.empty_keccak_tables, + ); let root_proof = self.root.circuit.prove(root_inputs)?; @@ -2085,6 +2099,13 @@ where for table in 0..NUM_TABLES { let (table_circuit, index_verifier_data) = &table_circuits[table]; + if KECCAK_TABLES_INDICES.contains(&table) && all_proof.empty_keccak_tables { + root_inputs.set_target( + self.root.index_verifier_data[table], + F::from_canonical_u8(*index_verifier_data), + ); + continue; + } let stark_proof = &all_proof.multi_proof.stark_proofs[table]; @@ -2113,8 +2134,10 @@ where anyhow::Error::msg("Invalid conversion when setting public values targets.") })?; - // TODO(sdeng): Set to false when this segment contains no Keccak operations. - root_inputs.set_bool_target(self.root.enable_keccak_tables, true); + root_inputs.set_bool_target( + self.root.enable_keccak_tables, + all_proof.empty_keccak_tables, + ); let root_proof = self.root.circuit.prove(root_inputs)?; diff --git a/evm_arithmetization/src/generation/mod.rs b/evm_arithmetization/src/generation/mod.rs index 3b24e46b1..c11696543 100644 --- a/evm_arithmetization/src/generation/mod.rs +++ b/evm_arithmetization/src/generation/mod.rs @@ -453,7 +453,7 @@ fn get_all_memory_address_and_values(memory_before: &MemoryState) -> Vec<(Memory res } -pub(crate) struct TablesWithPVsAndFinalMem { +pub struct TablesWithPVsAndFinalMem { pub tables: [Vec>; NUM_TABLES], pub empty_keccak_tables: bool, pub public_values: PublicValues, diff --git a/evm_arithmetization/src/prover.rs b/evm_arithmetization/src/prover.rs index 137bed656..8d2df1a0d 100644 --- a/evm_arithmetization/src/prover.rs +++ b/evm_arithmetization/src/prover.rs @@ -22,7 +22,6 @@ use starky::stark::Stark; use crate::all_stark::{AllStark, Table, KECCAK_TABLES_INDICES, NUM_TABLES}; use crate::cpu::kernel::aggregator::KERNEL; -use crate::cpu::kernel::constants::trie_type::PartialTrieType::Hash; use crate::generation::segments::GenerationSegmentData; use crate::generation::{generate_traces, GenerationInputs, TrimmedGenerationInputs}; use crate::get_challenges::observe_public_values; From bbba82057d13541de8f8a9c572300d124bd42817 Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Tue, 24 Sep 2024 14:38:19 -0700 Subject: [PATCH 18/91] refactor --- .../src/fixed_recursive_verifier.rs | 24 +++++++------------ evm_arithmetization/src/generation/mod.rs | 8 +++---- evm_arithmetization/src/proof.rs | 6 ++--- evm_arithmetization/src/prover.rs | 6 ++--- 4 files changed, 19 insertions(+), 25 deletions(-) diff --git a/evm_arithmetization/src/fixed_recursive_verifier.rs b/evm_arithmetization/src/fixed_recursive_verifier.rs index 4d01a09cf..bdf9a8fc0 100644 --- a/evm_arithmetization/src/fixed_recursive_verifier.rs +++ b/evm_arithmetization/src/fixed_recursive_verifier.rs @@ -132,7 +132,7 @@ where /// structure of aggregation proofs. cyclic_vk: VerifierCircuitTarget, /// We can skip verifying Keccak tables when they are not in use. - enable_keccak_tables: BoolTarget, + use_keccak_tables: BoolTarget, } impl RootCircuitData @@ -155,7 +155,7 @@ where } self.public_values.to_buffer(buffer)?; buffer.write_target_verifier_circuit(&self.cyclic_vk)?; - buffer.write_target_bool(self.enable_keccak_tables)?; + buffer.write_target_bool(self.use_keccak_tables)?; Ok(()) } @@ -175,7 +175,7 @@ where } let public_values = PublicValuesTarget::from_buffer(buffer)?; let cyclic_vk = buffer.read_target_verifier_circuit()?; - let enable_keccak_tables = buffer.read_target_bool()?; + let use_keccak_tables = buffer.read_target_bool()?; Ok(Self { circuit, @@ -183,7 +183,7 @@ where index_verifier_data: index_verifier_data.try_into().unwrap(), public_values, cyclic_vk, - enable_keccak_tables, + use_keccak_tables, }) } } @@ -986,7 +986,7 @@ where index_verifier_data, public_values, cyclic_vk, - enable_keccak_tables, + use_keccak_tables: enable_keccak_tables, } } @@ -1929,7 +1929,7 @@ where for table in 0..NUM_TABLES { let table_circuits = &self.by_table[table]; - if KECCAK_TABLES_INDICES.contains(&table) && all_proof.empty_keccak_tables { + if KECCAK_TABLES_INDICES.contains(&table) && !all_proof.use_keccak_tables { let index_verifier_data = table_circuits .by_stark_size .keys() @@ -1982,10 +1982,7 @@ where anyhow::Error::msg("Invalid conversion when setting public values targets.") })?; - root_inputs.set_bool_target( - self.root.enable_keccak_tables, - all_proof.empty_keccak_tables, - ); + root_inputs.set_bool_target(self.root.use_keccak_tables, all_proof.use_keccak_tables); let root_proof = self.root.circuit.prove(root_inputs)?; @@ -2099,7 +2096,7 @@ where for table in 0..NUM_TABLES { let (table_circuit, index_verifier_data) = &table_circuits[table]; - if KECCAK_TABLES_INDICES.contains(&table) && all_proof.empty_keccak_tables { + if KECCAK_TABLES_INDICES.contains(&table) && !all_proof.use_keccak_tables { root_inputs.set_target( self.root.index_verifier_data[table], F::from_canonical_u8(*index_verifier_data), @@ -2134,10 +2131,7 @@ where anyhow::Error::msg("Invalid conversion when setting public values targets.") })?; - root_inputs.set_bool_target( - self.root.enable_keccak_tables, - all_proof.empty_keccak_tables, - ); + root_inputs.set_bool_target(self.root.use_keccak_tables, all_proof.use_keccak_tables); let root_proof = self.root.circuit.prove(root_inputs)?; diff --git a/evm_arithmetization/src/generation/mod.rs b/evm_arithmetization/src/generation/mod.rs index c11696543..7ab1249d6 100644 --- a/evm_arithmetization/src/generation/mod.rs +++ b/evm_arithmetization/src/generation/mod.rs @@ -455,7 +455,7 @@ fn get_all_memory_address_and_values(memory_before: &MemoryState) -> Vec<(Memory pub struct TablesWithPVsAndFinalMem { pub tables: [Vec>; NUM_TABLES], - pub empty_keccak_tables: bool, + pub use_keccak_tables: bool, pub public_values: PublicValues, } @@ -554,8 +554,8 @@ pub fn generate_traces, const D: usize>( mem_after: MemCap::default(), }; - let empty_keccak_tables = - state.traces.keccak_inputs.is_empty() && state.traces.keccak_sponge_ops.is_empty(); + let use_keccak_tables = + !state.traces.keccak_inputs.is_empty() || !state.traces.keccak_sponge_ops.is_empty(); let tables = timed!( timing, @@ -572,7 +572,7 @@ pub fn generate_traces, const D: usize>( Ok(TablesWithPVsAndFinalMem { tables, - empty_keccak_tables, + use_keccak_tables, public_values, }) } diff --git a/evm_arithmetization/src/proof.rs b/evm_arithmetization/src/proof.rs index 76eef9f54..50364d306 100644 --- a/evm_arithmetization/src/proof.rs +++ b/evm_arithmetization/src/proof.rs @@ -32,9 +32,9 @@ pub struct AllProof, C: GenericConfig, co /// Public memory values used for the recursive proofs. pub public_values: PublicValues, /// A flag indicating whether the Keccak and KeccakSponge tables contain - /// only padding values (i.e., no meaningful data). This is set when no - /// actual Keccak operations were performed. - pub empty_keccak_tables: bool, + /// only padding values (i.e., no meaningful data). This is set to false + /// when no actual Keccak operations were performed. + pub use_keccak_tables: bool, } impl, C: GenericConfig, const D: usize> AllProof { diff --git a/evm_arithmetization/src/prover.rs b/evm_arithmetization/src/prover.rs index 8d2df1a0d..f1684f470 100644 --- a/evm_arithmetization/src/prover.rs +++ b/evm_arithmetization/src/prover.rs @@ -59,7 +59,7 @@ where all_stark, config, tables_with_pvs.tables, - tables_with_pvs.empty_keccak_tables, + tables_with_pvs.use_keccak_tables, &mut tables_with_pvs.public_values, timing, abort_signal, @@ -73,7 +73,7 @@ pub(crate) fn prove_with_traces( all_stark: &AllStark, config: &StarkConfig, trace_poly_values: [Vec>; NUM_TABLES], - empty_keccak_tables: bool, + use_keccak_tables: bool, public_values: &mut PublicValues, timing: &mut TimingTree, abort_signal: Option>, @@ -213,7 +213,7 @@ where ctl_challenges, }, public_values: public_values.clone(), - empty_keccak_tables, + use_keccak_tables, }) } From 8c4a3625a844ffde773f9a23b0dbccbbe2604f39 Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Tue, 24 Sep 2024 14:43:28 -0700 Subject: [PATCH 19/91] fix tests --- evm_arithmetization/src/prover.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/evm_arithmetization/src/prover.rs b/evm_arithmetization/src/prover.rs index f1684f470..f165bed05 100644 --- a/evm_arithmetization/src/prover.rs +++ b/evm_arithmetization/src/prover.rs @@ -117,7 +117,7 @@ where .collect::>(); let mut challenger = Challenger::::new(); for (i, cap) in trace_caps.iter().enumerate() { - if KECCAK_TABLES_INDICES.contains(&i) { + if KECCAK_TABLES_INDICES.contains(&i) && !use_keccak_tables { let zero_merkle_cap = cap.flatten().iter().map(|_| F::ZERO).collect::>(); challenger.observe_elements(&zero_merkle_cap); } else { From 5cdb37db99ba0bdd4a57582bcf0718f6a5aef434 Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Tue, 24 Sep 2024 15:16:50 -0700 Subject: [PATCH 20/91] rename --- .../src/fixed_recursive_verifier.rs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/evm_arithmetization/src/fixed_recursive_verifier.rs b/evm_arithmetization/src/fixed_recursive_verifier.rs index bdf9a8fc0..ec561414c 100644 --- a/evm_arithmetization/src/fixed_recursive_verifier.rs +++ b/evm_arithmetization/src/fixed_recursive_verifier.rs @@ -792,8 +792,8 @@ where let mut builder = CircuitBuilder::new(CircuitConfig::standard_recursion_config()); - let enable_keccak_tables = builder.add_virtual_bool_target_safe(); - let disable_keccak_tables = builder.not(enable_keccak_tables); + let use_keccak_tables = builder.add_virtual_bool_target_safe(); + let skip_keccak_tables = builder.not(use_keccak_tables); let public_values = add_virtual_public_values_public_input(&mut builder); let recursive_proofs = @@ -816,7 +816,7 @@ where for i in KECCAK_TABLES_INDICES { for h in &pis[i].trace_cap { for t in h { - let trace_cap_check = builder.mul(disable_keccak_tables.target, *t); + let trace_cap_check = builder.mul(skip_keccak_tables.target, *t); builder.assert_zero(trace_cap_check); } } @@ -843,8 +843,8 @@ where ctl_challenges.challenges[j].gamma, pi.ctl_challenges.challenges[j].gamma, ); - let beta_check = builder.mul(enable_keccak_tables.target, beta_diff); - let gamma_check = builder.mul(enable_keccak_tables.target, gamma_diff); + let beta_check = builder.mul(use_keccak_tables.target, beta_diff); + let gamma_check = builder.mul(use_keccak_tables.target, gamma_diff); builder.assert_zero(beta_check); builder.assert_zero(gamma_check); } else { @@ -881,7 +881,7 @@ where ) { let state_difference = builder.sub(before, after); let challenger_state_check = - builder.mul(disable_keccak_tables.target, state_difference); + builder.mul(skip_keccak_tables.target, state_difference); builder.assert_zero(challenger_state_check); } } @@ -906,7 +906,7 @@ where // When Keccak Tables are disabled, Keccak Tables' ctl_zs_first should be 0s. for &i in KECCAK_TABLES_INDICES.iter() { for &t in pis[i].ctl_zs_first.iter() { - let ctl_check = builder.mul(disable_keccak_tables.target, t); + let ctl_check = builder.mul(skip_keccak_tables.target, t); builder.assert_zero(ctl_check); } } @@ -943,7 +943,7 @@ where if KECCAK_TABLES_INDICES.contains(&i) { builder .conditionally_verify_proof_or_dummy::( - enable_keccak_tables, + use_keccak_tables, &recursive_proofs[i], &inner_verifier_data, inner_common_data[i], @@ -986,7 +986,7 @@ where index_verifier_data, public_values, cyclic_vk, - use_keccak_tables: enable_keccak_tables, + use_keccak_tables, } } From 2f0129e0028a5c2ee51f0b740bf298ff3a07bb42 Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Tue, 24 Sep 2024 16:27:38 -0700 Subject: [PATCH 21/91] fix pis --- .../src/fixed_recursive_verifier.rs | 102 +++++++++++------- 1 file changed, 65 insertions(+), 37 deletions(-) diff --git a/evm_arithmetization/src/fixed_recursive_verifier.rs b/evm_arithmetization/src/fixed_recursive_verifier.rs index ec561414c..a4f948c00 100644 --- a/evm_arithmetization/src/fixed_recursive_verifier.rs +++ b/evm_arithmetization/src/fixed_recursive_verifier.rs @@ -24,7 +24,7 @@ use plonky2::plonk::circuit_data::{ use plonky2::plonk::config::{AlgebraicHasher, GenericConfig, GenericHashOut}; use plonky2::plonk::proof::{ProofWithPublicInputs, ProofWithPublicInputsTarget}; use plonky2::recursion::cyclic_recursion::check_cyclic_proof_verifier_data; -use plonky2::recursion::dummy_circuit::cyclic_base_proof; +use plonky2::recursion::dummy_circuit::{cyclic_base_proof, dummy_circuit, dummy_proof}; use plonky2::util::serialization::{ Buffer, GateSerializer, IoResult, Read, WitnessGeneratorSerializer, Write, }; @@ -1934,36 +1934,52 @@ where .by_stark_size .keys() .min() - .expect("No valid size found in the shrinking circuits"); + .expect("No valid size in shrinking circuits"); root_inputs.set_target( self.root.index_verifier_data[table], F::from_canonical_usize(*index_verifier_data), ); - continue; + let common_date = &table_circuits + .by_stark_size + .get(index_verifier_data) + .expect("No valid size in shrinking circuits") + .shrinking_wrappers + .last() + .expect("No shrinking circuits") + .circuit + .common; + let dummy_circuit: CircuitData = dummy_circuit(common_date); + let dummy_pis = HashMap::new(); + let dummy_proof = dummy_proof(&dummy_circuit, dummy_pis) + .expect("Unable to generate dummy proofs"); + root_inputs + .set_proof_with_pis_target(&self.root.proof_with_pis[table], &dummy_proof); + } else { + let stark_proof = &all_proof.multi_proof.stark_proofs[table]; + let original_degree_bits = stark_proof.proof.recover_degree_bits(config); + let shrunk_proof = table_circuits + .by_stark_size + .get(&original_degree_bits) + .ok_or_else(|| { + anyhow!(format!( + "Missing preprocessed circuits for {:?} table with size {}.", + Table::all()[table], + original_degree_bits, + )) + })? + .shrink(stark_proof, &all_proof.multi_proof.ctl_challenges)?; + let index_verifier_data = table_circuits + .by_stark_size + .keys() + .position(|&size| size == original_degree_bits) + .unwrap(); + root_inputs.set_target( + self.root.index_verifier_data[table], + F::from_canonical_usize(index_verifier_data), + ); + root_inputs + .set_proof_with_pis_target(&self.root.proof_with_pis[table], &shrunk_proof); } - let stark_proof = &all_proof.multi_proof.stark_proofs[table]; - let original_degree_bits = stark_proof.proof.recover_degree_bits(config); - let shrunk_proof = table_circuits - .by_stark_size - .get(&original_degree_bits) - .ok_or_else(|| { - anyhow!(format!( - "Missing preprocessed circuits for {:?} table with size {}.", - Table::all()[table], - original_degree_bits, - )) - })? - .shrink(stark_proof, &all_proof.multi_proof.ctl_challenges)?; - let index_verifier_data = table_circuits - .by_stark_size - .keys() - .position(|&size| size == original_degree_bits) - .unwrap(); - root_inputs.set_target( - self.root.index_verifier_data[table], - F::from_canonical_usize(index_verifier_data), - ); - root_inputs.set_proof_with_pis_target(&self.root.proof_with_pis[table], &shrunk_proof); check_abort_signal(abort_signal.clone())?; } @@ -2101,18 +2117,30 @@ where self.root.index_verifier_data[table], F::from_canonical_u8(*index_verifier_data), ); - continue; - } - - let stark_proof = &all_proof.multi_proof.stark_proofs[table]; + let common_date = &table_circuit + .shrinking_wrappers + .last() + .expect("") + .circuit + .common; + let dummy_circuit: CircuitData = dummy_circuit(common_date); + let dummy_pis = HashMap::new(); + let dummy_proof = dummy_proof(&dummy_circuit, dummy_pis) + .expect("Unable to generate dummy proofs"); + root_inputs + .set_proof_with_pis_target(&self.root.proof_with_pis[table], &dummy_proof); + } else { + let stark_proof = &all_proof.multi_proof.stark_proofs[table]; - let shrunk_proof = - table_circuit.shrink(stark_proof, &all_proof.multi_proof.ctl_challenges)?; - root_inputs.set_target( - self.root.index_verifier_data[table], - F::from_canonical_u8(*index_verifier_data), - ); - root_inputs.set_proof_with_pis_target(&self.root.proof_with_pis[table], &shrunk_proof); + let shrunk_proof = + table_circuit.shrink(stark_proof, &all_proof.multi_proof.ctl_challenges)?; + root_inputs.set_target( + self.root.index_verifier_data[table], + F::from_canonical_u8(*index_verifier_data), + ); + root_inputs + .set_proof_with_pis_target(&self.root.proof_with_pis[table], &shrunk_proof); + } check_abort_signal(abort_signal.clone())?; } From 36ace10e5ceeef9a8937c5167b47b083cca5420c Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Tue, 24 Sep 2024 20:38:24 -0700 Subject: [PATCH 22/91] fix --- .../src/fixed_recursive_verifier.rs | 103 ++++++------------ 1 file changed, 32 insertions(+), 71 deletions(-) diff --git a/evm_arithmetization/src/fixed_recursive_verifier.rs b/evm_arithmetization/src/fixed_recursive_verifier.rs index 4beda37b6..8056f9ab3 100644 --- a/evm_arithmetization/src/fixed_recursive_verifier.rs +++ b/evm_arithmetization/src/fixed_recursive_verifier.rs @@ -7,7 +7,6 @@ use std::sync::Arc; use anyhow::anyhow; use hashbrown::HashMap; use itertools::{zip_eq, Itertools}; -use log::info; use mpt_trie::partial_trie::{HashedPartialTrie, Node, PartialTrie}; use plonky2::field::extension::Extendable; use plonky2::fri::FriParams; @@ -132,7 +131,7 @@ where /// structure of aggregation proofs. cyclic_vk: VerifierCircuitTarget, /// We can skip verifying Keccak tables when they are not in use. - enable_keccak_tables: BoolTarget, + use_keccak_tables: BoolTarget, } impl RootCircuitData @@ -155,7 +154,7 @@ where } self.public_values.to_buffer(buffer)?; buffer.write_target_verifier_circuit(&self.cyclic_vk)?; - buffer.write_target_bool(self.enable_keccak_tables)?; + buffer.write_target_bool(self.use_keccak_tables)?; Ok(()) } @@ -175,7 +174,7 @@ where } let public_values = PublicValuesTarget::from_buffer(buffer)?; let cyclic_vk = buffer.read_target_verifier_circuit()?; - let enable_keccak_tables = buffer.read_target_bool()?; + let use_keccak_tables = buffer.read_target_bool()?; Ok(Self { circuit, @@ -183,7 +182,7 @@ where index_verifier_data: index_verifier_data.try_into().unwrap(), public_values, cyclic_vk, - enable_keccak_tables, + use_keccak_tables, }) } } @@ -792,8 +791,8 @@ where let mut builder = CircuitBuilder::new(CircuitConfig::standard_recursion_config()); - let enable_keccak_tables = builder.add_virtual_bool_target_safe(); - let disable_keccak_tables = builder.not(enable_keccak_tables); + let use_keccak_tables = builder.add_virtual_bool_target_safe(); + let skip_keccak_tables = builder.not(use_keccak_tables); let public_values = add_virtual_public_values_public_input(&mut builder); let recursive_proofs = @@ -816,7 +815,7 @@ where for i in KECCAK_TABLES_INDICES { for h in &pis[i].trace_cap { for t in h { - let trace_cap_check = builder.mul(disable_keccak_tables.target, *t); + let trace_cap_check = builder.mul(skip_keccak_tables.target, *t); builder.assert_zero(trace_cap_check); } } @@ -843,8 +842,8 @@ where ctl_challenges.challenges[j].gamma, pi.ctl_challenges.challenges[j].gamma, ); - let beta_check = builder.mul(enable_keccak_tables.target, beta_diff); - let gamma_check = builder.mul(enable_keccak_tables.target, gamma_diff); + let beta_check = builder.mul(use_keccak_tables.target, beta_diff); + let gamma_check = builder.mul(use_keccak_tables.target, gamma_diff); builder.assert_zero(beta_check); builder.assert_zero(gamma_check); } else { @@ -865,24 +864,25 @@ where builder.connect(before, s); } // Check that the challenger state is consistent between proofs. + let mut prev_state = pis[0].challenger_state_after.as_ref().to_vec(); + let state_len = prev_state.len(); for i in 1..NUM_TABLES { - for (&before, &after) in zip_eq( - pis[i].challenger_state_before.as_ref(), - pis[i - 1].challenger_state_after.as_ref(), - ) { - builder.connect(before, after); - } - if KECCAK_TABLES_INDICES.contains(&i) { - // Ensure that the challenger state remains consistent before and after Keccak - // tables. - for (&before, &after) in zip_eq( - pis[i].challenger_state_before.as_ref(), - pis[i].challenger_state_after.as_ref(), - ) { - let state_difference = builder.sub(before, after); - let challenger_state_check = - builder.mul(disable_keccak_tables.target, state_difference); - builder.assert_zero(challenger_state_check); + let current_state_before = pis[i].challenger_state_before.as_ref(); + let current_state_after = pis[i].challenger_state_after.as_ref(); + for j in 0..state_len { + if KECCAK_TABLES_INDICES.contains(&i) { + // Ensure that the challenger state: + // 1) prev == current_before, when using keccak + let diff = builder.sub(prev_state[j], current_state_before[j]); + let check = builder.mul(use_keccak_tables.target, diff); + builder.assert_zero(check); + // 2) prev <- current_after, when using keccak + // 3) prev <- prev, when skipping using keccak + prev_state[j] = + builder.select(use_keccak_tables, current_state_after[j], prev_state[j]); + } else { + builder.connect(prev_state[j], current_state_before[j]); + prev_state[j] = current_state_after[j]; } } } @@ -906,7 +906,7 @@ where // When Keccak Tables are disabled, Keccak Tables' ctl_zs_first should be 0s. for &i in KECCAK_TABLES_INDICES.iter() { for &t in pis[i].ctl_zs_first.iter() { - let ctl_check = builder.mul(disable_keccak_tables.target, t); + let ctl_check = builder.mul(skip_keccak_tables.target, t); builder.assert_zero(ctl_check); } } @@ -943,7 +943,7 @@ where if KECCAK_TABLES_INDICES.contains(&i) { builder .conditionally_verify_proof_or_dummy::( - enable_keccak_tables, + use_keccak_tables, &recursive_proofs[i], &inner_verifier_data, inner_common_data[i], @@ -986,7 +986,7 @@ where index_verifier_data, public_values, cyclic_vk, - enable_keccak_tables, + use_keccak_tables, } } @@ -1885,45 +1885,6 @@ where timing, abort_signal.clone(), )?; - info!( - "Debugging trace_cap.0[0]: {:?}", - &all_proof.multi_proof.stark_proofs[*Table::Keccak] - .proof - .trace_cap - .0[0] - ); - - info!( - "Debugging openings.ctl_zs_first: {:?}", - &all_proof.multi_proof.stark_proofs[*Table::Keccak] - .proof - .openings - .ctl_zs_first - ); - - info!( - "Debugging trace_cap.0[0] for KeccakSponge: {:?}", - &all_proof.multi_proof.stark_proofs[*Table::KeccakSponge] - .proof - .trace_cap - .0[0] - ); - - info!( - "Debugging openings.ctl_zs_first for KeccakSponge: {:?}", - &all_proof.multi_proof.stark_proofs[*Table::KeccakSponge] - .proof - .openings - .ctl_zs_first - ); - - info!( - "Debugging openings.ctl_zs_first for Logic: {:?}", - &all_proof.multi_proof.stark_proofs[*Table::Logic] - .proof - .openings - .ctl_zs_first - ); let mut root_inputs = PartialWitness::new(); @@ -1971,7 +1932,7 @@ where })?; // TODO(sdeng): Set to false when this segment contains no Keccak operations. - root_inputs.set_bool_target(self.root.enable_keccak_tables, true); + root_inputs.set_bool_target(self.root.use_keccak_tables, true); let root_proof = self.root.circuit.prove(root_inputs)?; @@ -2114,7 +2075,7 @@ where })?; // TODO(sdeng): Set to false when this segment contains no Keccak operations. - root_inputs.set_bool_target(self.root.enable_keccak_tables, true); + root_inputs.set_bool_target(self.root.use_keccak_tables, true); let root_proof = self.root.circuit.prove(root_inputs)?; From 56deaa384a199a0b68806d72cebf6999eaf292f5 Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Tue, 24 Sep 2024 21:51:25 -0700 Subject: [PATCH 23/91] done --- evm_arithmetization/src/fixed_recursive_verifier.rs | 6 ------ evm_arithmetization/src/prover.rs | 9 +++++++++ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/evm_arithmetization/src/fixed_recursive_verifier.rs b/evm_arithmetization/src/fixed_recursive_verifier.rs index fb9c813e8..ee45d53e4 100644 --- a/evm_arithmetization/src/fixed_recursive_verifier.rs +++ b/evm_arithmetization/src/fixed_recursive_verifier.rs @@ -3112,12 +3112,7 @@ mod tests { let mut proofs_without_keccak = vec![]; - let skip_proofs_before_index = 3; for (i, segment_run) in segment_iterator.enumerate() { - if i < skip_proofs_before_index { - continue; - } - // Process and prove segment let (_, mut segment_data) = segment_run.map_err(|e: SegmentError| anyhow::format_err!(e))?; @@ -3136,7 +3131,6 @@ mod tests { ); proofs_without_keccak.push(segment_proof); - break; // Process only one proof } // Verify the generated segment proof diff --git a/evm_arithmetization/src/prover.rs b/evm_arithmetization/src/prover.rs index f165bed05..acd7fdcde 100644 --- a/evm_arithmetization/src/prover.rs +++ b/evm_arithmetization/src/prover.rs @@ -150,6 +150,7 @@ where config, &trace_poly_values, trace_commitments, + use_keccak_tables, ctl_data_per_table, &mut challenger, &ctl_challenges, @@ -237,6 +238,7 @@ fn prove_with_commitments( config: &StarkConfig, trace_poly_values: &[Vec>; NUM_TABLES], trace_commitments: Vec>, + use_keccak_tables: bool, ctl_data_per_table: [CtlData; NUM_TABLES], challenger: &mut Challenger, ctl_challenges: &GrandProductChallengeSet, @@ -270,8 +272,15 @@ where let (arithmetic_proof, _) = prove_table!(arithmetic_stark, Table::Arithmetic); let (byte_packing_proof, _) = prove_table!(byte_packing_stark, Table::BytePacking); let (cpu_proof, _) = prove_table!(cpu_stark, Table::Cpu); + let challenger_after_cpu = challenger.clone(); + // TODO: We still need Keccak proofs for CTLs, etc. let (keccak_proof, _) = prove_table!(keccak_stark, Table::Keccak); let (keccak_sponge_proof, _) = prove_table!(keccak_sponge_stark, Table::KeccakSponge); + if !use_keccak_tables { + // We need to connect the challenger state of Logic and CPU tables when the + // Keccak tables are not in use. + *challenger = challenger_after_cpu; + } let (logic_proof, _) = prove_table!(logic_stark, Table::Logic); let (memory_proof, _) = prove_table!(memory_stark, Table::Memory); let (mem_before_proof, mem_before_cap) = prove_table!(mem_before_stark, Table::MemBefore); From 7a3fe087d3752aa3c5f1c55d22178a7e507ebcfe Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Wed, 25 Sep 2024 10:14:27 -0700 Subject: [PATCH 24/91] polish the comments --- evm_arithmetization/src/fixed_recursive_verifier.rs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/evm_arithmetization/src/fixed_recursive_verifier.rs b/evm_arithmetization/src/fixed_recursive_verifier.rs index 8056f9ab3..a13386569 100644 --- a/evm_arithmetization/src/fixed_recursive_verifier.rs +++ b/evm_arithmetization/src/fixed_recursive_verifier.rs @@ -812,6 +812,7 @@ where } } + // Ensures that the trace cap is set to 0 when skipping Keccak tables. for i in KECCAK_TABLES_INDICES { for h in &pis[i].trace_cap { for t in h { @@ -871,13 +872,13 @@ where let current_state_after = pis[i].challenger_state_after.as_ref(); for j in 0..state_len { if KECCAK_TABLES_INDICES.contains(&i) { - // Ensure that the challenger state: - // 1) prev == current_before, when using keccak + // Ensure the challenger state: + // 1) prev == current_before when using Keccak let diff = builder.sub(prev_state[j], current_state_before[j]); let check = builder.mul(use_keccak_tables.target, diff); builder.assert_zero(check); - // 2) prev <- current_after, when using keccak - // 3) prev <- prev, when skipping using keccak + // 2) Update prev <- current_after when using Keccak + // 3) Keep prev <- prev when skipping Keccak prev_state[j] = builder.select(use_keccak_tables, current_state_after[j], prev_state[j]); } else { @@ -903,7 +904,8 @@ where }) .collect_vec(); - // When Keccak Tables are disabled, Keccak Tables' ctl_zs_first should be 0s. + // Ensure that when Keccak tables are skipped, the Keccak tables' ctl_zs_first + // are all zeros. for &i in KECCAK_TABLES_INDICES.iter() { for &t in pis[i].ctl_zs_first.iter() { let ctl_check = builder.mul(skip_keccak_tables.target, t); From 4d288f82ca06ddd26ead3d71932977c6788344f8 Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Wed, 25 Sep 2024 10:20:15 -0700 Subject: [PATCH 25/91] fix ci --- evm_arithmetization/src/fixed_recursive_verifier.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/evm_arithmetization/src/fixed_recursive_verifier.rs b/evm_arithmetization/src/fixed_recursive_verifier.rs index 8f23cc527..777597e05 100644 --- a/evm_arithmetization/src/fixed_recursive_verifier.rs +++ b/evm_arithmetization/src/fixed_recursive_verifier.rs @@ -3114,7 +3114,7 @@ mod tests { let mut proofs_without_keccak = vec![]; - for (i, segment_run) in segment_iterator.enumerate() { + for (_i, segment_run) in segment_iterator.enumerate() { // Process and prove segment let (_, mut segment_data) = segment_run.map_err(|e: SegmentError| anyhow::format_err!(e))?; From e6bed697cdc464db6f1ad9403ccd3fa35e6b40a2 Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Wed, 25 Sep 2024 10:27:09 -0700 Subject: [PATCH 26/91] add comments --- evm_arithmetization/src/fixed_recursive_verifier.rs | 2 ++ evm_arithmetization/src/prover.rs | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/evm_arithmetization/src/fixed_recursive_verifier.rs b/evm_arithmetization/src/fixed_recursive_verifier.rs index 777597e05..eb53f5e8b 100644 --- a/evm_arithmetization/src/fixed_recursive_verifier.rs +++ b/evm_arithmetization/src/fixed_recursive_verifier.rs @@ -1893,6 +1893,7 @@ where for table in 0..NUM_TABLES { let table_circuits = &self.by_table[table]; if KECCAK_TABLES_INDICES.contains(&table) && !all_proof.use_keccak_tables { + // generate and set a dummy `index_verifier_data` and `proof_with_pis` let index_verifier_data = table_circuits .by_stark_size .keys() @@ -2080,6 +2081,7 @@ where self.root.index_verifier_data[table], F::from_canonical_u8(*index_verifier_data), ); + // generate and set a dummy `proof_with_pis` let common_date = &table_circuit .shrinking_wrappers .last() diff --git a/evm_arithmetization/src/prover.rs b/evm_arithmetization/src/prover.rs index acd7fdcde..fa30e6041 100644 --- a/evm_arithmetization/src/prover.rs +++ b/evm_arithmetization/src/prover.rs @@ -118,6 +118,7 @@ where let mut challenger = Challenger::::new(); for (i, cap) in trace_caps.iter().enumerate() { if KECCAK_TABLES_INDICES.contains(&i) && !use_keccak_tables { + // Observe zero merkle caps when skipping Keccak tables. let zero_merkle_cap = cap.flatten().iter().map(|_| F::ZERO).collect::>(); challenger.observe_elements(&zero_merkle_cap); } else { @@ -273,7 +274,8 @@ where let (byte_packing_proof, _) = prove_table!(byte_packing_stark, Table::BytePacking); let (cpu_proof, _) = prove_table!(cpu_stark, Table::Cpu); let challenger_after_cpu = challenger.clone(); - // TODO: We still need Keccak proofs for CTLs, etc. + // TODO(sdeng): Keccak proofs are still required for CTLs, etc. Refactor the + // code and remove the unnecessary parts. let (keccak_proof, _) = prove_table!(keccak_stark, Table::Keccak); let (keccak_sponge_proof, _) = prove_table!(keccak_sponge_stark, Table::KeccakSponge); if !use_keccak_tables { From 0bd14481b0b131a5ba49bcd1124fa680266d20bf Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Wed, 25 Sep 2024 10:34:27 -0700 Subject: [PATCH 27/91] fix ci --- evm_arithmetization/src/fixed_recursive_verifier.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/evm_arithmetization/src/fixed_recursive_verifier.rs b/evm_arithmetization/src/fixed_recursive_verifier.rs index eb53f5e8b..45e4581f5 100644 --- a/evm_arithmetization/src/fixed_recursive_verifier.rs +++ b/evm_arithmetization/src/fixed_recursive_verifier.rs @@ -3116,7 +3116,7 @@ mod tests { let mut proofs_without_keccak = vec![]; - for (_i, segment_run) in segment_iterator.enumerate() { + for segment_run in segment_iterator { // Process and prove segment let (_, mut segment_data) = segment_run.map_err(|e: SegmentError| anyhow::format_err!(e))?; From 38009861d130e5fd6b8f88d8cafa2a73634a3c92 Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Wed, 25 Sep 2024 11:15:47 -0700 Subject: [PATCH 28/91] update tests --- .../src/fixed_recursive_verifier.rs | 20 ++----------------- 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/evm_arithmetization/src/fixed_recursive_verifier.rs b/evm_arithmetization/src/fixed_recursive_verifier.rs index 80f7f1b66..781635b6c 100644 --- a/evm_arithmetization/src/fixed_recursive_verifier.rs +++ b/evm_arithmetization/src/fixed_recursive_verifier.rs @@ -2973,17 +2973,7 @@ mod tests { "Create all recursive circuits", AllRecursiveCircuits::::new( &all_stark, - &[ - 16..17, - 8..9, - 9..10, - 4..5, - 8..9, - 4..5, - 17..18, - 17..18, - 17..18 - ], + &[16..17, 8..9, 9..10, 4..9, 8..9, 4..7, 17..18, 17..18, 7..18], &config, ) ); @@ -3001,12 +2991,7 @@ mod tests { let mut proofs_without_keccak = vec![]; - let skip_proofs_before_index = 3; - for (i, segment_run) in segment_iterator.enumerate() { - if i < skip_proofs_before_index { - continue; - } - + for segment_run in segment_iterator { // Process and prove segment let (_, mut segment_data) = segment_run.map_err(|e: SegmentError| anyhow::format_err!(e))?; @@ -3025,7 +3010,6 @@ mod tests { ); proofs_without_keccak.push(segment_proof); - break; // Process only one proof } // Verify the generated segment proof From 91dbeaa3ea4da6fbda43541645aba4e263b71d3b Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Wed, 25 Sep 2024 11:25:18 -0700 Subject: [PATCH 29/91] fix ci --- evm_arithmetization/src/testing_utils.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/evm_arithmetization/src/testing_utils.rs b/evm_arithmetization/src/testing_utils.rs index 528cae9ed..41b2a05fd 100644 --- a/evm_arithmetization/src/testing_utils.rs +++ b/evm_arithmetization/src/testing_utils.rs @@ -10,8 +10,7 @@ use mpt_trie::{ nibbles::Nibbles, partial_trie::{HashedPartialTrie, Node, PartialTrie}, }; -use plonky2::field::goldilocks_field::GoldilocksField; - +use plonky2::hash::hash_types::RichField; pub use crate::cpu::kernel::cancun_constants::*; pub use crate::cpu::kernel::constants::global_exit_root::*; use crate::generation::TrieInputs; @@ -168,10 +167,10 @@ pub fn scalable_contract_from_storage(storage_trie: &HashedPartialTrie) -> Accou /// Get `GenerationInputs` for a dummy payload, where the block has the given /// timestamp. -pub fn dummy_payload( +pub fn dummy_payload( timestamp: u64, is_first_payload: bool, -) -> Result> { +) -> Result> { let beneficiary = hex!("deadbeefdeadbeefdeadbeefdeadbeefdeadbeef"); let block_metadata = BlockMetadata { From bb790807d04bfe8c3f41d23926cd8890e8c9d918 Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Wed, 25 Sep 2024 11:26:52 -0700 Subject: [PATCH 30/91] fix ci --- evm_arithmetization/src/testing_utils.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/evm_arithmetization/src/testing_utils.rs b/evm_arithmetization/src/testing_utils.rs index 41b2a05fd..18f1f1186 100644 --- a/evm_arithmetization/src/testing_utils.rs +++ b/evm_arithmetization/src/testing_utils.rs @@ -11,6 +11,7 @@ use mpt_trie::{ partial_trie::{HashedPartialTrie, Node, PartialTrie}, }; use plonky2::hash::hash_types::RichField; + pub use crate::cpu::kernel::cancun_constants::*; pub use crate::cpu::kernel::constants::global_exit_root::*; use crate::generation::TrieInputs; From fd55d1a4cf42803664391253a77f855279f6e38a Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Wed, 25 Sep 2024 11:41:28 -0700 Subject: [PATCH 31/91] fix ci --- evm_arithmetization/src/fixed_recursive_verifier.rs | 3 +-- evm_arithmetization/src/testing_utils.rs | 6 +----- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/evm_arithmetization/src/fixed_recursive_verifier.rs b/evm_arithmetization/src/fixed_recursive_verifier.rs index 6a20bef48..41ab5498f 100644 --- a/evm_arithmetization/src/fixed_recursive_verifier.rs +++ b/evm_arithmetization/src/fixed_recursive_verifier.rs @@ -2992,8 +2992,7 @@ mod tests { for segment_run in segment_iterator { // Process and prove segment - let (_, mut segment_data) = - segment_run.map_err(|e: SegmentError| anyhow::format_err!(e))?; + let (_, mut segment_data) = segment_run?; let segment_proof = timed!( timing, log::Level::Info, diff --git a/evm_arithmetization/src/testing_utils.rs b/evm_arithmetization/src/testing_utils.rs index 18f1f1186..383c4aa10 100644 --- a/evm_arithmetization/src/testing_utils.rs +++ b/evm_arithmetization/src/testing_utils.rs @@ -10,7 +10,6 @@ use mpt_trie::{ nibbles::Nibbles, partial_trie::{HashedPartialTrie, Node, PartialTrie}, }; -use plonky2::hash::hash_types::RichField; pub use crate::cpu::kernel::cancun_constants::*; pub use crate::cpu::kernel::constants::global_exit_root::*; @@ -168,10 +167,7 @@ pub fn scalable_contract_from_storage(storage_trie: &HashedPartialTrie) -> Accou /// Get `GenerationInputs` for a dummy payload, where the block has the given /// timestamp. -pub fn dummy_payload( - timestamp: u64, - is_first_payload: bool, -) -> Result> { +pub fn dummy_payload(timestamp: u64, is_first_payload: bool) -> Result { let beneficiary = hex!("deadbeefdeadbeefdeadbeefdeadbeefdeadbeef"); let block_metadata = BlockMetadata { From 4de4c64f0cec61564c679bfda212242caabea82c Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Thu, 26 Sep 2024 15:39:47 -0700 Subject: [PATCH 32/91] empty payload and no keccak op check --- evm_arithmetization/src/arithmetic/mod.rs | 5 +- .../src/cpu/kernel/interpreter.rs | 14 ++- .../src/fixed_recursive_verifier.rs | 56 +++++----- .../src/generation/segments.rs | 11 ++ evm_arithmetization/src/logic.rs | 3 +- evm_arithmetization/src/testing_utils.rs | 70 ++++-------- evm_arithmetization/src/witness/operation.rs | 3 +- evm_arithmetization/tests/two_to_one_block.rs | 102 +++++++++++++++--- 8 files changed, 163 insertions(+), 101 deletions(-) diff --git a/evm_arithmetization/src/arithmetic/mod.rs b/evm_arithmetization/src/arithmetic/mod.rs index 0ca1bbe7a..ab65def9b 100644 --- a/evm_arithmetization/src/arithmetic/mod.rs +++ b/evm_arithmetization/src/arithmetic/mod.rs @@ -1,5 +1,6 @@ use ethereum_types::U256; use plonky2::field::types::PrimeField64; +use serde::{Deserialize, Serialize}; use self::columns::{ INPUT_REGISTER_0, INPUT_REGISTER_1, INPUT_REGISTER_2, OPCODE_COL, OUTPUT_REGISTER, @@ -24,7 +25,7 @@ pub(crate) mod columns; /// /// `Shl` and `Shr` are handled differently, by leveraging `Mul` and `Div` /// respectively. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)] pub(crate) enum BinaryOperator { Add, Mul, @@ -114,7 +115,7 @@ impl BinaryOperator { /// An enum representing different ternary operations. #[allow(clippy::enum_variant_names)] -#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)] pub(crate) enum TernaryOperator { AddMod, MulMod, diff --git a/evm_arithmetization/src/cpu/kernel/interpreter.rs b/evm_arithmetization/src/cpu/kernel/interpreter.rs index b831a921a..6c3cc73e0 100644 --- a/evm_arithmetization/src/cpu/kernel/interpreter.rs +++ b/evm_arithmetization/src/cpu/kernel/interpreter.rs @@ -54,8 +54,7 @@ pub(crate) struct Interpreter { /// halt_context pub(crate) halt_context: Option, /// Counts the number of appearances of each opcode. For debugging purposes. - #[allow(unused)] - pub(crate) opcode_count: [usize; 0x100], + pub(crate) opcode_count: HashMap, jumpdest_table: HashMap>, /// `true` if the we are currently carrying out a jumpdest analysis. pub(crate) is_jumpdest_analysis: bool, @@ -178,7 +177,7 @@ impl Interpreter { // while the label `halt` is the halting label in the kernel. halt_offsets: vec![DEFAULT_HALT_OFFSET, KERNEL.global_labels["halt_final"]], halt_context: None, - opcode_count: [0; 256], + opcode_count: HashMap::new(), jumpdest_table: HashMap::new(), is_jumpdest_analysis: false, clock: 0, @@ -209,7 +208,7 @@ impl Interpreter { generation_state: state.soft_clone(), halt_offsets: vec![halt_offset], halt_context: Some(halt_context), - opcode_count: [0; 256], + opcode_count: HashMap::new(), jumpdest_table: HashMap::new(), is_jumpdest_analysis: true, clock: 0, @@ -428,6 +427,10 @@ impl Interpreter { self.max_cpu_len_log } + pub(crate) fn reset_opcode_counts(&mut self) { + self.opcode_count = HashMap::new(); + } + pub(crate) fn code(&self) -> &MemorySegmentState { // The context is 0 if we are in kernel mode. &self.generation_state.memory.contexts[(1 - self.is_kernel() as usize) * self.context()] @@ -661,6 +664,9 @@ impl State for Interpreter { let op = decode(registers, opcode)?; + // Increment the opcode count + *self.opcode_count.entry(op).or_insert(0) += 1; + fill_op_flag(op, &mut row); self.fill_stack_fields(&mut row)?; diff --git a/evm_arithmetization/src/fixed_recursive_verifier.rs b/evm_arithmetization/src/fixed_recursive_verifier.rs index 41ab5498f..825ab4091 100644 --- a/evm_arithmetization/src/fixed_recursive_verifier.rs +++ b/evm_arithmetization/src/fixed_recursive_verifier.rs @@ -2951,7 +2951,8 @@ mod tests { use plonky2::timed; use super::*; - use crate::testing_utils::{dummy_payload, init_logger}; + use crate::testing_utils::{empty_payload, init_logger}; + use crate::witness::operation::Operation; type F = GoldilocksField; const D: usize = 2; @@ -2966,56 +2967,47 @@ mod tests { let all_stark = AllStark::::default(); let config = StarkConfig::standard_fast_config(); + // Generate a dummy payload for testing + let payload = empty_payload()?; + let max_cpu_len_log = Some(7); + let mut segment_iterator = SegmentDataIterator::::new(&payload, max_cpu_len_log); + let (_, mut segment_data) = segment_iterator.next().unwrap()?; + + let opcode_counts = &segment_data.opcode_counts; + assert!(!opcode_counts.contains_key(&Operation::KeccakGeneral)); + + // Process and prove segment let all_circuits = timed!( timing, log::Level::Info, "Create all recursive circuits", AllRecursiveCircuits::::new( &all_stark, - &[16..17, 8..9, 9..10, 4..9, 8..9, 4..7, 17..18, 17..18, 7..18], + &[16..17, 8..9, 7..8, 4..9, 8..9, 4..7, 17..18, 17..18, 17..18], &config, ) ); - // Generate a dummy payload for testing - let dummy_payload = timed!( + let segment_proof = timed!( timing, log::Level::Info, - "Generate dummy payload", - dummy_payload(100, true)? - ); - - let max_cpu_len_log = 9; - let segment_iterator = SegmentDataIterator::::new(&dummy_payload, Some(max_cpu_len_log)); - - let mut proofs_without_keccak = vec![]; - - for segment_run in segment_iterator { - // Process and prove segment - let (_, mut segment_data) = segment_run?; - let segment_proof = timed!( + "Prove segment", + all_circuits.prove_segment( + &all_stark, + &config, + payload.trim(), + &mut segment_data, timing, - log::Level::Info, - "Prove segment", - all_circuits.prove_segment( - &all_stark, - &config, - dummy_payload.trim(), - &mut segment_data, - timing, - None, - )? - ); - - proofs_without_keccak.push(segment_proof); - } + None, + )? + ); // Verify the generated segment proof timed!( timing, log::Level::Info, "Verify segment proof", - all_circuits.verify_root(proofs_without_keccak[0].proof_with_pis.clone())? + all_circuits.verify_root(segment_proof.proof_with_pis.clone())? ); // Print timing details diff --git a/evm_arithmetization/src/generation/segments.rs b/evm_arithmetization/src/generation/segments.rs index b3a129137..8203aafdb 100644 --- a/evm_arithmetization/src/generation/segments.rs +++ b/evm_arithmetization/src/generation/segments.rs @@ -1,6 +1,8 @@ //! Module defining the logic around proof segmentation into chunks, //! which allows what is commonly known as zk-continuations. +use std::collections::HashMap; + use anyhow::Result; use plonky2::hash::hash_types::RichField; use serde::{Deserialize, Serialize}; @@ -11,6 +13,7 @@ use crate::cpu::kernel::interpreter::{set_registers_and_run, ExtraSegmentData, I use crate::generation::state::State; use crate::generation::{collect_debug_tries, debug_inputs, ErrorWithTries, GenerationInputs}; use crate::witness::memory::MemoryState; +use crate::witness::operation::Operation; use crate::witness::state::RegistersState; /// Structure holding the data needed to initialize a segment. @@ -29,6 +32,8 @@ pub struct GenerationSegmentData { pub(crate) extra_data: ExtraSegmentData, /// Log of the maximal cpu length. pub(crate) max_cpu_len_log: Option, + /// Counts the number of appearances of each opcode. For debugging purposes. + pub(crate) opcode_counts: HashMap, } impl GenerationSegmentData { @@ -77,6 +82,7 @@ fn build_segment_data( accounts: interpreter.generation_state.accounts_pointers.clone(), storage: interpreter.generation_state.storage_pointers.clone(), }, + opcode_counts: interpreter.opcode_count.clone(), } } @@ -133,6 +139,9 @@ impl SegmentDataIterator { let segment_index = segment_data.segment_index; + // Reset opcode counts before executing the segment + self.interpreter.reset_opcode_counts(); + // Run the interpreter to get `registers_after` and the partial data for the // next segment. let execution_result = @@ -147,6 +156,8 @@ impl SegmentDataIterator { )); segment_data.registers_after = updated_registers; + segment_data.opcode_counts = self.interpreter.opcode_count.clone(); + Ok(Some(Box::new((segment_data, partial_segment_data)))) } else { let inputs = &self.interpreter.get_generation_state().inputs; diff --git a/evm_arithmetization/src/logic.rs b/evm_arithmetization/src/logic.rs index 0ba378b52..7d68e463e 100644 --- a/evm_arithmetization/src/logic.rs +++ b/evm_arithmetization/src/logic.rs @@ -11,6 +11,7 @@ use plonky2::hash::hash_types::RichField; use plonky2::iop::ext_target::ExtensionTarget; use plonky2::timed; use plonky2::util::timing::TimingTree; +use serde::{Deserialize, Serialize}; use starky::constraint_consumer::{ConstraintConsumer, RecursiveConstraintConsumer}; use starky::evaluation_frame::StarkEvaluationFrame; use starky::lookup::{Column, Filter}; @@ -118,7 +119,7 @@ pub(crate) struct LogicStark { } /// Logic operations. -#[derive(Copy, Clone, Debug, Eq, PartialEq)] +#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)] pub(crate) enum Op { And, Or, diff --git a/evm_arithmetization/src/testing_utils.rs b/evm_arithmetization/src/testing_utils.rs index 383c4aa10..5f1b1b041 100644 --- a/evm_arithmetization/src/testing_utils.rs +++ b/evm_arithmetization/src/testing_utils.rs @@ -165,70 +165,46 @@ pub fn scalable_contract_from_storage(storage_trie: &HashedPartialTrie) -> Accou } } -/// Get `GenerationInputs` for a dummy payload, where the block has the given -/// timestamp. -pub fn dummy_payload(timestamp: u64, is_first_payload: bool) -> Result { - let beneficiary = hex!("deadbeefdeadbeefdeadbeefdeadbeefdeadbeef"); - +pub fn empty_payload() -> Result { + // Set up default block metadata let block_metadata = BlockMetadata { - block_beneficiary: Address::from(beneficiary), - block_timestamp: timestamp.into(), - block_number: 1.into(), - block_difficulty: 0x020000.into(), - block_random: H256::from_uint(&0x020000.into()), - block_gaslimit: 0xff112233u32.into(), - block_chain_id: 1.into(), - block_base_fee: 0xa.into(), + block_beneficiary: Address::zero(), + block_timestamp: U256::zero(), + block_number: U256::one(), + block_difficulty: U256::zero(), + block_random: H256::zero(), + block_gaslimit: U256::zero(), + block_chain_id: U256::one(), + block_base_fee: U256::zero(), ..Default::default() }; - let (mut state_trie_before, mut storage_tries) = preinitialized_state_and_storage_tries()?; + // Initialize an empty state trie and storage tries + let state_trie_before = HashedPartialTrie::from(crate::Node::Empty); + let storage_tries = Vec::new(); let checkpoint_state_trie_root = state_trie_before.hash(); - let mut beacon_roots_account_storage = storage_tries[0].1.clone(); - - update_beacon_roots_account_storage( - &mut beacon_roots_account_storage, - block_metadata.block_timestamp, - block_metadata.parent_beacon_block_root, - )?; - let updated_beacon_roots_account = - beacon_roots_contract_from_storage(&beacon_roots_account_storage); - - if !is_first_payload { - // This isn't the first dummy payload being processed. We need to update the - // initial state trie to account for the update on the beacon roots contract. - state_trie_before.insert( - beacon_roots_account_nibbles(), - rlp::encode(&updated_beacon_roots_account).to_vec(), - )?; - storage_tries[0].1 = beacon_roots_account_storage; - } + // Prepare the tries without any transactions or receipts let tries_before = TrieInputs { - state_trie: state_trie_before, - storage_tries, - ..Default::default() + state_trie: state_trie_before.clone(), + storage_tries: storage_tries.clone(), + transactions_trie: HashedPartialTrie::from(crate::Node::Empty), + receipts_trie: HashedPartialTrie::from(crate::Node::Empty), }; - let expected_state_trie_after: HashedPartialTrie = { - let mut state_trie_after = HashedPartialTrie::from(crate::Node::Empty); - state_trie_after.insert( - beacon_roots_account_nibbles(), - rlp::encode(&updated_beacon_roots_account).to_vec(), - )?; - - state_trie_after - }; + // The expected state trie after execution remains the same as before + let expected_state_trie_after = state_trie_before; + // Compute the trie roots after execution let trie_roots_after = TrieRoots { state_root: expected_state_trie_after.hash(), transactions_root: tries_before.transactions_trie.hash(), receipts_root: tries_before.receipts_trie.hash(), }; + // Construct the GenerationInputs without any transactions or state changes let inputs = GenerationInputs { - tries: tries_before.clone(), - burn_addr: None, + tries: tries_before, trie_roots_after, checkpoint_state_trie_root, block_metadata, diff --git a/evm_arithmetization/src/witness/operation.rs b/evm_arithmetization/src/witness/operation.rs index 55f3d3c18..c8e2a5376 100644 --- a/evm_arithmetization/src/witness/operation.rs +++ b/evm_arithmetization/src/witness/operation.rs @@ -2,6 +2,7 @@ use ethereum_types::{BigEndianHash, U256}; use itertools::Itertools; use keccak_hash::keccak; use plonky2::hash::hash_types::RichField; +use serde::{Deserialize, Serialize}; use super::state::KERNEL_CONTEXT; use super::transition::Transition; @@ -29,7 +30,7 @@ use crate::witness::util::{ }; use crate::{arithmetic, logic}; -#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)] pub(crate) enum Operation { Iszero, Not, diff --git a/evm_arithmetization/tests/two_to_one_block.rs b/evm_arithmetization/tests/two_to_one_block.rs index 55489187d..db4363ae6 100644 --- a/evm_arithmetization/tests/two_to_one_block.rs +++ b/evm_arithmetization/tests/two_to_one_block.rs @@ -1,12 +1,22 @@ #![cfg(feature = "eth_mainnet")] +use ethereum_types::{Address, BigEndianHash, H256}; use evm_arithmetization::fixed_recursive_verifier::{ extract_block_final_public_values, extract_two_to_one_block_hash, }; -use evm_arithmetization::proof::{FinalPublicValues, PublicValues}; -use evm_arithmetization::testing_utils::{dummy_payload, init_logger}; -use evm_arithmetization::{AllRecursiveCircuits, AllStark, StarkConfig}; +use evm_arithmetization::generation::{GenerationInputs, TrieInputs}; +use evm_arithmetization::proof::{ + BlockMetadata, FinalPublicValues, PublicValues, TrieRoots, EMPTY_CONSOLIDATED_BLOCKHASH, +}; +use evm_arithmetization::testing_utils::{ + beacon_roots_account_nibbles, beacon_roots_contract_from_storage, init_logger, + preinitialized_state_and_storage_tries, update_beacon_roots_account_storage, +}; +use evm_arithmetization::{AllRecursiveCircuits, AllStark, Node, StarkConfig}; +use hex_literal::hex; +use mpt_trie::partial_trie::{HashedPartialTrie, PartialTrie}; use plonky2::field::goldilocks_field::GoldilocksField; +use plonky2::field::types::Field; use plonky2::hash::poseidon::PoseidonHash; use plonky2::plonk::config::{Hasher, PoseidonGoldilocksConfig}; use plonky2::plonk::proof::ProofWithPublicInputs; @@ -16,6 +26,80 @@ type F = GoldilocksField; const D: usize = 2; type C = PoseidonGoldilocksConfig; +/// Get `GenerationInputs` for a dummy payload, where the block has the given +/// timestamp. +fn dummy_payload(timestamp: u64, is_first_payload: bool) -> anyhow::Result> { + let beneficiary = hex!("deadbeefdeadbeefdeadbeefdeadbeefdeadbeef"); + + let block_metadata = BlockMetadata { + block_beneficiary: Address::from(beneficiary), + block_timestamp: timestamp.into(), + block_number: 1.into(), + block_difficulty: 0x020000.into(), + block_random: H256::from_uint(&0x020000.into()), + block_gaslimit: 0xff112233u32.into(), + block_chain_id: 1.into(), + block_base_fee: 0xa.into(), + ..Default::default() + }; + + let (mut state_trie_before, mut storage_tries) = preinitialized_state_and_storage_tries()?; + let checkpoint_state_trie_root = state_trie_before.hash(); + let mut beacon_roots_account_storage = storage_tries[0].1.clone(); + + update_beacon_roots_account_storage( + &mut beacon_roots_account_storage, + block_metadata.block_timestamp, + block_metadata.parent_beacon_block_root, + )?; + let updated_beacon_roots_account = + beacon_roots_contract_from_storage(&beacon_roots_account_storage); + + if !is_first_payload { + // This isn't the first dummy payload being processed. We need to update the + // initial state trie to account for the update on the beacon roots contract. + state_trie_before.insert( + beacon_roots_account_nibbles(), + rlp::encode(&updated_beacon_roots_account).to_vec(), + )?; + storage_tries[0].1 = beacon_roots_account_storage; + } + + let tries_before = TrieInputs { + state_trie: state_trie_before, + storage_tries, + ..Default::default() + }; + + let expected_state_trie_after: HashedPartialTrie = { + let mut state_trie_after = HashedPartialTrie::from(Node::Empty); + state_trie_after.insert( + beacon_roots_account_nibbles(), + rlp::encode(&updated_beacon_roots_account).to_vec(), + )?; + + state_trie_after + }; + + let trie_roots_after = TrieRoots { + state_root: expected_state_trie_after.hash(), + transactions_root: tries_before.transactions_trie.hash(), + receipts_root: tries_before.receipts_trie.hash(), + }; + + let inputs = GenerationInputs { + tries: tries_before.clone(), + burn_addr: None, + trie_roots_after, + checkpoint_state_trie_root, + checkpoint_consolidated_hash: EMPTY_CONSOLIDATED_BLOCKHASH.map(F::from_canonical_u64), + block_metadata, + ..Default::default() + }; + + Ok(inputs) +} + fn get_test_block_proof( timestamp: u64, all_circuits: &AllRecursiveCircuits, @@ -90,17 +174,7 @@ fn test_two_to_one_block_aggregation() -> anyhow::Result<()> { let all_circuits = AllRecursiveCircuits::new( &all_stark, - &[ - 16..17, - 8..9, - 12..13, - 9..10, - 8..9, - 6..7, - 17..18, - 17..18, - 7..8, - ], + &[16..17, 8..9, 12..13, 8..9, 8..9, 6..7, 17..18, 17..18, 7..8], &config, ); From e34c59a62a544601caa48bd28c8857ab47750c6d Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Thu, 26 Sep 2024 16:05:17 -0700 Subject: [PATCH 33/91] refactor --- evm_arithmetization/src/fixed_recursive_verifier.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/evm_arithmetization/src/fixed_recursive_verifier.rs b/evm_arithmetization/src/fixed_recursive_verifier.rs index 825ab4091..d3dd1f286 100644 --- a/evm_arithmetization/src/fixed_recursive_verifier.rs +++ b/evm_arithmetization/src/fixed_recursive_verifier.rs @@ -2961,7 +2961,6 @@ mod tests { #[test] #[ignore] fn test_segment_proof_generation_without_keccak() -> anyhow::Result<()> { - let timing = &mut TimingTree::new("Segment Proof Generation", log::Level::Info); init_logger(); let all_stark = AllStark::::default(); @@ -2976,6 +2975,10 @@ mod tests { let opcode_counts = &segment_data.opcode_counts; assert!(!opcode_counts.contains_key(&Operation::KeccakGeneral)); + let timing = &mut TimingTree::new( + "Segment Proof Generation Without Keccak Test", + log::Level::Info, + ); // Process and prove segment let all_circuits = timed!( timing, From be887e6f3a5c6d133cee46878476ca0f159e7887 Mon Sep 17 00:00:00 2001 From: Sai <135601871+sai-deng@users.noreply.github.com> Date: Mon, 30 Sep 2024 11:02:31 -0700 Subject: [PATCH 34/91] Update evm_arithmetization/src/fixed_recursive_verifier.rs Co-authored-by: Robin Salen <30937548+Nashtare@users.noreply.github.com> --- evm_arithmetization/src/fixed_recursive_verifier.rs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/evm_arithmetization/src/fixed_recursive_verifier.rs b/evm_arithmetization/src/fixed_recursive_verifier.rs index 8e15901d7..2c0b70574 100644 --- a/evm_arithmetization/src/fixed_recursive_verifier.rs +++ b/evm_arithmetization/src/fixed_recursive_verifier.rs @@ -835,18 +835,16 @@ where if KECCAK_TABLES_INDICES.contains(&i) { // Ensures that the correct CTL challenges are used in Keccak tables when // `enable_keccak_tables` is true. - let beta_diff = builder.sub( + builder.conditional_assert_eq( + use_keccak_tables.target, ctl_challenges.challenges[j].beta, pi.ctl_challenges.challenges[j].beta, ); - let gamma_diff = builder.sub( + builder.conditional_assert_eq( + use_keccak_tables.target, ctl_challenges.challenges[j].gamma, pi.ctl_challenges.challenges[j].gamma, ); - let beta_check = builder.mul(use_keccak_tables.target, beta_diff); - let gamma_check = builder.mul(use_keccak_tables.target, gamma_diff); - builder.assert_zero(beta_check); - builder.assert_zero(gamma_check); } else { builder.connect( ctl_challenges.challenges[j].beta, From d2ca0cc97a5431fc323296b0aab6ed7d63a7e1c2 Mon Sep 17 00:00:00 2001 From: Sai <135601871+sai-deng@users.noreply.github.com> Date: Mon, 30 Sep 2024 11:03:21 -0700 Subject: [PATCH 35/91] Update evm_arithmetization/src/fixed_recursive_verifier.rs Co-authored-by: Robin Salen <30937548+Nashtare@users.noreply.github.com> --- evm_arithmetization/src/fixed_recursive_verifier.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/evm_arithmetization/src/fixed_recursive_verifier.rs b/evm_arithmetization/src/fixed_recursive_verifier.rs index 2c0b70574..60c2eae42 100644 --- a/evm_arithmetization/src/fixed_recursive_verifier.rs +++ b/evm_arithmetization/src/fixed_recursive_verifier.rs @@ -872,9 +872,7 @@ where if KECCAK_TABLES_INDICES.contains(&i) { // Ensure the challenger state: // 1) prev == current_before when using Keccak - let diff = builder.sub(prev_state[j], current_state_before[j]); - let check = builder.mul(use_keccak_tables.target, diff); - builder.assert_zero(check); + builder. conditional_assert_eq(use_keccak_tables.target, prev_state[j], current_state_before[j]); // 2) Update prev <- current_after when using Keccak // 3) Keep prev <- prev when skipping Keccak prev_state[j] = From ce88ef8450c6e4b3263c0c73f9f13600da3e1bb4 Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Thu, 26 Sep 2024 15:39:47 -0700 Subject: [PATCH 36/91] empty payload and no keccak op check --- evm_arithmetization/src/arithmetic/mod.rs | 5 +- .../src/cpu/kernel/interpreter.rs | 14 ++- .../src/fixed_recursive_verifier.rs | 56 +++++----- .../src/generation/segments.rs | 11 ++ evm_arithmetization/src/logic.rs | 3 +- evm_arithmetization/src/testing_utils.rs | 70 ++++-------- evm_arithmetization/src/witness/operation.rs | 3 +- evm_arithmetization/tests/two_to_one_block.rs | 102 +++++++++++++++--- 8 files changed, 163 insertions(+), 101 deletions(-) diff --git a/evm_arithmetization/src/arithmetic/mod.rs b/evm_arithmetization/src/arithmetic/mod.rs index 0ca1bbe7a..ab65def9b 100644 --- a/evm_arithmetization/src/arithmetic/mod.rs +++ b/evm_arithmetization/src/arithmetic/mod.rs @@ -1,5 +1,6 @@ use ethereum_types::U256; use plonky2::field::types::PrimeField64; +use serde::{Deserialize, Serialize}; use self::columns::{ INPUT_REGISTER_0, INPUT_REGISTER_1, INPUT_REGISTER_2, OPCODE_COL, OUTPUT_REGISTER, @@ -24,7 +25,7 @@ pub(crate) mod columns; /// /// `Shl` and `Shr` are handled differently, by leveraging `Mul` and `Div` /// respectively. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)] pub(crate) enum BinaryOperator { Add, Mul, @@ -114,7 +115,7 @@ impl BinaryOperator { /// An enum representing different ternary operations. #[allow(clippy::enum_variant_names)] -#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)] pub(crate) enum TernaryOperator { AddMod, MulMod, diff --git a/evm_arithmetization/src/cpu/kernel/interpreter.rs b/evm_arithmetization/src/cpu/kernel/interpreter.rs index b831a921a..6c3cc73e0 100644 --- a/evm_arithmetization/src/cpu/kernel/interpreter.rs +++ b/evm_arithmetization/src/cpu/kernel/interpreter.rs @@ -54,8 +54,7 @@ pub(crate) struct Interpreter { /// halt_context pub(crate) halt_context: Option, /// Counts the number of appearances of each opcode. For debugging purposes. - #[allow(unused)] - pub(crate) opcode_count: [usize; 0x100], + pub(crate) opcode_count: HashMap, jumpdest_table: HashMap>, /// `true` if the we are currently carrying out a jumpdest analysis. pub(crate) is_jumpdest_analysis: bool, @@ -178,7 +177,7 @@ impl Interpreter { // while the label `halt` is the halting label in the kernel. halt_offsets: vec![DEFAULT_HALT_OFFSET, KERNEL.global_labels["halt_final"]], halt_context: None, - opcode_count: [0; 256], + opcode_count: HashMap::new(), jumpdest_table: HashMap::new(), is_jumpdest_analysis: false, clock: 0, @@ -209,7 +208,7 @@ impl Interpreter { generation_state: state.soft_clone(), halt_offsets: vec![halt_offset], halt_context: Some(halt_context), - opcode_count: [0; 256], + opcode_count: HashMap::new(), jumpdest_table: HashMap::new(), is_jumpdest_analysis: true, clock: 0, @@ -428,6 +427,10 @@ impl Interpreter { self.max_cpu_len_log } + pub(crate) fn reset_opcode_counts(&mut self) { + self.opcode_count = HashMap::new(); + } + pub(crate) fn code(&self) -> &MemorySegmentState { // The context is 0 if we are in kernel mode. &self.generation_state.memory.contexts[(1 - self.is_kernel() as usize) * self.context()] @@ -661,6 +664,9 @@ impl State for Interpreter { let op = decode(registers, opcode)?; + // Increment the opcode count + *self.opcode_count.entry(op).or_insert(0) += 1; + fill_op_flag(op, &mut row); self.fill_stack_fields(&mut row)?; diff --git a/evm_arithmetization/src/fixed_recursive_verifier.rs b/evm_arithmetization/src/fixed_recursive_verifier.rs index 60c2eae42..92853a9c0 100644 --- a/evm_arithmetization/src/fixed_recursive_verifier.rs +++ b/evm_arithmetization/src/fixed_recursive_verifier.rs @@ -3025,7 +3025,8 @@ mod tests { use plonky2::timed; use super::*; - use crate::testing_utils::{dummy_payload, init_logger}; + use crate::testing_utils::{empty_payload, init_logger}; + use crate::witness::operation::Operation; type F = GoldilocksField; const D: usize = 2; @@ -3040,56 +3041,47 @@ mod tests { let all_stark = AllStark::::default(); let config = StarkConfig::standard_fast_config(); + // Generate a dummy payload for testing + let payload = empty_payload()?; + let max_cpu_len_log = Some(7); + let mut segment_iterator = SegmentDataIterator::::new(&payload, max_cpu_len_log); + let (_, mut segment_data) = segment_iterator.next().unwrap()?; + + let opcode_counts = &segment_data.opcode_counts; + assert!(!opcode_counts.contains_key(&Operation::KeccakGeneral)); + + // Process and prove segment let all_circuits = timed!( timing, log::Level::Info, "Create all recursive circuits", AllRecursiveCircuits::::new( &all_stark, - &[16..17, 8..9, 9..10, 4..9, 8..9, 4..7, 17..18, 17..18, 7..18], + &[16..17, 8..9, 7..8, 4..9, 8..9, 4..7, 17..18, 17..18, 17..18], &config, ) ); - // Generate a dummy payload for testing - let dummy_payload = timed!( + let segment_proof = timed!( timing, log::Level::Info, - "Generate dummy payload", - dummy_payload(100, true)? - ); - - let max_cpu_len_log = 9; - let segment_iterator = SegmentDataIterator::::new(&dummy_payload, Some(max_cpu_len_log)); - - let mut proofs_without_keccak = vec![]; - - for segment_run in segment_iterator { - // Process and prove segment - let (_, mut segment_data) = segment_run?; - let segment_proof = timed!( + "Prove segment", + all_circuits.prove_segment( + &all_stark, + &config, + payload.trim(), + &mut segment_data, timing, - log::Level::Info, - "Prove segment", - all_circuits.prove_segment( - &all_stark, - &config, - dummy_payload.trim(), - &mut segment_data, - timing, - None, - )? - ); - - proofs_without_keccak.push(segment_proof); - } + None, + )? + ); // Verify the generated segment proof timed!( timing, log::Level::Info, "Verify segment proof", - all_circuits.verify_root(proofs_without_keccak[0].proof_with_pis.clone())? + all_circuits.verify_root(segment_proof.proof_with_pis.clone())? ); // Print timing details diff --git a/evm_arithmetization/src/generation/segments.rs b/evm_arithmetization/src/generation/segments.rs index b3a129137..8203aafdb 100644 --- a/evm_arithmetization/src/generation/segments.rs +++ b/evm_arithmetization/src/generation/segments.rs @@ -1,6 +1,8 @@ //! Module defining the logic around proof segmentation into chunks, //! which allows what is commonly known as zk-continuations. +use std::collections::HashMap; + use anyhow::Result; use plonky2::hash::hash_types::RichField; use serde::{Deserialize, Serialize}; @@ -11,6 +13,7 @@ use crate::cpu::kernel::interpreter::{set_registers_and_run, ExtraSegmentData, I use crate::generation::state::State; use crate::generation::{collect_debug_tries, debug_inputs, ErrorWithTries, GenerationInputs}; use crate::witness::memory::MemoryState; +use crate::witness::operation::Operation; use crate::witness::state::RegistersState; /// Structure holding the data needed to initialize a segment. @@ -29,6 +32,8 @@ pub struct GenerationSegmentData { pub(crate) extra_data: ExtraSegmentData, /// Log of the maximal cpu length. pub(crate) max_cpu_len_log: Option, + /// Counts the number of appearances of each opcode. For debugging purposes. + pub(crate) opcode_counts: HashMap, } impl GenerationSegmentData { @@ -77,6 +82,7 @@ fn build_segment_data( accounts: interpreter.generation_state.accounts_pointers.clone(), storage: interpreter.generation_state.storage_pointers.clone(), }, + opcode_counts: interpreter.opcode_count.clone(), } } @@ -133,6 +139,9 @@ impl SegmentDataIterator { let segment_index = segment_data.segment_index; + // Reset opcode counts before executing the segment + self.interpreter.reset_opcode_counts(); + // Run the interpreter to get `registers_after` and the partial data for the // next segment. let execution_result = @@ -147,6 +156,8 @@ impl SegmentDataIterator { )); segment_data.registers_after = updated_registers; + segment_data.opcode_counts = self.interpreter.opcode_count.clone(); + Ok(Some(Box::new((segment_data, partial_segment_data)))) } else { let inputs = &self.interpreter.get_generation_state().inputs; diff --git a/evm_arithmetization/src/logic.rs b/evm_arithmetization/src/logic.rs index 0ba378b52..7d68e463e 100644 --- a/evm_arithmetization/src/logic.rs +++ b/evm_arithmetization/src/logic.rs @@ -11,6 +11,7 @@ use plonky2::hash::hash_types::RichField; use plonky2::iop::ext_target::ExtensionTarget; use plonky2::timed; use plonky2::util::timing::TimingTree; +use serde::{Deserialize, Serialize}; use starky::constraint_consumer::{ConstraintConsumer, RecursiveConstraintConsumer}; use starky::evaluation_frame::StarkEvaluationFrame; use starky::lookup::{Column, Filter}; @@ -118,7 +119,7 @@ pub(crate) struct LogicStark { } /// Logic operations. -#[derive(Copy, Clone, Debug, Eq, PartialEq)] +#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)] pub(crate) enum Op { And, Or, diff --git a/evm_arithmetization/src/testing_utils.rs b/evm_arithmetization/src/testing_utils.rs index 383c4aa10..5f1b1b041 100644 --- a/evm_arithmetization/src/testing_utils.rs +++ b/evm_arithmetization/src/testing_utils.rs @@ -165,70 +165,46 @@ pub fn scalable_contract_from_storage(storage_trie: &HashedPartialTrie) -> Accou } } -/// Get `GenerationInputs` for a dummy payload, where the block has the given -/// timestamp. -pub fn dummy_payload(timestamp: u64, is_first_payload: bool) -> Result { - let beneficiary = hex!("deadbeefdeadbeefdeadbeefdeadbeefdeadbeef"); - +pub fn empty_payload() -> Result { + // Set up default block metadata let block_metadata = BlockMetadata { - block_beneficiary: Address::from(beneficiary), - block_timestamp: timestamp.into(), - block_number: 1.into(), - block_difficulty: 0x020000.into(), - block_random: H256::from_uint(&0x020000.into()), - block_gaslimit: 0xff112233u32.into(), - block_chain_id: 1.into(), - block_base_fee: 0xa.into(), + block_beneficiary: Address::zero(), + block_timestamp: U256::zero(), + block_number: U256::one(), + block_difficulty: U256::zero(), + block_random: H256::zero(), + block_gaslimit: U256::zero(), + block_chain_id: U256::one(), + block_base_fee: U256::zero(), ..Default::default() }; - let (mut state_trie_before, mut storage_tries) = preinitialized_state_and_storage_tries()?; + // Initialize an empty state trie and storage tries + let state_trie_before = HashedPartialTrie::from(crate::Node::Empty); + let storage_tries = Vec::new(); let checkpoint_state_trie_root = state_trie_before.hash(); - let mut beacon_roots_account_storage = storage_tries[0].1.clone(); - - update_beacon_roots_account_storage( - &mut beacon_roots_account_storage, - block_metadata.block_timestamp, - block_metadata.parent_beacon_block_root, - )?; - let updated_beacon_roots_account = - beacon_roots_contract_from_storage(&beacon_roots_account_storage); - - if !is_first_payload { - // This isn't the first dummy payload being processed. We need to update the - // initial state trie to account for the update on the beacon roots contract. - state_trie_before.insert( - beacon_roots_account_nibbles(), - rlp::encode(&updated_beacon_roots_account).to_vec(), - )?; - storage_tries[0].1 = beacon_roots_account_storage; - } + // Prepare the tries without any transactions or receipts let tries_before = TrieInputs { - state_trie: state_trie_before, - storage_tries, - ..Default::default() + state_trie: state_trie_before.clone(), + storage_tries: storage_tries.clone(), + transactions_trie: HashedPartialTrie::from(crate::Node::Empty), + receipts_trie: HashedPartialTrie::from(crate::Node::Empty), }; - let expected_state_trie_after: HashedPartialTrie = { - let mut state_trie_after = HashedPartialTrie::from(crate::Node::Empty); - state_trie_after.insert( - beacon_roots_account_nibbles(), - rlp::encode(&updated_beacon_roots_account).to_vec(), - )?; - - state_trie_after - }; + // The expected state trie after execution remains the same as before + let expected_state_trie_after = state_trie_before; + // Compute the trie roots after execution let trie_roots_after = TrieRoots { state_root: expected_state_trie_after.hash(), transactions_root: tries_before.transactions_trie.hash(), receipts_root: tries_before.receipts_trie.hash(), }; + // Construct the GenerationInputs without any transactions or state changes let inputs = GenerationInputs { - tries: tries_before.clone(), - burn_addr: None, + tries: tries_before, trie_roots_after, checkpoint_state_trie_root, block_metadata, diff --git a/evm_arithmetization/src/witness/operation.rs b/evm_arithmetization/src/witness/operation.rs index 55f3d3c18..c8e2a5376 100644 --- a/evm_arithmetization/src/witness/operation.rs +++ b/evm_arithmetization/src/witness/operation.rs @@ -2,6 +2,7 @@ use ethereum_types::{BigEndianHash, U256}; use itertools::Itertools; use keccak_hash::keccak; use plonky2::hash::hash_types::RichField; +use serde::{Deserialize, Serialize}; use super::state::KERNEL_CONTEXT; use super::transition::Transition; @@ -29,7 +30,7 @@ use crate::witness::util::{ }; use crate::{arithmetic, logic}; -#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)] pub(crate) enum Operation { Iszero, Not, diff --git a/evm_arithmetization/tests/two_to_one_block.rs b/evm_arithmetization/tests/two_to_one_block.rs index 55489187d..db4363ae6 100644 --- a/evm_arithmetization/tests/two_to_one_block.rs +++ b/evm_arithmetization/tests/two_to_one_block.rs @@ -1,12 +1,22 @@ #![cfg(feature = "eth_mainnet")] +use ethereum_types::{Address, BigEndianHash, H256}; use evm_arithmetization::fixed_recursive_verifier::{ extract_block_final_public_values, extract_two_to_one_block_hash, }; -use evm_arithmetization::proof::{FinalPublicValues, PublicValues}; -use evm_arithmetization::testing_utils::{dummy_payload, init_logger}; -use evm_arithmetization::{AllRecursiveCircuits, AllStark, StarkConfig}; +use evm_arithmetization::generation::{GenerationInputs, TrieInputs}; +use evm_arithmetization::proof::{ + BlockMetadata, FinalPublicValues, PublicValues, TrieRoots, EMPTY_CONSOLIDATED_BLOCKHASH, +}; +use evm_arithmetization::testing_utils::{ + beacon_roots_account_nibbles, beacon_roots_contract_from_storage, init_logger, + preinitialized_state_and_storage_tries, update_beacon_roots_account_storage, +}; +use evm_arithmetization::{AllRecursiveCircuits, AllStark, Node, StarkConfig}; +use hex_literal::hex; +use mpt_trie::partial_trie::{HashedPartialTrie, PartialTrie}; use plonky2::field::goldilocks_field::GoldilocksField; +use plonky2::field::types::Field; use plonky2::hash::poseidon::PoseidonHash; use plonky2::plonk::config::{Hasher, PoseidonGoldilocksConfig}; use plonky2::plonk::proof::ProofWithPublicInputs; @@ -16,6 +26,80 @@ type F = GoldilocksField; const D: usize = 2; type C = PoseidonGoldilocksConfig; +/// Get `GenerationInputs` for a dummy payload, where the block has the given +/// timestamp. +fn dummy_payload(timestamp: u64, is_first_payload: bool) -> anyhow::Result> { + let beneficiary = hex!("deadbeefdeadbeefdeadbeefdeadbeefdeadbeef"); + + let block_metadata = BlockMetadata { + block_beneficiary: Address::from(beneficiary), + block_timestamp: timestamp.into(), + block_number: 1.into(), + block_difficulty: 0x020000.into(), + block_random: H256::from_uint(&0x020000.into()), + block_gaslimit: 0xff112233u32.into(), + block_chain_id: 1.into(), + block_base_fee: 0xa.into(), + ..Default::default() + }; + + let (mut state_trie_before, mut storage_tries) = preinitialized_state_and_storage_tries()?; + let checkpoint_state_trie_root = state_trie_before.hash(); + let mut beacon_roots_account_storage = storage_tries[0].1.clone(); + + update_beacon_roots_account_storage( + &mut beacon_roots_account_storage, + block_metadata.block_timestamp, + block_metadata.parent_beacon_block_root, + )?; + let updated_beacon_roots_account = + beacon_roots_contract_from_storage(&beacon_roots_account_storage); + + if !is_first_payload { + // This isn't the first dummy payload being processed. We need to update the + // initial state trie to account for the update on the beacon roots contract. + state_trie_before.insert( + beacon_roots_account_nibbles(), + rlp::encode(&updated_beacon_roots_account).to_vec(), + )?; + storage_tries[0].1 = beacon_roots_account_storage; + } + + let tries_before = TrieInputs { + state_trie: state_trie_before, + storage_tries, + ..Default::default() + }; + + let expected_state_trie_after: HashedPartialTrie = { + let mut state_trie_after = HashedPartialTrie::from(Node::Empty); + state_trie_after.insert( + beacon_roots_account_nibbles(), + rlp::encode(&updated_beacon_roots_account).to_vec(), + )?; + + state_trie_after + }; + + let trie_roots_after = TrieRoots { + state_root: expected_state_trie_after.hash(), + transactions_root: tries_before.transactions_trie.hash(), + receipts_root: tries_before.receipts_trie.hash(), + }; + + let inputs = GenerationInputs { + tries: tries_before.clone(), + burn_addr: None, + trie_roots_after, + checkpoint_state_trie_root, + checkpoint_consolidated_hash: EMPTY_CONSOLIDATED_BLOCKHASH.map(F::from_canonical_u64), + block_metadata, + ..Default::default() + }; + + Ok(inputs) +} + fn get_test_block_proof( timestamp: u64, all_circuits: &AllRecursiveCircuits, @@ -90,17 +174,7 @@ fn test_two_to_one_block_aggregation() -> anyhow::Result<()> { let all_circuits = AllRecursiveCircuits::new( &all_stark, - &[ - 16..17, - 8..9, - 12..13, - 9..10, - 8..9, - 6..7, - 17..18, - 17..18, - 7..8, - ], + &[16..17, 8..9, 12..13, 8..9, 8..9, 6..7, 17..18, 17..18, 7..8], &config, ); From a23eaf03c25fe4f2d14369b8a9f219e83e37ee51 Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Mon, 30 Sep 2024 11:52:04 -0700 Subject: [PATCH 37/91] fmt --- evm_arithmetization/src/fixed_recursive_verifier.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/evm_arithmetization/src/fixed_recursive_verifier.rs b/evm_arithmetization/src/fixed_recursive_verifier.rs index 92853a9c0..9c3b9f42f 100644 --- a/evm_arithmetization/src/fixed_recursive_verifier.rs +++ b/evm_arithmetization/src/fixed_recursive_verifier.rs @@ -872,7 +872,11 @@ where if KECCAK_TABLES_INDICES.contains(&i) { // Ensure the challenger state: // 1) prev == current_before when using Keccak - builder. conditional_assert_eq(use_keccak_tables.target, prev_state[j], current_state_before[j]); + builder.conditional_assert_eq( + use_keccak_tables.target, + prev_state[j], + current_state_before[j], + ); // 2) Update prev <- current_after when using Keccak // 3) Keep prev <- prev when skipping Keccak prev_state[j] = From 04f51d5bf6091bad50c6edf9bc872a99957b555b Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Tue, 1 Oct 2024 10:29:50 -0700 Subject: [PATCH 38/91] wip --- Cargo.lock | 13 ++++--------- Cargo.toml | 6 +++--- .../src/fixed_recursive_verifier.rs | 8 ++++++-- evm_arithmetization/src/get_challenges.rs | 8 +++++++- evm_arithmetization/src/proof.rs | 18 +++++++++++++++++- evm_arithmetization/src/prover.rs | 6 +++--- 6 files changed, 40 insertions(+), 19 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 03223ee5f..061b3dc6d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3625,7 +3625,6 @@ checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" [[package]] name = "plonky2" version = "0.2.2" -source = "git+https://github.com/0xPolygonZero/plonky2.git?rev=dc77c77f2b06500e16ad4d7f1c2b057903602eed#dc77c77f2b06500e16ad4d7f1c2b057903602eed" dependencies = [ "ahash", "anyhow", @@ -3636,7 +3635,7 @@ dependencies = [ "log", "num", "plonky2_field", - "plonky2_maybe_rayon 0.2.0 (git+https://github.com/0xPolygonZero/plonky2.git?rev=dc77c77f2b06500e16ad4d7f1c2b057903602eed)", + "plonky2_maybe_rayon 0.2.0", "plonky2_util", "rand", "rand_chacha", @@ -3649,7 +3648,6 @@ dependencies = [ [[package]] name = "plonky2_field" version = "0.2.2" -source = "git+https://github.com/0xPolygonZero/plonky2.git?rev=dc77c77f2b06500e16ad4d7f1c2b057903602eed#dc77c77f2b06500e16ad4d7f1c2b057903602eed" dependencies = [ "anyhow", "itertools 0.11.0", @@ -3664,8 +3662,6 @@ dependencies = [ [[package]] name = "plonky2_maybe_rayon" version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92ff44a90aaca13e10e7ddf8fab815ba1b404c3f7c3ca82aaf11c46beabaa923" dependencies = [ "rayon", ] @@ -3673,7 +3669,8 @@ dependencies = [ [[package]] name = "plonky2_maybe_rayon" version = "0.2.0" -source = "git+https://github.com/0xPolygonZero/plonky2.git?rev=dc77c77f2b06500e16ad4d7f1c2b057903602eed#dc77c77f2b06500e16ad4d7f1c2b057903602eed" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92ff44a90aaca13e10e7ddf8fab815ba1b404c3f7c3ca82aaf11c46beabaa923" dependencies = [ "rayon", ] @@ -3681,7 +3678,6 @@ dependencies = [ [[package]] name = "plonky2_util" version = "0.2.0" -source = "git+https://github.com/0xPolygonZero/plonky2.git?rev=dc77c77f2b06500e16ad4d7f1c2b057903602eed#dc77c77f2b06500e16ad4d7f1c2b057903602eed" [[package]] name = "plotters" @@ -4644,7 +4640,6 @@ checksum = "8acdd7dbfcfb5dd6e46c63512508bf71c2043f70b8f143813ad75cb5e8a589f2" [[package]] name = "starky" version = "0.4.0" -source = "git+https://github.com/0xPolygonZero/plonky2.git?rev=dc77c77f2b06500e16ad4d7f1c2b057903602eed#dc77c77f2b06500e16ad4d7f1c2b057903602eed" dependencies = [ "ahash", "anyhow", @@ -4653,7 +4648,7 @@ dependencies = [ "log", "num-bigint", "plonky2", - "plonky2_maybe_rayon 0.2.0 (git+https://github.com/0xPolygonZero/plonky2.git?rev=dc77c77f2b06500e16ad4d7f1c2b057903602eed)", + "plonky2_maybe_rayon 0.2.0", "plonky2_util", ] diff --git a/Cargo.toml b/Cargo.toml index 9cbc60fd4..e67377430 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -110,10 +110,10 @@ zk_evm_proc_macro = { path = "proc_macro", version = "0.1.0" } zero = { path = "zero", default-features = false } # plonky2-related dependencies -plonky2 = { git = "https://github.com/0xPolygonZero/plonky2.git", rev = "dc77c77f2b06500e16ad4d7f1c2b057903602eed" } +plonky2 = { path = "../plonky2/plonky2" } plonky2_maybe_rayon = "0.2.0" -plonky2_util = { git = "https://github.com/0xPolygonZero/plonky2.git", rev = "dc77c77f2b06500e16ad4d7f1c2b057903602eed" } -starky = { git = "https://github.com/0xPolygonZero/plonky2.git", rev = "dc77c77f2b06500e16ad4d7f1c2b057903602eed" } +plonky2_util = { path = "../plonky2/util" } +starky = { path = "../plonky2/starky" } # proc macro related dependencies proc-macro2 = "1.0" diff --git a/evm_arithmetization/src/fixed_recursive_verifier.rs b/evm_arithmetization/src/fixed_recursive_verifier.rs index 45b2ed13e..0e323b84b 100644 --- a/evm_arithmetization/src/fixed_recursive_verifier.rs +++ b/evm_arithmetization/src/fixed_recursive_verifier.rs @@ -1919,7 +1919,9 @@ where root_inputs .set_proof_with_pis_target(&self.root.proof_with_pis[table], &dummy_proof); } else { - let stark_proof = &all_proof.multi_proof.stark_proofs[table]; + let stark_proof = &all_proof.multi_proof.stark_proofs[table] + .as_ref() + .expect("Unable to get stark proof"); let original_degree_bits = stark_proof.proof.recover_degree_bits(config); let shrunk_proof = table_circuits .by_stark_size @@ -2094,7 +2096,9 @@ where root_inputs .set_proof_with_pis_target(&self.root.proof_with_pis[table], &dummy_proof); } else { - let stark_proof = &all_proof.multi_proof.stark_proofs[table]; + let stark_proof = &all_proof.multi_proof.stark_proofs[table] + .as_ref() + .expect("Unable to get stark proof"); let shrunk_proof = table_circuit.shrink(stark_proof, &all_proof.multi_proof.ctl_challenges)?; diff --git a/evm_arithmetization/src/get_challenges.rs b/evm_arithmetization/src/get_challenges.rs index 5f0df7cfa..348a246ce 100644 --- a/evm_arithmetization/src/get_challenges.rs +++ b/evm_arithmetization/src/get_challenges.rs @@ -1,6 +1,7 @@ use ethereum_types::{BigEndianHash, H256, U256}; use plonky2::field::extension::Extendable; use plonky2::hash::hash_types::RichField; +use plonky2::hash::merkle_tree::MerkleCap; use plonky2::iop::challenger::{Challenger, RecursiveChallenger}; use plonky2::plonk::config::{AlgebraicHasher, GenericConfig}; use starky::config::StarkConfig; @@ -258,7 +259,12 @@ impl, C: GenericConfig, const D: usize> A let stark_proofs = &self.multi_proof.stark_proofs; for proof in stark_proofs { - challenger.observe_cap(&proof.proof.trace_cap); + if let Some(proof) = proof { + challenger.observe_cap(&proof.proof.trace_cap); + } else { + let zero_cap = vec![F::ZERO; config.fri_config.num_cap_elements()]; + challenger.observe_elements(&zero_cap); + } } observe_public_values::(&mut challenger, &self.public_values)?; diff --git a/evm_arithmetization/src/proof.rs b/evm_arithmetization/src/proof.rs index 50364d306..5b38249f2 100644 --- a/evm_arithmetization/src/proof.rs +++ b/evm_arithmetization/src/proof.rs @@ -11,7 +11,7 @@ use plonky2::util::serialization::{Buffer, IoResult, Read, Write}; use serde::{Deserialize, Serialize}; use starky::config::StarkConfig; use starky::lookup::GrandProductChallengeSet; -use starky::proof::{MultiProof, StarkProofChallenges}; +use starky::proof::{StarkProofChallenges, StarkProofWithMetadata}; use crate::all_stark::NUM_TABLES; use crate::util::{get_h160, get_h256, get_u256, h256_limbs, h2u}; @@ -22,6 +22,22 @@ pub(crate) const DEFAULT_CAP_HEIGHT: usize = 4; /// Number of elements contained in a Merkle cap with default height. pub(crate) const DEFAULT_CAP_LEN: usize = 1 << DEFAULT_CAP_HEIGHT; +/// A combination of STARK proofs for independent statements operating on +/// possibly shared variables, along with Cross-Table Lookup (CTL) challenges to +/// assert consistency of common variables across tables. +#[derive(Debug, Clone)] +pub struct MultiProof< + F: RichField + Extendable, + C: GenericConfig, + const D: usize, + const N: usize, +> { + /// Proofs for all the different STARK modules. + pub stark_proofs: [Option>; N], + /// Cross-table lookup challenges. + pub ctl_challenges: GrandProductChallengeSet, +} + /// A STARK proof for each table, plus some metadata used to create recursive /// wrapper proofs. #[derive(Debug, Clone)] diff --git a/evm_arithmetization/src/prover.rs b/evm_arithmetization/src/prover.rs index 0f3726ad2..020893f2c 100644 --- a/evm_arithmetization/src/prover.rs +++ b/evm_arithmetization/src/prover.rs @@ -16,7 +16,7 @@ use plonky2::util::timing::TimingTree; use starky::config::StarkConfig; use starky::cross_table_lookup::{get_ctl_data, CtlData}; use starky::lookup::GrandProductChallengeSet; -use starky::proof::{MultiProof, StarkProofWithMetadata}; +use starky::proof::StarkProofWithMetadata; use starky::prover::prove_with_commitment; use starky::stark::Stark; @@ -25,7 +25,7 @@ use crate::cpu::kernel::aggregator::KERNEL; use crate::generation::segments::GenerationSegmentData; use crate::generation::{generate_traces, GenerationInputs, TrimmedGenerationInputs}; use crate::get_challenges::observe_public_values; -use crate::proof::{AllProof, MemCap, PublicValues, DEFAULT_CAP_LEN}; +use crate::proof::{AllProof, MemCap, MultiProof, PublicValues, DEFAULT_CAP_LEN}; /// Generate traces, then create all STARK proofs. pub fn prove( @@ -220,7 +220,7 @@ where } type ProofWithMemCaps = ( - [StarkProofWithMetadata; NUM_TABLES], + [Option>; NUM_TABLES], MerkleCap, MerkleCap, ); From c822a70394130124451849c8f44606c71969b7b5 Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Tue, 1 Oct 2024 15:29:10 -0700 Subject: [PATCH 39/91] fix test --- evm_arithmetization/src/fixed_recursive_verifier.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/evm_arithmetization/src/fixed_recursive_verifier.rs b/evm_arithmetization/src/fixed_recursive_verifier.rs index 89813d530..adab855c9 100644 --- a/evm_arithmetization/src/fixed_recursive_verifier.rs +++ b/evm_arithmetization/src/fixed_recursive_verifier.rs @@ -3064,7 +3064,7 @@ mod tests { timing, log::Level::Info, "Verify segment proof", - all_circuits.verify_root(segment_proof.proof_with_pis.clone())? + all_circuits.verify_root(segment_proof.proof_with_pvs.intern.clone())? ); // Print timing details From 12c61a85e4126421beaa98c0cc6b1b7f3325beb3 Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Tue, 1 Oct 2024 15:39:13 -0700 Subject: [PATCH 40/91] fix ci with "cdk_erigon" --- evm_arithmetization/src/fixed_recursive_verifier.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/evm_arithmetization/src/fixed_recursive_verifier.rs b/evm_arithmetization/src/fixed_recursive_verifier.rs index 9e1a36e0a..791e1408c 100644 --- a/evm_arithmetization/src/fixed_recursive_verifier.rs +++ b/evm_arithmetization/src/fixed_recursive_verifier.rs @@ -3124,6 +3124,7 @@ where } #[cfg(test)] +#[cfg(not(feature = "cdk_erigon"))] mod tests { use plonky2::field::goldilocks_field::GoldilocksField; use plonky2::plonk::config::PoseidonGoldilocksConfig; From 447cfb93cdf3dbdbc8a7de170c97c2b50fa2f455 Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Tue, 1 Oct 2024 15:40:17 -0700 Subject: [PATCH 41/91] fix ci error with "cdk_erigon" --- evm_arithmetization/src/fixed_recursive_verifier.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/evm_arithmetization/src/fixed_recursive_verifier.rs b/evm_arithmetization/src/fixed_recursive_verifier.rs index adab855c9..4432011d6 100644 --- a/evm_arithmetization/src/fixed_recursive_verifier.rs +++ b/evm_arithmetization/src/fixed_recursive_verifier.rs @@ -2999,6 +2999,7 @@ where } #[cfg(test)] +#[cfg(not(feature = "cdk_erigon"))] mod tests { use plonky2::field::goldilocks_field::GoldilocksField; use plonky2::plonk::config::PoseidonGoldilocksConfig; From ac5a87ab5b8b3412da07f41b698e58c9a46d33a4 Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Wed, 2 Oct 2024 09:05:13 -0700 Subject: [PATCH 42/91] wip --- .../src/fixed_recursive_verifier.rs | 1 - evm_arithmetization/src/get_challenges.rs | 17 ++++++----- evm_arithmetization/src/proof.rs | 10 +++++-- evm_arithmetization/src/prover.rs | 30 +++++++++++-------- evm_arithmetization/src/verifier.rs | 16 +++++++++- 5 files changed, 50 insertions(+), 24 deletions(-) diff --git a/evm_arithmetization/src/fixed_recursive_verifier.rs b/evm_arithmetization/src/fixed_recursive_verifier.rs index b7f55971f..62a470faf 100644 --- a/evm_arithmetization/src/fixed_recursive_verifier.rs +++ b/evm_arithmetization/src/fixed_recursive_verifier.rs @@ -3128,7 +3128,6 @@ where } #[cfg(test)] -#[cfg(not(feature = "cdk_erigon"))] mod tests { use plonky2::field::goldilocks_field::GoldilocksField; use plonky2::plonk::config::PoseidonGoldilocksConfig; diff --git a/evm_arithmetization/src/get_challenges.rs b/evm_arithmetization/src/get_challenges.rs index 348a246ce..c0de10f6d 100644 --- a/evm_arithmetization/src/get_challenges.rs +++ b/evm_arithmetization/src/get_challenges.rs @@ -1,7 +1,6 @@ use ethereum_types::{BigEndianHash, H256, U256}; use plonky2::field::extension::Extendable; use plonky2::hash::hash_types::RichField; -use plonky2::hash::merkle_tree::MerkleCap; use plonky2::iop::challenger::{Challenger, RecursiveChallenger}; use plonky2::plonk::config::{AlgebraicHasher, GenericConfig}; use starky::config::StarkConfig; @@ -275,12 +274,16 @@ impl, C: GenericConfig, const D: usize> A Ok(AllProofChallenges { stark_challenges: core::array::from_fn(|i| { challenger.compact(); - stark_proofs[i].proof.get_challenges( - &mut challenger, - Some(&ctl_challenges), - true, - config, - ) + if let Some(stark_proof) = &stark_proofs[i] { + Some(stark_proof.proof.get_challenges( + &mut challenger, + Some(&ctl_challenges), + true, + config, + )) + } else { + None + } }), ctl_challenges, }) diff --git a/evm_arithmetization/src/proof.rs b/evm_arithmetization/src/proof.rs index 5b38249f2..cca97e3b1 100644 --- a/evm_arithmetization/src/proof.rs +++ b/evm_arithmetization/src/proof.rs @@ -55,15 +55,19 @@ pub struct AllProof, C: GenericConfig, co impl, C: GenericConfig, const D: usize> AllProof { /// Returns the degree (i.e. the trace length) of each STARK. - pub fn degree_bits(&self, config: &StarkConfig) -> [usize; NUM_TABLES] { - self.multi_proof.recover_degree_bits(config) + pub fn degree_bits(&self, config: &StarkConfig) -> [Option; NUM_TABLES] { + core::array::from_fn(|i| { + self.multi_proof.stark_proofs[i] + .as_ref() + .map(|proof| proof.proof.recover_degree_bits(config)) + }) } } /// Randomness for all STARKs. pub(crate) struct AllProofChallenges, const D: usize> { /// Randomness used in each STARK proof. - pub stark_challenges: [StarkProofChallenges; NUM_TABLES], + pub stark_challenges: [Option>; NUM_TABLES], /// Randomness used for cross-table lookups. It is shared by all STARKs. pub ctl_challenges: GrandProductChallengeSet, } diff --git a/evm_arithmetization/src/prover.rs b/evm_arithmetization/src/prover.rs index 020893f2c..535f722dc 100644 --- a/evm_arithmetization/src/prover.rs +++ b/evm_arithmetization/src/prover.rs @@ -274,10 +274,16 @@ where let (byte_packing_proof, _) = prove_table!(byte_packing_stark, Table::BytePacking); let (cpu_proof, _) = prove_table!(cpu_stark, Table::Cpu); let challenger_after_cpu = challenger.clone(); - // TODO(sdeng): Keccak proofs are still required for CTLs, etc. Refactor the - // code and remove the unnecessary parts. - let (keccak_proof, _) = prove_table!(keccak_stark, Table::Keccak); - let (keccak_sponge_proof, _) = prove_table!(keccak_sponge_stark, Table::KeccakSponge); + let keccak_proof = if use_keccak_tables { + Some(prove_table!(keccak_stark, Table::Keccak).0) + } else { + None + }; + let keccak_sponge_proof = if use_keccak_tables { + Some(prove_table!(keccak_sponge_stark, Table::KeccakSponge).0) + } else { + None + }; if !use_keccak_tables { // We need to connect the challenger state of Logic and CPU tables when the // Keccak tables are not in use. @@ -293,17 +299,17 @@ where Ok(( [ - arithmetic_proof, - byte_packing_proof, - cpu_proof, + Some(arithmetic_proof), + Some(byte_packing_proof), + Some(cpu_proof), keccak_proof, keccak_sponge_proof, - logic_proof, - memory_proof, - mem_before_proof, - mem_after_proof, + Some(logic_proof), + Some(memory_proof), + Some(mem_before_proof), + Some(mem_after_proof), #[cfg(feature = "cdk_erigon")] - poseidon_proof, + Some(poseidon_proof), ], mem_before_cap, mem_after_cap, diff --git a/evm_arithmetization/src/verifier.rs b/evm_arithmetization/src/verifier.rs index caa01859e..2516edbc9 100644 --- a/evm_arithmetization/src/verifier.rs +++ b/evm_arithmetization/src/verifier.rs @@ -146,8 +146,22 @@ fn verify_proof, C: GenericConfig, const cross_table_lookups, } = all_stark; + let auxiliary_polys = core::array::from_fn(|i| { + all_proof.multi_proof.stark_proofs[i] + .as_ref() + .map(|proof| &proof.proof.openings.auxiliary_polys) + .unwrap_or(&None) + }); + let auxiliary_polys_next = core::array::from_fn(|i| { + all_proof.multi_proof.stark_proofs[i] + .as_ref() + .map(|proof| &proof.proof.openings.auxiliary_polys_next) + .unwrap_or(&None) + }); + let ctl_vars_per_table = get_ctl_vars_from_proofs( - &all_proof.multi_proof, + &auxiliary_polys, + &auxiliary_polys_next, cross_table_lookups, &ctl_challenges, &num_lookup_columns, From 09a1a4a2f3dfddbfb082be554e9e10a9dc970e82 Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Wed, 2 Oct 2024 09:13:09 -0700 Subject: [PATCH 43/91] enable test --- evm_arithmetization/src/fixed_recursive_verifier.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/evm_arithmetization/src/fixed_recursive_verifier.rs b/evm_arithmetization/src/fixed_recursive_verifier.rs index 791e1408c..8330b8963 100644 --- a/evm_arithmetization/src/fixed_recursive_verifier.rs +++ b/evm_arithmetization/src/fixed_recursive_verifier.rs @@ -3139,7 +3139,6 @@ mod tests { type C = PoseidonGoldilocksConfig; #[test] - #[ignore] fn test_segment_proof_generation_without_keccak() -> anyhow::Result<()> { init_logger(); From 4a69e8abbe1789e33cd86101c9338030f601456c Mon Sep 17 00:00:00 2001 From: Sai <135601871+sai-deng@users.noreply.github.com> Date: Wed, 2 Oct 2024 09:22:40 -0700 Subject: [PATCH 44/91] Update evm_arithmetization/src/fixed_recursive_verifier.rs Co-authored-by: Linda Guiga <101227802+LindaGuiga@users.noreply.github.com> --- evm_arithmetization/src/fixed_recursive_verifier.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/evm_arithmetization/src/fixed_recursive_verifier.rs b/evm_arithmetization/src/fixed_recursive_verifier.rs index 8330b8963..14d09c3b2 100644 --- a/evm_arithmetization/src/fixed_recursive_verifier.rs +++ b/evm_arithmetization/src/fixed_recursive_verifier.rs @@ -2112,7 +2112,7 @@ where let common_date = &table_circuit .shrinking_wrappers .last() - .expect("") + .expect("No shrinking circuits") .circuit .common; let dummy_circuit: CircuitData = dummy_circuit(common_date); From 81d3681924e72f957727b396197f4888b59b8031 Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Wed, 2 Oct 2024 09:24:45 -0700 Subject: [PATCH 45/91] address comments --- evm_arithmetization/src/fixed_recursive_verifier.rs | 4 ++-- evm_arithmetization/src/generation/mod.rs | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/evm_arithmetization/src/fixed_recursive_verifier.rs b/evm_arithmetization/src/fixed_recursive_verifier.rs index 14d09c3b2..92ef707ec 100644 --- a/evm_arithmetization/src/fixed_recursive_verifier.rs +++ b/evm_arithmetization/src/fixed_recursive_verifier.rs @@ -1928,7 +1928,7 @@ where self.root.index_verifier_data[table], F::from_canonical_usize(*index_verifier_data), ); - let common_date = &table_circuits + let common_data = &table_circuits .by_stark_size .get(index_verifier_data) .expect("No valid size in shrinking circuits") @@ -1937,7 +1937,7 @@ where .expect("No shrinking circuits") .circuit .common; - let dummy_circuit: CircuitData = dummy_circuit(common_date); + let dummy_circuit: CircuitData = dummy_circuit(common_data); let dummy_pis = HashMap::new(); let dummy_proof = dummy_proof(&dummy_circuit, dummy_pis) .expect("Unable to generate dummy proofs"); diff --git a/evm_arithmetization/src/generation/mod.rs b/evm_arithmetization/src/generation/mod.rs index 11e881c09..8c5846d9f 100644 --- a/evm_arithmetization/src/generation/mod.rs +++ b/evm_arithmetization/src/generation/mod.rs @@ -486,7 +486,7 @@ fn get_all_memory_address_and_values(memory_before: &MemoryState) -> Vec<(Memory res } -pub struct TablesWithPVsAndFinalMem { +pub struct TablesWithPVs { pub tables: [Vec>; NUM_TABLES], pub use_keccak_tables: bool, pub public_values: PublicValues, @@ -498,7 +498,7 @@ pub fn generate_traces, const D: usize>( config: &StarkConfig, segment_data: &mut GenerationSegmentData, timing: &mut TimingTree, -) -> anyhow::Result> { +) -> anyhow::Result> { let mut state = GenerationState::::new_with_segment_data(inputs, segment_data) .map_err(|err| anyhow!("Failed to parse all the initial prover inputs: {:?}", err))?; @@ -601,7 +601,7 @@ pub fn generate_traces, const D: usize>( ) ); - Ok(TablesWithPVsAndFinalMem { + Ok(TablesWithPVs { tables, use_keccak_tables, public_values, From 79aeb72ff5ab3ff94ba4de83e7188f4c8357b25b Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Wed, 2 Oct 2024 09:39:46 -0700 Subject: [PATCH 46/91] sync changes into get_challenges --- evm_arithmetization/src/get_challenges.rs | 18 +++++++++++++-- evm_arithmetization/src/verifier.rs | 27 ++++++++++++++++++----- 2 files changed, 38 insertions(+), 7 deletions(-) diff --git a/evm_arithmetization/src/get_challenges.rs b/evm_arithmetization/src/get_challenges.rs index 5f0df7cfa..bc3e318ff 100644 --- a/evm_arithmetization/src/get_challenges.rs +++ b/evm_arithmetization/src/get_challenges.rs @@ -6,6 +6,7 @@ use plonky2::plonk::config::{AlgebraicHasher, GenericConfig}; use starky::config::StarkConfig; use starky::lookup::get_grand_product_challenge_set; +use crate::all_stark::KECCAK_TABLES_INDICES; use crate::proof::*; use crate::util::{h256_limbs, u256_limbs, u256_to_u32, u256_to_u64}; use crate::witness::errors::ProgramError; @@ -252,13 +253,26 @@ impl, C: GenericConfig, const D: usize> A pub(crate) fn get_challenges( &self, config: &StarkConfig, + use_keccak_tables: bool, ) -> Result, ProgramError> { let mut challenger = Challenger::::new(); let stark_proofs = &self.multi_proof.stark_proofs; - for proof in stark_proofs { - challenger.observe_cap(&proof.proof.trace_cap); + for (i, proof) in stark_proofs.iter().enumerate() { + if KECCAK_TABLES_INDICES.contains(&i) && !use_keccak_tables { + // Observe zero merkle caps when skipping Keccak tables. + let zero_merkle_cap = proof + .proof + .trace_cap + .flatten() + .iter() + .map(|_| F::ZERO) + .collect::>(); + challenger.observe_elements(&zero_merkle_cap); + } else { + challenger.observe_cap(&proof.proof.trace_cap); + } } observe_public_values::(&mut challenger, &self.public_values)?; diff --git a/evm_arithmetization/src/verifier.rs b/evm_arithmetization/src/verifier.rs index caa01859e..788a974dd 100644 --- a/evm_arithmetization/src/verifier.rs +++ b/evm_arithmetization/src/verifier.rs @@ -120,13 +120,14 @@ fn verify_proof, C: GenericConfig, const all_stark: &AllStark, all_proof: AllProof, config: &StarkConfig, + use_keccak_tables: bool, is_initial: bool, ) -> Result<()> { let AllProofChallenges { stark_challenges, ctl_challenges, } = all_proof - .get_challenges(config) + .get_challenges(config, use_keccak_tables) .map_err(|_| anyhow::Error::msg("Invalid sampling of proof challenges."))?; let num_lookup_columns = all_stark.num_lookups_helper_columns(config); @@ -172,8 +173,10 @@ fn verify_proof, C: GenericConfig, const verify_table!(arithmetic_stark, Table::Arithmetic); verify_table!(byte_packing_stark, Table::BytePacking); verify_table!(cpu_stark, Table::Cpu); - verify_table!(keccak_stark, Table::Keccak); - verify_table!(keccak_sponge_stark, Table::KeccakSponge); + if use_keccak_tables { + verify_table!(keccak_stark, Table::Keccak); + verify_table!(keccak_sponge_stark, Table::KeccakSponge); + } verify_table!(logic_stark, Table::Logic); verify_table!(memory_stark, Table::Memory); verify_table!(mem_before_stark, Table::MemBefore); @@ -423,10 +426,24 @@ pub mod testing { ) -> Result<()> { assert!(!all_proofs.is_empty()); - verify_proof(all_stark, all_proofs[0].clone(), config, true)?; + // TODO(sdeng): determine the value of `use_keccak_tables` from `all_proof`. + let use_keccak_tables = true; + verify_proof( + all_stark, + all_proofs[0].clone(), + config, + use_keccak_tables, + true, + )?; for all_proof in &all_proofs[1..] { - verify_proof(all_stark, all_proof.clone(), config, false)?; + verify_proof( + all_stark, + all_proof.clone(), + config, + use_keccak_tables, + false, + )?; } Ok(()) From 732020d51511c477778b1642e493a389c957ef1e Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Wed, 2 Oct 2024 14:32:36 -0700 Subject: [PATCH 47/91] wip --- evm_arithmetization/src/get_challenges.rs | 2 ++ evm_arithmetization/src/proof.rs | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/evm_arithmetization/src/get_challenges.rs b/evm_arithmetization/src/get_challenges.rs index bc3e318ff..4bd42db54 100644 --- a/evm_arithmetization/src/get_challenges.rs +++ b/evm_arithmetization/src/get_challenges.rs @@ -280,6 +280,8 @@ impl, C: GenericConfig, const D: usize> A let ctl_challenges = get_grand_product_challenge_set(&mut challenger, config.num_challenges); + let stark_challenges = + Ok(AllProofChallenges { stark_challenges: core::array::from_fn(|i| { challenger.compact(); diff --git a/evm_arithmetization/src/proof.rs b/evm_arithmetization/src/proof.rs index 50364d306..ec27293e8 100644 --- a/evm_arithmetization/src/proof.rs +++ b/evm_arithmetization/src/proof.rs @@ -47,7 +47,7 @@ impl, C: GenericConfig, const D: usize> A /// Randomness for all STARKs. pub(crate) struct AllProofChallenges, const D: usize> { /// Randomness used in each STARK proof. - pub stark_challenges: [StarkProofChallenges; NUM_TABLES], + pub stark_challenges: [Option>; NUM_TABLES], /// Randomness used for cross-table lookups. It is shared by all STARKs. pub ctl_challenges: GrandProductChallengeSet, } From 9f8ec9e0b29a0decbc3119c85dcd6b38ac6d52ac Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Wed, 2 Oct 2024 14:42:08 -0700 Subject: [PATCH 48/91] fix --- evm_arithmetization/src/get_challenges.rs | 20 +++++++++++--------- evm_arithmetization/src/verifier.rs | 4 +++- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/evm_arithmetization/src/get_challenges.rs b/evm_arithmetization/src/get_challenges.rs index 4bd42db54..d5f076e7e 100644 --- a/evm_arithmetization/src/get_challenges.rs +++ b/evm_arithmetization/src/get_challenges.rs @@ -280,17 +280,19 @@ impl, C: GenericConfig, const D: usize> A let ctl_challenges = get_grand_product_challenge_set(&mut challenger, config.num_challenges); - let stark_challenges = - Ok(AllProofChallenges { stark_challenges: core::array::from_fn(|i| { - challenger.compact(); - stark_proofs[i].proof.get_challenges( - &mut challenger, - Some(&ctl_challenges), - true, - config, - ) + if KECCAK_TABLES_INDICES.contains(&i) && !use_keccak_tables { + challenger.compact(); + Some(stark_proofs[i].proof.get_challenges( + &mut challenger, + Some(&ctl_challenges), + true, + config, + )) + } else { + None + } }), ctl_challenges, }) diff --git a/evm_arithmetization/src/verifier.rs b/evm_arithmetization/src/verifier.rs index 788a974dd..b53c2b077 100644 --- a/evm_arithmetization/src/verifier.rs +++ b/evm_arithmetization/src/verifier.rs @@ -162,7 +162,9 @@ fn verify_proof, C: GenericConfig, const verify_stark_proof_with_challenges( $stark, &stark_proofs[*$table].proof, - &stark_challenges[*$table], + &stark_challenges[*$table] + .as_ref() + .expect("Missing challenges"), Some(&ctl_vars_per_table[*$table]), &[], config, From 503085bfb8d7bafd19bf33a0c63377bf13f05bc3 Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Wed, 2 Oct 2024 14:55:37 -0700 Subject: [PATCH 49/91] fix --- evm_arithmetization/src/verifier.rs | 23 ++++------------------- 1 file changed, 4 insertions(+), 19 deletions(-) diff --git a/evm_arithmetization/src/verifier.rs b/evm_arithmetization/src/verifier.rs index b53c2b077..efac7ebb1 100644 --- a/evm_arithmetization/src/verifier.rs +++ b/evm_arithmetization/src/verifier.rs @@ -120,14 +120,13 @@ fn verify_proof, C: GenericConfig, const all_stark: &AllStark, all_proof: AllProof, config: &StarkConfig, - use_keccak_tables: bool, is_initial: bool, ) -> Result<()> { let AllProofChallenges { stark_challenges, ctl_challenges, } = all_proof - .get_challenges(config, use_keccak_tables) + .get_challenges(config, all_proof.use_keccak_tables) .map_err(|_| anyhow::Error::msg("Invalid sampling of proof challenges."))?; let num_lookup_columns = all_stark.num_lookups_helper_columns(config); @@ -175,7 +174,7 @@ fn verify_proof, C: GenericConfig, const verify_table!(arithmetic_stark, Table::Arithmetic); verify_table!(byte_packing_stark, Table::BytePacking); verify_table!(cpu_stark, Table::Cpu); - if use_keccak_tables { + if all_proof.use_keccak_tables { verify_table!(keccak_stark, Table::Keccak); verify_table!(keccak_sponge_stark, Table::KeccakSponge); } @@ -428,24 +427,10 @@ pub mod testing { ) -> Result<()> { assert!(!all_proofs.is_empty()); - // TODO(sdeng): determine the value of `use_keccak_tables` from `all_proof`. - let use_keccak_tables = true; - verify_proof( - all_stark, - all_proofs[0].clone(), - config, - use_keccak_tables, - true, - )?; + verify_proof(all_stark, all_proofs[0].clone(), config, true)?; for all_proof in &all_proofs[1..] { - verify_proof( - all_stark, - all_proof.clone(), - config, - use_keccak_tables, - false, - )?; + verify_proof(all_stark, all_proof.clone(), config, false)?; } Ok(()) From 09a3f7c914513f344aa45beb5e1305db202e6e60 Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Wed, 2 Oct 2024 17:12:13 -0700 Subject: [PATCH 50/91] wip --- evm_arithmetization/src/verifier.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/evm_arithmetization/src/verifier.rs b/evm_arithmetization/src/verifier.rs index 8210128ed..75b1201de 100644 --- a/evm_arithmetization/src/verifier.rs +++ b/evm_arithmetization/src/verifier.rs @@ -126,7 +126,7 @@ fn verify_proof, C: GenericConfig, const stark_challenges, ctl_challenges, } = all_proof - .get_challenges(config, all_proof.use_keccak_tables) + .get_challenges(config) .map_err(|_| anyhow::Error::msg("Invalid sampling of proof challenges."))?; let num_lookup_columns = all_stark.num_lookups_helper_columns(config); From a36c5f78dc14da33fd1eeafbb2f9a8d3bf7ca6c4 Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Wed, 2 Oct 2024 17:17:56 -0700 Subject: [PATCH 51/91] fix tests --- evm_arithmetization/src/get_challenges.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/evm_arithmetization/src/get_challenges.rs b/evm_arithmetization/src/get_challenges.rs index d5f076e7e..c18f9e920 100644 --- a/evm_arithmetization/src/get_challenges.rs +++ b/evm_arithmetization/src/get_challenges.rs @@ -253,14 +253,13 @@ impl, C: GenericConfig, const D: usize> A pub(crate) fn get_challenges( &self, config: &StarkConfig, - use_keccak_tables: bool, ) -> Result, ProgramError> { let mut challenger = Challenger::::new(); let stark_proofs = &self.multi_proof.stark_proofs; for (i, proof) in stark_proofs.iter().enumerate() { - if KECCAK_TABLES_INDICES.contains(&i) && !use_keccak_tables { + if KECCAK_TABLES_INDICES.contains(&i) && !self.use_keccak_tables { // Observe zero merkle caps when skipping Keccak tables. let zero_merkle_cap = proof .proof @@ -282,7 +281,9 @@ impl, C: GenericConfig, const D: usize> A Ok(AllProofChallenges { stark_challenges: core::array::from_fn(|i| { - if KECCAK_TABLES_INDICES.contains(&i) && !use_keccak_tables { + if KECCAK_TABLES_INDICES.contains(&i) && !self.use_keccak_tables { + None + } else { challenger.compact(); Some(stark_proofs[i].proof.get_challenges( &mut challenger, @@ -290,8 +291,6 @@ impl, C: GenericConfig, const D: usize> A true, config, )) - } else { - None } }), ctl_challenges, From fb59562d66ce975c6b7affce0659055a35152efa Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Wed, 2 Oct 2024 18:19:59 -0700 Subject: [PATCH 52/91] wip --- evm_arithmetization/src/verifier.rs | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/evm_arithmetization/src/verifier.rs b/evm_arithmetization/src/verifier.rs index 75b1201de..89a36b1fd 100644 --- a/evm_arithmetization/src/verifier.rs +++ b/evm_arithmetization/src/verifier.rs @@ -174,7 +174,10 @@ fn verify_proof, C: GenericConfig, const ($stark:ident, $table:expr) => { verify_stark_proof_with_challenges( $stark, - &stark_proofs[*$table].proof, + &stark_proofs[*$table] + .as_ref() + .expect("Missing stark_proof") + .proof, &stark_challenges[*$table] .as_ref() .expect("Missing challenges"), @@ -218,10 +221,18 @@ fn verify_proof, C: GenericConfig, const verify_cross_table_lookups::( cross_table_lookups, - all_proof - .multi_proof - .stark_proofs - .map(|p| p.proof.openings.ctl_zs_first.unwrap()), + core::array::from_fn(|i| { + if let Some(stark_proof) = &stark_proofs[i] { + stark_proof + .proof + .openings + .ctl_zs_first + .as_ref() + .expect("Missing ctl_zs") + } else { + + } + }), Some(&extra_looking_sums), config, ) From 640bae1a6f8c12e25d630b3c9f25e8d1571ce167 Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Wed, 2 Oct 2024 18:20:54 -0700 Subject: [PATCH 53/91] fix --- evm_arithmetization/src/verifier.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/evm_arithmetization/src/verifier.rs b/evm_arithmetization/src/verifier.rs index efac7ebb1..434100bbd 100644 --- a/evm_arithmetization/src/verifier.rs +++ b/evm_arithmetization/src/verifier.rs @@ -126,7 +126,7 @@ fn verify_proof, C: GenericConfig, const stark_challenges, ctl_challenges, } = all_proof - .get_challenges(config, all_proof.use_keccak_tables) + .get_challenges(config) .map_err(|_| anyhow::Error::msg("Invalid sampling of proof challenges."))?; let num_lookup_columns = all_stark.num_lookups_helper_columns(config); From 3eadb91700f7eacf381e0b3d6627b703674d11fa Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Thu, 3 Oct 2024 16:19:40 -0700 Subject: [PATCH 54/91] done --- Cargo.lock | 13 +++++++++---- Cargo.toml | 6 +++--- evm_arithmetization/src/verifier.rs | 29 ++++++++++++++++++++++++++--- zero/src/prover_state/mod.rs | 5 ++++- 4 files changed, 42 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6540ce734..ef268f919 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3599,6 +3599,7 @@ checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" [[package]] name = "plonky2" version = "0.2.2" +source = "git+https://github.com/0xPolygonZero/plonky2.git?rev=cd0fa567938f8f5f5066493cd862a8913fbd1fdf#cd0fa567938f8f5f5066493cd862a8913fbd1fdf" dependencies = [ "ahash", "anyhow", @@ -3609,7 +3610,7 @@ dependencies = [ "log", "num", "plonky2_field", - "plonky2_maybe_rayon 0.2.0", + "plonky2_maybe_rayon 0.2.0 (git+https://github.com/0xPolygonZero/plonky2.git?rev=cd0fa567938f8f5f5066493cd862a8913fbd1fdf)", "plonky2_util", "rand", "rand_chacha", @@ -3622,6 +3623,7 @@ dependencies = [ [[package]] name = "plonky2_field" version = "0.2.2" +source = "git+https://github.com/0xPolygonZero/plonky2.git?rev=cd0fa567938f8f5f5066493cd862a8913fbd1fdf#cd0fa567938f8f5f5066493cd862a8913fbd1fdf" dependencies = [ "anyhow", "itertools 0.11.0", @@ -3636,6 +3638,8 @@ dependencies = [ [[package]] name = "plonky2_maybe_rayon" version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92ff44a90aaca13e10e7ddf8fab815ba1b404c3f7c3ca82aaf11c46beabaa923" dependencies = [ "rayon", ] @@ -3643,8 +3647,7 @@ dependencies = [ [[package]] name = "plonky2_maybe_rayon" version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92ff44a90aaca13e10e7ddf8fab815ba1b404c3f7c3ca82aaf11c46beabaa923" +source = "git+https://github.com/0xPolygonZero/plonky2.git?rev=cd0fa567938f8f5f5066493cd862a8913fbd1fdf#cd0fa567938f8f5f5066493cd862a8913fbd1fdf" dependencies = [ "rayon", ] @@ -3652,6 +3655,7 @@ dependencies = [ [[package]] name = "plonky2_util" version = "0.2.0" +source = "git+https://github.com/0xPolygonZero/plonky2.git?rev=cd0fa567938f8f5f5066493cd862a8913fbd1fdf#cd0fa567938f8f5f5066493cd862a8913fbd1fdf" [[package]] name = "plotters" @@ -4601,6 +4605,7 @@ checksum = "8acdd7dbfcfb5dd6e46c63512508bf71c2043f70b8f143813ad75cb5e8a589f2" [[package]] name = "starky" version = "0.4.0" +source = "git+https://github.com/0xPolygonZero/plonky2.git?rev=cd0fa567938f8f5f5066493cd862a8913fbd1fdf#cd0fa567938f8f5f5066493cd862a8913fbd1fdf" dependencies = [ "ahash", "anyhow", @@ -4609,7 +4614,7 @@ dependencies = [ "log", "num-bigint", "plonky2", - "plonky2_maybe_rayon 0.2.0", + "plonky2_maybe_rayon 0.2.0 (git+https://github.com/0xPolygonZero/plonky2.git?rev=cd0fa567938f8f5f5066493cd862a8913fbd1fdf)", "plonky2_util", ] diff --git a/Cargo.toml b/Cargo.toml index 0c9bf3071..9cf1dc1f4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -114,10 +114,10 @@ zk_evm_proc_macro = { path = "proc_macro", version = "0.1.0" } zero = { path = "zero", default-features = false } # plonky2-related dependencies -plonky2 = { path = "../plonky2/plonky2" } +plonky2 = { git = "https://github.com/0xPolygonZero/plonky2.git", rev = "cd0fa567938f8f5f5066493cd862a8913fbd1fdf" } plonky2_maybe_rayon = "0.2.0" -plonky2_util = { path = "../plonky2/util" } -starky = { path = "../plonky2/starky" } +plonky2_util = { git = "https://github.com/0xPolygonZero/plonky2.git", rev = "cd0fa567938f8f5f5066493cd862a8913fbd1fdf" } +starky = { git = "https://github.com/0xPolygonZero/plonky2.git", rev = "cd0fa567938f8f5f5066493cd862a8913fbd1fdf" } [workspace.lints.clippy] too_long_first_doc_paragraph = "allow" diff --git a/evm_arithmetization/src/verifier.rs b/evm_arithmetization/src/verifier.rs index 89a36b1fd..965e23294 100644 --- a/evm_arithmetization/src/verifier.rs +++ b/evm_arithmetization/src/verifier.rs @@ -10,7 +10,9 @@ use plonky2::plonk::config::{GenericConfig, GenericHashOut}; use plonky2::util::timing::TimingTree; use plonky2::util::transpose; use starky::config::StarkConfig; -use starky::cross_table_lookup::{get_ctl_vars_from_proofs, verify_cross_table_lookups}; +use starky::cross_table_lookup::{ + get_ctl_vars_from_proofs, verify_cross_table_lookups, CrossTableLookup, +}; use starky::lookup::GrandProductChallenge; use starky::stark::Stark; use starky::verifier::verify_stark_proof_with_challenges; @@ -159,7 +161,7 @@ fn verify_proof, C: GenericConfig, const .unwrap_or(&None) }); - let ctl_vars_per_table = get_ctl_vars_from_proofs( + let ctl_vars_per_table = get_ctl_vars_from_proofs::( &auxiliary_polys, &auxiliary_polys_next, cross_table_lookups, @@ -219,6 +221,8 @@ fn verify_proof, C: GenericConfig, const .map(|i| get_memory_extra_looking_sum(&public_values, ctl_challenges.challenges[i])) .collect_vec(); + let all_ctls = &all_stark.cross_table_lookups; + verify_cross_table_lookups::( cross_table_lookups, core::array::from_fn(|i| { @@ -229,8 +233,27 @@ fn verify_proof, C: GenericConfig, const .ctl_zs_first .as_ref() .expect("Missing ctl_zs") + .clone() } else { - + if i == *Table::Keccak { + let (_, n, _) = CrossTableLookup::num_ctl_helpers_zs_all( + all_ctls, + *Table::Keccak, + config.num_challenges, + keccak_stark.constraint_degree(), + ); + vec![F::ZERO; n] + } else if i == *Table::KeccakSponge { + let (_, n, _) = CrossTableLookup::num_ctl_helpers_zs_all( + all_ctls, + *Table::KeccakSponge, + config.num_challenges, + keccak_sponge_stark.constraint_degree(), + ); + vec![F::ZERO; n] + } else { + panic!("Unable to find stark_proof"); + } } }), Some(&extra_looking_sums), diff --git a/zero/src/prover_state/mod.rs b/zero/src/prover_state/mod.rs index edbad02ba..4ccfe48f8 100644 --- a/zero/src/prover_state/mod.rs +++ b/zero/src/prover_state/mod.rs @@ -175,7 +175,10 @@ impl ProverStateManager { config: &StarkConfig, all_proof: &AllProof, ) -> anyhow::Result<[(RecursiveCircuitsForTableSize, u8); NUM_TABLES]> { - let degrees = all_proof.degree_bits(config); + let default_degree = 5; // default value for None + let degrees = all_proof + .degree_bits(config) + .map(|opt| opt.unwrap_or(default_degree)); /// Given a recursive circuit index (e.g., Arithmetic / 0), return a /// tuple containing the loaded table at the specified size and From d04d0ffe3630d0c7acb51c809c7aa8bbe167e444 Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Thu, 3 Oct 2024 16:24:46 -0700 Subject: [PATCH 55/91] fix clippy --- evm_arithmetization/src/verifier.rs | 36 ++++++++++++++--------------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/evm_arithmetization/src/verifier.rs b/evm_arithmetization/src/verifier.rs index 965e23294..acee503c8 100644 --- a/evm_arithmetization/src/verifier.rs +++ b/evm_arithmetization/src/verifier.rs @@ -234,26 +234,24 @@ fn verify_proof, C: GenericConfig, const .as_ref() .expect("Missing ctl_zs") .clone() + } else if i == *Table::Keccak { + let (_, n, _) = CrossTableLookup::num_ctl_helpers_zs_all( + all_ctls, + *Table::Keccak, + config.num_challenges, + keccak_stark.constraint_degree(), + ); + vec![F::ZERO; n] + } else if i == *Table::KeccakSponge { + let (_, n, _) = CrossTableLookup::num_ctl_helpers_zs_all( + all_ctls, + *Table::KeccakSponge, + config.num_challenges, + keccak_sponge_stark.constraint_degree(), + ); + vec![F::ZERO; n] } else { - if i == *Table::Keccak { - let (_, n, _) = CrossTableLookup::num_ctl_helpers_zs_all( - all_ctls, - *Table::Keccak, - config.num_challenges, - keccak_stark.constraint_degree(), - ); - vec![F::ZERO; n] - } else if i == *Table::KeccakSponge { - let (_, n, _) = CrossTableLookup::num_ctl_helpers_zs_all( - all_ctls, - *Table::KeccakSponge, - config.num_challenges, - keccak_sponge_stark.constraint_degree(), - ); - vec![F::ZERO; n] - } else { - panic!("Unable to find stark_proof"); - } + panic!("Unable to find stark_proof"); } }), Some(&extra_looking_sums), From 87e91021c9eb018c865bddb24d5c6f43d36dbead Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Thu, 3 Oct 2024 16:27:07 -0700 Subject: [PATCH 56/91] fix build --- evm_arithmetization/src/fixed_recursive_verifier.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/evm_arithmetization/src/fixed_recursive_verifier.rs b/evm_arithmetization/src/fixed_recursive_verifier.rs index 12567aabd..4b442d20e 100644 --- a/evm_arithmetization/src/fixed_recursive_verifier.rs +++ b/evm_arithmetization/src/fixed_recursive_verifier.rs @@ -3128,6 +3128,7 @@ where } #[cfg(test)] +#[cfg(not(feature = "cdk_erigon"))] mod tests { use plonky2::field::goldilocks_field::GoldilocksField; use plonky2::plonk::config::PoseidonGoldilocksConfig; From 86eccafc3e02419b39ccc26947a49514ec816b12 Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Thu, 3 Oct 2024 19:32:17 -0700 Subject: [PATCH 57/91] change default degree --- zero/src/prover_state/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zero/src/prover_state/mod.rs b/zero/src/prover_state/mod.rs index 4ccfe48f8..aaaa7a371 100644 --- a/zero/src/prover_state/mod.rs +++ b/zero/src/prover_state/mod.rs @@ -175,7 +175,7 @@ impl ProverStateManager { config: &StarkConfig, all_proof: &AllProof, ) -> anyhow::Result<[(RecursiveCircuitsForTableSize, u8); NUM_TABLES]> { - let default_degree = 5; // default value for None + let default_degree = 8; // default value for None let degrees = all_proof .degree_bits(config) .map(|opt| opt.unwrap_or(default_degree)); From e1a5bb010f75dc1e8800936b6d8b34f9a5864882 Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Fri, 4 Oct 2024 15:18:31 -0700 Subject: [PATCH 58/91] wip --- evm_arithmetization/src/arithmetic/mod.rs | 4 +- evm_arithmetization/src/generation/mod.rs | 2 +- .../src/generation/segments.rs | 2 +- evm_arithmetization/src/logic.rs | 2 +- evm_arithmetization/src/witness/mod.rs | 2 +- evm_arithmetization/src/witness/operation.rs | 2 +- evm_arithmetization/tests/add11_yml.rs | 46 ++++++++++++++++++- 7 files changed, 51 insertions(+), 9 deletions(-) diff --git a/evm_arithmetization/src/arithmetic/mod.rs b/evm_arithmetization/src/arithmetic/mod.rs index ab65def9b..39ae32a1a 100644 --- a/evm_arithmetization/src/arithmetic/mod.rs +++ b/evm_arithmetization/src/arithmetic/mod.rs @@ -26,7 +26,7 @@ pub(crate) mod columns; /// `Shl` and `Shr` are handled differently, by leveraging `Mul` and `Div` /// respectively. #[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)] -pub(crate) enum BinaryOperator { +pub enum BinaryOperator { Add, Mul, Sub, @@ -116,7 +116,7 @@ impl BinaryOperator { /// An enum representing different ternary operations. #[allow(clippy::enum_variant_names)] #[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)] -pub(crate) enum TernaryOperator { +pub enum TernaryOperator { AddMod, MulMod, SubMod, diff --git a/evm_arithmetization/src/generation/mod.rs b/evm_arithmetization/src/generation/mod.rs index 8c5846d9f..6e4c23c50 100644 --- a/evm_arithmetization/src/generation/mod.rs +++ b/evm_arithmetization/src/generation/mod.rs @@ -229,7 +229,7 @@ impl GenerationInputs { /// Outputs a trimmed version of the `GenerationInputs`, that do not contain /// the fields that have already been processed during pre-initialization, /// namely: the input tries, the signed transaction, and the withdrawals. - pub(crate) fn trim(&self) -> TrimmedGenerationInputs { + pub fn trim(&self) -> TrimmedGenerationInputs { let txn_hashes = self .signed_txns .iter() diff --git a/evm_arithmetization/src/generation/segments.rs b/evm_arithmetization/src/generation/segments.rs index 36bac8f8a..440c14169 100644 --- a/evm_arithmetization/src/generation/segments.rs +++ b/evm_arithmetization/src/generation/segments.rs @@ -33,7 +33,7 @@ pub struct GenerationSegmentData { /// Log of the maximal cpu length. pub(crate) max_cpu_len_log: Option, /// Counts the number of appearances of each opcode. For debugging purposes. - pub(crate) opcode_counts: HashMap, + pub opcode_counts: HashMap, } impl GenerationSegmentData { diff --git a/evm_arithmetization/src/logic.rs b/evm_arithmetization/src/logic.rs index 7d68e463e..19d68cfd3 100644 --- a/evm_arithmetization/src/logic.rs +++ b/evm_arithmetization/src/logic.rs @@ -120,7 +120,7 @@ pub(crate) struct LogicStark { /// Logic operations. #[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)] -pub(crate) enum Op { +pub enum Op { And, Or, Xor, diff --git a/evm_arithmetization/src/witness/mod.rs b/evm_arithmetization/src/witness/mod.rs index a38a55229..74ff47644 100644 --- a/evm_arithmetization/src/witness/mod.rs +++ b/evm_arithmetization/src/witness/mod.rs @@ -1,7 +1,7 @@ pub(crate) mod errors; pub(crate) mod gas; pub(crate) mod memory; -pub(crate) mod operation; +pub mod operation; pub(crate) mod state; pub(crate) mod traces; pub mod transition; diff --git a/evm_arithmetization/src/witness/operation.rs b/evm_arithmetization/src/witness/operation.rs index c8e2a5376..f29dfab5c 100644 --- a/evm_arithmetization/src/witness/operation.rs +++ b/evm_arithmetization/src/witness/operation.rs @@ -31,7 +31,7 @@ use crate::witness::util::{ use crate::{arithmetic, logic}; #[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)] -pub(crate) enum Operation { +pub enum Operation { Iszero, Not, /// (syscall number, minimum stack length, increases stack length) diff --git a/evm_arithmetization/tests/add11_yml.rs b/evm_arithmetization/tests/add11_yml.rs index dd9bfb1ce..569a97fc2 100644 --- a/evm_arithmetization/tests/add11_yml.rs +++ b/evm_arithmetization/tests/add11_yml.rs @@ -8,14 +8,17 @@ use ethereum_types::{Address, BigEndianHash, H256}; use evm_arithmetization::generation::mpt::{AccountRlp, LegacyReceiptRlp}; use evm_arithmetization::generation::TrieInputs; use evm_arithmetization::proof::{BlockHashes, BlockMetadata, TrieRoots}; +use evm_arithmetization::prover::prove; use evm_arithmetization::prover::testing::prove_all_segments; use evm_arithmetization::testing_utils::{ beacon_roots_account_nibbles, beacon_roots_contract_from_storage, init_logger, preinitialized_state_and_storage_tries, update_beacon_roots_account_storage, }; use evm_arithmetization::verifier::testing::verify_all_proofs; +use evm_arithmetization::witness::operation::Operation::KeccakGeneral; use evm_arithmetization::{ - AllStark, GenerationInputs, Node, StarkConfig, EMPTY_CONSOLIDATED_BLOCKHASH, + AllStark, GenerationInputs, Node, SegmentDataIterator, StarkConfig, + EMPTY_CONSOLIDATED_BLOCKHASH, }; use hex_literal::hex; use keccak_hash::keccak; @@ -23,7 +26,7 @@ use mpt_trie::nibbles::Nibbles; use mpt_trie::partial_trie::{HashedPartialTrie, PartialTrie}; use plonky2::field::goldilocks_field::GoldilocksField; use plonky2::field::types::Field; -use plonky2::plonk::config::KeccakGoldilocksConfig; +use plonky2::plonk::config::{KeccakGoldilocksConfig, PoseidonGoldilocksConfig}; use plonky2::util::timing::TimingTree; type F = GoldilocksField; @@ -202,6 +205,7 @@ fn get_generation_inputs() -> GenerationInputs { }, } } + /// The `add11_yml` test case from https://github.com/ethereum/tests #[test] fn add11_yml() -> anyhow::Result<()> { @@ -228,3 +232,41 @@ fn add11_yml() -> anyhow::Result<()> { verify_all_proofs(&all_stark, &proofs, &config) } + +/// This test focuses on testing zkVM proofs with some empty tables. +#[test] +fn add11_yml_first_segment() -> anyhow::Result<()> { + type F = GoldilocksField; + const D: usize = 2; + type C = PoseidonGoldilocksConfig; + + init_logger(); + + let all_stark = AllStark::::default(); + let config = StarkConfig::standard_fast_config(); + let inputs = get_generation_inputs(); + + let mut timing = TimingTree::new("prove", log::Level::Debug); + + let max_cpu_len_log = 7; + let mut segment_iterator = SegmentDataIterator::::new(&inputs, Some(max_cpu_len_log)); + let mut proofs = vec![]; + let (_, mut segment_data) = segment_iterator.next().unwrap()?; + + let opcode_counts = &segment_data.opcode_counts; + assert!(!opcode_counts.contains_key(&KeccakGeneral)); + + let proof = prove::( + &all_stark, + &config, + inputs.trim(), + &mut segment_data, + &mut timing, + None, + )?; + proofs.push(proof); + + timing.filter(Duration::from_millis(100)).print(); + + verify_all_proofs(&all_stark, &proofs, &config) +} From b89359e8367923b5414480b4ad95214c2cf0c874 Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Sat, 5 Oct 2024 15:33:37 -0700 Subject: [PATCH 59/91] refactor get ctl_vars from proof --- Cargo.lock | 13 +++---- Cargo.toml | 6 ++-- evm_arithmetization/src/verifier.rs | 55 +++++++++++++---------------- 3 files changed, 32 insertions(+), 42 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ef268f919..6540ce734 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3599,7 +3599,6 @@ checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" [[package]] name = "plonky2" version = "0.2.2" -source = "git+https://github.com/0xPolygonZero/plonky2.git?rev=cd0fa567938f8f5f5066493cd862a8913fbd1fdf#cd0fa567938f8f5f5066493cd862a8913fbd1fdf" dependencies = [ "ahash", "anyhow", @@ -3610,7 +3609,7 @@ dependencies = [ "log", "num", "plonky2_field", - "plonky2_maybe_rayon 0.2.0 (git+https://github.com/0xPolygonZero/plonky2.git?rev=cd0fa567938f8f5f5066493cd862a8913fbd1fdf)", + "plonky2_maybe_rayon 0.2.0", "plonky2_util", "rand", "rand_chacha", @@ -3623,7 +3622,6 @@ dependencies = [ [[package]] name = "plonky2_field" version = "0.2.2" -source = "git+https://github.com/0xPolygonZero/plonky2.git?rev=cd0fa567938f8f5f5066493cd862a8913fbd1fdf#cd0fa567938f8f5f5066493cd862a8913fbd1fdf" dependencies = [ "anyhow", "itertools 0.11.0", @@ -3638,8 +3636,6 @@ dependencies = [ [[package]] name = "plonky2_maybe_rayon" version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92ff44a90aaca13e10e7ddf8fab815ba1b404c3f7c3ca82aaf11c46beabaa923" dependencies = [ "rayon", ] @@ -3647,7 +3643,8 @@ dependencies = [ [[package]] name = "plonky2_maybe_rayon" version = "0.2.0" -source = "git+https://github.com/0xPolygonZero/plonky2.git?rev=cd0fa567938f8f5f5066493cd862a8913fbd1fdf#cd0fa567938f8f5f5066493cd862a8913fbd1fdf" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92ff44a90aaca13e10e7ddf8fab815ba1b404c3f7c3ca82aaf11c46beabaa923" dependencies = [ "rayon", ] @@ -3655,7 +3652,6 @@ dependencies = [ [[package]] name = "plonky2_util" version = "0.2.0" -source = "git+https://github.com/0xPolygonZero/plonky2.git?rev=cd0fa567938f8f5f5066493cd862a8913fbd1fdf#cd0fa567938f8f5f5066493cd862a8913fbd1fdf" [[package]] name = "plotters" @@ -4605,7 +4601,6 @@ checksum = "8acdd7dbfcfb5dd6e46c63512508bf71c2043f70b8f143813ad75cb5e8a589f2" [[package]] name = "starky" version = "0.4.0" -source = "git+https://github.com/0xPolygonZero/plonky2.git?rev=cd0fa567938f8f5f5066493cd862a8913fbd1fdf#cd0fa567938f8f5f5066493cd862a8913fbd1fdf" dependencies = [ "ahash", "anyhow", @@ -4614,7 +4609,7 @@ dependencies = [ "log", "num-bigint", "plonky2", - "plonky2_maybe_rayon 0.2.0 (git+https://github.com/0xPolygonZero/plonky2.git?rev=cd0fa567938f8f5f5066493cd862a8913fbd1fdf)", + "plonky2_maybe_rayon 0.2.0", "plonky2_util", ] diff --git a/Cargo.toml b/Cargo.toml index 9cf1dc1f4..0c9bf3071 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -114,10 +114,10 @@ zk_evm_proc_macro = { path = "proc_macro", version = "0.1.0" } zero = { path = "zero", default-features = false } # plonky2-related dependencies -plonky2 = { git = "https://github.com/0xPolygonZero/plonky2.git", rev = "cd0fa567938f8f5f5066493cd862a8913fbd1fdf" } +plonky2 = { path = "../plonky2/plonky2" } plonky2_maybe_rayon = "0.2.0" -plonky2_util = { git = "https://github.com/0xPolygonZero/plonky2.git", rev = "cd0fa567938f8f5f5066493cd862a8913fbd1fdf" } -starky = { git = "https://github.com/0xPolygonZero/plonky2.git", rev = "cd0fa567938f8f5f5066493cd862a8913fbd1fdf" } +plonky2_util = { path = "../plonky2/util" } +starky = { path = "../plonky2/starky" } [workspace.lints.clippy] too_long_first_doc_paragraph = "allow" diff --git a/evm_arithmetization/src/verifier.rs b/evm_arithmetization/src/verifier.rs index acee503c8..07f4e947a 100644 --- a/evm_arithmetization/src/verifier.rs +++ b/evm_arithmetization/src/verifier.rs @@ -10,9 +10,7 @@ use plonky2::plonk::config::{GenericConfig, GenericHashOut}; use plonky2::util::timing::TimingTree; use plonky2::util::transpose; use starky::config::StarkConfig; -use starky::cross_table_lookup::{ - get_ctl_vars_from_proofs, verify_cross_table_lookups, CrossTableLookup, -}; +use starky::cross_table_lookup::{verify_cross_table_lookups, CrossTableLookup, CtlCheckVars}; use starky::lookup::GrandProductChallenge; use starky::stark::Stark; use starky::verifier::verify_stark_proof_with_challenges; @@ -148,42 +146,39 @@ fn verify_proof, C: GenericConfig, const cross_table_lookups, } = all_stark; - let auxiliary_polys = core::array::from_fn(|i| { - all_proof.multi_proof.stark_proofs[i] - .as_ref() - .map(|proof| &proof.proof.openings.auxiliary_polys) - .unwrap_or(&None) - }); - let auxiliary_polys_next = core::array::from_fn(|i| { - all_proof.multi_proof.stark_proofs[i] - .as_ref() - .map(|proof| &proof.proof.openings.auxiliary_polys_next) - .unwrap_or(&None) - }); - - let ctl_vars_per_table = get_ctl_vars_from_proofs::( - &auxiliary_polys, - &auxiliary_polys_next, - cross_table_lookups, - &ctl_challenges, - &num_lookup_columns, - all_stark.arithmetic_stark.constraint_degree(), - ); - let stark_proofs = &all_proof.multi_proof.stark_proofs; macro_rules! verify_table { ($stark:ident, $table:expr) => { + let stark_proof = &stark_proofs[*$table] + .as_ref() + .expect("Missing stark_proof") + .proof; + let ctl_vars = { + let (total_num_helpers, _, num_helpers_by_ctl) = + CrossTableLookup::num_ctl_helpers_zs_all( + &all_stark.cross_table_lookups, + *$table, + config.num_challenges, + $stark.constraint_degree(), + ); + CtlCheckVars::from_proof( + *$table, + &stark_proof, + &all_stark.cross_table_lookups, + &ctl_challenges, + num_lookup_columns[*$table], + total_num_helpers, + &num_helpers_by_ctl, + ) + }; verify_stark_proof_with_challenges( $stark, - &stark_proofs[*$table] - .as_ref() - .expect("Missing stark_proof") - .proof, + stark_proof, &stark_challenges[*$table] .as_ref() .expect("Missing challenges"), - Some(&ctl_vars_per_table[*$table]), + Some(&ctl_vars), &[], config, )?; From 179e6bf37bbac1f4c9d1e6deec311a317ad2d966 Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Sat, 5 Oct 2024 16:05:09 -0700 Subject: [PATCH 60/91] wip --- Cargo.lock | 13 +++++++++---- Cargo.toml | 6 +++--- evm_arithmetization/src/verifier.rs | 2 ++ 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6540ce734..d68775370 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3599,6 +3599,7 @@ checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" [[package]] name = "plonky2" version = "0.2.2" +source = "git+https://github.com/0xPolygonZero/plonky2.git?rev=8463effe0dd1472a52906cd12ffb047885db42ee#8463effe0dd1472a52906cd12ffb047885db42ee" dependencies = [ "ahash", "anyhow", @@ -3609,7 +3610,7 @@ dependencies = [ "log", "num", "plonky2_field", - "plonky2_maybe_rayon 0.2.0", + "plonky2_maybe_rayon 0.2.0 (git+https://github.com/0xPolygonZero/plonky2.git?rev=8463effe0dd1472a52906cd12ffb047885db42ee)", "plonky2_util", "rand", "rand_chacha", @@ -3622,6 +3623,7 @@ dependencies = [ [[package]] name = "plonky2_field" version = "0.2.2" +source = "git+https://github.com/0xPolygonZero/plonky2.git?rev=8463effe0dd1472a52906cd12ffb047885db42ee#8463effe0dd1472a52906cd12ffb047885db42ee" dependencies = [ "anyhow", "itertools 0.11.0", @@ -3636,6 +3638,8 @@ dependencies = [ [[package]] name = "plonky2_maybe_rayon" version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92ff44a90aaca13e10e7ddf8fab815ba1b404c3f7c3ca82aaf11c46beabaa923" dependencies = [ "rayon", ] @@ -3643,8 +3647,7 @@ dependencies = [ [[package]] name = "plonky2_maybe_rayon" version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92ff44a90aaca13e10e7ddf8fab815ba1b404c3f7c3ca82aaf11c46beabaa923" +source = "git+https://github.com/0xPolygonZero/plonky2.git?rev=8463effe0dd1472a52906cd12ffb047885db42ee#8463effe0dd1472a52906cd12ffb047885db42ee" dependencies = [ "rayon", ] @@ -3652,6 +3655,7 @@ dependencies = [ [[package]] name = "plonky2_util" version = "0.2.0" +source = "git+https://github.com/0xPolygonZero/plonky2.git?rev=8463effe0dd1472a52906cd12ffb047885db42ee#8463effe0dd1472a52906cd12ffb047885db42ee" [[package]] name = "plotters" @@ -4601,6 +4605,7 @@ checksum = "8acdd7dbfcfb5dd6e46c63512508bf71c2043f70b8f143813ad75cb5e8a589f2" [[package]] name = "starky" version = "0.4.0" +source = "git+https://github.com/0xPolygonZero/plonky2.git?rev=8463effe0dd1472a52906cd12ffb047885db42ee#8463effe0dd1472a52906cd12ffb047885db42ee" dependencies = [ "ahash", "anyhow", @@ -4609,7 +4614,7 @@ dependencies = [ "log", "num-bigint", "plonky2", - "plonky2_maybe_rayon 0.2.0", + "plonky2_maybe_rayon 0.2.0 (git+https://github.com/0xPolygonZero/plonky2.git?rev=8463effe0dd1472a52906cd12ffb047885db42ee)", "plonky2_util", ] diff --git a/Cargo.toml b/Cargo.toml index 0c9bf3071..1f952c7b4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -114,10 +114,10 @@ zk_evm_proc_macro = { path = "proc_macro", version = "0.1.0" } zero = { path = "zero", default-features = false } # plonky2-related dependencies -plonky2 = { path = "../plonky2/plonky2" } +plonky2 = { git = "https://github.com/0xPolygonZero/plonky2.git", rev = "8463effe0dd1472a52906cd12ffb047885db42ee" } plonky2_maybe_rayon = "0.2.0" -plonky2_util = { path = "../plonky2/util" } -starky = { path = "../plonky2/starky" } +plonky2_util = { git = "https://github.com/0xPolygonZero/plonky2.git", rev = "8463effe0dd1472a52906cd12ffb047885db42ee" } +starky = { git = "https://github.com/0xPolygonZero/plonky2.git", rev = "8463effe0dd1472a52906cd12ffb047885db42ee" } [workspace.lints.clippy] too_long_first_doc_paragraph = "allow" diff --git a/evm_arithmetization/src/verifier.rs b/evm_arithmetization/src/verifier.rs index 07f4e947a..b6ef375a2 100644 --- a/evm_arithmetization/src/verifier.rs +++ b/evm_arithmetization/src/verifier.rs @@ -1,6 +1,7 @@ use anyhow::{ensure, Result}; use ethereum_types::{BigEndianHash, U256}; use itertools::Itertools; +use log::info; use plonky2::field::extension::Extendable; use plonky2::field::polynomial::PolynomialValues; use plonky2::fri::oracle::PolynomialBatch; @@ -172,6 +173,7 @@ fn verify_proof, C: GenericConfig, const &num_helpers_by_ctl, ) }; + info!("verify table: {}", *$table); verify_stark_proof_with_challenges( $stark, stark_proof, From c022343425d35a32d8645e4022fd917a80b85f5c Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Sat, 5 Oct 2024 16:34:21 -0700 Subject: [PATCH 61/91] fix tests --- evm_arithmetization/src/get_challenges.rs | 5 +++-- evm_arithmetization/src/prover.rs | 6 ------ evm_arithmetization/src/verifier.rs | 2 -- 3 files changed, 3 insertions(+), 10 deletions(-) diff --git a/evm_arithmetization/src/get_challenges.rs b/evm_arithmetization/src/get_challenges.rs index ab537bfff..1e1f58cca 100644 --- a/evm_arithmetization/src/get_challenges.rs +++ b/evm_arithmetization/src/get_challenges.rs @@ -1,6 +1,6 @@ use ethereum_types::{BigEndianHash, H256, U256}; use plonky2::field::extension::Extendable; -use plonky2::hash::hash_types::RichField; +use plonky2::hash::hash_types::{RichField, NUM_HASH_OUT_ELTS}; use plonky2::iop::challenger::{Challenger, RecursiveChallenger}; use plonky2::plonk::config::{AlgebraicHasher, GenericConfig}; use starky::config::StarkConfig; @@ -263,7 +263,8 @@ impl, C: GenericConfig, const D: usize> A challenger.observe_cap(&stark_proof.proof.trace_cap); } else { assert!(KECCAK_TABLES_INDICES.contains(&i) && !self.use_keccak_tables); - let zero_cap = vec![F::ZERO; config.fri_config.num_cap_elements()]; + let zero_cap = + vec![F::ZERO; config.fri_config.num_cap_elements() * NUM_HASH_OUT_ELTS]; challenger.observe_elements(&zero_cap); } } diff --git a/evm_arithmetization/src/prover.rs b/evm_arithmetization/src/prover.rs index 535f722dc..4a8fdc315 100644 --- a/evm_arithmetization/src/prover.rs +++ b/evm_arithmetization/src/prover.rs @@ -273,7 +273,6 @@ where let (arithmetic_proof, _) = prove_table!(arithmetic_stark, Table::Arithmetic); let (byte_packing_proof, _) = prove_table!(byte_packing_stark, Table::BytePacking); let (cpu_proof, _) = prove_table!(cpu_stark, Table::Cpu); - let challenger_after_cpu = challenger.clone(); let keccak_proof = if use_keccak_tables { Some(prove_table!(keccak_stark, Table::Keccak).0) } else { @@ -284,11 +283,6 @@ where } else { None }; - if !use_keccak_tables { - // We need to connect the challenger state of Logic and CPU tables when the - // Keccak tables are not in use. - *challenger = challenger_after_cpu; - } let (logic_proof, _) = prove_table!(logic_stark, Table::Logic); let (memory_proof, _) = prove_table!(memory_stark, Table::Memory); let (mem_before_proof, mem_before_cap) = prove_table!(mem_before_stark, Table::MemBefore); diff --git a/evm_arithmetization/src/verifier.rs b/evm_arithmetization/src/verifier.rs index b6ef375a2..07f4e947a 100644 --- a/evm_arithmetization/src/verifier.rs +++ b/evm_arithmetization/src/verifier.rs @@ -1,7 +1,6 @@ use anyhow::{ensure, Result}; use ethereum_types::{BigEndianHash, U256}; use itertools::Itertools; -use log::info; use plonky2::field::extension::Extendable; use plonky2::field::polynomial::PolynomialValues; use plonky2::fri::oracle::PolynomialBatch; @@ -173,7 +172,6 @@ fn verify_proof, C: GenericConfig, const &num_helpers_by_ctl, ) }; - info!("verify table: {}", *$table); verify_stark_proof_with_challenges( $stark, stark_proof, From 051702da6d43c5e563432896509ddad3c56765e2 Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Sun, 6 Oct 2024 09:05:03 -0700 Subject: [PATCH 62/91] address comments --- .../src/fixed_recursive_verifier.rs | 22 +++++++++---------- evm_arithmetization/src/generation/mod.rs | 3 +-- evm_arithmetization/src/prover.rs | 2 +- 3 files changed, 12 insertions(+), 15 deletions(-) diff --git a/evm_arithmetization/src/fixed_recursive_verifier.rs b/evm_arithmetization/src/fixed_recursive_verifier.rs index 92ef707ec..11784944f 100644 --- a/evm_arithmetization/src/fixed_recursive_verifier.rs +++ b/evm_arithmetization/src/fixed_recursive_verifier.rs @@ -1919,25 +1919,23 @@ where let table_circuits = &self.by_table[table]; if KECCAK_TABLES_INDICES.contains(&table) && !all_proof.use_keccak_tables { // generate and set a dummy `index_verifier_data` and `proof_with_pis` - let index_verifier_data = table_circuits - .by_stark_size - .keys() - .min() - .expect("No valid size in shrinking circuits"); + let index_verifier_data = + table_circuits.by_stark_size.keys().min().ok_or_else(|| { + anyhow::format_err!("No valid size in shrinking circuits") + })?; root_inputs.set_target( self.root.index_verifier_data[table], F::from_canonical_usize(*index_verifier_data), ); - let common_data = &table_circuits + let table_circuit = table_circuits .by_stark_size .get(index_verifier_data) - .expect("No valid size in shrinking circuits") + .ok_or_else(|| anyhow::format_err!("No valid size in shrinking circuits"))? .shrinking_wrappers .last() - .expect("No shrinking circuits") - .circuit - .common; - let dummy_circuit: CircuitData = dummy_circuit(common_data); + .ok_or_else(|| anyhow::format_err!("No shrinking circuits"))?; + let dummy_circuit: CircuitData = + dummy_circuit(&table_circuit.circuit.common); let dummy_pis = HashMap::new(); let dummy_proof = dummy_proof(&dummy_circuit, dummy_pis) .expect("Unable to generate dummy proofs"); @@ -2112,7 +2110,7 @@ where let common_date = &table_circuit .shrinking_wrappers .last() - .expect("No shrinking circuits") + .ok_or_else(|| anyhow::format_err!("No shrinking circuits"))? .circuit .common; let dummy_circuit: CircuitData = dummy_circuit(common_date); diff --git a/evm_arithmetization/src/generation/mod.rs b/evm_arithmetization/src/generation/mod.rs index 8c5846d9f..2c7751016 100644 --- a/evm_arithmetization/src/generation/mod.rs +++ b/evm_arithmetization/src/generation/mod.rs @@ -585,8 +585,7 @@ pub fn generate_traces, const D: usize>( mem_after: MemCap::default(), }; - let use_keccak_tables = - !state.traces.keccak_inputs.is_empty() || !state.traces.keccak_sponge_ops.is_empty(); + let use_keccak_tables = !state.traces.keccak_inputs.is_empty(); let tables = timed!( timing, diff --git a/evm_arithmetization/src/prover.rs b/evm_arithmetization/src/prover.rs index 0f3726ad2..1068f07fd 100644 --- a/evm_arithmetization/src/prover.rs +++ b/evm_arithmetization/src/prover.rs @@ -279,7 +279,7 @@ where let (keccak_proof, _) = prove_table!(keccak_stark, Table::Keccak); let (keccak_sponge_proof, _) = prove_table!(keccak_sponge_stark, Table::KeccakSponge); if !use_keccak_tables { - // We need to connect the challenger state of Logic and CPU tables when the + // We need to connect the challenger state of CPU and Logic tables when the // Keccak tables are not in use. *challenger = challenger_after_cpu; } From 246afda24be3f7acc0e6beef943d359be1c31278 Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Mon, 7 Oct 2024 10:05:07 -0700 Subject: [PATCH 63/91] longer zero bin build timeout --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 09280fae2..030731bdd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -96,7 +96,7 @@ jobs: test_zero_bin: name: Test zero_bin runs-on: ubuntu-latest - timeout-minutes: 5 + timeout-minutes: 10 if: ${{ ! contains(toJSON(github.event.commits.*.message), '[skip-ci]') }} steps: - name: Checkout sources From eca94e07e6e4665b118bfeb7e2683b4419eadb38 Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Mon, 7 Oct 2024 10:29:40 -0700 Subject: [PATCH 64/91] temporally force run test zero bin ci --- .github/workflows/ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 030731bdd..d41701eb5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -97,7 +97,6 @@ jobs: name: Test zero_bin runs-on: ubuntu-latest timeout-minutes: 10 - if: ${{ ! contains(toJSON(github.event.commits.*.message), '[skip-ci]') }} steps: - name: Checkout sources uses: actions/checkout@v4 From fc126912958528cc870b60c2016584c951dd5f24 Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Mon, 7 Oct 2024 10:33:02 -0700 Subject: [PATCH 65/91] temporally disable pr checking --- .github/workflows/pr_checking.yml | 68 +++++++++++++++---------------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/.github/workflows/pr_checking.yml b/.github/workflows/pr_checking.yml index 497a17208..27e0cbf7b 100644 --- a/.github/workflows/pr_checking.yml +++ b/.github/workflows/pr_checking.yml @@ -16,41 +16,41 @@ jobs: id: setup_keywords run: echo "RESTRICTED_KEYWORDS=$(echo '${{ secrets.RESTRICTED_KEYWORDS }}' | jq -r '.[]' | tr '\n' ' ')" >> $GITHUB_ENV - - name: Check for spam PR - id: check - run: | - # Initialize variables to track spam presence - title_is_spam=false - description_is_spam=false - - # Check title for spam - for keyword in $RESTRICTED_KEYWORDS; do - if echo "${{ github.event.pull_request.title }}" | grep -i -q "$keyword"; then - title_is_spam=true - break - fi - done - - # Check description for spam - for keyword in $RESTRICTED_KEYWORDS; do - if echo "${{ github.event.pull_request.body }}" | grep -i -q "$keyword"; then - description_is_spam=true - break - fi - done - - # Set the output based on the presence of spam - if [ "$title_is_spam" = true ] || [ "$description_is_spam" = true ]; then - echo "is_spam=true" >> $GITHUB_ENV - else - echo "is_spam=false" >> $GITHUB_ENV - fi +# - name: Check for spam PR +# id: check +# run: | +# # Initialize variables to track spam presence +# title_is_spam=false +# description_is_spam=false +# +# # Check title for spam +# for keyword in $RESTRICTED_KEYWORDS; do +# if echo "${{ github.event.pull_request.title }}" | grep -i -q "$keyword"; then +# title_is_spam=true +# break +# fi +# done +# +# # Check description for spam +# for keyword in $RESTRICTED_KEYWORDS; do +# if echo "${{ github.event.pull_request.body }}" | grep -i -q "$keyword"; then +# description_is_spam=true +# break +# fi +# done +# +# # Set the output based on the presence of spam +# if [ "$title_is_spam" = true ] || [ "$description_is_spam" = true ]; then +# echo "is_spam=true" >> $GITHUB_ENV +# else +# echo "is_spam=false" >> $GITHUB_ENV +# fi - name: Checkout repository uses: actions/checkout@v4 - - name: Close PR if spam are found and author is not a contributor or member - if: ${{ env.is_spam == 'true' && github.event.pull_request.author_association == 'FIRST_TIME_CONTRIBUTOR' }} - run: gh pr close ${{ github.event.pull_request.number }} --comment "Spam detected" - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} +# - name: Close PR if spam are found and author is not a contributor or member +# if: ${{ env.is_spam == 'true' && github.event.pull_request.author_association == 'FIRST_TIME_CONTRIBUTOR' }} +# run: gh pr close ${{ github.event.pull_request.number }} --comment "Spam detected" +# env: +# GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 36af7e622b566bbb56c323fb62856d22621e19ac Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Mon, 7 Oct 2024 10:37:12 -0700 Subject: [PATCH 66/91] revert ci changes --- .github/workflows/ci.yml | 3 +- .github/workflows/pr_checking.yml | 68 +++++++++++++++---------------- 2 files changed, 36 insertions(+), 35 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d41701eb5..09280fae2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -96,7 +96,8 @@ jobs: test_zero_bin: name: Test zero_bin runs-on: ubuntu-latest - timeout-minutes: 10 + timeout-minutes: 5 + if: ${{ ! contains(toJSON(github.event.commits.*.message), '[skip-ci]') }} steps: - name: Checkout sources uses: actions/checkout@v4 diff --git a/.github/workflows/pr_checking.yml b/.github/workflows/pr_checking.yml index 27e0cbf7b..497a17208 100644 --- a/.github/workflows/pr_checking.yml +++ b/.github/workflows/pr_checking.yml @@ -16,41 +16,41 @@ jobs: id: setup_keywords run: echo "RESTRICTED_KEYWORDS=$(echo '${{ secrets.RESTRICTED_KEYWORDS }}' | jq -r '.[]' | tr '\n' ' ')" >> $GITHUB_ENV -# - name: Check for spam PR -# id: check -# run: | -# # Initialize variables to track spam presence -# title_is_spam=false -# description_is_spam=false -# -# # Check title for spam -# for keyword in $RESTRICTED_KEYWORDS; do -# if echo "${{ github.event.pull_request.title }}" | grep -i -q "$keyword"; then -# title_is_spam=true -# break -# fi -# done -# -# # Check description for spam -# for keyword in $RESTRICTED_KEYWORDS; do -# if echo "${{ github.event.pull_request.body }}" | grep -i -q "$keyword"; then -# description_is_spam=true -# break -# fi -# done -# -# # Set the output based on the presence of spam -# if [ "$title_is_spam" = true ] || [ "$description_is_spam" = true ]; then -# echo "is_spam=true" >> $GITHUB_ENV -# else -# echo "is_spam=false" >> $GITHUB_ENV -# fi + - name: Check for spam PR + id: check + run: | + # Initialize variables to track spam presence + title_is_spam=false + description_is_spam=false + + # Check title for spam + for keyword in $RESTRICTED_KEYWORDS; do + if echo "${{ github.event.pull_request.title }}" | grep -i -q "$keyword"; then + title_is_spam=true + break + fi + done + + # Check description for spam + for keyword in $RESTRICTED_KEYWORDS; do + if echo "${{ github.event.pull_request.body }}" | grep -i -q "$keyword"; then + description_is_spam=true + break + fi + done + + # Set the output based on the presence of spam + if [ "$title_is_spam" = true ] || [ "$description_is_spam" = true ]; then + echo "is_spam=true" >> $GITHUB_ENV + else + echo "is_spam=false" >> $GITHUB_ENV + fi - name: Checkout repository uses: actions/checkout@v4 -# - name: Close PR if spam are found and author is not a contributor or member -# if: ${{ env.is_spam == 'true' && github.event.pull_request.author_association == 'FIRST_TIME_CONTRIBUTOR' }} -# run: gh pr close ${{ github.event.pull_request.number }} --comment "Spam detected" -# env: -# GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Close PR if spam are found and author is not a contributor or member + if: ${{ env.is_spam == 'true' && github.event.pull_request.author_association == 'FIRST_TIME_CONTRIBUTOR' }} + run: gh pr close ${{ github.event.pull_request.number }} --comment "Spam detected" + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 13575f61bcec92aa960e8a3f1b1beeb373b71747 Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Mon, 7 Oct 2024 10:48:50 -0700 Subject: [PATCH 67/91] fix build after merge --- .../src/fixed_recursive_verifier.rs | 2 +- evm_arithmetization/src/get_challenges.rs | 9 +- evm_arithmetization/src/proof.rs | 2 +- evm_arithmetization/src/verifier.rs | 171 +++++++++--------- 4 files changed, 92 insertions(+), 92 deletions(-) diff --git a/evm_arithmetization/src/fixed_recursive_verifier.rs b/evm_arithmetization/src/fixed_recursive_verifier.rs index c41d45927..185ffa9b5 100644 --- a/evm_arithmetization/src/fixed_recursive_verifier.rs +++ b/evm_arithmetization/src/fixed_recursive_verifier.rs @@ -1943,7 +1943,7 @@ where .set_proof_with_pis_target(&self.root.proof_with_pis[table], &dummy_proof); } else { let stark_proof = &all_proof.multi_proof.stark_proofs[table] - .as_ref() + .as_ref() .expect("Unable to get stark proof"); let original_degree_bits = stark_proof.proof.recover_degree_bits(config); let shrunk_proof = table_circuits diff --git a/evm_arithmetization/src/get_challenges.rs b/evm_arithmetization/src/get_challenges.rs index db14d4ca3..c7471471b 100644 --- a/evm_arithmetization/src/get_challenges.rs +++ b/evm_arithmetization/src/get_challenges.rs @@ -1,10 +1,9 @@ use ethereum_types::{BigEndianHash, H256, U256}; use plonky2::field::extension::Extendable; -use plonky2::hash::hash_types::{RichField, NUM_HASH_OUT_ELTS}; +use plonky2::hash::hash_types::RichField; use plonky2::iop::challenger::{Challenger, RecursiveChallenger}; use plonky2::plonk::config::{AlgebraicHasher, GenericConfig}; -use crate::all_stark::KECCAK_TABLES_INDICES; use crate::proof::*; use crate::util::{h256_limbs, u256_limbs, u256_to_u32, u256_to_u64}; use crate::witness::errors::ProgramError; @@ -248,7 +247,7 @@ pub(crate) fn observe_public_values_target< pub mod testing { use plonky2::field::extension::Extendable; - use plonky2::hash::hash_types::RichField; + use plonky2::hash::hash_types::{RichField, NUM_HASH_OUT_ELTS}; use plonky2::iop::challenger::Challenger; use plonky2::plonk::config::GenericConfig; use starky::config::StarkConfig; @@ -269,7 +268,7 @@ pub mod testing { pub ctl_challenges: GrandProductChallengeSet, } - impl, C: GenericConfig, const D: usize> AllProof { + impl, C: GenericConfig, const D: usize> AllProof { /// Computes all Fiat-Shamir challenges used in the STARK proof. pub(crate) fn get_challenges( &self, @@ -313,4 +312,4 @@ pub mod testing { }) } } -} \ No newline at end of file +} diff --git a/evm_arithmetization/src/proof.rs b/evm_arithmetization/src/proof.rs index 450c9f916..b0de028af 100644 --- a/evm_arithmetization/src/proof.rs +++ b/evm_arithmetization/src/proof.rs @@ -11,7 +11,7 @@ use plonky2::util::serialization::{Buffer, IoResult, Read, Write}; use serde::{Deserialize, Serialize}; use starky::config::StarkConfig; use starky::lookup::GrandProductChallengeSet; -use starky::proof::{StarkProofChallenges, StarkProofWithMetadata}; +use starky::proof::StarkProofWithMetadata; use crate::all_stark::NUM_TABLES; use crate::util::{get_h160, get_h256, get_u256, h256_limbs, h2u}; diff --git a/evm_arithmetization/src/verifier.rs b/evm_arithmetization/src/verifier.rs index 02e440a3b..81e9b502b 100644 --- a/evm_arithmetization/src/verifier.rs +++ b/evm_arithmetization/src/verifier.rs @@ -7,7 +7,6 @@ use plonky2::hash::merkle_tree::MerkleCap; use plonky2::plonk::config::GenericConfig; use plonky2::util::timing::TimingTree; use plonky2::util::transpose; -use starky::stark::Stark; use crate::cpu::kernel::aggregator::KERNEL; use crate::memory::segments::Segment; @@ -86,7 +85,9 @@ pub mod testing { use plonky2::hash::hash_types::RichField; use plonky2::plonk::config::{GenericConfig, GenericHashOut}; use starky::config::StarkConfig; - use starky::cross_table_lookup::{get_ctl_vars_from_proofs, verify_cross_table_lookups}; + use starky::cross_table_lookup::verify_cross_table_lookups; + use starky::cross_table_lookup::CrossTableLookup; + use starky::cross_table_lookup::CtlCheckVars; use starky::lookup::GrandProductChallenge; use starky::stark::Stark; use starky::verifier::verify_stark_proof_with_challenges; @@ -185,56 +186,56 @@ pub mod testing { cross_table_lookups, } = all_stark; - let stark_proofs = &all_proof.multi_proof.stark_proofs; - - macro_rules! verify_table { - ($stark:ident, $table:expr) => { - let stark_proof = &stark_proofs[*$table] - .as_ref() - .expect("Missing stark_proof") - .proof; - let ctl_vars = { - let (total_num_helpers, _, num_helpers_by_ctl) = - CrossTableLookup::num_ctl_helpers_zs_all( - &all_stark.cross_table_lookups, + let stark_proofs = &all_proof.multi_proof.stark_proofs; + + macro_rules! verify_table { + ($stark:ident, $table:expr) => { + let stark_proof = &stark_proofs[*$table] + .as_ref() + .expect("Missing stark_proof") + .proof; + let ctl_vars = { + let (total_num_helpers, _, num_helpers_by_ctl) = + CrossTableLookup::num_ctl_helpers_zs_all( + &all_stark.cross_table_lookups, + *$table, + config.num_challenges, + $stark.constraint_degree(), + ); + CtlCheckVars::from_proof( *$table, - config.num_challenges, - $stark.constraint_degree(), - ); - CtlCheckVars::from_proof( - *$table, - &stark_proof, - &all_stark.cross_table_lookups, - &ctl_challenges, - num_lookup_columns[*$table], - total_num_helpers, - &num_helpers_by_ctl, - ) + &stark_proof, + &all_stark.cross_table_lookups, + &ctl_challenges, + num_lookup_columns[*$table], + total_num_helpers, + &num_helpers_by_ctl, + ) + }; + verify_stark_proof_with_challenges( + $stark, + stark_proof, + &stark_challenges[*$table] + .as_ref() + .expect("Missing challenges"), + Some(&ctl_vars), + &[], + config, + )?; }; - verify_stark_proof_with_challenges( - $stark, - stark_proof, - &stark_challenges[*$table] - .as_ref() - .expect("Missing challenges"), - Some(&ctl_vars), - &[], - config, - )?; - }; - } + } - verify_table!(arithmetic_stark, Table::Arithmetic); - verify_table!(byte_packing_stark, Table::BytePacking); - verify_table!(cpu_stark, Table::Cpu); - if all_proof.use_keccak_tables { - verify_table!(keccak_stark, Table::Keccak); - verify_table!(keccak_sponge_stark, Table::KeccakSponge); - } - verify_table!(logic_stark, Table::Logic); - verify_table!(memory_stark, Table::Memory); - verify_table!(mem_before_stark, Table::MemBefore); - verify_table!(mem_after_stark, Table::MemAfter); + verify_table!(arithmetic_stark, Table::Arithmetic); + verify_table!(byte_packing_stark, Table::BytePacking); + verify_table!(cpu_stark, Table::Cpu); + if all_proof.use_keccak_tables { + verify_table!(keccak_stark, Table::Keccak); + verify_table!(keccak_sponge_stark, Table::KeccakSponge); + } + verify_table!(logic_stark, Table::Logic); + verify_table!(memory_stark, Table::Memory); + verify_table!(mem_before_stark, Table::MemBefore); + verify_table!(mem_after_stark, Table::MemAfter); #[cfg(feature = "cdk_erigon")] verify_table!(poseidon_stark, Table::Poseidon); @@ -255,43 +256,43 @@ pub mod testing { .map(|i| get_memory_extra_looking_sum(&public_values, ctl_challenges.challenges[i])) .collect_vec(); - let all_ctls = &all_stark.cross_table_lookups; + let all_ctls = &all_stark.cross_table_lookups; - verify_cross_table_lookups::( - cross_table_lookups, - core::array::from_fn(|i| { - if let Some(stark_proof) = &stark_proofs[i] { - stark_proof - .proof - .openings - .ctl_zs_first - .as_ref() - .expect("Missing ctl_zs") - .clone() - } else if i == *Table::Keccak { - let (_, n, _) = CrossTableLookup::num_ctl_helpers_zs_all( - all_ctls, - *Table::Keccak, - config.num_challenges, - keccak_stark.constraint_degree(), - ); - vec![F::ZERO; n] - } else if i == *Table::KeccakSponge { - let (_, n, _) = CrossTableLookup::num_ctl_helpers_zs_all( - all_ctls, - *Table::KeccakSponge, - config.num_challenges, - keccak_sponge_stark.constraint_degree(), - ); - vec![F::ZERO; n] - } else { - panic!("Unable to find stark_proof"); - } - }), - Some(&extra_looking_sums), - config, - ) -} + verify_cross_table_lookups::( + cross_table_lookups, + core::array::from_fn(|i| { + if let Some(stark_proof) = &stark_proofs[i] { + stark_proof + .proof + .openings + .ctl_zs_first + .as_ref() + .expect("Missing ctl_zs") + .clone() + } else if i == *Table::Keccak { + let (_, n, _) = CrossTableLookup::num_ctl_helpers_zs_all( + all_ctls, + *Table::Keccak, + config.num_challenges, + keccak_stark.constraint_degree(), + ); + vec![F::ZERO; n] + } else if i == *Table::KeccakSponge { + let (_, n, _) = CrossTableLookup::num_ctl_helpers_zs_all( + all_ctls, + *Table::KeccakSponge, + config.num_challenges, + keccak_sponge_stark.constraint_degree(), + ); + vec![F::ZERO; n] + } else { + panic!("Unable to find stark_proof"); + } + }), + Some(&extra_looking_sums), + config, + ) + } /// Computes the extra product to multiply to the looked value. It contains /// memory operations not in the CPU trace: From 782eeb1f443e40265f40929ac1e82ee1fd69b140 Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Mon, 7 Oct 2024 10:55:42 -0700 Subject: [PATCH 68/91] do not use remove --- evm_arithmetization/src/fixed_recursive_verifier.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/evm_arithmetization/src/fixed_recursive_verifier.rs b/evm_arithmetization/src/fixed_recursive_verifier.rs index 185ffa9b5..b3ef33359 100644 --- a/evm_arithmetization/src/fixed_recursive_verifier.rs +++ b/evm_arithmetization/src/fixed_recursive_verifier.rs @@ -1938,13 +1938,13 @@ where dummy_circuit(&table_circuit.circuit.common); let dummy_pis = HashMap::new(); let dummy_proof = dummy_proof(&dummy_circuit, dummy_pis) - .expect("Unable to generate dummy proofs"); + .ok_or_else(|| anyhow::format_err!("Unable to generate dummy proofs"))?; root_inputs .set_proof_with_pis_target(&self.root.proof_with_pis[table], &dummy_proof); } else { let stark_proof = &all_proof.multi_proof.stark_proofs[table] .as_ref() - .expect("Unable to get stark proof"); + .ok_or_else(|| anyhow::format_err!("Unable to get stark proof"))?; let original_degree_bits = stark_proof.proof.recover_degree_bits(config); let shrunk_proof = table_circuits .by_stark_size @@ -2076,7 +2076,7 @@ where let dummy_circuit: CircuitData = dummy_circuit(common_data); let dummy_pis = HashMap::new(); let dummy_proof = dummy_proof(&dummy_circuit, dummy_pis) - .expect("Unable to generate dummy proofs"); + .ok_or_else(|| anyhow::format_err!("Unable to generate dummy proofs"))?; root_inputs .set_proof_with_pis_target(&self.root.proof_with_pis[table], &dummy_proof); } else { From 6766fdd38226de0fd43c47d41b79149e27d3bd32 Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Mon, 7 Oct 2024 10:57:28 -0700 Subject: [PATCH 69/91] longer zero bin timeout --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 09280fae2..030731bdd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -96,7 +96,7 @@ jobs: test_zero_bin: name: Test zero_bin runs-on: ubuntu-latest - timeout-minutes: 5 + timeout-minutes: 10 if: ${{ ! contains(toJSON(github.event.commits.*.message), '[skip-ci]') }} steps: - name: Checkout sources From 082cc89f7795f5523e39e1c552d72fe33fb4295d Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Mon, 7 Oct 2024 11:17:10 -0700 Subject: [PATCH 70/91] fix build --- evm_arithmetization/src/fixed_recursive_verifier.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/evm_arithmetization/src/fixed_recursive_verifier.rs b/evm_arithmetization/src/fixed_recursive_verifier.rs index b3ef33359..5b423905a 100644 --- a/evm_arithmetization/src/fixed_recursive_verifier.rs +++ b/evm_arithmetization/src/fixed_recursive_verifier.rs @@ -1938,7 +1938,7 @@ where dummy_circuit(&table_circuit.circuit.common); let dummy_pis = HashMap::new(); let dummy_proof = dummy_proof(&dummy_circuit, dummy_pis) - .ok_or_else(|| anyhow::format_err!("Unable to generate dummy proofs"))?; + .or_else(|_| Err(anyhow::format_err!("Unable to generate dummy proofs")))?; root_inputs .set_proof_with_pis_target(&self.root.proof_with_pis[table], &dummy_proof); } else { @@ -2076,7 +2076,7 @@ where let dummy_circuit: CircuitData = dummy_circuit(common_data); let dummy_pis = HashMap::new(); let dummy_proof = dummy_proof(&dummy_circuit, dummy_pis) - .ok_or_else(|| anyhow::format_err!("Unable to generate dummy proofs"))?; + .or_else(|_| Err(anyhow::format_err!("Unable to generate dummy proofs")))?; root_inputs .set_proof_with_pis_target(&self.root.proof_with_pis[table], &dummy_proof); } else { From ecb99436259af648e970a70a5d951250ea9822ca Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Mon, 7 Oct 2024 11:22:37 -0700 Subject: [PATCH 71/91] fix clippy --- evm_arithmetization/src/fixed_recursive_verifier.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/evm_arithmetization/src/fixed_recursive_verifier.rs b/evm_arithmetization/src/fixed_recursive_verifier.rs index 5b423905a..b0dd689ce 100644 --- a/evm_arithmetization/src/fixed_recursive_verifier.rs +++ b/evm_arithmetization/src/fixed_recursive_verifier.rs @@ -1938,9 +1938,9 @@ where dummy_circuit(&table_circuit.circuit.common); let dummy_pis = HashMap::new(); let dummy_proof = dummy_proof(&dummy_circuit, dummy_pis) - .or_else(|_| Err(anyhow::format_err!("Unable to generate dummy proofs")))?; + .map_err(|_| anyhow::format_err!("Unable to generate dummy proofs")); root_inputs - .set_proof_with_pis_target(&self.root.proof_with_pis[table], &dummy_proof); + .set_proof_with_pis_target(&self.root.proof_with_pis[table], &dummy_proof?); } else { let stark_proof = &all_proof.multi_proof.stark_proofs[table] .as_ref() @@ -2076,9 +2076,9 @@ where let dummy_circuit: CircuitData = dummy_circuit(common_data); let dummy_pis = HashMap::new(); let dummy_proof = dummy_proof(&dummy_circuit, dummy_pis) - .or_else(|_| Err(anyhow::format_err!("Unable to generate dummy proofs")))?; + .map_err(|_| anyhow::format_err!("Unable to generate dummy proofs")); root_inputs - .set_proof_with_pis_target(&self.root.proof_with_pis[table], &dummy_proof); + .set_proof_with_pis_target(&self.root.proof_with_pis[table], &dummy_proof?); } else { let stark_proof = &all_proof.multi_proof.stark_proofs[table] .as_ref() From 78fce60faf684a3a3e448d8f2c9fed2f71662957 Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Mon, 7 Oct 2024 11:38:40 -0700 Subject: [PATCH 72/91] revert zero bin timeout --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 030731bdd..09280fae2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -96,7 +96,7 @@ jobs: test_zero_bin: name: Test zero_bin runs-on: ubuntu-latest - timeout-minutes: 10 + timeout-minutes: 5 if: ${{ ! contains(toJSON(github.event.commits.*.message), '[skip-ci]') }} steps: - name: Checkout sources From 1a673e26c87442ffaa084779bcb4ce3659be7c3a Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Tue, 8 Oct 2024 10:05:24 -0700 Subject: [PATCH 73/91] address comments --- evm_arithmetization/src/arithmetic/mod.rs | 4 +- .../src/fixed_recursive_verifier.rs | 13 +----- .../src/generation/segments.rs | 2 +- evm_arithmetization/src/logic.rs | 2 +- evm_arithmetization/src/testing_utils.rs | 22 +++++++++- evm_arithmetization/src/witness/operation.rs | 2 +- evm_arithmetization/tests/add11_yml.rs | 38 ----------------- evm_arithmetization/tests/empty_tables.rs | 42 +++++++++++++++++++ zero/src/prover_state/mod.rs | 6 ++- 9 files changed, 73 insertions(+), 58 deletions(-) create mode 100644 evm_arithmetization/tests/empty_tables.rs diff --git a/evm_arithmetization/src/arithmetic/mod.rs b/evm_arithmetization/src/arithmetic/mod.rs index 39ae32a1a..ab65def9b 100644 --- a/evm_arithmetization/src/arithmetic/mod.rs +++ b/evm_arithmetization/src/arithmetic/mod.rs @@ -26,7 +26,7 @@ pub(crate) mod columns; /// `Shl` and `Shr` are handled differently, by leveraging `Mul` and `Div` /// respectively. #[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)] -pub enum BinaryOperator { +pub(crate) enum BinaryOperator { Add, Mul, Sub, @@ -116,7 +116,7 @@ impl BinaryOperator { /// An enum representing different ternary operations. #[allow(clippy::enum_variant_names)] #[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)] -pub enum TernaryOperator { +pub(crate) enum TernaryOperator { AddMod, MulMod, SubMod, diff --git a/evm_arithmetization/src/fixed_recursive_verifier.rs b/evm_arithmetization/src/fixed_recursive_verifier.rs index b0dd689ce..37415aa39 100644 --- a/evm_arithmetization/src/fixed_recursive_verifier.rs +++ b/evm_arithmetization/src/fixed_recursive_verifier.rs @@ -3144,8 +3144,7 @@ mod tests { use plonky2::timed; use super::*; - use crate::testing_utils::{empty_payload, init_logger}; - use crate::witness::operation::Operation; + use crate::testing_utils::{init_logger, segment_without_keccak}; type F = GoldilocksField; const D: usize = 2; @@ -3158,15 +3157,6 @@ mod tests { let all_stark = AllStark::::default(); let config = StarkConfig::standard_fast_config(); - // Generate a dummy payload for testing - let payload = empty_payload()?; - let max_cpu_len_log = Some(7); - let mut segment_iterator = SegmentDataIterator::::new(&payload, max_cpu_len_log); - let (_, mut segment_data) = segment_iterator.next().unwrap()?; - - let opcode_counts = &segment_data.opcode_counts; - assert!(!opcode_counts.contains_key(&Operation::KeccakGeneral)); - let timing = &mut TimingTree::new( "Segment Proof Generation Without Keccak Test", log::Level::Info, @@ -3183,6 +3173,7 @@ mod tests { ) ); + let (payload, mut segment_data) = segment_without_keccak()?; let segment_proof = timed!( timing, log::Level::Info, diff --git a/evm_arithmetization/src/generation/segments.rs b/evm_arithmetization/src/generation/segments.rs index 440c14169..36bac8f8a 100644 --- a/evm_arithmetization/src/generation/segments.rs +++ b/evm_arithmetization/src/generation/segments.rs @@ -33,7 +33,7 @@ pub struct GenerationSegmentData { /// Log of the maximal cpu length. pub(crate) max_cpu_len_log: Option, /// Counts the number of appearances of each opcode. For debugging purposes. - pub opcode_counts: HashMap, + pub(crate) opcode_counts: HashMap, } impl GenerationSegmentData { diff --git a/evm_arithmetization/src/logic.rs b/evm_arithmetization/src/logic.rs index 19d68cfd3..7d68e463e 100644 --- a/evm_arithmetization/src/logic.rs +++ b/evm_arithmetization/src/logic.rs @@ -120,7 +120,7 @@ pub(crate) struct LogicStark { /// Logic operations. #[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)] -pub enum Op { +pub(crate) enum Op { And, Or, Xor, diff --git a/evm_arithmetization/src/testing_utils.rs b/evm_arithmetization/src/testing_utils.rs index 5f1b1b041..26b98a3b8 100644 --- a/evm_arithmetization/src/testing_utils.rs +++ b/evm_arithmetization/src/testing_utils.rs @@ -10,12 +10,17 @@ use mpt_trie::{ nibbles::Nibbles, partial_trie::{HashedPartialTrie, Node, PartialTrie}, }; +use plonky2::field::goldilocks_field::GoldilocksField; pub use crate::cpu::kernel::cancun_constants::*; pub use crate::cpu::kernel::constants::global_exit_root::*; use crate::generation::TrieInputs; use crate::proof::TrieRoots; -use crate::{generation::mpt::AccountRlp, proof::BlockMetadata, util::h2u, GenerationInputs}; +use crate::witness::operation::Operation; +use crate::{ + generation::mpt::AccountRlp, proof::BlockMetadata, util::h2u, GenerationInputs, + GenerationSegmentData, SegmentDataIterator, +}; pub const EMPTY_NODE_HASH: H256 = H256(hex!( "56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421" @@ -165,7 +170,7 @@ pub fn scalable_contract_from_storage(storage_trie: &HashedPartialTrie) -> Accou } } -pub fn empty_payload() -> Result { +fn empty_payload() -> Result { // Set up default block metadata let block_metadata = BlockMetadata { block_beneficiary: Address::zero(), @@ -213,3 +218,16 @@ pub fn empty_payload() -> Result { Ok(inputs) } + +pub fn segment_without_keccak() -> Result<(GenerationInputs, GenerationSegmentData)> { + let payload = empty_payload()?; + let max_cpu_len_log = Some(7); + let mut segment_iterator = + SegmentDataIterator::::new(&payload, max_cpu_len_log); + let (_, segment_data) = segment_iterator.next().unwrap()?; + + let opcode_counts = &segment_data.opcode_counts; + assert!(!opcode_counts.contains_key(&Operation::KeccakGeneral)); + + Ok((payload, segment_data)) +} diff --git a/evm_arithmetization/src/witness/operation.rs b/evm_arithmetization/src/witness/operation.rs index f29dfab5c..c8e2a5376 100644 --- a/evm_arithmetization/src/witness/operation.rs +++ b/evm_arithmetization/src/witness/operation.rs @@ -31,7 +31,7 @@ use crate::witness::util::{ use crate::{arithmetic, logic}; #[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)] -pub enum Operation { +pub(crate) enum Operation { Iszero, Not, /// (syscall number, minimum stack length, increases stack length) diff --git a/evm_arithmetization/tests/add11_yml.rs b/evm_arithmetization/tests/add11_yml.rs index 569a97fc2..127a41b87 100644 --- a/evm_arithmetization/tests/add11_yml.rs +++ b/evm_arithmetization/tests/add11_yml.rs @@ -232,41 +232,3 @@ fn add11_yml() -> anyhow::Result<()> { verify_all_proofs(&all_stark, &proofs, &config) } - -/// This test focuses on testing zkVM proofs with some empty tables. -#[test] -fn add11_yml_first_segment() -> anyhow::Result<()> { - type F = GoldilocksField; - const D: usize = 2; - type C = PoseidonGoldilocksConfig; - - init_logger(); - - let all_stark = AllStark::::default(); - let config = StarkConfig::standard_fast_config(); - let inputs = get_generation_inputs(); - - let mut timing = TimingTree::new("prove", log::Level::Debug); - - let max_cpu_len_log = 7; - let mut segment_iterator = SegmentDataIterator::::new(&inputs, Some(max_cpu_len_log)); - let mut proofs = vec![]; - let (_, mut segment_data) = segment_iterator.next().unwrap()?; - - let opcode_counts = &segment_data.opcode_counts; - assert!(!opcode_counts.contains_key(&KeccakGeneral)); - - let proof = prove::( - &all_stark, - &config, - inputs.trim(), - &mut segment_data, - &mut timing, - None, - )?; - proofs.push(proof); - - timing.filter(Duration::from_millis(100)).print(); - - verify_all_proofs(&all_stark, &proofs, &config) -} diff --git a/evm_arithmetization/tests/empty_tables.rs b/evm_arithmetization/tests/empty_tables.rs new file mode 100644 index 000000000..3d24ccee0 --- /dev/null +++ b/evm_arithmetization/tests/empty_tables.rs @@ -0,0 +1,42 @@ +#![cfg(feature = "eth_mainnet")] + +use std::time::Duration; + +use evm_arithmetization::prover::prove; +use evm_arithmetization::testing_utils::{init_logger, segment_without_keccak}; +use evm_arithmetization::verifier::testing::verify_all_proofs; +use evm_arithmetization::AllStark; +use plonky2::field::goldilocks_field::GoldilocksField; +use plonky2::plonk::config::PoseidonGoldilocksConfig; +use plonky2::util::timing::TimingTree; +use starky::config::StarkConfig; + +/// This test focuses on testing zkVM proofs with some empty tables. +#[test] +fn empty_tables() -> anyhow::Result<()> { + type F = GoldilocksField; + const D: usize = 2; + type C = PoseidonGoldilocksConfig; + + init_logger(); + + let all_stark = AllStark::::default(); + let config = StarkConfig::standard_fast_config(); + let mut proofs = vec![]; + let mut timing = TimingTree::new("prove", log::Level::Debug); + + let (payload, mut segment_data) = segment_without_keccak()?; + let proof = prove::( + &all_stark, + &config, + payload.trim(), + &mut segment_data, + &mut timing, + None, + )?; + proofs.push(proof); + + timing.filter(Duration::from_millis(100)).print(); + + verify_all_proofs(&all_stark, &proofs, &config) +} diff --git a/zero/src/prover_state/mod.rs b/zero/src/prover_state/mod.rs index aaaa7a371..d1d6c5983 100644 --- a/zero/src/prover_state/mod.rs +++ b/zero/src/prover_state/mod.rs @@ -31,6 +31,9 @@ use crate::prover_state::persistence::{ VerifierResource, }; +// Default degree of the recursive circuit when a proof is missing from AllProof +const DEFAULT_CIRCUIT_DEGREE: usize = 8; + pub mod circuit; pub mod cli; pub mod persistence; @@ -175,10 +178,9 @@ impl ProverStateManager { config: &StarkConfig, all_proof: &AllProof, ) -> anyhow::Result<[(RecursiveCircuitsForTableSize, u8); NUM_TABLES]> { - let default_degree = 8; // default value for None let degrees = all_proof .degree_bits(config) - .map(|opt| opt.unwrap_or(default_degree)); + .map(|opt| opt.unwrap_or(DEFAULT_CIRCUIT_DEGREE)); /// Given a recursive circuit index (e.g., Arithmetic / 0), return a /// tuple containing the loaded table at the specified size and From 8660749a00dff11c0dd95f99d61867a706c454b7 Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Tue, 8 Oct 2024 10:08:47 -0700 Subject: [PATCH 74/91] address comments --- evm_arithmetization/src/generation/mod.rs | 2 +- evm_arithmetization/src/witness/mod.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/evm_arithmetization/src/generation/mod.rs b/evm_arithmetization/src/generation/mod.rs index c19d3df17..a4994f686 100644 --- a/evm_arithmetization/src/generation/mod.rs +++ b/evm_arithmetization/src/generation/mod.rs @@ -229,7 +229,7 @@ impl GenerationInputs { /// Outputs a trimmed version of the `GenerationInputs`, that do not contain /// the fields that have already been processed during pre-initialization, /// namely: the input tries, the signed transaction, and the withdrawals. - pub fn trim(&self) -> TrimmedGenerationInputs { + pub(crate) fn trim(&self) -> TrimmedGenerationInputs { let txn_hashes = self .signed_txns .iter() diff --git a/evm_arithmetization/src/witness/mod.rs b/evm_arithmetization/src/witness/mod.rs index 74ff47644..a38a55229 100644 --- a/evm_arithmetization/src/witness/mod.rs +++ b/evm_arithmetization/src/witness/mod.rs @@ -1,7 +1,7 @@ pub(crate) mod errors; pub(crate) mod gas; pub(crate) mod memory; -pub mod operation; +pub(crate) mod operation; pub(crate) mod state; pub(crate) mod traces; pub mod transition; From 46c06841c96f2c2517d4681f7be7b5aa907f8fa8 Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Tue, 8 Oct 2024 10:12:59 -0700 Subject: [PATCH 75/91] clippy --- evm_arithmetization/tests/add11_yml.rs | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/evm_arithmetization/tests/add11_yml.rs b/evm_arithmetization/tests/add11_yml.rs index 127a41b87..e39762a49 100644 --- a/evm_arithmetization/tests/add11_yml.rs +++ b/evm_arithmetization/tests/add11_yml.rs @@ -8,17 +8,14 @@ use ethereum_types::{Address, BigEndianHash, H256}; use evm_arithmetization::generation::mpt::{AccountRlp, LegacyReceiptRlp}; use evm_arithmetization::generation::TrieInputs; use evm_arithmetization::proof::{BlockHashes, BlockMetadata, TrieRoots}; -use evm_arithmetization::prover::prove; use evm_arithmetization::prover::testing::prove_all_segments; use evm_arithmetization::testing_utils::{ beacon_roots_account_nibbles, beacon_roots_contract_from_storage, init_logger, preinitialized_state_and_storage_tries, update_beacon_roots_account_storage, }; use evm_arithmetization::verifier::testing::verify_all_proofs; -use evm_arithmetization::witness::operation::Operation::KeccakGeneral; use evm_arithmetization::{ - AllStark, GenerationInputs, Node, SegmentDataIterator, StarkConfig, - EMPTY_CONSOLIDATED_BLOCKHASH, + AllStark, GenerationInputs, Node, StarkConfig, EMPTY_CONSOLIDATED_BLOCKHASH, }; use hex_literal::hex; use keccak_hash::keccak; @@ -26,7 +23,7 @@ use mpt_trie::nibbles::Nibbles; use mpt_trie::partial_trie::{HashedPartialTrie, PartialTrie}; use plonky2::field::goldilocks_field::GoldilocksField; use plonky2::field::types::Field; -use plonky2::plonk::config::{KeccakGoldilocksConfig, PoseidonGoldilocksConfig}; +use plonky2::plonk::config::KeccakGoldilocksConfig; use plonky2::util::timing::TimingTree; type F = GoldilocksField; @@ -113,7 +110,7 @@ fn get_generation_inputs() -> GenerationInputs { block_metadata.block_timestamp, block_metadata.parent_beacon_block_root, ) - .unwrap(); + .unwrap(); let beacon_roots_account = beacon_roots_contract_from_storage(&beacon_roots_account_storage); @@ -134,7 +131,7 @@ fn get_generation_inputs() -> GenerationInputs { nibbles: Nibbles::from_h256_be(keccak([0u8; 32])), value: vec![2], }) - .hash(), + .hash(), ..AccountRlp::default() }; @@ -177,7 +174,7 @@ fn get_generation_inputs() -> GenerationInputs { nibbles: Nibbles::from_str("0x80").unwrap(), value: txn.to_vec(), } - .into(); + .into(); let trie_roots_after = TrieRoots { state_root: expected_state_trie_after.hash(), @@ -205,7 +202,6 @@ fn get_generation_inputs() -> GenerationInputs { }, } } - /// The `add11_yml` test case from https://github.com/ethereum/tests #[test] fn add11_yml() -> anyhow::Result<()> { From a4715b73e84327452fb00b1438d7425a96b9c79f Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Tue, 8 Oct 2024 10:19:51 -0700 Subject: [PATCH 76/91] fix build --- evm_arithmetization/src/fixed_recursive_verifier.rs | 2 +- evm_arithmetization/src/testing_utils.rs | 11 +++++++---- evm_arithmetization/tests/add11_yml.rs | 6 +++--- evm_arithmetization/tests/empty_tables.rs | 2 +- 4 files changed, 12 insertions(+), 9 deletions(-) diff --git a/evm_arithmetization/src/fixed_recursive_verifier.rs b/evm_arithmetization/src/fixed_recursive_verifier.rs index 37415aa39..7cc105d44 100644 --- a/evm_arithmetization/src/fixed_recursive_verifier.rs +++ b/evm_arithmetization/src/fixed_recursive_verifier.rs @@ -3181,7 +3181,7 @@ mod tests { all_circuits.prove_segment( &all_stark, &config, - payload.trim(), + payload, &mut segment_data, timing, None, diff --git a/evm_arithmetization/src/testing_utils.rs b/evm_arithmetization/src/testing_utils.rs index 26b98a3b8..dc6460f7e 100644 --- a/evm_arithmetization/src/testing_utils.rs +++ b/evm_arithmetization/src/testing_utils.rs @@ -14,7 +14,7 @@ use plonky2::field::goldilocks_field::GoldilocksField; pub use crate::cpu::kernel::cancun_constants::*; pub use crate::cpu::kernel::constants::global_exit_root::*; -use crate::generation::TrieInputs; +use crate::generation::{TrieInputs, TrimmedGenerationInputs}; use crate::proof::TrieRoots; use crate::witness::operation::Operation; use crate::{ @@ -219,15 +219,18 @@ fn empty_payload() -> Result { Ok(inputs) } -pub fn segment_without_keccak() -> Result<(GenerationInputs, GenerationSegmentData)> { +pub fn segment_without_keccak() -> Result<( + TrimmedGenerationInputs, + GenerationSegmentData, +)> { let payload = empty_payload()?; let max_cpu_len_log = Some(7); let mut segment_iterator = SegmentDataIterator::::new(&payload, max_cpu_len_log); - let (_, segment_data) = segment_iterator.next().unwrap()?; + let (trimmed_inputs, segment_data) = segment_iterator.next().unwrap()?; let opcode_counts = &segment_data.opcode_counts; assert!(!opcode_counts.contains_key(&Operation::KeccakGeneral)); - Ok((payload, segment_data)) + Ok((trimmed_inputs, segment_data)) } diff --git a/evm_arithmetization/tests/add11_yml.rs b/evm_arithmetization/tests/add11_yml.rs index e39762a49..dd9bfb1ce 100644 --- a/evm_arithmetization/tests/add11_yml.rs +++ b/evm_arithmetization/tests/add11_yml.rs @@ -110,7 +110,7 @@ fn get_generation_inputs() -> GenerationInputs { block_metadata.block_timestamp, block_metadata.parent_beacon_block_root, ) - .unwrap(); + .unwrap(); let beacon_roots_account = beacon_roots_contract_from_storage(&beacon_roots_account_storage); @@ -131,7 +131,7 @@ fn get_generation_inputs() -> GenerationInputs { nibbles: Nibbles::from_h256_be(keccak([0u8; 32])), value: vec![2], }) - .hash(), + .hash(), ..AccountRlp::default() }; @@ -174,7 +174,7 @@ fn get_generation_inputs() -> GenerationInputs { nibbles: Nibbles::from_str("0x80").unwrap(), value: txn.to_vec(), } - .into(); + .into(); let trie_roots_after = TrieRoots { state_root: expected_state_trie_after.hash(), diff --git a/evm_arithmetization/tests/empty_tables.rs b/evm_arithmetization/tests/empty_tables.rs index 3d24ccee0..9cb3e6fa9 100644 --- a/evm_arithmetization/tests/empty_tables.rs +++ b/evm_arithmetization/tests/empty_tables.rs @@ -29,7 +29,7 @@ fn empty_tables() -> anyhow::Result<()> { let proof = prove::( &all_stark, &config, - payload.trim(), + payload, &mut segment_data, &mut timing, None, From a260fba13195b4b6197afe1a613c34111cd2730f Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Wed, 9 Oct 2024 14:56:54 -0700 Subject: [PATCH 77/91] merge tests --- .../src/fixed_recursive_verifier.rs | 78 +++---------------- evm_arithmetization/tests/empty_tables.rs | 69 ++++++++++++---- 2 files changed, 64 insertions(+), 83 deletions(-) diff --git a/evm_arithmetization/src/fixed_recursive_verifier.rs b/evm_arithmetization/src/fixed_recursive_verifier.rs index 7cc105d44..17969c1b7 100644 --- a/evm_arithmetization/src/fixed_recursive_verifier.rs +++ b/evm_arithmetization/src/fixed_recursive_verifier.rs @@ -1903,7 +1903,6 @@ where abort_signal: Option>, ) -> anyhow::Result> { features_check(&generation_inputs); - let all_proof = prove::( all_stark, config, @@ -1912,7 +1911,15 @@ where timing, abort_signal.clone(), )?; + self.prove_segment_with_all_proofs(&all_proof, config, abort_signal.clone()) + } + pub fn prove_segment_with_all_proofs( + &self, + all_proof: &AllProof, + config: &StarkConfig, + abort_signal: Option>, + ) -> anyhow::Result> { let mut root_inputs = PartialWitness::new(); for table in 0..NUM_TABLES { @@ -1995,7 +2002,7 @@ where is_agg: false, is_dummy: false, proof_with_pvs: ProofWithPublicValues { - public_values: all_proof.public_values, + public_values: all_proof.public_values.clone(), intern: root_proof, }, }) @@ -3135,70 +3142,3 @@ pub mod testing { } } } - -#[cfg(test)] -#[cfg(not(feature = "cdk_erigon"))] -mod tests { - use plonky2::field::goldilocks_field::GoldilocksField; - use plonky2::plonk::config::PoseidonGoldilocksConfig; - use plonky2::timed; - - use super::*; - use crate::testing_utils::{init_logger, segment_without_keccak}; - - type F = GoldilocksField; - const D: usize = 2; - type C = PoseidonGoldilocksConfig; - - #[test] - fn test_segment_proof_generation_without_keccak() -> anyhow::Result<()> { - init_logger(); - - let all_stark = AllStark::::default(); - let config = StarkConfig::standard_fast_config(); - - let timing = &mut TimingTree::new( - "Segment Proof Generation Without Keccak Test", - log::Level::Info, - ); - // Process and prove segment - let all_circuits = timed!( - timing, - log::Level::Info, - "Create all recursive circuits", - AllRecursiveCircuits::::new( - &all_stark, - &[16..17, 8..9, 7..8, 4..9, 8..9, 4..7, 17..18, 17..18, 17..18], - &config, - ) - ); - - let (payload, mut segment_data) = segment_without_keccak()?; - let segment_proof = timed!( - timing, - log::Level::Info, - "Prove segment", - all_circuits.prove_segment( - &all_stark, - &config, - payload, - &mut segment_data, - timing, - None, - )? - ); - - // Verify the generated segment proof - timed!( - timing, - log::Level::Info, - "Verify segment proof", - all_circuits.verify_root(segment_proof.proof_with_pvs.intern.clone())? - ); - - // Print timing details - timing.print(); - - Ok(()) - } -} diff --git a/evm_arithmetization/tests/empty_tables.rs b/evm_arithmetization/tests/empty_tables.rs index 9cb3e6fa9..18fe6166d 100644 --- a/evm_arithmetization/tests/empty_tables.rs +++ b/evm_arithmetization/tests/empty_tables.rs @@ -1,13 +1,13 @@ #![cfg(feature = "eth_mainnet")] -use std::time::Duration; - +use evm_arithmetization::fixed_recursive_verifier::AllRecursiveCircuits; use evm_arithmetization::prover::prove; use evm_arithmetization::testing_utils::{init_logger, segment_without_keccak}; use evm_arithmetization::verifier::testing::verify_all_proofs; use evm_arithmetization::AllStark; use plonky2::field::goldilocks_field::GoldilocksField; use plonky2::plonk::config::PoseidonGoldilocksConfig; +use plonky2::timed; use plonky2::util::timing::TimingTree; use starky::config::StarkConfig; @@ -22,21 +22,62 @@ fn empty_tables() -> anyhow::Result<()> { let all_stark = AllStark::::default(); let config = StarkConfig::standard_fast_config(); - let mut proofs = vec![]; - let mut timing = TimingTree::new("prove", log::Level::Debug); + let mut timing = &mut TimingTree::new( + "Empty Table Test", + log::Level::Info, + ); + // Generate segment data let (payload, mut segment_data) = segment_without_keccak()?; - let proof = prove::( - &all_stark, - &config, - payload, - &mut segment_data, - &mut timing, - None, - )?; + + // Create all STARK proofs + let mut proofs = vec![]; + let proof = timed!( + timing, + log::Level::Info, + "Create all STARK proofs", + prove::( + &all_stark, + &config, + payload, + &mut segment_data, + &mut timing, + None, + )? + ); proofs.push(proof); - timing.filter(Duration::from_millis(100)).print(); + // Verify the generated STARK proofs + verify_all_proofs(&all_stark, &proofs, &config)?; + + // Process and generate segment proof + let all_circuits = timed!( + timing, + log::Level::Info, + "Create all recursive circuits", + AllRecursiveCircuits::::new( + &all_stark, + &[16..17, 8..9, 7..8, 4..9, 8..9, 4..7, 17..18, 17..18, 17..18], + &config, + ) + ); + let segment_proof = timed!( + timing, + log::Level::Info, + "Prove segment", + all_circuits.prove_segment_with_all_proofs(&proofs[0], &config, None)? + ); + + // Verify the generated segment proof + timed!( + timing, + log::Level::Info, + "Verify segment proof", + all_circuits.verify_root(segment_proof.proof_with_pvs.intern.clone())? + ); + + // Print timing details + timing.print(); - verify_all_proofs(&all_stark, &proofs, &config) + Ok(()) } From 27590f897fb84e15b1c52ca77e527e06c1018536 Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Wed, 9 Oct 2024 14:59:07 -0700 Subject: [PATCH 78/91] clippy --- evm_arithmetization/tests/empty_tables.rs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/evm_arithmetization/tests/empty_tables.rs b/evm_arithmetization/tests/empty_tables.rs index 18fe6166d..78aedd137 100644 --- a/evm_arithmetization/tests/empty_tables.rs +++ b/evm_arithmetization/tests/empty_tables.rs @@ -22,10 +22,7 @@ fn empty_tables() -> anyhow::Result<()> { let all_stark = AllStark::::default(); let config = StarkConfig::standard_fast_config(); - let mut timing = &mut TimingTree::new( - "Empty Table Test", - log::Level::Info, - ); + let timing = &mut TimingTree::new("Empty Table Test", log::Level::Info); // Generate segment data let (payload, mut segment_data) = segment_without_keccak()?; @@ -41,7 +38,7 @@ fn empty_tables() -> anyhow::Result<()> { &config, payload, &mut segment_data, - &mut timing, + timing, None, )? ); From 3dd6d61f6eba200af28ea98cc2ed4f5e740793b0 Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Thu, 10 Oct 2024 13:39:40 -0700 Subject: [PATCH 79/91] build --- .../src/fixed_recursive_verifier.rs | 131 +++++++++++++++--- evm_arithmetization/src/lib.rs | 2 + evm_arithmetization/src/testing_utils.rs | 2 +- evm_arithmetization/tests/empty_tables.rs | 4 +- 4 files changed, 118 insertions(+), 21 deletions(-) diff --git a/evm_arithmetization/src/fixed_recursive_verifier.rs b/evm_arithmetization/src/fixed_recursive_verifier.rs index 17969c1b7..b5e8e7cd5 100644 --- a/evm_arithmetization/src/fixed_recursive_verifier.rs +++ b/evm_arithmetization/src/fixed_recursive_verifier.rs @@ -1,4 +1,4 @@ -use core::mem::{self, MaybeUninit}; +use core::mem::MaybeUninit; use core::ops::Range; use std::collections::BTreeMap; use std::sync::atomic::AtomicBool; @@ -25,7 +25,7 @@ use plonky2::plonk::proof::{ProofWithPublicInputs, ProofWithPublicInputsTarget}; use plonky2::recursion::cyclic_recursion::check_cyclic_proof_verifier_data; use plonky2::recursion::dummy_circuit::{cyclic_base_proof, dummy_circuit, dummy_proof}; use plonky2::util::serialization::{ - Buffer, GateSerializer, IoResult, Read, WitnessGeneratorSerializer, Write, + Buffer, GateSerializer, IoError, IoResult, Read, WitnessGeneratorSerializer, Write, }; use plonky2::util::timing::TimingTree; use plonky2_util::log2_ceil; @@ -664,29 +664,39 @@ where true => (0..NUM_TABLES) .map(|_| RecursiveCircuitsForTable { by_stark_size: BTreeMap::default(), + dummy_proof_height: None, + dummy_proof: None, }) .collect_vec() .try_into() .unwrap(), false => { - // Tricky use of MaybeUninit to remove the need for implementing Debug - // for all underlying types, necessary to convert a by_table Vec to an array. + // Initialize an uninitialized array of MaybeUninit let mut by_table: [MaybeUninit>; NUM_TABLES] = - unsafe { MaybeUninit::uninit().assume_init() }; - for table in &mut by_table[..] { - let value = RecursiveCircuitsForTable::from_buffer( + MaybeUninit::uninit_array(); + + for i in 0..NUM_TABLES { + let value = match RecursiveCircuitsForTable::from_buffer( &mut buffer, gate_serializer, generator_serializer, - )?; - *table = MaybeUninit::new(value); - } - unsafe { - mem::transmute::< - [std::mem::MaybeUninit>; NUM_TABLES], - [RecursiveCircuitsForTable; NUM_TABLES], - >(by_table) + ) { + Ok(v) => v, + Err(e) => { + // If there's an error, deinitialize any initialized elements + for initialized in by_table[..i].iter_mut() { + unsafe { std::ptr::drop_in_place(initialized.as_mut_ptr()) }; + } + return Err(e); + } + }; + + // Initialize the current element + by_table[i] = MaybeUninit::new(value); } + + // Safely convert the array of MaybeUninit to a fully initialized array + unsafe { MaybeUninit::array_assume_init(by_table) } } }; @@ -735,6 +745,7 @@ where degree_bits_ranges[*$table_enum].clone(), &all_stark.cross_table_lookups, stark_config, + true, ) }; } @@ -2759,6 +2770,10 @@ where /// A map from `log_2(height)` to a chain of shrinking recursion circuits /// starting at that height. pub by_stark_size: BTreeMap>, + /// The initial height for generating the dummy proof. + pub dummy_proof_height: Option, + /// The dummy proof after applying shrinking recursion. + pub dummy_proof: Option>, } impl RecursiveCircuitsForTable @@ -2778,6 +2793,27 @@ where buffer.write_usize(size)?; table.to_buffer(buffer, gate_serializer, generator_serializer)?; } + + // Write dummy_proof_height if it exists + match self.dummy_proof_height { + Some(height) => { + buffer.write_bool(true)?; + buffer.write_usize(height)?; + } + None => buffer.write_bool(false)?, + } + + // Write dummy_proof if it exists using `to_bytes` + match &self.dummy_proof { + Some(proof) => { + buffer.write_bool(true)?; + let proof_bytes = proof.to_bytes(); + buffer.write_usize(proof_bytes.len())?; + buffer.extend_from_slice(&proof_bytes); + } + None => buffer.write_bool(false)?, + } + Ok(()) } @@ -2797,7 +2833,40 @@ where )?; by_stark_size.insert(key, table); } - Ok(Self { by_stark_size }) + + // Read dummy_proof_height if present + let dummy_proof_height = if buffer.read_bool()? { + Some(buffer.read_usize()?) + } else { + None + }; + + // Read dummy_proof if present + let dummy_proof = if buffer.read_bool()? { + let dummy_proof_height = dummy_proof_height.ok_or_else(|| IoError)?; + let common_data = &by_stark_size + .get(&dummy_proof_height) + .ok_or_else(|| IoError)? + .shrinking_wrappers + .last() + .ok_or_else(|| IoError)? + .circuit + .common; + let proof_len = buffer.read_usize()?; + let mut proof_bytes = vec![0u8; proof_len]; + buffer.read_exact(&mut proof_bytes)?; + let proof = + ProofWithPublicInputs::from_bytes(proof_bytes, common_data).map_err(|_| IoError)?; + Some(proof) + } else { + None + }; + + Ok(Self { + by_stark_size, + dummy_proof_height, + dummy_proof, + }) } fn new>( @@ -2806,8 +2875,10 @@ where degree_bits_range: Range, all_ctls: &[CrossTableLookup], stark_config: &StarkConfig, + is_optional_table: bool, ) -> Self { - let by_stark_size = degree_bits_range + let by_stark_size: BTreeMap = degree_bits_range + .clone() .map(|degree_bits| { ( degree_bits, @@ -2821,7 +2892,31 @@ where ) }) .collect(); - Self { by_stark_size } + let dummy_proof_height = if is_optional_table { + Some(degree_bits_range.start) + } else { + None + }; + let dummy_proof = if is_optional_table { + let dummy_proof_height = dummy_proof_height.expect("Failed to get dummy_proof_height"); + let dummy_circuit = &by_stark_size + .get(&dummy_proof_height) + .expect("Failed to get dummy_proof_height in by_stark_size") + .shrinking_wrappers + .last() + .expect("No shrinking_wrappers available") + .circuit; + let dummy_pis = HashMap::new(); + Some(dummy_proof(&dummy_circuit, dummy_pis).expect("Unable to generate dummy proofs")) + } else { + None + }; + + Self { + by_stark_size, + dummy_proof_height, + dummy_proof, + } } /// For each initial `degree_bits`, get the final circuit at the end of that diff --git a/evm_arithmetization/src/lib.rs b/evm_arithmetization/src/lib.rs index 41b7f093a..26b030429 100644 --- a/evm_arithmetization/src/lib.rs +++ b/evm_arithmetization/src/lib.rs @@ -252,6 +252,8 @@ #![allow(clippy::too_many_arguments)] #![allow(clippy::field_reassign_with_default)] #![feature(let_chains)] +#![feature(maybe_uninit_uninit_array)] +#![feature(maybe_uninit_array_assume_init)] zk_evm_common::check_chain_features!(); diff --git a/evm_arithmetization/src/testing_utils.rs b/evm_arithmetization/src/testing_utils.rs index dc6460f7e..039dc2504 100644 --- a/evm_arithmetization/src/testing_utils.rs +++ b/evm_arithmetization/src/testing_utils.rs @@ -219,7 +219,7 @@ fn empty_payload() -> Result { Ok(inputs) } -pub fn segment_without_keccak() -> Result<( +pub fn segment_with_empty_tables() -> Result<( TrimmedGenerationInputs, GenerationSegmentData, )> { diff --git a/evm_arithmetization/tests/empty_tables.rs b/evm_arithmetization/tests/empty_tables.rs index 78aedd137..9fcf4a74a 100644 --- a/evm_arithmetization/tests/empty_tables.rs +++ b/evm_arithmetization/tests/empty_tables.rs @@ -2,7 +2,7 @@ use evm_arithmetization::fixed_recursive_verifier::AllRecursiveCircuits; use evm_arithmetization::prover::prove; -use evm_arithmetization::testing_utils::{init_logger, segment_without_keccak}; +use evm_arithmetization::testing_utils::{init_logger, segment_with_empty_tables}; use evm_arithmetization::verifier::testing::verify_all_proofs; use evm_arithmetization::AllStark; use plonky2::field::goldilocks_field::GoldilocksField; @@ -25,7 +25,7 @@ fn empty_tables() -> anyhow::Result<()> { let timing = &mut TimingTree::new("Empty Table Test", log::Level::Info); // Generate segment data - let (payload, mut segment_data) = segment_without_keccak()?; + let (payload, mut segment_data) = segment_with_empty_tables()?; // Create all STARK proofs let mut proofs = vec![]; From 0aa31ed9c4bd6a005993db9bd7a69cdacc7c7ce0 Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Thu, 10 Oct 2024 13:41:31 -0700 Subject: [PATCH 80/91] polish code --- .../src/fixed_recursive_verifier.rs | 23 ++++++++----------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/evm_arithmetization/src/fixed_recursive_verifier.rs b/evm_arithmetization/src/fixed_recursive_verifier.rs index b5e8e7cd5..6397c69ad 100644 --- a/evm_arithmetization/src/fixed_recursive_verifier.rs +++ b/evm_arithmetization/src/fixed_recursive_verifier.rs @@ -2877,7 +2877,7 @@ where stark_config: &StarkConfig, is_optional_table: bool, ) -> Self { - let by_stark_size: BTreeMap = degree_bits_range + let by_stark_size = degree_bits_range .clone() .map(|degree_bits| { ( @@ -2891,26 +2891,21 @@ where ), ) }) - .collect(); - let dummy_proof_height = if is_optional_table { - Some(degree_bits_range.start) - } else { - None - }; - let dummy_proof = if is_optional_table { - let dummy_proof_height = dummy_proof_height.expect("Failed to get dummy_proof_height"); + .collect::>(); + + let dummy_proof_height = is_optional_table.then_some(degree_bits_range.start); + + let dummy_proof = dummy_proof_height.map(|height| { let dummy_circuit = &by_stark_size - .get(&dummy_proof_height) + .get(&height) .expect("Failed to get dummy_proof_height in by_stark_size") .shrinking_wrappers .last() .expect("No shrinking_wrappers available") .circuit; let dummy_pis = HashMap::new(); - Some(dummy_proof(&dummy_circuit, dummy_pis).expect("Unable to generate dummy proofs")) - } else { - None - }; + dummy_proof(dummy_circuit, dummy_pis).expect("Unable to generate dummy proofs") + }); Self { by_stark_size, From 96f499b1b1915bb826218cf26401ec60160a4675 Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Thu, 10 Oct 2024 15:48:39 -0700 Subject: [PATCH 81/91] refactor --- .../src/fixed_recursive_verifier.rs | 117 ++++++++---------- zero/src/prover_state/persistence.rs | 34 ++++- 2 files changed, 81 insertions(+), 70 deletions(-) diff --git a/evm_arithmetization/src/fixed_recursive_verifier.rs b/evm_arithmetization/src/fixed_recursive_verifier.rs index 6397c69ad..64c269bab 100644 --- a/evm_arithmetization/src/fixed_recursive_verifier.rs +++ b/evm_arithmetization/src/fixed_recursive_verifier.rs @@ -664,8 +664,7 @@ where true => (0..NUM_TABLES) .map(|_| RecursiveCircuitsForTable { by_stark_size: BTreeMap::default(), - dummy_proof_height: None, - dummy_proof: None, + dummy_proof_data: None, }) .collect_vec() .try_into() @@ -1936,29 +1935,18 @@ where for table in 0..NUM_TABLES { let table_circuits = &self.by_table[table]; if KECCAK_TABLES_INDICES.contains(&table) && !all_proof.use_keccak_tables { - // generate and set a dummy `index_verifier_data` and `proof_with_pis` - let index_verifier_data = - table_circuits.by_stark_size.keys().min().ok_or_else(|| { - anyhow::format_err!("No valid size in shrinking circuits") - })?; + let dummy_proof_data = table_circuits + .dummy_proof_data + .as_ref() + .ok_or_else(|| anyhow::format_err!("No dummy_proof_data"))?; root_inputs.set_target( self.root.index_verifier_data[table], - F::from_canonical_usize(*index_verifier_data), + F::from_canonical_usize(dummy_proof_data.init_degree), + ); + root_inputs.set_proof_with_pis_target( + &self.root.proof_with_pis[table], + &dummy_proof_data.proof, ); - let table_circuit = table_circuits - .by_stark_size - .get(index_verifier_data) - .ok_or_else(|| anyhow::format_err!("No valid size in shrinking circuits"))? - .shrinking_wrappers - .last() - .ok_or_else(|| anyhow::format_err!("No shrinking circuits"))?; - let dummy_circuit: CircuitData = - dummy_circuit(&table_circuit.circuit.common); - let dummy_pis = HashMap::new(); - let dummy_proof = dummy_proof(&dummy_circuit, dummy_pis) - .map_err(|_| anyhow::format_err!("Unable to generate dummy proofs")); - root_inputs - .set_proof_with_pis_target(&self.root.proof_with_pis[table], &dummy_proof?); } else { let stark_proof = &all_proof.multi_proof.stark_proofs[table] .as_ref() @@ -2758,6 +2746,20 @@ where agg_inputs.set_proof_with_pis_target(&agg_child.base_proof, proof); } } + +/// A struct that encapsulates both the init degree and the shrunk proof. +#[derive(Eq, PartialEq, Debug)] +pub struct ShrunkProofData +where + F: RichField + Extendable, + C: GenericConfig, +{ + /// The initial degree for generating the proof. + pub init_degree: usize, + /// The proof after applying shrinking recursion. + pub proof: ProofWithPublicInputs, +} + /// A map between initial degree sizes and their associated shrinking recursion /// circuits. #[derive(Eq, PartialEq, Debug)] @@ -2770,10 +2772,9 @@ where /// A map from `log_2(height)` to a chain of shrinking recursion circuits /// starting at that height. pub by_stark_size: BTreeMap>, - /// The initial height for generating the dummy proof. - pub dummy_proof_height: Option, - /// The dummy proof after applying shrinking recursion. - pub dummy_proof: Option>, + + /// A dummy proof for the root circuit when the table is empty. + pub dummy_proof_data: Option>, } impl RecursiveCircuitsForTable @@ -2794,20 +2795,12 @@ where table.to_buffer(buffer, gate_serializer, generator_serializer)?; } - // Write dummy_proof_height if it exists - match self.dummy_proof_height { - Some(height) => { + // Serialize dummy_proof if it exists + match &self.dummy_proof_data { + Some(dummy_proof_data) => { buffer.write_bool(true)?; - buffer.write_usize(height)?; - } - None => buffer.write_bool(false)?, - } - - // Write dummy_proof if it exists using `to_bytes` - match &self.dummy_proof { - Some(proof) => { - buffer.write_bool(true)?; - let proof_bytes = proof.to_bytes(); + buffer.write_usize(dummy_proof_data.init_degree)?; + let proof_bytes = dummy_proof_data.proof.to_bytes(); buffer.write_usize(proof_bytes.len())?; buffer.extend_from_slice(&proof_bytes); } @@ -2834,18 +2827,11 @@ where by_stark_size.insert(key, table); } - // Read dummy_proof_height if present - let dummy_proof_height = if buffer.read_bool()? { - Some(buffer.read_usize()?) - } else { - None - }; - // Read dummy_proof if present - let dummy_proof = if buffer.read_bool()? { - let dummy_proof_height = dummy_proof_height.ok_or_else(|| IoError)?; + let dummy_proof_data = if buffer.read_bool()? { + let init_degree = buffer.read_usize()?; let common_data = &by_stark_size - .get(&dummy_proof_height) + .get(&init_degree) .ok_or_else(|| IoError)? .shrinking_wrappers .last() @@ -2857,15 +2843,14 @@ where buffer.read_exact(&mut proof_bytes)?; let proof = ProofWithPublicInputs::from_bytes(proof_bytes, common_data).map_err(|_| IoError)?; - Some(proof) + Some(ShrunkProofData { init_degree, proof }) } else { None }; Ok(Self { by_stark_size, - dummy_proof_height, - dummy_proof, + dummy_proof_data, }) } @@ -2893,24 +2878,24 @@ where }) .collect::>(); - let dummy_proof_height = is_optional_table.then_some(degree_bits_range.start); - - let dummy_proof = dummy_proof_height.map(|height| { - let dummy_circuit = &by_stark_size - .get(&height) - .expect("Failed to get dummy_proof_height in by_stark_size") - .shrinking_wrappers - .last() - .expect("No shrinking_wrappers available") - .circuit; - let dummy_pis = HashMap::new(); - dummy_proof(dummy_circuit, dummy_pis).expect("Unable to generate dummy proofs") + let dummy_proof_data = is_optional_table.then_some(ShrunkProofData { + init_degree: degree_bits_range.start, + proof: { + let dummy_circuit = &by_stark_size + .get(°ree_bits_range.start) + .expect("Failed to get dummy_proof_height in by_stark_size") + .shrinking_wrappers + .last() + .expect("No shrinking_wrappers available") + .circuit; + let dummy_pis = HashMap::new(); + dummy_proof(dummy_circuit, dummy_pis).expect("Unable to generate dummy proofs") + }, }); Self { by_stark_size, - dummy_proof_height, - dummy_proof, + dummy_proof_data, } } diff --git a/zero/src/prover_state/persistence.rs b/zero/src/prover_state/persistence.rs index fbd7be2d0..f2a46a341 100644 --- a/zero/src/prover_state/persistence.rs +++ b/zero/src/prover_state/persistence.rs @@ -8,10 +8,7 @@ use std::{ use alloy::hex; use anyhow::anyhow; use directories::ProjectDirs; -use evm_arithmetization::{ - cpu::kernel::aggregator::KERNEL, AllRecursiveCircuits, RecursionConfig, - RecursiveCircuitsForTableSize, VerifierData, EXTENSION_DEGREE, -}; +use evm_arithmetization::{cpu::kernel::aggregator::KERNEL, AllRecursiveCircuits, ProofWithPublicInputs, RecursionConfig, RecursiveCircuitsForTableSize, VerifierData, EXTENSION_DEGREE}; use once_cell::sync::Lazy; use plonky2::util::serialization::{ Buffer, DefaultGateSerializer, DefaultGeneratorSerializer, IoError, @@ -232,6 +229,35 @@ impl DiskResource for RecursiveCircuitResource { } } +#[derive(Debug, Default)] +pub(crate) struct DummyProofResource; + +impl DiskResource for DummyProofResource { + type Resource = ProofWithPublicInputs; + type Error = IoError; + type PathConstrutor = (Circuit, usize); + + fn path((circuit_type, size): &Self::PathConstrutor) -> impl AsRef { + format!( + "{}/{}_dummy_proof_{}_{}", + circuit_dir(), + PROVER_STATE_FILE_PREFIX, + circuit_type.as_short_str(), + size + ) + } + + fn serialize(r: &Self::Resource) -> Result, DiskResourceError> { + Ok(r.to_bytes()) // Assuming `to_bytes` is implemented for `ProofWithPublicInputs` + } + + fn deserialize( + bytes: &[u8], + ) -> Result> { + ProofWithPublicInputs::from_bytes(bytes.to_vec(), &common_data).map_err(DiskResourceError::Serialization) + } +} + /// An individual circuit table with a specific size. #[derive(Debug, Default)] pub(crate) struct VerifierResource; From 8cec513fcae1bb1c8b339d4da09ef4f831e3868b Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Thu, 10 Oct 2024 21:24:08 -0700 Subject: [PATCH 82/91] wip --- .../src/fixed_recursive_verifier.rs | 173 +++++++++--------- zero/src/prover_state/persistence.rs | 34 +--- 2 files changed, 87 insertions(+), 120 deletions(-) diff --git a/evm_arithmetization/src/fixed_recursive_verifier.rs b/evm_arithmetization/src/fixed_recursive_verifier.rs index 64c269bab..e3aa8005f 100644 --- a/evm_arithmetization/src/fixed_recursive_verifier.rs +++ b/evm_arithmetization/src/fixed_recursive_verifier.rs @@ -131,6 +131,8 @@ where /// Holds chains of circuits for each table and for each initial /// `degree_bits`. pub by_table: [RecursiveCircuitsForTable; NUM_TABLES], + /// Dummy proofs of each table for the root circuit. + pub table_dummy_proofs: [Option>; NUM_TABLES], } /// Data for the EVM root circuit, which is used to combine each STARK's shrunk @@ -569,6 +571,43 @@ where } } +/// A struct that encapsulates both the init degree and the shrunk proof. +#[derive(Eq, PartialEq, Debug)] +pub struct ShrunkProofData +where + F: RichField + Extendable, + C: GenericConfig, +{ + /// The initial degree for generating the proof. + pub init_degree: usize, + /// The proof bytes after applying shrinking recursion. + pub proof_bytes: Vec, +} + +impl ShrunkProofData +where + F: RichField + Extendable, + C: GenericConfig, +{ + fn to_buffer(&self, buffer: &mut Vec) -> IoResult<()> { + buffer.write_usize(self.init_degree); + buffer.write_usize(self.proof_bytes.len()); + buffer.write_all(&self.proof_bytes); + Ok(()) + } + + fn from_buffer(buffer: &mut Buffer) -> IoResult { + let init_degree = buffer.read_usize()?; + let proof_bytes_len = buffer.read_usize()?; + let mut proof_bytes = vec![0u8; proof_bytes_len]; + buffer.read_exact(&mut proof_bytes); + Ok(Self { + init_degree, + proof_bytes, + }) + } +} + impl AllRecursiveCircuits where F: RichField + Extendable, @@ -612,6 +651,15 @@ where table.to_buffer(&mut buffer, gate_serializer, generator_serializer)?; } } + for table in &self.table_dummy_proofs { + match table { + Some(dummy_proof_data) => { + buffer.write_bool(true)?; + dummy_proof_data.to_buffer(&mut buffer)? + }, + None => buffer.write_bool(false)?, + } + } Ok(buffer) } @@ -664,7 +712,6 @@ where true => (0..NUM_TABLES) .map(|_| RecursiveCircuitsForTable { by_stark_size: BTreeMap::default(), - dummy_proof_data: None, }) .collect_vec() .try_into() @@ -699,6 +746,22 @@ where } }; + let table_dummy_proofs = core::array::from_fn(|i| { + if buffer.read_bool()? { + let init_degree = buffer.read_usize()?; + let proof_bytes_len = buffer.read_usize()?; + let mut proof_bytes = vec![0u8; proof_bytes_len]; + buffer.read_exact(&mut proof_bytes)?; + + Some(ShrunkProofData { + init_degree, + proof_bytes, + }) + } else { + None + } + }); + Ok(Self { root, segment_aggregation, @@ -707,6 +770,7 @@ where block_wrapper, two_to_one_block, by_table, + table_dummy_proofs, }) } @@ -2067,35 +2131,34 @@ where for table in 0..NUM_TABLES { let (table_circuit, index_verifier_data) = &table_circuits[table]; + root_inputs.set_target( + self.root.index_verifier_data[table], + F::from_canonical_u8(*index_verifier_data), + ); if KECCAK_TABLES_INDICES.contains(&table) && !all_proof.use_keccak_tables { - root_inputs.set_target( - self.root.index_verifier_data[table], - F::from_canonical_u8(*index_verifier_data), - ); - // generate and set a dummy `proof_with_pis` + let dummy_proof = self.table_dummy_proofs[table] + .as_ref() + .ok_or_else(|| anyhow::format_err!("Unable to get dummpy proof"))?; + assert_eq!(dummy_proof.init_degree, *index_verifier_data as usize); + let common_data = &table_circuit .shrinking_wrappers .last() - .ok_or_else(|| anyhow::format_err!("No shrinking circuits"))? + .ok_or_else(|| anyhow::format_err!("Unable to get circuit"))? .circuit .common; - let dummy_circuit: CircuitData = dummy_circuit(common_data); - let dummy_pis = HashMap::new(); - let dummy_proof = dummy_proof(&dummy_circuit, dummy_pis) - .map_err(|_| anyhow::format_err!("Unable to generate dummy proofs")); - root_inputs - .set_proof_with_pis_target(&self.root.proof_with_pis[table], &dummy_proof?); + let proof = ProofWithPublicInputs::from_bytes( + dummy_proof.proof_bytes.clone(), + common_data, + )?; + root_inputs.set_proof_with_pis_target(&self.root.proof_with_pis[table], &proof); } else { - let stark_proof = &all_proof.multi_proof.stark_proofs[table] + let stark_proof = all_proof.multi_proof.stark_proofs[table] .as_ref() .ok_or_else(|| anyhow::format_err!("Unable to get stark proof"))?; let shrunk_proof = table_circuit.shrink(stark_proof, &all_proof.multi_proof.ctl_challenges)?; - root_inputs.set_target( - self.root.index_verifier_data[table], - F::from_canonical_u8(*index_verifier_data), - ); root_inputs .set_proof_with_pis_target(&self.root.proof_with_pis[table], &shrunk_proof); } @@ -2747,19 +2810,6 @@ where } } -/// A struct that encapsulates both the init degree and the shrunk proof. -#[derive(Eq, PartialEq, Debug)] -pub struct ShrunkProofData -where - F: RichField + Extendable, - C: GenericConfig, -{ - /// The initial degree for generating the proof. - pub init_degree: usize, - /// The proof after applying shrinking recursion. - pub proof: ProofWithPublicInputs, -} - /// A map between initial degree sizes and their associated shrinking recursion /// circuits. #[derive(Eq, PartialEq, Debug)] @@ -2772,9 +2822,6 @@ where /// A map from `log_2(height)` to a chain of shrinking recursion circuits /// starting at that height. pub by_stark_size: BTreeMap>, - - /// A dummy proof for the root circuit when the table is empty. - pub dummy_proof_data: Option>, } impl RecursiveCircuitsForTable @@ -2795,18 +2842,6 @@ where table.to_buffer(buffer, gate_serializer, generator_serializer)?; } - // Serialize dummy_proof if it exists - match &self.dummy_proof_data { - Some(dummy_proof_data) => { - buffer.write_bool(true)?; - buffer.write_usize(dummy_proof_data.init_degree)?; - let proof_bytes = dummy_proof_data.proof.to_bytes(); - buffer.write_usize(proof_bytes.len())?; - buffer.extend_from_slice(&proof_bytes); - } - None => buffer.write_bool(false)?, - } - Ok(()) } @@ -2827,31 +2862,7 @@ where by_stark_size.insert(key, table); } - // Read dummy_proof if present - let dummy_proof_data = if buffer.read_bool()? { - let init_degree = buffer.read_usize()?; - let common_data = &by_stark_size - .get(&init_degree) - .ok_or_else(|| IoError)? - .shrinking_wrappers - .last() - .ok_or_else(|| IoError)? - .circuit - .common; - let proof_len = buffer.read_usize()?; - let mut proof_bytes = vec![0u8; proof_len]; - buffer.read_exact(&mut proof_bytes)?; - let proof = - ProofWithPublicInputs::from_bytes(proof_bytes, common_data).map_err(|_| IoError)?; - Some(ShrunkProofData { init_degree, proof }) - } else { - None - }; - - Ok(Self { - by_stark_size, - dummy_proof_data, - }) + Ok(Self { by_stark_size }) } fn new>( @@ -2878,25 +2889,7 @@ where }) .collect::>(); - let dummy_proof_data = is_optional_table.then_some(ShrunkProofData { - init_degree: degree_bits_range.start, - proof: { - let dummy_circuit = &by_stark_size - .get(°ree_bits_range.start) - .expect("Failed to get dummy_proof_height in by_stark_size") - .shrinking_wrappers - .last() - .expect("No shrinking_wrappers available") - .circuit; - let dummy_pis = HashMap::new(); - dummy_proof(dummy_circuit, dummy_pis).expect("Unable to generate dummy proofs") - }, - }); - - Self { - by_stark_size, - dummy_proof_data, - } + Self { by_stark_size } } /// For each initial `degree_bits`, get the final circuit at the end of that diff --git a/zero/src/prover_state/persistence.rs b/zero/src/prover_state/persistence.rs index f2a46a341..fbd7be2d0 100644 --- a/zero/src/prover_state/persistence.rs +++ b/zero/src/prover_state/persistence.rs @@ -8,7 +8,10 @@ use std::{ use alloy::hex; use anyhow::anyhow; use directories::ProjectDirs; -use evm_arithmetization::{cpu::kernel::aggregator::KERNEL, AllRecursiveCircuits, ProofWithPublicInputs, RecursionConfig, RecursiveCircuitsForTableSize, VerifierData, EXTENSION_DEGREE}; +use evm_arithmetization::{ + cpu::kernel::aggregator::KERNEL, AllRecursiveCircuits, RecursionConfig, + RecursiveCircuitsForTableSize, VerifierData, EXTENSION_DEGREE, +}; use once_cell::sync::Lazy; use plonky2::util::serialization::{ Buffer, DefaultGateSerializer, DefaultGeneratorSerializer, IoError, @@ -229,35 +232,6 @@ impl DiskResource for RecursiveCircuitResource { } } -#[derive(Debug, Default)] -pub(crate) struct DummyProofResource; - -impl DiskResource for DummyProofResource { - type Resource = ProofWithPublicInputs; - type Error = IoError; - type PathConstrutor = (Circuit, usize); - - fn path((circuit_type, size): &Self::PathConstrutor) -> impl AsRef { - format!( - "{}/{}_dummy_proof_{}_{}", - circuit_dir(), - PROVER_STATE_FILE_PREFIX, - circuit_type.as_short_str(), - size - ) - } - - fn serialize(r: &Self::Resource) -> Result, DiskResourceError> { - Ok(r.to_bytes()) // Assuming `to_bytes` is implemented for `ProofWithPublicInputs` - } - - fn deserialize( - bytes: &[u8], - ) -> Result> { - ProofWithPublicInputs::from_bytes(bytes.to_vec(), &common_data).map_err(DiskResourceError::Serialization) - } -} - /// An individual circuit table with a specific size. #[derive(Debug, Default)] pub(crate) struct VerifierResource; From 20b567b513ee35bf325f09968883b5dd72060949 Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Thu, 10 Oct 2024 21:50:44 -0700 Subject: [PATCH 83/91] build --- .../src/fixed_recursive_verifier.rs | 95 +++++++++++-------- 1 file changed, 57 insertions(+), 38 deletions(-) diff --git a/evm_arithmetization/src/fixed_recursive_verifier.rs b/evm_arithmetization/src/fixed_recursive_verifier.rs index e3aa8005f..b9b8c3b87 100644 --- a/evm_arithmetization/src/fixed_recursive_verifier.rs +++ b/evm_arithmetization/src/fixed_recursive_verifier.rs @@ -23,9 +23,9 @@ use plonky2::plonk::circuit_data::{ use plonky2::plonk::config::{AlgebraicHasher, GenericConfig, GenericHashOut}; use plonky2::plonk::proof::{ProofWithPublicInputs, ProofWithPublicInputsTarget}; use plonky2::recursion::cyclic_recursion::check_cyclic_proof_verifier_data; -use plonky2::recursion::dummy_circuit::{cyclic_base_proof, dummy_circuit, dummy_proof}; +use plonky2::recursion::dummy_circuit::{cyclic_base_proof, dummy_proof}; use plonky2::util::serialization::{ - Buffer, GateSerializer, IoError, IoResult, Read, WitnessGeneratorSerializer, Write, + Buffer, GateSerializer, IoResult, Read, WitnessGeneratorSerializer, Write, }; use plonky2::util::timing::TimingTree; use plonky2_util::log2_ceil; @@ -132,7 +132,7 @@ where /// `degree_bits`. pub by_table: [RecursiveCircuitsForTable; NUM_TABLES], /// Dummy proofs of each table for the root circuit. - pub table_dummy_proofs: [Option>; NUM_TABLES], + pub table_dummy_proofs: [Option; NUM_TABLES], } /// Data for the EVM root circuit, which is used to combine each STARK's shrunk @@ -573,26 +573,18 @@ where /// A struct that encapsulates both the init degree and the shrunk proof. #[derive(Eq, PartialEq, Debug)] -pub struct ShrunkProofData -where - F: RichField + Extendable, - C: GenericConfig, -{ +pub struct ShrunkProofData { /// The initial degree for generating the proof. pub init_degree: usize, /// The proof bytes after applying shrinking recursion. pub proof_bytes: Vec, } -impl ShrunkProofData -where - F: RichField + Extendable, - C: GenericConfig, -{ +impl ShrunkProofData { fn to_buffer(&self, buffer: &mut Vec) -> IoResult<()> { - buffer.write_usize(self.init_degree); - buffer.write_usize(self.proof_bytes.len()); - buffer.write_all(&self.proof_bytes); + buffer.write_usize(self.init_degree)?; + buffer.write_usize(self.proof_bytes.len())?; + buffer.write_all(&self.proof_bytes)?; Ok(()) } @@ -600,7 +592,7 @@ where let init_degree = buffer.read_usize()?; let proof_bytes_len = buffer.read_usize()?; let mut proof_bytes = vec![0u8; proof_bytes_len]; - buffer.read_exact(&mut proof_bytes); + buffer.read_exact(&mut proof_bytes)?; Ok(Self { init_degree, proof_bytes, @@ -656,7 +648,7 @@ where Some(dummy_proof_data) => { buffer.write_bool(true)?; dummy_proof_data.to_buffer(&mut buffer)? - }, + } None => buffer.write_bool(false)?, } } @@ -746,17 +738,9 @@ where } }; - let table_dummy_proofs = core::array::from_fn(|i| { - if buffer.read_bool()? { - let init_degree = buffer.read_usize()?; - let proof_bytes_len = buffer.read_usize()?; - let mut proof_bytes = vec![0u8; proof_bytes_len]; - buffer.read_exact(&mut proof_bytes)?; - - Some(ShrunkProofData { - init_degree, - proof_bytes, - }) + let table_dummy_proofs = core::array::from_fn(|_| { + if buffer.read_bool().ok()? { + Some(ShrunkProofData::from_buffer(&mut buffer).ok()?) } else { None } @@ -808,7 +792,6 @@ where degree_bits_ranges[*$table_enum].clone(), &all_stark.cross_table_lookups, stark_config, - true, ) }; } @@ -848,6 +831,33 @@ where let block_wrapper = Self::create_block_wrapper_circuit(&block); let two_to_one_block = Self::create_two_to_one_block_circuit(&block_wrapper); + // TODO(sdeng): enable more optional Tables + let table_dummy_proofs = core::array::from_fn(|i| { + if KECCAK_TABLES_INDICES.contains(&i) { + let init_degree = degree_bits_ranges[i] + .clone() + .last() + .expect("Unable to get the initial degree bit"); + let circuit = &by_table[i] + .by_stark_size + .get(&init_degree) + .expect("Unable to get the shrinking circuits") + .shrinking_wrappers + .last() + .expect("Unable to get the last shrinking circuit") + .circuit; + let pis = HashMap::new(); + let proof = dummy_proof(circuit, pis).expect("Unable to generate dummy proof"); + let proof_bytes = proof.to_bytes(); + Some(ShrunkProofData { + init_degree, + proof_bytes, + }) + } else { + None + } + }); + Self { root, segment_aggregation, @@ -856,6 +866,7 @@ where block_wrapper, two_to_one_block, by_table, + table_dummy_proofs, } } @@ -1999,18 +2010,27 @@ where for table in 0..NUM_TABLES { let table_circuits = &self.by_table[table]; if KECCAK_TABLES_INDICES.contains(&table) && !all_proof.use_keccak_tables { - let dummy_proof_data = table_circuits - .dummy_proof_data + let dummy_proof_data = self.table_dummy_proofs[table] .as_ref() .ok_or_else(|| anyhow::format_err!("No dummy_proof_data"))?; root_inputs.set_target( self.root.index_verifier_data[table], F::from_canonical_usize(dummy_proof_data.init_degree), ); - root_inputs.set_proof_with_pis_target( - &self.root.proof_with_pis[table], - &dummy_proof_data.proof, - ); + let common_data = &table_circuits + .by_stark_size + .get(&dummy_proof_data.init_degree) + .ok_or_else(|| anyhow::format_err!("Unable to get shrinking circuits"))? + .shrinking_wrappers + .last() + .ok_or_else(|| anyhow::format_err!("Unable to get circuit"))? + .circuit + .common; + let proof: ProofWithPublicInputs = ProofWithPublicInputs::from_bytes( + dummy_proof_data.proof_bytes.clone(), + common_data, + )?; + root_inputs.set_proof_with_pis_target(&self.root.proof_with_pis[table], &proof); } else { let stark_proof = &all_proof.multi_proof.stark_proofs[table] .as_ref() @@ -2147,7 +2167,7 @@ where .ok_or_else(|| anyhow::format_err!("Unable to get circuit"))? .circuit .common; - let proof = ProofWithPublicInputs::from_bytes( + let proof: ProofWithPublicInputs = ProofWithPublicInputs::from_bytes( dummy_proof.proof_bytes.clone(), common_data, )?; @@ -2871,7 +2891,6 @@ where degree_bits_range: Range, all_ctls: &[CrossTableLookup], stark_config: &StarkConfig, - is_optional_table: bool, ) -> Self { let by_stark_size = degree_bits_range .clone() From 911bd4b50395c71c307500d098e746cc21332602 Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Thu, 10 Oct 2024 22:36:43 -0700 Subject: [PATCH 84/91] fix --- zero/src/prover_state/mod.rs | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/zero/src/prover_state/mod.rs b/zero/src/prover_state/mod.rs index d1d6c5983..a46373bdd 100644 --- a/zero/src/prover_state/mod.rs +++ b/zero/src/prover_state/mod.rs @@ -31,9 +31,6 @@ use crate::prover_state::persistence::{ VerifierResource, }; -// Default degree of the recursive circuit when a proof is missing from AllProof -const DEFAULT_CIRCUIT_DEGREE: usize = 8; - pub mod circuit; pub mod cli; pub mod persistence; @@ -180,7 +177,17 @@ impl ProverStateManager { ) -> anyhow::Result<[(RecursiveCircuitsForTableSize, u8); NUM_TABLES]> { let degrees = all_proof .degree_bits(config) - .map(|opt| opt.unwrap_or(DEFAULT_CIRCUIT_DEGREE)); + .iter() + .enumerate() + .map(|(i, opt)| { + opt.unwrap_or_else(|| { + p_state().state.table_dummy_proofs[i] + .as_ref() + .expect("Unable to get table dummy proof data") + .init_degree + }) + }) + .collect::>(); /// Given a recursive circuit index (e.g., Arithmetic / 0), return a /// tuple containing the loaded table at the specified size and From 9f2283e79b45c50ce40b3f9f9dfdab34fd223d0f Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Fri, 11 Oct 2024 09:17:04 -0700 Subject: [PATCH 85/91] try to fix --- .../src/fixed_recursive_verifier.rs | 139 ++++++++---------- 1 file changed, 63 insertions(+), 76 deletions(-) diff --git a/evm_arithmetization/src/fixed_recursive_verifier.rs b/evm_arithmetization/src/fixed_recursive_verifier.rs index b9b8c3b87..99657dcbd 100644 --- a/evm_arithmetization/src/fixed_recursive_verifier.rs +++ b/evm_arithmetization/src/fixed_recursive_verifier.rs @@ -1,4 +1,4 @@ -use core::mem::MaybeUninit; +use core::mem::{self, MaybeUninit}; use core::ops::Range; use std::collections::BTreeMap; use std::sync::atomic::AtomicBool; @@ -132,7 +132,7 @@ where /// `degree_bits`. pub by_table: [RecursiveCircuitsForTable; NUM_TABLES], /// Dummy proofs of each table for the root circuit. - pub table_dummy_proofs: [Option; NUM_TABLES], + pub table_dummy_proofs: [Option>; NUM_TABLES], } /// Data for the EVM root circuit, which is used to combine each STARK's shrunk @@ -573,29 +573,43 @@ where /// A struct that encapsulates both the init degree and the shrunk proof. #[derive(Eq, PartialEq, Debug)] -pub struct ShrunkProofData { +pub struct ShrunkProofData, C: GenericConfig, const D: usize> +{ /// The initial degree for generating the proof. pub init_degree: usize, - /// The proof bytes after applying shrinking recursion. - pub proof_bytes: Vec, + + /// The Common Circuit Data from last shrinking circuit. + pub common_circuit_data: CommonCircuitData, + + /// The proof after applying shrinking recursion. + pub proof: ProofWithPublicInputs, } -impl ShrunkProofData { - fn to_buffer(&self, buffer: &mut Vec) -> IoResult<()> { +impl, C: GenericConfig, const D: usize> + ShrunkProofData +{ + fn to_buffer( + &self, + buffer: &mut Vec, + gate_serializer: &dyn GateSerializer, + ) -> IoResult<()> { buffer.write_usize(self.init_degree)?; - buffer.write_usize(self.proof_bytes.len())?; - buffer.write_all(&self.proof_bytes)?; + buffer.write_common_circuit_data(&self.common_circuit_data, gate_serializer)?; + buffer.write_proof_with_public_inputs(&self.proof)?; Ok(()) } - fn from_buffer(buffer: &mut Buffer) -> IoResult { + fn from_buffer( + buffer: &mut Buffer, + gate_serializer: &dyn GateSerializer, + ) -> IoResult { let init_degree = buffer.read_usize()?; - let proof_bytes_len = buffer.read_usize()?; - let mut proof_bytes = vec![0u8; proof_bytes_len]; - buffer.read_exact(&mut proof_bytes)?; + let common_circuit_data = buffer.read_common_circuit_data(gate_serializer)?; + let proof = buffer.read_proof_with_public_inputs(&common_circuit_data)?; Ok(Self { init_degree, - proof_bytes, + common_circuit_data, + proof, }) } } @@ -647,7 +661,7 @@ where match table { Some(dummy_proof_data) => { buffer.write_bool(true)?; - dummy_proof_data.to_buffer(&mut buffer)? + dummy_proof_data.to_buffer(&mut buffer, gate_serializer)? } None => buffer.write_bool(false)?, } @@ -709,38 +723,30 @@ where .try_into() .unwrap(), false => { - // Initialize an uninitialized array of MaybeUninit + // Tricky use of MaybeUninit to remove the need for implementing Debug + // for all underlying types, necessary to convert a by_table Vec to an array. let mut by_table: [MaybeUninit>; NUM_TABLES] = - MaybeUninit::uninit_array(); - - for i in 0..NUM_TABLES { - let value = match RecursiveCircuitsForTable::from_buffer( + unsafe { MaybeUninit::uninit().assume_init() }; + for table in &mut by_table[..] { + let value = RecursiveCircuitsForTable::from_buffer( &mut buffer, gate_serializer, generator_serializer, - ) { - Ok(v) => v, - Err(e) => { - // If there's an error, deinitialize any initialized elements - for initialized in by_table[..i].iter_mut() { - unsafe { std::ptr::drop_in_place(initialized.as_mut_ptr()) }; - } - return Err(e); - } - }; - - // Initialize the current element - by_table[i] = MaybeUninit::new(value); + )?; + *table = MaybeUninit::new(value); + } + unsafe { + mem::transmute::< + [std::mem::MaybeUninit>; NUM_TABLES], + [RecursiveCircuitsForTable; NUM_TABLES], + >(by_table) } - - // Safely convert the array of MaybeUninit to a fully initialized array - unsafe { MaybeUninit::array_assume_init(by_table) } } }; let table_dummy_proofs = core::array::from_fn(|_| { if buffer.read_bool().ok()? { - Some(ShrunkProofData::from_buffer(&mut buffer).ok()?) + Some(ShrunkProofData::from_buffer(&mut buffer, gate_serializer).ok()?) } else { None } @@ -834,10 +840,7 @@ where // TODO(sdeng): enable more optional Tables let table_dummy_proofs = core::array::from_fn(|i| { if KECCAK_TABLES_INDICES.contains(&i) { - let init_degree = degree_bits_ranges[i] - .clone() - .last() - .expect("Unable to get the initial degree bit"); + let init_degree = degree_bits_ranges[i].start; let circuit = &by_table[i] .by_stark_size .get(&init_degree) @@ -848,10 +851,10 @@ where .circuit; let pis = HashMap::new(); let proof = dummy_proof(circuit, pis).expect("Unable to generate dummy proof"); - let proof_bytes = proof.to_bytes(); Some(ShrunkProofData { init_degree, - proof_bytes, + common_circuit_data: circuit.common.clone(), + proof, }) } else { None @@ -2017,20 +2020,10 @@ where self.root.index_verifier_data[table], F::from_canonical_usize(dummy_proof_data.init_degree), ); - let common_data = &table_circuits - .by_stark_size - .get(&dummy_proof_data.init_degree) - .ok_or_else(|| anyhow::format_err!("Unable to get shrinking circuits"))? - .shrinking_wrappers - .last() - .ok_or_else(|| anyhow::format_err!("Unable to get circuit"))? - .circuit - .common; - let proof: ProofWithPublicInputs = ProofWithPublicInputs::from_bytes( - dummy_proof_data.proof_bytes.clone(), - common_data, - )?; - root_inputs.set_proof_with_pis_target(&self.root.proof_with_pis[table], &proof); + root_inputs.set_proof_with_pis_target( + &self.root.proof_with_pis[table], + &dummy_proof_data.proof, + ); } else { let stark_proof = &all_proof.multi_proof.stark_proofs[table] .as_ref() @@ -2150,33 +2143,27 @@ where let mut root_inputs = PartialWitness::new(); for table in 0..NUM_TABLES { - let (table_circuit, index_verifier_data) = &table_circuits[table]; - root_inputs.set_target( - self.root.index_verifier_data[table], - F::from_canonical_u8(*index_verifier_data), - ); if KECCAK_TABLES_INDICES.contains(&table) && !all_proof.use_keccak_tables { let dummy_proof = self.table_dummy_proofs[table] .as_ref() .ok_or_else(|| anyhow::format_err!("Unable to get dummpy proof"))?; - assert_eq!(dummy_proof.init_degree, *index_verifier_data as usize); - - let common_data = &table_circuit - .shrinking_wrappers - .last() - .ok_or_else(|| anyhow::format_err!("Unable to get circuit"))? - .circuit - .common; - let proof: ProofWithPublicInputs = ProofWithPublicInputs::from_bytes( - dummy_proof.proof_bytes.clone(), - common_data, - )?; - root_inputs.set_proof_with_pis_target(&self.root.proof_with_pis[table], &proof); + root_inputs.set_target( + self.root.index_verifier_data[table], + F::from_canonical_usize(dummy_proof.init_degree), + ); + root_inputs.set_proof_with_pis_target( + &self.root.proof_with_pis[table], + &dummy_proof.proof, + ); } else { + let (table_circuit, index_verifier_data) = &table_circuits[table]; + root_inputs.set_target( + self.root.index_verifier_data[table], + F::from_canonical_u8(*index_verifier_data), + ); let stark_proof = all_proof.multi_proof.stark_proofs[table] .as_ref() .ok_or_else(|| anyhow::format_err!("Unable to get stark proof"))?; - let shrunk_proof = table_circuit.shrink(stark_proof, &all_proof.multi_proof.ctl_challenges)?; root_inputs From a7be2b9c71bb4bfa61196c7ffe3fbc766f4957ec Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Fri, 11 Oct 2024 09:35:24 -0700 Subject: [PATCH 86/91] wip --- .../src/fixed_recursive_verifier.rs | 9 ++- evm_arithmetization/src/lib.rs | 2 - zero/src/lib.rs | 2 + zero/src/prover_state/mod.rs | 77 +++++++------------ 4 files changed, 34 insertions(+), 56 deletions(-) diff --git a/evm_arithmetization/src/fixed_recursive_verifier.rs b/evm_arithmetization/src/fixed_recursive_verifier.rs index 99657dcbd..dd63387bd 100644 --- a/evm_arithmetization/src/fixed_recursive_verifier.rs +++ b/evm_arithmetization/src/fixed_recursive_verifier.rs @@ -2137,7 +2137,7 @@ where pub fn prove_segment_after_initial_stark( &self, all_proof: AllProof, - table_circuits: &[(RecursiveCircuitsForTableSize, u8); NUM_TABLES], + table_circuits: &[Option<(RecursiveCircuitsForTableSize, u8)>; NUM_TABLES], abort_signal: Option>, ) -> anyhow::Result> { let mut root_inputs = PartialWitness::new(); @@ -2156,7 +2156,9 @@ where &dummy_proof.proof, ); } else { - let (table_circuit, index_verifier_data) = &table_circuits[table]; + let (table_circuit, index_verifier_data) = &table_circuits[table] + .as_ref() + .ok_or_else(|| anyhow::format_err!("Unable to get circuits"))?; root_inputs.set_target( self.root.index_verifier_data[table], F::from_canonical_u8(*index_verifier_data), @@ -2880,7 +2882,6 @@ where stark_config: &StarkConfig, ) -> Self { let by_stark_size = degree_bits_range - .clone() .map(|degree_bits| { ( degree_bits, @@ -2893,7 +2894,7 @@ where ), ) }) - .collect::>(); + .collect(); Self { by_stark_size } } diff --git a/evm_arithmetization/src/lib.rs b/evm_arithmetization/src/lib.rs index 26b030429..41b7f093a 100644 --- a/evm_arithmetization/src/lib.rs +++ b/evm_arithmetization/src/lib.rs @@ -252,8 +252,6 @@ #![allow(clippy::too_many_arguments)] #![allow(clippy::field_reassign_with_default)] #![feature(let_chains)] -#![feature(maybe_uninit_uninit_array)] -#![feature(maybe_uninit_array_assume_init)] zk_evm_common::check_chain_features!(); diff --git a/zero/src/lib.rs b/zero/src/lib.rs index c2ca63f6a..6de5dee98 100644 --- a/zero/src/lib.rs +++ b/zero/src/lib.rs @@ -1,3 +1,5 @@ +#![feature(array_try_from_fn)] + zk_evm_common::check_chain_features!(); pub mod block_interval; diff --git a/zero/src/prover_state/mod.rs b/zero/src/prover_state/mod.rs index a46373bdd..d5bf2c741 100644 --- a/zero/src/prover_state/mod.rs +++ b/zero/src/prover_state/mod.rs @@ -174,57 +174,34 @@ impl ProverStateManager { &self, config: &StarkConfig, all_proof: &AllProof, - ) -> anyhow::Result<[(RecursiveCircuitsForTableSize, u8); NUM_TABLES]> { - let degrees = all_proof - .degree_bits(config) - .iter() - .enumerate() - .map(|(i, opt)| { - opt.unwrap_or_else(|| { - p_state().state.table_dummy_proofs[i] - .as_ref() - .expect("Unable to get table dummy proof data") - .init_degree - }) - }) - .collect::>(); - - /// Given a recursive circuit index (e.g., Arithmetic / 0), return a - /// tuple containing the loaded table at the specified size and - /// its offset relative to the configured range used to pre-process the - /// circuits. - macro_rules! circuit { - ($circuit_index:expr) => { - ( - RecursiveCircuitResource::get(&( - $circuit_index.into(), - degrees[$circuit_index], - )) - .map_err(|e| { - let circuit: $crate::prover_state::circuit::Circuit = $circuit_index.into(); - let size = degrees[$circuit_index]; - anyhow::Error::from(e).context(format!( - "Attempting to load circuit: {circuit:?} at size: {size}" - )) - })?, - (degrees[$circuit_index] - self.circuit_config[$circuit_index].start) as u8, - ) - }; - } + ) -> anyhow::Result<[Option<(RecursiveCircuitsForTableSize, u8)>; NUM_TABLES]> { + let degrees = all_proof.degree_bits(config); + + // Given a recursive circuit index (e.g., Arithmetic / 0), return a + // tuple containing the loaded table at the specified size and + // its offset relative to the configured range used to pre-process the + // circuits. + let circuits = std::array::try_from_fn( + |i| -> anyhow::Result> { + match degrees[i] { + Some(size) => { + let circuit_resource = RecursiveCircuitResource::get(&(i.into(), size)) + .map_err(|e| { + anyhow::Error::from(e).context(format!( + "Attempting to load circuit: {i} at size: {size}" + )) + })?; + Ok(Some(( + circuit_resource, + (size - self.circuit_config[i].start) as u8, + ))) + } + None => Ok(None), + } + }, + )?; - Ok([ - circuit!(0), - circuit!(1), - circuit!(2), - circuit!(3), - circuit!(4), - circuit!(5), - circuit!(6), - circuit!(7), - circuit!(8), - #[cfg(feature = "cdk_erigon")] - circuit!(9), - ]) + Ok(circuits) } /// Generate a segment proof using the specified input, loading From 9021124bbaf11c83e573d8a5d66c2700e36d226c Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Fri, 11 Oct 2024 09:53:48 -0700 Subject: [PATCH 87/91] done --- .../src/fixed_recursive_verifier.rs | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/evm_arithmetization/src/fixed_recursive_verifier.rs b/evm_arithmetization/src/fixed_recursive_verifier.rs index dd63387bd..a8a96f8ea 100644 --- a/evm_arithmetization/src/fixed_recursive_verifier.rs +++ b/evm_arithmetization/src/fixed_recursive_verifier.rs @@ -23,7 +23,7 @@ use plonky2::plonk::circuit_data::{ use plonky2::plonk::config::{AlgebraicHasher, GenericConfig, GenericHashOut}; use plonky2::plonk::proof::{ProofWithPublicInputs, ProofWithPublicInputsTarget}; use plonky2::recursion::cyclic_recursion::check_cyclic_proof_verifier_data; -use plonky2::recursion::dummy_circuit::{cyclic_base_proof, dummy_proof}; +use plonky2::recursion::dummy_circuit::{cyclic_base_proof, dummy_circuit, dummy_proof}; use plonky2::util::serialization::{ Buffer, GateSerializer, IoResult, Read, WitnessGeneratorSerializer, Write, }; @@ -841,19 +841,23 @@ where let table_dummy_proofs = core::array::from_fn(|i| { if KECCAK_TABLES_INDICES.contains(&i) { let init_degree = degree_bits_ranges[i].start; - let circuit = &by_table[i] + let common_circuit_data = by_table[i] .by_stark_size .get(&init_degree) .expect("Unable to get the shrinking circuits") .shrinking_wrappers .last() .expect("Unable to get the last shrinking circuit") - .circuit; - let pis = HashMap::new(); - let proof = dummy_proof(circuit, pis).expect("Unable to generate dummy proof"); + .circuit + .common + .clone(); + let dummy_circuit: CircuitData = dummy_circuit(&common_circuit_data); + let dummy_pis = HashMap::new(); + let proof = dummy_proof(&dummy_circuit, dummy_pis) + .expect("Unable to generate dummy proofs"); Some(ShrunkProofData { init_degree, - common_circuit_data: circuit.common.clone(), + common_circuit_data, proof, }) } else { From 7374f050ec0fd2c3028a41f56eb3af1cf54c0660 Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Fri, 11 Oct 2024 10:19:06 -0700 Subject: [PATCH 88/91] fix --- evm_arithmetization/tests/empty_tables.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/evm_arithmetization/tests/empty_tables.rs b/evm_arithmetization/tests/empty_tables.rs index 9fcf4a74a..fc8cf309f 100644 --- a/evm_arithmetization/tests/empty_tables.rs +++ b/evm_arithmetization/tests/empty_tables.rs @@ -54,7 +54,7 @@ fn empty_tables() -> anyhow::Result<()> { "Create all recursive circuits", AllRecursiveCircuits::::new( &all_stark, - &[16..17, 8..9, 7..8, 4..9, 8..9, 4..7, 17..18, 17..18, 17..18], + &[16..17, 8..9, 7..8, 4..9, 8..9, 4..7, 16..17, 17..18, 17..18], &config, ) ); From 557d221fc0e132b364a3415b0fab19ca17eeffaf Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Fri, 11 Oct 2024 10:20:47 -0700 Subject: [PATCH 89/91] fix tests --- evm_arithmetization/tests/empty_tables.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/evm_arithmetization/tests/empty_tables.rs b/evm_arithmetization/tests/empty_tables.rs index fc8cf309f..966445dfd 100644 --- a/evm_arithmetization/tests/empty_tables.rs +++ b/evm_arithmetization/tests/empty_tables.rs @@ -54,7 +54,7 @@ fn empty_tables() -> anyhow::Result<()> { "Create all recursive circuits", AllRecursiveCircuits::::new( &all_stark, - &[16..17, 8..9, 7..8, 4..9, 8..9, 4..7, 16..17, 17..18, 17..18], + &[16..17, 8..9, 7..8, 4..9, 8..9, 4..7, 16..17, 16..17, 16..17], &config, ) ); From 564b950a8d646b1da8062058f582082ab010fb63 Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Tue, 15 Oct 2024 14:43:44 -0700 Subject: [PATCH 90/91] address comments --- .../src/fixed_recursive_verifier.rs | 2 +- zero/src/lib.rs | 2 -- zero/src/prover_state/mod.rs | 34 ++++++++----------- 3 files changed, 16 insertions(+), 22 deletions(-) diff --git a/evm_arithmetization/src/fixed_recursive_verifier.rs b/evm_arithmetization/src/fixed_recursive_verifier.rs index a8a96f8ea..b28f4adaf 100644 --- a/evm_arithmetization/src/fixed_recursive_verifier.rs +++ b/evm_arithmetization/src/fixed_recursive_verifier.rs @@ -578,7 +578,7 @@ pub struct ShrunkProofData, C: GenericConfig, /// The proof after applying shrinking recursion. diff --git a/zero/src/lib.rs b/zero/src/lib.rs index 6de5dee98..c2ca63f6a 100644 --- a/zero/src/lib.rs +++ b/zero/src/lib.rs @@ -1,5 +1,3 @@ -#![feature(array_try_from_fn)] - zk_evm_common::check_chain_features!(); pub mod block_interval; diff --git a/zero/src/prover_state/mod.rs b/zero/src/prover_state/mod.rs index d5bf2c741..8cdfb45ce 100644 --- a/zero/src/prover_state/mod.rs +++ b/zero/src/prover_state/mod.rs @@ -181,25 +181,21 @@ impl ProverStateManager { // tuple containing the loaded table at the specified size and // its offset relative to the configured range used to pre-process the // circuits. - let circuits = std::array::try_from_fn( - |i| -> anyhow::Result> { - match degrees[i] { - Some(size) => { - let circuit_resource = RecursiveCircuitResource::get(&(i.into(), size)) - .map_err(|e| { - anyhow::Error::from(e).context(format!( - "Attempting to load circuit: {i} at size: {size}" - )) - })?; - Ok(Some(( - circuit_resource, - (size - self.circuit_config[i].start) as u8, - ))) - } - None => Ok(None), - } - }, - )?; + let circuits = core::array::from_fn(|i| match degrees[i] { + Some(size) => RecursiveCircuitResource::get(&(i.into(), size)) + .map(|circuit_resource| { + Some(( + circuit_resource, + (size - self.circuit_config[i].start) as u8, + )) + }) + .map_err(|e| { + anyhow::Error::from(e) + .context(format!("Attempting to load circuit: {i} at size: {size}")) + }) + .unwrap_or(None), + None => None, + }); Ok(circuits) } From c8f03fc9edc5ce4a103f9ad50a29c6b4b90fc800 Mon Sep 17 00:00:00 2001 From: Sai Deng Date: Tue, 15 Oct 2024 15:00:37 -0700 Subject: [PATCH 91/91] fix a bug with index_verifier_data --- .../src/fixed_recursive_verifier.rs | 17 ++--------------- evm_arithmetization/tests/empty_tables.rs | 2 +- 2 files changed, 3 insertions(+), 16 deletions(-) diff --git a/evm_arithmetization/src/fixed_recursive_verifier.rs b/evm_arithmetization/src/fixed_recursive_verifier.rs index b28f4adaf..96ecce6d3 100644 --- a/evm_arithmetization/src/fixed_recursive_verifier.rs +++ b/evm_arithmetization/src/fixed_recursive_verifier.rs @@ -575,9 +575,6 @@ where #[derive(Eq, PartialEq, Debug)] pub struct ShrunkProofData, C: GenericConfig, const D: usize> { - /// The initial degree for generating the proof. - pub init_degree: usize, - /// The [`CommonCircuitData`] of the last shrinking circuit. pub common_circuit_data: CommonCircuitData, @@ -593,7 +590,6 @@ impl, C: GenericConfig, const D: usize> buffer: &mut Vec, gate_serializer: &dyn GateSerializer, ) -> IoResult<()> { - buffer.write_usize(self.init_degree)?; buffer.write_common_circuit_data(&self.common_circuit_data, gate_serializer)?; buffer.write_proof_with_public_inputs(&self.proof)?; Ok(()) @@ -603,11 +599,9 @@ impl, C: GenericConfig, const D: usize> buffer: &mut Buffer, gate_serializer: &dyn GateSerializer, ) -> IoResult { - let init_degree = buffer.read_usize()?; let common_circuit_data = buffer.read_common_circuit_data(gate_serializer)?; let proof = buffer.read_proof_with_public_inputs(&common_circuit_data)?; Ok(Self { - init_degree, common_circuit_data, proof, }) @@ -856,7 +850,6 @@ where let proof = dummy_proof(&dummy_circuit, dummy_pis) .expect("Unable to generate dummy proofs"); Some(ShrunkProofData { - init_degree, common_circuit_data, proof, }) @@ -2020,10 +2013,7 @@ where let dummy_proof_data = self.table_dummy_proofs[table] .as_ref() .ok_or_else(|| anyhow::format_err!("No dummy_proof_data"))?; - root_inputs.set_target( - self.root.index_verifier_data[table], - F::from_canonical_usize(dummy_proof_data.init_degree), - ); + root_inputs.set_target(self.root.index_verifier_data[table], F::ZERO); root_inputs.set_proof_with_pis_target( &self.root.proof_with_pis[table], &dummy_proof_data.proof, @@ -2151,10 +2141,7 @@ where let dummy_proof = self.table_dummy_proofs[table] .as_ref() .ok_or_else(|| anyhow::format_err!("Unable to get dummpy proof"))?; - root_inputs.set_target( - self.root.index_verifier_data[table], - F::from_canonical_usize(dummy_proof.init_degree), - ); + root_inputs.set_target(self.root.index_verifier_data[table], F::ZERO); root_inputs.set_proof_with_pis_target( &self.root.proof_with_pis[table], &dummy_proof.proof, diff --git a/evm_arithmetization/tests/empty_tables.rs b/evm_arithmetization/tests/empty_tables.rs index 966445dfd..d25901e24 100644 --- a/evm_arithmetization/tests/empty_tables.rs +++ b/evm_arithmetization/tests/empty_tables.rs @@ -54,7 +54,7 @@ fn empty_tables() -> anyhow::Result<()> { "Create all recursive circuits", AllRecursiveCircuits::::new( &all_stark, - &[16..17, 8..9, 7..8, 4..9, 8..9, 4..7, 16..17, 16..17, 16..17], + &[16..17, 8..9, 7..8, 4..6, 8..9, 4..5, 16..17, 16..17, 16..17], &config, ) );