diff --git a/docs/docs/reference/developer_references/smart_contract_reference/storage/private_state.md b/docs/docs/reference/developer_references/smart_contract_reference/storage/private_state.md index 66e9ccf9821..19e803910ba 100644 --- a/docs/docs/reference/developer_references/smart_contract_reference/storage/private_state.md +++ b/docs/docs/reference/developer_references/smart_contract_reference/storage/private_state.md @@ -269,9 +269,21 @@ When the `limit` is set to a non-zero value, the data oracle will return a maxim This setting enables us to skip the first `offset` notes. It's particularly useful for pagination. +### `preprocessor: fn ([Option; MAX_NOTE_HASH_READ_REQUESTS_PER_CALL], PREPROCESSOR_ARGS) -> [Option; MAX_NOTE_HASH_READ_REQUESTS_PER_CALL]` + +Developers have the option to provide a custom preprocessor. +This allows specific logic to be applied to notes that meet the criteria outlined above. +The preprocessor takes the notes returned from the oracle and `preprocessor_args` as its parameters. + +An important distinction from the filter function described below is that preprocessor is applied first and unlike filter it is applied in an unconstrained context. + +### `preprocessor_args: PREPROCESSOR_ARGS` + +`preprocessor_args` provides a means to furnish additional data or context to the custom preprocessor. + ### `filter: fn ([Option; MAX_NOTE_HASH_READ_REQUESTS_PER_CALL], FILTER_ARGS) -> [Option; MAX_NOTE_HASH_READ_REQUESTS_PER_CALL]` -Developers have the option to provide a custom filter. This allows specific logic to be applied to notes that meet the criteria outlined above. The filter takes the notes returned from the oracle and `filter_args` as its parameters. +Just like preprocessor just applied in a constrained context (correct execution is proven) and applied after the preprocessor. ### `filter_args: FILTER_ARGS` diff --git a/noir-projects/aztec-nr/aztec/src/note/note_getter/mod.nr b/noir-projects/aztec-nr/aztec/src/note/note_getter/mod.nr index 22c9857230d..f806dae9d27 100644 --- a/noir-projects/aztec-nr/aztec/src/note/note_getter/mod.nr +++ b/noir-projects/aztec-nr/aztec/src/note/note_getter/mod.nr @@ -103,21 +103,29 @@ pub fn get_note( note } -pub fn get_notes( +pub fn get_notes( context: &mut PrivateContext, storage_slot: Field, - options: NoteGetterOptions + options: NoteGetterOptions ) -> BoundedVec where Note: NoteInterface + Eq { let opt_notes = get_notes_internal(storage_slot, options); constrain_get_notes_internal(context, storage_slot, opt_notes, options) } -fn constrain_get_notes_internal( +unconstrained fn apply_preprocessor( + notes: [Option; MAX_NOTE_HASH_READ_REQUESTS_PER_CALL], + preprocessor: fn([Option; MAX_NOTE_HASH_READ_REQUESTS_PER_CALL], PREPROCESSOR_ARGS) -> [Option; MAX_NOTE_HASH_READ_REQUESTS_PER_CALL], + preprocessor_args: PREPROCESSOR_ARGS +) -> [Option; MAX_NOTE_HASH_READ_REQUESTS_PER_CALL] { + preprocessor(notes, preprocessor_args) +} + +fn constrain_get_notes_internal( context: &mut PrivateContext, storage_slot: Field, opt_notes: [Option; MAX_NOTE_HASH_READ_REQUESTS_PER_CALL], - options: NoteGetterOptions + options: NoteGetterOptions ) -> BoundedVec where Note: NoteInterface + Eq { // The filter is applied first to avoid pushing note read requests for notes we're not interested in. Note that // while the filter function can technically mutate the contents of the notes (as opposed to simply removing some), @@ -181,9 +189,9 @@ unconstrained fn get_note_internal(storage_slot: F )[0].unwrap() // Notice: we don't allow dummies to be returned from get_note (singular). } -unconstrained fn get_notes_internal( +unconstrained fn get_notes_internal( storage_slot: Field, - options: NoteGetterOptions + options: NoteGetterOptions ) -> [Option; MAX_NOTE_HASH_READ_REQUESTS_PER_CALL] where Note: NoteInterface { // This function simply performs some transformations from NoteGetterOptions into the types required by the oracle. @@ -192,7 +200,7 @@ unconstrained fn get_notes_internal( let placeholder_fields = [0; GET_NOTES_ORACLE_RETURN_LENGTH]; let placeholder_note_length = [0; N]; - oracle::notes::get_notes( + let opt_notes = oracle::notes::get_notes( storage_slot, num_selects, select_by_indexes, @@ -210,7 +218,9 @@ unconstrained fn get_notes_internal( placeholder_opt_notes, placeholder_fields, placeholder_note_length - ) + ); + + apply_preprocessor(opt_notes, options.preprocessor, options.preprocessor_args) } unconstrained pub fn view_notes( diff --git a/noir-projects/aztec-nr/aztec/src/note/note_getter_options.nr b/noir-projects/aztec-nr/aztec/src/note/note_getter_options.nr index 7bc535b052f..61049c8f19e 100644 --- a/noir-projects/aztec-nr/aztec/src/note/note_getter_options.nr +++ b/noir-projects/aztec-nr/aztec/src/note/note_getter_options.nr @@ -78,46 +78,77 @@ fn return_all_notes( } // docs:start:NoteGetterOptions -struct NoteGetterOptions { +struct NoteGetterOptions { selects: BoundedVec, N>, sorts: BoundedVec, N>, limit: u32, offset: u32, + // Preprocessor and filter functions are used to filter notes. The preprocessor is applied before the filter and + // unlike filter it is applied in an unconstrained context. + preprocessor: fn ([Option; MAX_NOTE_HASH_READ_REQUESTS_PER_CALL], PREPROCESSOR_ARGS) -> [Option; MAX_NOTE_HASH_READ_REQUESTS_PER_CALL], + preprocessor_args: PREPROCESSOR_ARGS, filter: fn ([Option; MAX_NOTE_HASH_READ_REQUESTS_PER_CALL], FILTER_ARGS) -> [Option; MAX_NOTE_HASH_READ_REQUESTS_PER_CALL], filter_args: FILTER_ARGS, status: u8, } // docs:end:NoteGetterOptions -// When retrieving notes using the NoteGetterOptions, the configurations are applied in a specific sequence to ensure precise and controlled data retrieval. +// When retrieving notes using the NoteGetterOptions, the configurations are applied in a specific sequence to ensure +// precise and controlled data retrieval. // The database-level configurations are applied first: -// `selects` to specify fields, `sorts` to establish sorting criteria, `offset` to skip items, and `limit` to cap the result size. -// And finally, a custom filter to refine the outcome further. -impl NoteGetterOptions { +// `selects` to specify fields, `sorts` to establish sorting criteria, `offset` to skip items, and `limit` to cap +// the result size. +// And finally, a custom preprocessor and filter to refine the outcome further. +impl NoteGetterOptions { // This function initializes a NoteGetterOptions that simply returns the maximum number of notes allowed in a call. - pub fn new() -> NoteGetterOptions where Note: NoteInterface { + pub fn new() -> NoteGetterOptions where Note: NoteInterface { NoteGetterOptions { selects: BoundedVec::new(), sorts: BoundedVec::new(), limit: MAX_NOTE_HASH_READ_REQUESTS_PER_CALL as u32, offset: 0, + preprocessor: return_all_notes, + preprocessor_args: 0, filter: return_all_notes, filter_args: 0, status: NoteStatus.ACTIVE } } - // This function initializes a NoteGetterOptions with a filter, which takes the notes returned from the database and filter_args as its parameters. + // This function initializes a NoteGetterOptions with a preprocessor, which takes the notes returned from + // the database and preprocessor_args as its parameters. + // `preprocessor_args` allows you to provide additional data or context to the custom preprocessor. + pub fn with_preprocessor( + preprocessor: fn([Option; MAX_NOTE_HASH_READ_REQUESTS_PER_CALL], PREPROCESSOR_ARGS) -> [Option; MAX_NOTE_HASH_READ_REQUESTS_PER_CALL], + preprocessor_args: PREPROCESSOR_ARGS + ) -> NoteGetterOptions where Note: NoteInterface { + NoteGetterOptions { + selects: BoundedVec::new(), + sorts: BoundedVec::new(), + limit: MAX_NOTE_HASH_READ_REQUESTS_PER_CALL as u32, + offset: 0, + preprocessor, + preprocessor_args, + filter: return_all_notes, + filter_args: 0, + status: NoteStatus.ACTIVE + } + } + + // This function initializes a NoteGetterOptions with a filter, which takes + // the notes returned from the database and filter_args as its parameters. // `filter_args` allows you to provide additional data or context to the custom filter. pub fn with_filter( filter: fn([Option; MAX_NOTE_HASH_READ_REQUESTS_PER_CALL], FILTER_ARGS) -> [Option; MAX_NOTE_HASH_READ_REQUESTS_PER_CALL], filter_args: FILTER_ARGS - ) -> Self where Note: NoteInterface { + ) -> NoteGetterOptions where Note: NoteInterface { NoteGetterOptions { selects: BoundedVec::new(), sorts: BoundedVec::new(), limit: MAX_NOTE_HASH_READ_REQUESTS_PER_CALL as u32, offset: 0, + preprocessor: return_all_notes, + preprocessor_args: 0, filter, filter_args, status: NoteStatus.ACTIVE diff --git a/noir-projects/aztec-nr/aztec/src/state_vars/private_set.nr b/noir-projects/aztec-nr/aztec/src/state_vars/private_set.nr index 4765e94368f..9051cb86f85 100644 --- a/noir-projects/aztec-nr/aztec/src/state_vars/private_set.nr +++ b/noir-projects/aztec-nr/aztec/src/state_vars/private_set.nr @@ -41,9 +41,9 @@ impl PrivateSet where N } // docs:end:insert - pub fn pop_notes( + pub fn pop_notes( self, - options: NoteGetterOptions + options: NoteGetterOptions ) -> BoundedVec { let notes = get_notes(self.context, self.storage_slot, options); // We iterate in a range 0..options.limit instead of 0..notes.len() because options.limit is known at compile @@ -73,9 +73,9 @@ impl PrivateSet where N /// Note that if you later on remove the note it's much better to use `pop_notes` as `pop_notes` results /// in significantly less constrains due to avoiding 1 read request check. - pub fn get_notes( + pub fn get_notes( self, - options: NoteGetterOptions + options: NoteGetterOptions ) -> BoundedVec { get_notes(self.context, self.storage_slot, options) } diff --git a/noir-projects/aztec-nr/aztec/src/utils/point.nr b/noir-projects/aztec-nr/aztec/src/utils/point.nr index 20c6af687f8..249df9e2b0b 100644 --- a/noir-projects/aztec-nr/aztec/src/utils/point.nr +++ b/noir-projects/aztec-nr/aztec/src/utils/point.nr @@ -9,6 +9,8 @@ global BN254_FR_MODULUS_DIV_2: Field = 10944121435919637611123202872628637544274 /// We don't serialize the point at infinity flag because this function is used in situations where we do not want /// to waste the extra byte (encrypted log). pub fn point_to_bytes(pk: Point) -> [u8; 32] { + // Note that there is 1 more free bit in the 32 bytes (254 bits currently occupied by the x coordinate, 1 bit for + // the "sign") so it's possible to use that last bit as an "is_infinite" flag if desired in the future. assert(!pk.is_infinite, "Cannot serialize point at infinity as bytes."); let mut result = pk.x.to_be_bytes(32); diff --git a/noir-projects/aztec-nr/value-note/src/utils.nr b/noir-projects/aztec-nr/value-note/src/utils.nr index 107562ef2b6..0f67f82eddf 100644 --- a/noir-projects/aztec-nr/value-note/src/utils.nr +++ b/noir-projects/aztec-nr/value-note/src/utils.nr @@ -5,7 +5,7 @@ use crate::{filter::filter_notes_min_sum, value_note::{ValueNote, VALUE_NOTE_LEN // Sort the note values (0th field) in descending order. // Pick the fewest notes whose sum is equal to or greater than `amount`. -pub fn create_note_getter_options_for_decreasing_balance(amount: Field) -> NoteGetterOptions { +pub fn create_note_getter_options_for_decreasing_balance(amount: Field) -> NoteGetterOptions { NoteGetterOptions::with_filter(filter_notes_min_sum, amount).sort(ValueNote::properties().value, SortOrder.DESC) } diff --git a/noir-projects/noir-contracts/contracts/docs_example_contract/src/options.nr b/noir-projects/noir-contracts/contracts/docs_example_contract/src/options.nr index df478482f1f..51e8f090500 100644 --- a/noir-projects/noir-contracts/contracts/docs_example_contract/src/options.nr +++ b/noir-projects/noir-contracts/contracts/docs_example_contract/src/options.nr @@ -10,7 +10,7 @@ use dep::aztec::note::note_getter_options::{Sort, SortOrder}; pub fn create_points_card_getter_options( points: Field, offset: u32 -) -> NoteGetterOptions { +) -> NoteGetterOptions { let mut options = NoteGetterOptions::new(); options.select(CardNote::properties().points, points, Option::none()).sort(CardNote::properties().points, SortOrder.DESC).set_offset(offset) } @@ -21,7 +21,7 @@ pub fn create_exact_card_getter_options( points: u8, secret: Field, account_npk_m_hash: Field -) -> NoteGetterOptions { +) -> NoteGetterOptions { let mut options = NoteGetterOptions::new(); options.select(CardNote::properties().points, points as Field, Option::none()).select(CardNote::properties().randomness, secret, Option::none()).select( CardNote::properties().npk_m_hash, @@ -49,13 +49,13 @@ pub fn filter_min_points( // docs:end:state_vars-OptionFilter // docs:start:state_vars-NoteGetterOptionsFilter -pub fn create_cards_with_min_points_getter_options(min_points: u8) -> NoteGetterOptions { +pub fn create_cards_with_min_points_getter_options(min_points: u8) -> NoteGetterOptions { NoteGetterOptions::with_filter(filter_min_points, min_points).sort(CardNote::properties().points, SortOrder.ASC) } // docs:end:state_vars-NoteGetterOptionsFilter // docs:start:state_vars-NoteGetterOptionsPickOne -pub fn create_largest_card_getter_options() -> NoteGetterOptions { +pub fn create_largest_card_getter_options() -> NoteGetterOptions { let mut options = NoteGetterOptions::new(); options.sort(CardNote::properties().points, SortOrder.DESC).set_limit(1) } diff --git a/noir-projects/noir-contracts/contracts/token_contract/src/types/balances_map.nr b/noir-projects/noir-contracts/contracts/token_contract/src/types/balances_map.nr index 6a582c259f7..b7975db3c5a 100644 --- a/noir-projects/noir-contracts/contracts/token_contract/src/types/balances_map.nr +++ b/noir-projects/noir-contracts/contracts/token_contract/src/types/balances_map.nr @@ -107,7 +107,12 @@ impl BalancesMap { target_amount: U128, max_notes: u32 ) -> U128 where T: NoteInterface + OwnedNote + Eq { - let options = NoteGetterOptions::with_filter(filter_notes_min_sum, target_amount).set_limit(max_notes); + // We are using a preprocessor here (filter applied in an unconstrained context) instead of a filter because + // we do not need to prove correct execution of the preprocessor. + // Because the `min_sum` notes is not constrained, users could choose to e.g. not call it. However, all this + // might result in is simply higher DA costs due to more nullifiers being emitted. Since we don't care + // about proving optimal note usage, we can save these constraints and make the circuit smaller. + let options = NoteGetterOptions::with_preprocessor(preprocess_notes_min_sum, target_amount).set_limit(max_notes); let notes = self.map.at(owner).pop_notes(options); let mut subtracted = U128::from_integer(0); @@ -124,10 +129,10 @@ impl BalancesMap { // Computes the partial sum of the notes array, stopping once 'min_sum' is reached. This can be used to minimize the // number of notes read that add to some value, e.g. when transferring some amount of tokens. -// The filter does not check if total sum is larger or equal to 'min_sum' - all it does is remove extra notes if it does -// reach that value. -// Note that proper usage of this filter requires for notes to be sorted in descending order. -pub fn filter_notes_min_sum( +// The preprocessor (a filter applied in an unconstrained context) does not check if total sum is larger or equal to +// 'min_sum' - all it does is remove extra notes if it does reach that value. +// Note that proper usage of this preprocessor requires for notes to be sorted in descending order. +pub fn preprocess_notes_min_sum( notes: [Option; MAX_NOTE_HASH_READ_REQUESTS_PER_CALL], min_sum: U128 ) -> [Option; MAX_NOTE_HASH_READ_REQUESTS_PER_CALL] where T: NoteInterface + OwnedNote {