From d3375eefbfb829650aff6ac7b036b7b87fce1c85 Mon Sep 17 00:00:00 2001 From: wlmyng <127570466+wlmyng@users.noreply.github.com> Date: Fri, 21 Jun 2024 10:27:16 -0700 Subject: [PATCH] [3/n][gql-performance] add min_tx_sequence_number and max_tx_sequence_number to checkpoints table (#18244) ## Description Per title - add `min_tx_sequence_number` and `max_tx_sequence_number` to `checkpoints` table to support queries mapping `cp` to a `tx` range. This is so that when given a `checkpoint_sequence_number` range, we can map it to the corresponding `tx_sequence_number` range to facilitate transactions queries. The two new columns are nullable, because it is possible for a checkpoint to not contain any transactions. In practice, we'll typically have at least a commit prologue txn. The nullable columns do not impact the queries we intend to run on the table. If through a CTE, a null value will eventually resolve to FALSE when we attempt to use the min or max tx sequence number in a query against the transactions and tx lookup tables. If we make multiple roundtrips, we can stop early if we see that the range is empty for a checkpoint. ## Test plan Existing tests --- ## Release notes Check each box that your changes affect. If none of the boxes relate to your changes, release notes aren't required. For each box you select, include information after the relevant heading that describes the impact of your changes that a user might notice and any actions they must take to implement updates. - [ ] Protocol: - [ ] Nodes (Validators and Full nodes): - [ ] Indexer: - [ ] JSON-RPC: - [ ] GraphQL: - [ ] CLI: - [ ] Rust SDK: --- .../pg/2023-08-19-044044_checkpoints/up.sql | 20 ++++++++++--------- .../src/handlers/checkpoint_handler.rs | 3 +++ crates/sui-indexer/src/indexer_reader.rs | 8 ++++---- crates/sui-indexer/src/models/checkpoints.rs | 4 ++++ crates/sui-indexer/src/schema/pg.rs | 6 +++++- crates/sui-indexer/src/types.rs | 6 ++++++ 6 files changed, 33 insertions(+), 14 deletions(-) diff --git a/crates/sui-indexer/migrations/pg/2023-08-19-044044_checkpoints/up.sql b/crates/sui-indexer/migrations/pg/2023-08-19-044044_checkpoints/up.sql index 8de09a7ff88e8..8f47ef0eeb4a4 100644 --- a/crates/sui-indexer/migrations/pg/2023-08-19-044044_checkpoints/up.sql +++ b/crates/sui-indexer/migrations/pg/2023-08-19-044044_checkpoints/up.sql @@ -1,15 +1,15 @@ CREATE TABLE checkpoints ( - sequence_number bigint PRIMARY KEY, - checkpoint_digest bytea NOT NULL, - epoch bigint NOT NULL, + sequence_number BIGINT PRIMARY KEY, + checkpoint_digest BYTEA NOT NULL, + epoch BIGINT NOT NULL, -- total transactions in the network at the end of this checkpoint (including itself) - network_total_transactions bigint NOT NULL, - previous_checkpoint_digest bytea, + network_total_transactions BIGINT NOT NULL, + previous_checkpoint_digest BYTEA, -- if this checkpoitn is the last checkpoint of an epoch end_of_epoch boolean NOT NULL, -- array of TranscationDigest in bytes included in this checkpoint - tx_digests bytea[] NOT NULL, + tx_digests BYTEA[] NOT NULL, timestamp_ms BIGINT NOT NULL, total_gas_cost BIGINT NOT NULL, computation_cost BIGINT NOT NULL, @@ -17,11 +17,13 @@ CREATE TABLE checkpoints storage_rebate BIGINT NOT NULL, non_refundable_storage_fee BIGINT NOT NULL, -- bcs serialized Vec bytes - checkpoint_commitments bytea NOT NULL, + checkpoint_commitments BYTEA NOT NULL, -- bcs serialized AggregateAuthoritySignature bytes - validator_signature bytea NOT NULL, + validator_signature BYTEA NOT NULL, -- bcs serialzied EndOfEpochData bytes, if the checkpoint marks end of an epoch - end_of_epoch_data bytea + end_of_epoch_data BYTEA, + min_tx_sequence_number BIGINT, + max_tx_sequence_number BIGINT ); CREATE INDEX checkpoints_epoch ON checkpoints (epoch, sequence_number); diff --git a/crates/sui-indexer/src/handlers/checkpoint_handler.rs b/crates/sui-indexer/src/handlers/checkpoint_handler.rs index 2ce65bce4376d..06772c78d99dd 100644 --- a/crates/sui-indexer/src/handlers/checkpoint_handler.rs +++ b/crates/sui-indexer/src/handlers/checkpoint_handler.rs @@ -315,6 +315,8 @@ where &checkpoint_summary, &checkpoint_contents, successful_tx_num as usize, + db_transactions.first().map(|t| t.tx_sequence_number), + db_transactions.last().map(|t| t.tx_sequence_number), ), db_transactions, db_events, @@ -397,6 +399,7 @@ where checkpoint_seq, tx_digest, sender_signed_data.digest() ))); } + let tx = sender_signed_data.transaction_data(); let events = events .as_ref() diff --git a/crates/sui-indexer/src/indexer_reader.rs b/crates/sui-indexer/src/indexer_reader.rs index 00408945140bd..f05098a764185 100644 --- a/crates/sui-indexer/src/indexer_reader.rs +++ b/crates/sui-indexer/src/indexer_reader.rs @@ -777,14 +777,14 @@ impl IndexerReader { let package = Hex::encode(package.to_vec()); match (module, function) { (Some(module), Some(function)) => ( - "tx_calls".into(), + "tx_calls_fun".into(), format!( "package = '\\x{}'::bytea AND module = '{}' AND func = '{}'", package, module, function ), ), (Some(module), None) => ( - "tx_calls".into(), + "tx_calls_mod".into(), format!( "package = '\\x{}'::bytea AND module = '{}'", package, module @@ -792,11 +792,11 @@ impl IndexerReader { ), (None, Some(_)) => { return Err(IndexerError::InvalidArgumentError( - "Function cannot be present wihtout Module.".into(), + "Function cannot be present without Module.".into(), )); } (None, None) => ( - "tx_calls".into(), + "tx_calls_pkg".into(), format!("package = '\\x{}'::bytea", package), ), } diff --git a/crates/sui-indexer/src/models/checkpoints.rs b/crates/sui-indexer/src/models/checkpoints.rs index b94d8e7e44106..220bf6e60271d 100644 --- a/crates/sui-indexer/src/models/checkpoints.rs +++ b/crates/sui-indexer/src/models/checkpoints.rs @@ -36,6 +36,8 @@ pub struct StoredCheckpoint { pub checkpoint_commitments: Vec, pub validator_signature: Vec, pub end_of_epoch_data: Option>, + pub min_tx_sequence_number: Option, + pub max_tx_sequence_number: Option, } impl From<&IndexedCheckpoint> for StoredCheckpoint { @@ -77,6 +79,8 @@ impl From<&IndexedCheckpoint> for StoredCheckpoint { .as_ref() .map(|d| bcs::to_bytes(d).unwrap()), end_of_epoch: c.end_of_epoch_data.is_some(), + min_tx_sequence_number: c.min_tx_sequence_number.map(|t| t as i64), + max_tx_sequence_number: c.max_tx_sequence_number.map(|t| t as i64), } } } diff --git a/crates/sui-indexer/src/schema/pg.rs b/crates/sui-indexer/src/schema/pg.rs index 6d298ff5ed948..f1de68d061bc3 100644 --- a/crates/sui-indexer/src/schema/pg.rs +++ b/crates/sui-indexer/src/schema/pg.rs @@ -20,6 +20,8 @@ diesel::table! { checkpoint_commitments -> Bytea, validator_signature -> Bytea, end_of_epoch_data -> Nullable, + min_tx_sequence_number -> Nullable, + max_tx_sequence_number -> Nullable } } @@ -323,7 +325,9 @@ macro_rules! for_all_tables { packages, transactions, transactions_partition_0, - tx_calls, + tx_calls_pkg, + tx_calls_mod, + tx_calls_fun, tx_changed_objects, tx_digests, tx_input_objects, diff --git a/crates/sui-indexer/src/types.rs b/crates/sui-indexer/src/types.rs index db6ad9f155f0c..fab9511bd3dec 100644 --- a/crates/sui-indexer/src/types.rs +++ b/crates/sui-indexer/src/types.rs @@ -45,6 +45,8 @@ pub struct IndexedCheckpoint { pub successful_tx_num: usize, pub end_of_epoch_data: Option, pub end_of_epoch: bool, + pub min_tx_sequence_number: Option, + pub max_tx_sequence_number: Option, } impl IndexedCheckpoint { @@ -52,6 +54,8 @@ impl IndexedCheckpoint { checkpoint: &sui_types::messages_checkpoint::CertifiedCheckpointSummary, contents: &sui_types::messages_checkpoint::CheckpointContents, successful_tx_num: usize, + min_tx_sequence_number: Option, + max_tx_sequence_number: Option, ) -> Self { let total_gas_cost = checkpoint.epoch_rolling_gas_cost_summary.computation_cost as i64 + checkpoint.epoch_rolling_gas_cost_summary.storage_cost as i64 @@ -78,6 +82,8 @@ impl IndexedCheckpoint { timestamp_ms: checkpoint.timestamp_ms, validator_signature: auth_sig.clone(), checkpoint_commitments: checkpoint.checkpoint_commitments.clone(), + min_tx_sequence_number, + max_tx_sequence_number, } } }