From 265185f648b2c676bd9d055922691920ef335db6 Mon Sep 17 00:00:00 2001 From: Emma Zhong Date: Sun, 21 Jul 2024 00:05:08 -0700 Subject: [PATCH 1/5] [gql][indexer] index chain identifier into its own table --- .../tests/epoch/chain_identifier.exp | 14 ++++++++++ .../tests/epoch/chain_identifier.move | 12 +++++++++ .../src/types/chain_identifier.rs | 12 +++------ .../down.sql | 2 ++ .../2024-07-13-003534_chain_identifier/up.sql | 6 +++++ crates/sui-indexer/src/models/checkpoints.rs | 8 +++++- crates/sui-indexer/src/schema/mod.rs | 2 ++ crates/sui-indexer/src/schema/pg.rs | 9 +++++-- .../sui-indexer/src/store/pg_indexer_store.rs | 26 ++++++++++++++++--- 9 files changed, 77 insertions(+), 14 deletions(-) create mode 100644 crates/sui-graphql-e2e-tests/tests/epoch/chain_identifier.exp create mode 100644 crates/sui-graphql-e2e-tests/tests/epoch/chain_identifier.move create mode 100644 crates/sui-indexer/migrations/pg/2024-07-13-003534_chain_identifier/down.sql create mode 100644 crates/sui-indexer/migrations/pg/2024-07-13-003534_chain_identifier/up.sql diff --git a/crates/sui-graphql-e2e-tests/tests/epoch/chain_identifier.exp b/crates/sui-graphql-e2e-tests/tests/epoch/chain_identifier.exp new file mode 100644 index 0000000000000..e2e35372a3759 --- /dev/null +++ b/crates/sui-graphql-e2e-tests/tests/epoch/chain_identifier.exp @@ -0,0 +1,14 @@ +processed 3 tasks + +init: +C: object(0,0) + +task 1 'create-checkpoint'. lines 8-8: +Checkpoint created: 1 + +task 2 'run-graphql'. lines 10-13: +Response: { + "data": { + "chainIdentifier": "8f58ad28" + } +} diff --git a/crates/sui-graphql-e2e-tests/tests/epoch/chain_identifier.move b/crates/sui-graphql-e2e-tests/tests/epoch/chain_identifier.move new file mode 100644 index 0000000000000..22f11bcf6277e --- /dev/null +++ b/crates/sui-graphql-e2e-tests/tests/epoch/chain_identifier.move @@ -0,0 +1,12 @@ +// Copyright (c) Mysten Labs, Inc. +// SPDX-License-Identifier: Apache-2.0 + +//# init --protocol-version 48 --simulator --accounts C + + +//# create-checkpoint + +//# run-graphql +{ + chainIdentifier +} \ No newline at end of file diff --git a/crates/sui-graphql-rpc/src/types/chain_identifier.rs b/crates/sui-graphql-rpc/src/types/chain_identifier.rs index 1072d0a580f1a..50fb3d8adaf81 100644 --- a/crates/sui-graphql-rpc/src/types/chain_identifier.rs +++ b/crates/sui-graphql-rpc/src/types/chain_identifier.rs @@ -6,8 +6,8 @@ use crate::{ error::Error, }; use async_graphql::*; -use diesel::{ExpressionMethods, QueryDsl}; -use sui_indexer::schema::checkpoints; +use diesel::QueryDsl; +use sui_indexer::schema::chain_identifier; use sui_types::{ digests::ChainIdentifier as NativeChainIdentifier, messages_checkpoint::CheckpointDigest, }; @@ -17,15 +17,11 @@ pub(crate) struct ChainIdentifier; impl ChainIdentifier { /// Query the Chain Identifier from the DB. pub(crate) async fn query(db: &Db) -> Result { - use checkpoints::dsl; + use chain_identifier::dsl; let digest_bytes = db .execute(move |conn| { - conn.first(move || { - dsl::checkpoints - .select(dsl::checkpoint_digest) - .order_by(dsl::sequence_number.asc()) - }) + conn.first(move || dsl::chain_identifier.select(dsl::checkpoint_digest)) }) .await .map_err(|e| Error::Internal(format!("Failed to fetch genesis digest: {e}")))?; diff --git a/crates/sui-indexer/migrations/pg/2024-07-13-003534_chain_identifier/down.sql b/crates/sui-indexer/migrations/pg/2024-07-13-003534_chain_identifier/down.sql new file mode 100644 index 0000000000000..57f1de973b1d2 --- /dev/null +++ b/crates/sui-indexer/migrations/pg/2024-07-13-003534_chain_identifier/down.sql @@ -0,0 +1,2 @@ +-- This file should undo anything in `up.sql` +DROP TABLE IF EXISTS chain_identifier; diff --git a/crates/sui-indexer/migrations/pg/2024-07-13-003534_chain_identifier/up.sql b/crates/sui-indexer/migrations/pg/2024-07-13-003534_chain_identifier/up.sql new file mode 100644 index 0000000000000..205e3a89f63e5 --- /dev/null +++ b/crates/sui-indexer/migrations/pg/2024-07-13-003534_chain_identifier/up.sql @@ -0,0 +1,6 @@ +-- Your SQL goes here +CREATE TABLE chain_identifier +( + checkpoint_digest BYTEA NOT NULL, + PRIMARY KEY(checkpoint_digest) +); diff --git a/crates/sui-indexer/src/models/checkpoints.rs b/crates/sui-indexer/src/models/checkpoints.rs index ea3579eb624d9..260fcfb5944f2 100644 --- a/crates/sui-indexer/src/models/checkpoints.rs +++ b/crates/sui-indexer/src/models/checkpoints.rs @@ -9,9 +9,15 @@ use sui_types::digests::CheckpointDigest; use sui_types::gas::GasCostSummary; use crate::errors::IndexerError; -use crate::schema::{checkpoints, pruner_cp_watermark}; +use crate::schema::{chain_identifier, checkpoints, pruner_cp_watermark}; use crate::types::IndexedCheckpoint; +#[derive(Queryable, Insertable, Selectable, Debug, Clone, Default)] +#[diesel(table_name = chain_identifier)] +pub struct StoredChainIdentifier { + pub checkpoint_digest: Vec, +} + #[derive(Queryable, Insertable, Selectable, Debug, Clone, Default)] #[diesel(table_name = checkpoints)] pub struct StoredCheckpoint { diff --git a/crates/sui-indexer/src/schema/mod.rs b/crates/sui-indexer/src/schema/mod.rs index 2d14fe31d765f..7d47b0de2ee4c 100644 --- a/crates/sui-indexer/src/schema/mod.rs +++ b/crates/sui-indexer/src/schema/mod.rs @@ -12,6 +12,7 @@ mod pg; #[cfg(feature = "postgres-feature")] mod inner { + pub use crate::schema::pg::chain_identifier; pub use crate::schema::pg::checkpoints; pub use crate::schema::pg::display; pub use crate::schema::pg::epochs; @@ -51,6 +52,7 @@ mod inner { pub use crate::schema::mysql::tx_senders; } +pub use inner::chain_identifier; pub use inner::checkpoints; pub use inner::display; pub use inner::epochs; diff --git a/crates/sui-indexer/src/schema/pg.rs b/crates/sui-indexer/src/schema/pg.rs index 68975e8584c12..7b2bb44b63bdc 100644 --- a/crates/sui-indexer/src/schema/pg.rs +++ b/crates/sui-indexer/src/schema/pg.rs @@ -1,7 +1,11 @@ -// Copyright (c) Mysten Labs, Inc. -// SPDX-License-Identifier: Apache-2.0 // @generated automatically by Diesel CLI. +diesel::table! { + chain_identifier (checkpoint_digest) { + checkpoint_digest -> Bytea, + } +} + diesel::table! { checkpoints (sequence_number) { sequence_number -> Int8, @@ -284,6 +288,7 @@ diesel::table! { macro_rules! for_all_tables { ($action:path) => { $action!( + chain_identifier, checkpoints, pruner_cp_watermark, display, diff --git a/crates/sui-indexer/src/store/pg_indexer_store.rs b/crates/sui-indexer/src/store/pg_indexer_store.rs index fc4537cf73084..83a841196d353 100644 --- a/crates/sui-indexer/src/store/pg_indexer_store.rs +++ b/crates/sui-indexer/src/store/pg_indexer_store.rs @@ -27,6 +27,7 @@ use crate::errors::{Context, IndexerError}; use crate::handlers::EpochToCommit; use crate::handlers::TransactionObjectChangesToCommit; use crate::metrics::IndexerMetrics; +use crate::models::checkpoints::StoredChainIdentifier; use crate::models::checkpoints::StoredCheckpoint; use crate::models::checkpoints::StoredCpTx; use crate::models::display::StoredDisplay; @@ -39,9 +40,9 @@ use crate::models::objects::{ use crate::models::packages::StoredPackage; use crate::models::transactions::StoredTransaction; use crate::schema::{ - checkpoints, display, epochs, events, objects, objects_history, objects_snapshot, packages, - pruner_cp_watermark, transactions, tx_calls, tx_changed_objects, tx_digests, tx_input_objects, - tx_recipients, tx_senders, + chain_identifier, checkpoints, display, epochs, events, objects, objects_history, + objects_snapshot, packages, pruner_cp_watermark, transactions, tx_calls, tx_changed_objects, + tx_digests, tx_input_objects, tx_recipients, tx_senders, }; use crate::types::{IndexedCheckpoint, IndexedEvent, IndexedPackage, IndexedTransaction, TxIndex}; use crate::{ @@ -601,6 +602,25 @@ impl PgIndexerStore { if checkpoints.is_empty() { return Ok(()); } + // If the first checkpoint has sequence number 0, we need to persist the digest as + // chain identifier. + let first_checkpoint = checkpoints.first().unwrap(); + if first_checkpoint.sequence_number == 0 { + transactional_blocking_with_retry!( + &self.blocking_cp, + |conn| { + let checkpoint_digest = + first_checkpoint.checkpoint_digest.into_inner().to_vec(); + insert_or_ignore_into!( + chain_identifier::table, + StoredChainIdentifier { checkpoint_digest }, + conn + ); + Ok::<(), IndexerError>(()) + }, + PG_DB_COMMIT_SLEEP_DURATION + )?; + } let guard = self .metrics .checkpoint_db_commit_latency_checkpoints From 59eb509035f35eb0992f123b485449480a23cb64 Mon Sep 17 00:00:00 2001 From: stefan-mysten <135084671+stefan-mysten@users.noreply.github.com> Date: Tue, 30 Jul 2024 03:38:51 -0700 Subject: [PATCH 2/5] Fixes --- .../sui-graphql-e2e-tests/tests/epoch/chain_identifier.exp | 4 ++-- crates/sui-indexer/src/schema/mod.rs | 2 +- crates/sui-indexer/src/schema/mysql.rs | 7 +++++++ crates/sui-indexer/src/schema/pg.rs | 2 ++ 4 files changed, 12 insertions(+), 3 deletions(-) diff --git a/crates/sui-graphql-e2e-tests/tests/epoch/chain_identifier.exp b/crates/sui-graphql-e2e-tests/tests/epoch/chain_identifier.exp index e2e35372a3759..224aa3881be83 100644 --- a/crates/sui-graphql-e2e-tests/tests/epoch/chain_identifier.exp +++ b/crates/sui-graphql-e2e-tests/tests/epoch/chain_identifier.exp @@ -3,10 +3,10 @@ processed 3 tasks init: C: object(0,0) -task 1 'create-checkpoint'. lines 8-8: +task 1 'create-checkpoint'. lines 7-7: Checkpoint created: 1 -task 2 'run-graphql'. lines 10-13: +task 2 'run-graphql'. lines 9-12: Response: { "data": { "chainIdentifier": "8f58ad28" diff --git a/crates/sui-indexer/src/schema/mod.rs b/crates/sui-indexer/src/schema/mod.rs index 7d47b0de2ee4c..a6fa8227f5a4a 100644 --- a/crates/sui-indexer/src/schema/mod.rs +++ b/crates/sui-indexer/src/schema/mod.rs @@ -34,6 +34,7 @@ mod inner { #[cfg(feature = "mysql-feature")] #[cfg(not(feature = "postgres-feature"))] mod inner { + pub use crate::schema::mysql::chain_identifier; pub use crate::schema::mysql::checkpoints; pub use crate::schema::mysql::display; pub use crate::schema::mysql::epochs; @@ -44,7 +45,6 @@ mod inner { pub use crate::schema::mysql::packages; pub use crate::schema::mysql::pruner_cp_watermark; pub use crate::schema::mysql::transactions; - pub use crate::schema::mysql::tx_calls; pub use crate::schema::mysql::tx_changed_objects; pub use crate::schema::mysql::tx_digests; pub use crate::schema::mysql::tx_input_objects; diff --git a/crates/sui-indexer/src/schema/mysql.rs b/crates/sui-indexer/src/schema/mysql.rs index fe5590929eed9..10cdc089c8884 100644 --- a/crates/sui-indexer/src/schema/mysql.rs +++ b/crates/sui-indexer/src/schema/mysql.rs @@ -2,6 +2,12 @@ // SPDX-License-Identifier: Apache-2.0 // @generated automatically by Diesel CLI. +diesel::table! { + chain_identifier (checkpoint_digest) { + checkpoint_digest -> Blob, + } +} + diesel::table! { checkpoints (sequence_number) { sequence_number -> Bigint, @@ -227,6 +233,7 @@ diesel::table! { macro_rules! for_all_tables { ($action:path) => { $action!( + chain_identifier, checkpoints, epochs, events, diff --git a/crates/sui-indexer/src/schema/pg.rs b/crates/sui-indexer/src/schema/pg.rs index 7b2bb44b63bdc..322ad8b6a030a 100644 --- a/crates/sui-indexer/src/schema/pg.rs +++ b/crates/sui-indexer/src/schema/pg.rs @@ -1,3 +1,5 @@ +// Copyright (c) Mysten Labs, Inc. +// SPDX-License-Identifier: Apache-2.0 // @generated automatically by Diesel CLI. diesel::table! { From 1d4da940662c440b2a259c14a64124ea6fa316b1 Mon Sep 17 00:00:00 2001 From: Ashok Menon Date: Tue, 30 Jul 2024 12:35:27 +0100 Subject: [PATCH 3/5] fixup: Update the transactional test format Now it includes the command that was run. --- .../sui-graphql-e2e-tests/tests/epoch/chain_identifier.exp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/crates/sui-graphql-e2e-tests/tests/epoch/chain_identifier.exp b/crates/sui-graphql-e2e-tests/tests/epoch/chain_identifier.exp index 224aa3881be83..c53d33a750392 100644 --- a/crates/sui-graphql-e2e-tests/tests/epoch/chain_identifier.exp +++ b/crates/sui-graphql-e2e-tests/tests/epoch/chain_identifier.exp @@ -3,10 +3,12 @@ processed 3 tasks init: C: object(0,0) -task 1 'create-checkpoint'. lines 7-7: +task 1, line 7: +//# create-checkpoint Checkpoint created: 1 -task 2 'run-graphql'. lines 9-12: +task 2, lines 9-12: +//# run-graphql Response: { "data": { "chainIdentifier": "8f58ad28" From afd47765531832b18fc6a6e65eadb082f0da259b Mon Sep 17 00:00:00 2001 From: Ashok Menon Date: Tue, 30 Jul 2024 13:11:00 +0100 Subject: [PATCH 4/5] fixup: reinstate mysql::tx_calls Not sure how that got deleted... --- crates/sui-indexer/src/schema/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/sui-indexer/src/schema/mod.rs b/crates/sui-indexer/src/schema/mod.rs index a6fa8227f5a4a..d1d408d76a307 100644 --- a/crates/sui-indexer/src/schema/mod.rs +++ b/crates/sui-indexer/src/schema/mod.rs @@ -45,6 +45,7 @@ mod inner { pub use crate::schema::mysql::packages; pub use crate::schema::mysql::pruner_cp_watermark; pub use crate::schema::mysql::transactions; + pub use crate::schema::mysql::tx_calls; pub use crate::schema::mysql::tx_changed_objects; pub use crate::schema::mysql::tx_digests; pub use crate::schema::mysql::tx_input_objects; From f13f81999a6b1e34d20137df35cdba66df15d366 Mon Sep 17 00:00:00 2001 From: Ashok Menon Date: Tue, 30 Jul 2024 13:15:44 +0100 Subject: [PATCH 5/5] Address comment about unwrap --- crates/sui-indexer/src/store/pg_indexer_store.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/sui-indexer/src/store/pg_indexer_store.rs b/crates/sui-indexer/src/store/pg_indexer_store.rs index 83a841196d353..39e16c2e394ab 100644 --- a/crates/sui-indexer/src/store/pg_indexer_store.rs +++ b/crates/sui-indexer/src/store/pg_indexer_store.rs @@ -599,12 +599,12 @@ impl PgIndexerStore { } fn persist_checkpoints(&self, checkpoints: Vec) -> Result<(), IndexerError> { - if checkpoints.is_empty() { + let Some(first_checkpoint) = checkpoints.first() else { return Ok(()); - } + }; + // If the first checkpoint has sequence number 0, we need to persist the digest as // chain identifier. - let first_checkpoint = checkpoints.first().unwrap(); if first_checkpoint.sequence_number == 0 { transactional_blocking_with_retry!( &self.blocking_cp,