From d6d0752e9cfe058022a20212297c50b17b7fa1a0 Mon Sep 17 00:00:00 2001 From: Ashok Menon Date: Mon, 13 May 2024 00:53:01 +0100 Subject: [PATCH 1/3] [Indexer/TiDB] Standardize prefix lengths for blob indices ## Description Pick standard prefix lengths for well-known types that make good use of details we know about their structure: - Object IDs, addresses and digests are all 32 byte quantities. - Move Identifiers cannot be larger than 128 bytes. - Fully-qualified types are unlikely to be more than 256 bytes (this is the only case where we have to approximate). ## Test plan ``` sui-indexer$ diesel database reset --database-url=$DB --migration-dir='migrations/mysql' sui-indexer$ cargo run \ --no-default-features --features mysql-feature \ -- --db-url $DB --reset-db \ --fullnode-sync-worker \ --rpc-client-url "$MAINNET" ``` Ensure data being filled from mainnet by the indexer is well-formed. --- .../mysql/2024-03-22-052019_events/up.sql | 8 ++--- .../mysql/2024-03-25-203621_objects/up.sql | 34 +++++++++---------- .../2024-03-26-004025_transactions/up.sql | 2 +- .../2024-04-24-180008_checkpoints/up.sql | 2 +- .../mysql/2024-04-24-180418_tx_indices/up.sql | 16 ++++----- .../mysql/2024-04-24-180727_display/up.sql | 2 +- 6 files changed, 32 insertions(+), 32 deletions(-) diff --git a/crates/sui-indexer/migrations/mysql/2024-03-22-052019_events/up.sql b/crates/sui-indexer/migrations/mysql/2024-03-22-052019_events/up.sql index ea46f67373fb9..bca338b64c195 100644 --- a/crates/sui-indexer/migrations/mysql/2024-03-22-052019_events/up.sql +++ b/crates/sui-indexer/migrations/mysql/2024-03-22-052019_events/up.sql @@ -27,8 +27,8 @@ CREATE TABLE events ) PARTITION BY RANGE (checkpoint_sequence_number) ( PARTITION events_partition_0 VALUES LESS THAN MAXVALUE ); -CREATE INDEX events_package ON events (package(255), tx_sequence_number, event_sequence_number); -CREATE INDEX events_package_module ON events (package(255), module(255), tx_sequence_number, event_sequence_number); -CREATE INDEX events_event_type ON events (event_type(255), tx_sequence_number, event_sequence_number); -CREATE INDEX events_type_package_module_name ON events (event_type_package(128), event_type_module(128), event_type_name(128), tx_sequence_number, event_sequence_number); +CREATE INDEX events_package ON events (package(32), tx_sequence_number, event_sequence_number); +CREATE INDEX events_package_module ON events (package(32), module(128), tx_sequence_number, event_sequence_number); +CREATE INDEX events_event_type ON events (event_type(256), tx_sequence_number, event_sequence_number); +CREATE INDEX events_type_package_module_name ON events (event_type_package(32), event_type_module(128), event_type_name(128), tx_sequence_number, event_sequence_number); CREATE INDEX events_checkpoint_sequence_number ON events (checkpoint_sequence_number); diff --git a/crates/sui-indexer/migrations/mysql/2024-03-25-203621_objects/up.sql b/crates/sui-indexer/migrations/mysql/2024-03-25-203621_objects/up.sql index bdefccc67e342..801b4ed55f023 100644 --- a/crates/sui-indexer/migrations/mysql/2024-03-25-203621_objects/up.sql +++ b/crates/sui-indexer/migrations/mysql/2024-03-25-203621_objects/up.sql @@ -32,15 +32,15 @@ CREATE TABLE objects ( df_object_type TEXT, -- object_id in DynamicFieldInfo. df_object_id BLOB, - CONSTRAINT objects_pk PRIMARY KEY (object_id(128)) + CONSTRAINT objects_pk PRIMARY KEY (object_id(32)) ); -- OwnerType: 1: Address, 2: Object, see types.rs -CREATE INDEX objects_owner ON objects (owner_type, owner_id(128)); -CREATE INDEX objects_coin ON objects (owner_id(128), coin_type(128)); +CREATE INDEX objects_owner ON objects (owner_type, owner_id(32)); +CREATE INDEX objects_coin ON objects (owner_id(32), coin_type(256)); CREATE INDEX objects_checkpoint_sequence_number ON objects (checkpoint_sequence_number); -CREATE INDEX objects_package_module_name_full_type ON objects (object_type_package(128), object_type_module(128), object_type_name(128), object_type(128)); -CREATE INDEX objects_owner_package_module_name_full_type ON objects (owner_id(128), object_type_package(128), object_type_module(128), object_type_name(128), object_type(128)); +CREATE INDEX objects_package_module_name_full_type ON objects (object_type_package(32), object_type_module(128), object_type_name(128), object_type(256)); +CREATE INDEX objects_owner_package_module_name_full_type ON objects (owner_id(32), object_type_package(32), object_type_module(128), object_type_name(128), object_type(256)); -- similar to objects table, except that -- 1. the primary key to store multiple object versions and partitions by checkpoint_sequence_number @@ -66,16 +66,16 @@ CREATE TABLE objects_history ( df_name BLOB, df_object_type TEXT, df_object_id BLOB, - CONSTRAINT objects_history_pk PRIMARY KEY (checkpoint_sequence_number, object_id(128), object_version) + CONSTRAINT objects_history_pk PRIMARY KEY (checkpoint_sequence_number, object_id(32), object_version) ) PARTITION BY RANGE (checkpoint_sequence_number) ( PARTITION objects_history_partition_0 VALUES LESS THAN MAXVALUE ); -CREATE INDEX objects_history_id_version ON objects_history (object_id(128), object_version, checkpoint_sequence_number); -CREATE INDEX objects_history_owner ON objects_history (checkpoint_sequence_number, owner_type, owner_id(128)); -CREATE INDEX objects_history_coin ON objects_history (checkpoint_sequence_number, owner_id(128), coin_type(128)); -CREATE INDEX objects_history_type ON objects_history (checkpoint_sequence_number, object_type(128)); -CREATE INDEX objects_history_package_module_name_full_type ON objects_history (checkpoint_sequence_number, object_type_package(128), object_type_module(128), object_type_name(128), object_type(128)); -CREATE INDEX objects_history_owner_package_module_name_full_type ON objects_history (checkpoint_sequence_number, owner_id(128), object_type_package(128), object_type_module(128), object_type_name(128), object_type(128)); +CREATE INDEX objects_history_id_version ON objects_history (object_id(32), object_version, checkpoint_sequence_number); +CREATE INDEX objects_history_owner ON objects_history (checkpoint_sequence_number, owner_type, owner_id(32)); +CREATE INDEX objects_history_coin ON objects_history (checkpoint_sequence_number, owner_id(32), coin_type(256)); +CREATE INDEX objects_history_type ON objects_history (checkpoint_sequence_number, object_type(256)); +CREATE INDEX objects_history_package_module_name_full_type ON objects_history (checkpoint_sequence_number, object_type_package(32), object_type_module(128), object_type_name(128), object_type(256)); +CREATE INDEX objects_history_owner_package_module_name_full_type ON objects_history (checkpoint_sequence_number, owner_id(32), object_type_package(32), object_type_module(128), object_type_name(128), object_type(256)); -- snapshot table by folding objects_history table until certain checkpoint, -- effectively the snapshot of objects at the same checkpoint, @@ -99,10 +99,10 @@ CREATE TABLE objects_snapshot ( df_name BLOB, df_object_type TEXT, df_object_id BLOB, - CONSTRAINT objects_snapshot_pk PRIMARY KEY (object_id(128)) + CONSTRAINT objects_snapshot_pk PRIMARY KEY (object_id(32)) ); CREATE INDEX objects_snapshot_checkpoint_sequence_number ON objects_snapshot (checkpoint_sequence_number); -CREATE INDEX objects_snapshot_owner ON objects_snapshot (owner_type, owner_id(128), object_id(128)); -CREATE INDEX objects_snapshot_coin ON objects_snapshot (owner_id(128), coin_type(128), object_id(128)); -CREATE INDEX objects_snapshot_package_module_name_full_type ON objects_snapshot (object_type_package(128), object_type_module(128), object_type_name(128), object_type(128)); -CREATE INDEX objects_snapshot_owner_package_module_name_full_type ON objects_snapshot (owner_id(128), object_type_package(128), object_type_module(128), object_type_name(128), object_type(128)); +CREATE INDEX objects_snapshot_owner ON objects_snapshot (owner_type, owner_id(32), object_id(32)); +CREATE INDEX objects_snapshot_coin ON objects_snapshot (owner_id(32), coin_type(256), object_id(32)); +CREATE INDEX objects_snapshot_package_module_name_full_type ON objects_snapshot (object_type_package(32), object_type_module(128), object_type_name(128), object_type(256)); +CREATE INDEX objects_snapshot_owner_package_module_name_full_type ON objects_snapshot (owner_id(32), object_type_package(32), object_type_module(128), object_type_name(128), object_type(256)); diff --git a/crates/sui-indexer/migrations/mysql/2024-03-26-004025_transactions/up.sql b/crates/sui-indexer/migrations/mysql/2024-03-26-004025_transactions/up.sql index 791c8105a0d58..e18f1f368fa42 100644 --- a/crates/sui-indexer/migrations/mysql/2024-03-26-004025_transactions/up.sql +++ b/crates/sui-indexer/migrations/mysql/2024-03-26-004025_transactions/up.sql @@ -23,7 +23,7 @@ CREATE TABLE transactions ( PARTITION transactions_partition_0 VALUES LESS THAN MAXVALUE ); -CREATE INDEX transactions_transaction_digest ON transactions (transaction_digest(255)); +CREATE INDEX transactions_transaction_digest ON transactions (transaction_digest(32)); CREATE INDEX transactions_checkpoint_sequence_number ON transactions (checkpoint_sequence_number); -- only create index for system transactions (0). See types.rs CREATE INDEX transactions_transaction_kind ON transactions (transaction_kind); diff --git a/crates/sui-indexer/migrations/mysql/2024-04-24-180008_checkpoints/up.sql b/crates/sui-indexer/migrations/mysql/2024-04-24-180008_checkpoints/up.sql index 76bcce28b1f56..7ab0fa8a3a5cd 100644 --- a/crates/sui-indexer/migrations/mysql/2024-04-24-180008_checkpoints/up.sql +++ b/crates/sui-indexer/migrations/mysql/2024-04-24-180008_checkpoints/up.sql @@ -25,4 +25,4 @@ CREATE TABLE checkpoints ); CREATE INDEX checkpoints_epoch ON checkpoints (epoch, sequence_number); -CREATE INDEX checkpoints_digest ON checkpoints (checkpoint_digest(255)); +CREATE INDEX checkpoints_digest ON checkpoints (checkpoint_digest(32)); diff --git a/crates/sui-indexer/migrations/mysql/2024-04-24-180418_tx_indices/up.sql b/crates/sui-indexer/migrations/mysql/2024-04-24-180418_tx_indices/up.sql index d1d0e428fdc19..13feddcd18bf0 100644 --- a/crates/sui-indexer/migrations/mysql/2024-04-24-180418_tx_indices/up.sql +++ b/crates/sui-indexer/migrations/mysql/2024-04-24-180418_tx_indices/up.sql @@ -4,7 +4,7 @@ CREATE TABLE tx_senders ( tx_sequence_number BIGINT NOT NULL, -- SuiAddress in bytes. sender BLOB NOT NULL, - PRIMARY KEY(sender(255), tx_sequence_number, cp_sequence_number) + PRIMARY KEY(sender(32), tx_sequence_number, cp_sequence_number) ); CREATE INDEX tx_senders_tx_sequence_number_index ON tx_senders (tx_sequence_number, cp_sequence_number); @@ -14,7 +14,7 @@ CREATE TABLE tx_recipients ( tx_sequence_number BIGINT NOT NULL, -- SuiAddress in bytes. recipient BLOB NOT NULL, - PRIMARY KEY(recipient(255), tx_sequence_number) + PRIMARY KEY(recipient(32), tx_sequence_number) ); CREATE INDEX tx_recipients_tx_sequence_number_index ON tx_recipients (tx_sequence_number, cp_sequence_number); @@ -23,7 +23,7 @@ CREATE TABLE tx_input_objects ( tx_sequence_number BIGINT NOT NULL, -- Object ID in bytes. object_id BLOB NOT NULL, - PRIMARY KEY(object_id(255), tx_sequence_number, cp_sequence_number) + PRIMARY KEY(object_id(32), tx_sequence_number, cp_sequence_number) ); CREATE TABLE tx_changed_objects ( @@ -31,7 +31,7 @@ CREATE TABLE tx_changed_objects ( tx_sequence_number BIGINT NOT NULL, -- Object Id in bytes. object_id BLOB NOT NULL, - PRIMARY KEY(object_id(255), tx_sequence_number) + PRIMARY KEY(object_id(32), tx_sequence_number) ); CREATE TABLE tx_calls ( @@ -42,16 +42,16 @@ CREATE TABLE tx_calls ( func TEXT NOT NULL, -- 1. Using Primary Key as a unique index. -- 2. Diesel does not like tables with no primary key. - PRIMARY KEY(package(255), tx_sequence_number, cp_sequence_number) + PRIMARY KEY(package(32), tx_sequence_number, cp_sequence_number) ); -CREATE INDEX tx_calls_module ON tx_calls (package(255), module(255), tx_sequence_number, cp_sequence_number); -CREATE INDEX tx_calls_func ON tx_calls (package(255), module(255), func(255), tx_sequence_number, cp_sequence_number); +CREATE INDEX tx_calls_module ON tx_calls (package(32), module(128), tx_sequence_number, cp_sequence_number); +CREATE INDEX tx_calls_func ON tx_calls (package(32), module(128), func(128), tx_sequence_number, cp_sequence_number); CREATE INDEX tx_calls_tx_sequence_number ON tx_calls (tx_sequence_number, cp_sequence_number); CREATE TABLE tx_digests ( tx_digest BLOB NOT NULL, cp_sequence_number BIGINT NOT NULL, tx_sequence_number BIGINT NOT NULL, - PRIMARY KEY(tx_digest(255)) + PRIMARY KEY(tx_digest(32)) ); diff --git a/crates/sui-indexer/migrations/mysql/2024-04-24-180727_display/up.sql b/crates/sui-indexer/migrations/mysql/2024-04-24-180727_display/up.sql index f30a9d0171a5e..82fce6bb86ea5 100644 --- a/crates/sui-indexer/migrations/mysql/2024-04-24-180727_display/up.sql +++ b/crates/sui-indexer/migrations/mysql/2024-04-24-180727_display/up.sql @@ -4,5 +4,5 @@ CREATE TABLE display id BLOB NOT NULL, version SMALLINT NOT NULL, bcs MEDIUMBLOB NOT NULL, - primary key (object_type(255)) + primary key (object_type(256)) ); From 5cb39a50709d73e2a63efa56a251aa9383293c99 Mon Sep 17 00:00:00 2001 From: Ashok Menon Date: Sun, 12 May 2024 17:31:38 +0100 Subject: [PATCH 2/3] [Indexer/Schema] Remove query cost function ## Description We no longer use the query_cost function anywhere (it has been replaced by a diesel extension that enables us to call `EXPLAIN` on a query), so cleaning it up to avoid having to maintain it (and create a MySQL compatible version of it). ## Test plan ``` sui-indexer$ diesel setup --database-url=$DB --migration-dir='migrations/mysql' sui-indexer$ diesel database reset --database-url=$DB --migration-dir='migrations/mysql' sui$ cargo nextest run -p sui-indexer sui$ cargo nextest run -p sui-graphql-rpc sui$ cargo nextest run -p sui-graphql-e2e-tests --features pg_integration ``` --- .../down.sql | 2 -- .../2023-10-24-160139_query_cost_function/up.sql | 15 --------------- 2 files changed, 17 deletions(-) delete mode 100644 crates/sui-indexer/migrations/pg/2023-10-24-160139_query_cost_function/down.sql delete mode 100644 crates/sui-indexer/migrations/pg/2023-10-24-160139_query_cost_function/up.sql diff --git a/crates/sui-indexer/migrations/pg/2023-10-24-160139_query_cost_function/down.sql b/crates/sui-indexer/migrations/pg/2023-10-24-160139_query_cost_function/down.sql deleted file mode 100644 index 742f7001fc58f..0000000000000 --- a/crates/sui-indexer/migrations/pg/2023-10-24-160139_query_cost_function/down.sql +++ /dev/null @@ -1,2 +0,0 @@ --- This file should undo anything in `up.sql` -DROP FUNCTION IF EXISTS query_cost(TEXT); diff --git a/crates/sui-indexer/migrations/pg/2023-10-24-160139_query_cost_function/up.sql b/crates/sui-indexer/migrations/pg/2023-10-24-160139_query_cost_function/up.sql deleted file mode 100644 index 7296c688baa7f..0000000000000 --- a/crates/sui-indexer/migrations/pg/2023-10-24-160139_query_cost_function/up.sql +++ /dev/null @@ -1,15 +0,0 @@ --- Your SQL goes here -CREATE OR REPLACE FUNCTION query_cost( - query_in text, - cost OUT float8 - ) RETURNS float8 AS -$$DECLARE - p json; -BEGIN - /* get execution plan in JSON */ - EXECUTE 'EXPLAIN (FORMAT JSON) ' || query_in INTO p; - /* extract total cost */ - SELECT p->0->'Plan'->>'Total Cost' - INTO cost; - RETURN; -END;$$ LANGUAGE plpgsql STRICT; From a2a281d6456d107e3d69acd688419d748277bc73 Mon Sep 17 00:00:00 2001 From: Ashok Menon Date: Mon, 29 Apr 2024 00:27:31 +0100 Subject: [PATCH 3/3] [GraphQL][EASY] Remove MovePackage.checkpoint_viewed_at ## Description The same data is available as `MovePackage.super_.checkpoint_viewed_at` and has been exposed as `MovePackage.checkpoint_viewed_at_impl()`. ## Test plan ``` sui$ cargo nextest run -p sui-graphql-e2e-tests --features pg_integration ``` --- crates/sui-graphql-rpc/src/types/move_package.rs | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/crates/sui-graphql-rpc/src/types/move_package.rs b/crates/sui-graphql-rpc/src/types/move_package.rs index 0bf275a97ac86..6f73d32a8ed23 100644 --- a/crates/sui-graphql-rpc/src/types/move_package.rs +++ b/crates/sui-graphql-rpc/src/types/move_package.rs @@ -32,9 +32,6 @@ pub(crate) struct MovePackage { /// Move-object-specific data, extracted from the native representation at /// `graphql_object.native_object.data`. pub native: NativeMovePackage, - - /// The checkpoint sequence number this package was viewed at. - pub checkpoint_viewed_at: u64, } /// Information used by a package to link to a specific version of its dependency. @@ -276,7 +273,8 @@ impl MovePackage { let page = Page::from_params(ctx.data_unchecked(), first, after, last, before)?; let cursor_viewed_at = page.validate_cursor_consistency()?; - let checkpoint_viewed_at = cursor_viewed_at.unwrap_or(self.checkpoint_viewed_at); + let checkpoint_viewed_at = + cursor_viewed_at.unwrap_or_else(|| self.checkpoint_viewed_at_impl()); let parsed = self.parsed_package()?; let module_range = parsed.modules().range::(( @@ -391,6 +389,12 @@ impl MovePackage { .map_err(|e| Error::Internal(format!("Error reading package: {e}"))) } + /// This package was viewed at a snapshot of the chain state at this checkpoint (identified by + /// its sequence number). + fn checkpoint_viewed_at_impl(&self) -> u64 { + self.super_.checkpoint_viewed_at + } + pub(crate) fn module_impl(&self, name: &str) -> Result, Error> { use PackageCacheError as E; match ( @@ -401,7 +405,7 @@ impl MovePackage { storage_id: self.super_.address, native: native.clone(), parsed: parsed.clone(), - checkpoint_viewed_at: self.checkpoint_viewed_at, + checkpoint_viewed_at: self.checkpoint_viewed_at_impl(), })), (None, _) | (_, Err(E::ModuleNotFound(_, _))) => Ok(None), @@ -438,7 +442,6 @@ impl TryFrom<&Object> for MovePackage { Ok(Self { super_: object.clone(), native: move_package.clone(), - checkpoint_viewed_at: object.checkpoint_viewed_at, }) } else { Err(MovePackageDowncastError)