indexer 2024-08: merge idx-breaking-change-park to main (#19005)

## Description 

title, this is another attempt of
#18899
which got reverted as it triggered some CI issues, the issues have been
resolved by #18993

## Test plan 

How did you test the new or updated feature?

---

## Release notes

Check each box that your changes affect. If none of the boxes relate to
your changes, release notes aren't required.

For each box you select, include information after the relevant heading
that describes the impact of your changes that a user might notice and
any actions they must take to implement updates.

- [ ] Protocol: 
- [ ] Nodes (Validators and Full nodes): 
- [ ] Indexer: 
- [ ] JSON-RPC: 
- [ ] GraphQL: 
- [ ] CLI: 
- [ ] Rust SDK:
- [ ] REST API:

---------

Co-authored-by: Emma Zhong <emma@mystenlabs.com>
Co-authored-by: Ashok Menon <ashok@mystenlabs.com>
Co-authored-by: wlmyng <127570466+wlmyng@users.noreply.github.com>
Co-authored-by: Emma Zhong <emmazhongjy@gmail.com>

crates/sui-graphql-rpc/src/types/transaction_block.rs

@@ -8,14 +8,14 @@ use async_graphql::{
     dataloader::Loader,
     *,
 };
-use diesel::{BoolExpressionMethods, ExpressionMethods, JoinOnDsl, QueryDsl, SelectableHelper};
+use diesel::{ExpressionMethods, JoinOnDsl, QueryDsl, SelectableHelper};
 use fastcrypto::encoding::{Base58, Encoding};
 use serde::{Deserialize, Serialize};
 use sui_indexer::{
     models::transactions::StoredTransaction,
     schema::{
-        transactions, tx_calls, tx_changed_objects, tx_digests, tx_input_objects, tx_recipients,
-        tx_senders,
+        transactions, tx_calls_fun, tx_changed_objects, tx_digests, tx_input_objects,
+        tx_recipients, tx_senders,
     },
 };
 use sui_types::{
@@ -318,15 +318,15 @@ impl TransactionBlock {
                         let mut query = tx::dsl::transactions.into_boxed();
 
                         if let Some(f) = &filter.function {
-                            let sub_query = tx_calls::dsl::tx_calls
-                                .select(tx_calls::dsl::tx_sequence_number)
+                            let sub_query = tx_calls_fun::dsl::tx_calls_fun
+                                .select(tx_calls_fun::dsl::tx_sequence_number)
                                 .into_boxed();
 
                             query = query.filter(tx::dsl::tx_sequence_number.eq_any(f.apply(
                                 sub_query,
-                                tx_calls::dsl::package,
-                                tx_calls::dsl::module,
-                                tx_calls::dsl::func,
+                                tx_calls_fun::dsl::package,
+                                tx_calls_fun::dsl::module,
+                                tx_calls_fun::dsl::func,
                             )));
                         }
 
@@ -507,9 +507,7 @@ impl Loader<DigestKey> for Db {
         let transactions: Vec<StoredTransaction> = self
             .execute(move |conn| {
                 conn.results(move || {
-                    let join = ds::cp_sequence_number
-                        .eq(tx::checkpoint_sequence_number)
-                        .and(ds::tx_sequence_number.eq(tx::tx_sequence_number));
+                    let join = ds::tx_sequence_number.eq(tx::tx_sequence_number);
 
                     tx::transactions
                         .inner_join(ds::tx_digests.on(join))

crates/sui-indexer/migrations/mysql/2024-04-24-180249_packages/up.sql

@@ -1,7 +1,14 @@
 CREATE TABLE packages
 (
-    package_id                   blob          NOT NULL,
+    package_id                   BLOB          NOT NULL,
+    original_id                  BLOB          NOT NULL,
+    package_version              BIGINT        NOT NULL,
     -- bcs serialized MovePackage
-    move_package                 MEDIUMBLOB          NOT NULL,
-        CONSTRAINT packages_pk PRIMARY KEY (package_id(255))
+    move_package                 MEDIUMBLOB    NOT NULL,
+    checkpoint_sequence_number   BIGINT        NOT NULL,
+    CONSTRAINT packages_pk PRIMARY KEY (package_id(32), original_id(32), package_version),
+    CONSTRAINT packages_unique_package_id UNIQUE (package_id(32))
 );
+
+CREATE INDEX packages_cp_id_version ON packages (checkpoint_sequence_number, original_id(32), package_version);
+CREATE INDEX packages_id_version_cp ON packages (original_id(32), package_version, checkpoint_sequence_number);

crates/sui-indexer/migrations/mysql/2024-05-05-155158_obj_indices/down.sql

@@ -0,0 +1 @@
+DROP TABLE IF EXISTS objects_version;

crates/sui-indexer/migrations/mysql/2024-05-05-155158_obj_indices/up.sql

@@ -0,0 +1,9 @@
+-- The Postgres version of this table is partitioned by the first byte
+-- of object_id, but this kind of partition is not easily supported in
+-- MySQL, so this variant is unpartitioned for now.
+CREATE TABLE objects_version (
+    object_id           BLOB          NOT NULL,
+    object_version      BIGINT        NOT NULL,
+    cp_sequence_number  BIGINT        NOT NULL,
+    PRIMARY KEY (object_id(32), object_version)
+)

crates/sui-indexer/migrations/pg/2023-08-19-044020_events/up.sql

@@ -1,3 +1,4 @@
+-- TODO: modify queries in indexer reader to take advantage of the new indices
 CREATE TABLE events
 (
     tx_sequence_number          BIGINT       NOT NULL,
@@ -23,8 +24,8 @@ CREATE TABLE events
     timestamp_ms                BIGINT       NOT NULL,
     -- bcs of the Event contents (Event.contents)
     bcs                         BYTEA        NOT NULL,
-    PRIMARY KEY(tx_sequence_number, event_sequence_number, checkpoint_sequence_number)
-) PARTITION BY RANGE (checkpoint_sequence_number);
+    PRIMARY KEY(tx_sequence_number, event_sequence_number)
+) PARTITION BY RANGE (tx_sequence_number);
 CREATE TABLE events_partition_0 PARTITION OF events FOR VALUES FROM (0) TO (MAXVALUE);
 CREATE INDEX events_package ON events (package, tx_sequence_number, event_sequence_number);
 CREATE INDEX events_package_module ON events (package, module, tx_sequence_number, event_sequence_number);

crates/sui-indexer/migrations/pg/2023-08-19-044026_transactions/down.sql

@@ -1,2 +1,3 @@
 -- This file should undo anything in `up.sql`
 DROP TABLE IF EXISTS transactions;
+DROP TABLE IF EXISTS transactions_partition_0;

crates/sui-indexer/migrations/pg/2023-08-19-044026_transactions/up.sql

@@ -18,10 +18,6 @@ CREATE TABLE transactions (
     -- number of successful commands in this transaction, bound by number of command
     -- in a programmaable transaction.
     success_command_count       smallint     NOT NULL,
-    PRIMARY KEY (tx_sequence_number, checkpoint_sequence_number)
-) PARTITION BY RANGE (checkpoint_sequence_number);
+    PRIMARY KEY (tx_sequence_number)
+) PARTITION BY RANGE (tx_sequence_number);
 CREATE TABLE transactions_partition_0 PARTITION OF transactions FOR VALUES FROM (0) TO (MAXVALUE);
-CREATE INDEX transactions_transaction_digest ON transactions (transaction_digest);
-CREATE INDEX transactions_checkpoint_sequence_number ON transactions (checkpoint_sequence_number);
--- only create index for system transactions (0). See types.rs
-CREATE INDEX transactions_transaction_kind ON transactions (transaction_kind) WHERE transaction_kind = 0;

crates/sui-indexer/migrations/pg/2023-08-19-044044_checkpoints/up.sql

@@ -1,27 +1,29 @@
 CREATE TABLE checkpoints
 (
-    sequence_number                     bigint       PRIMARY KEY,
-    checkpoint_digest                   bytea        NOT NULL,
-    epoch                               bigint       NOT NULL,
+    sequence_number                     BIGINT       PRIMARY KEY,
+    checkpoint_digest                   BYTEA        NOT NULL,
+    epoch                               BIGINT       NOT NULL,
     -- total transactions in the network at the end of this checkpoint (including itself)
-    network_total_transactions          bigint       NOT NULL,
-    previous_checkpoint_digest          bytea,
+    network_total_transactions          BIGINT       NOT NULL,
+    previous_checkpoint_digest          BYTEA,
     -- if this checkpoitn is the last checkpoint of an epoch
     end_of_epoch                        boolean      NOT NULL,
     -- array of TranscationDigest in bytes included in this checkpoint
-    tx_digests                          bytea[]      NOT NULL,
+    tx_digests                          BYTEA[]      NOT NULL,
     timestamp_ms                        BIGINT       NOT NULL,
     total_gas_cost                      BIGINT       NOT NULL,
     computation_cost                    BIGINT       NOT NULL,
     storage_cost                        BIGINT       NOT NULL,
     storage_rebate                      BIGINT       NOT NULL,
     non_refundable_storage_fee          BIGINT       NOT NULL,
     -- bcs serialized Vec<CheckpointCommitment> bytes
-    checkpoint_commitments              bytea        NOT NULL,
+    checkpoint_commitments              BYTEA        NOT NULL,
     -- bcs serialized AggregateAuthoritySignature bytes
-    validator_signature                 bytea        NOT NULL,
+    validator_signature                 BYTEA        NOT NULL,
     -- bcs serialzied EndOfEpochData bytes, if the checkpoint marks end of an epoch
-    end_of_epoch_data                   bytea
+    end_of_epoch_data                   BYTEA,
+    min_tx_sequence_number              BIGINT,
+    max_tx_sequence_number              BIGINT
 );
 
 CREATE INDEX checkpoints_epoch ON checkpoints (epoch, sequence_number);

crates/sui-indexer/migrations/pg/2023-08-19-060729_packages/up.sql

@@ -1,6 +1,14 @@
-CREATE TABLE packages 
+CREATE TABLE packages
 (
-    package_id                   bytea          PRIMARY KEY,
+    package_id                   bytea          NOT NULL,
+    original_id                  bytea          NOT NULL,
+    package_version              bigint         NOT NULL,
     -- bcs serialized MovePackage
-    move_package                 bytea          NOT NULL
+    move_package                 bytea          NOT NULL,
+    checkpoint_sequence_number   bigint         NOT NULL,
+    CONSTRAINT packages_pkey PRIMARY KEY (package_id, original_id, package_version),
+    CONSTRAINT packages_unique_package_id UNIQUE (package_id)
 );
+
+CREATE INDEX packages_cp_id_version ON packages (checkpoint_sequence_number, original_id, package_version);
+CREATE INDEX packages_id_version_cp ON packages (original_id, package_version, checkpoint_sequence_number);

crates/sui-indexer/migrations/pg/2023-10-06-204335_tx_indices/down.sql

@@ -2,5 +2,8 @@ DROP TABLE IF EXISTS tx_senders;
 DROP TABLE IF EXISTS tx_recipients;
 DROP TABLE IF EXISTS tx_input_objects;
 DROP TABLE IF EXISTS tx_changed_objects;
-DROP TABLE IF EXISTS tx_calls;
+DROP TABLE IF EXISTS tx_calls_pkg;
+DROP TABLE IF EXISTS tx_calls_mod;
+DROP TABLE IF EXISTS tx_calls_fun;
 DROP TABLE IF EXISTS tx_digests;
+DROP TABLE IF EXISTS tx_kinds;

crates/sui-indexer/migrations/pg/2023-10-06-204335_tx_indices/up.sql

@@ -1,57 +1,70 @@
 CREATE TABLE tx_senders (
-    cp_sequence_number          BIGINT       NOT NULL,
     tx_sequence_number          BIGINT       NOT NULL,
-    -- SuiAddress in bytes.
     sender                      BYTEA        NOT NULL,
-    PRIMARY KEY(sender, tx_sequence_number, cp_sequence_number)
+    PRIMARY KEY(sender, tx_sequence_number)
 );
-CREATE INDEX tx_senders_tx_sequence_number_index ON tx_senders (tx_sequence_number, cp_sequence_number);
 
 CREATE TABLE tx_recipients (
-    cp_sequence_number          BIGINT       NOT NULL,
     tx_sequence_number          BIGINT       NOT NULL,
-    -- SuiAddress in bytes.
     recipient                   BYTEA        NOT NULL,
-    PRIMARY KEY(recipient, tx_sequence_number, cp_sequence_number)
+    sender                      BYTEA        NOT NULL,
+    PRIMARY KEY(recipient, tx_sequence_number)
 );
-CREATE INDEX tx_recipients_tx_sequence_number_index ON tx_recipients (tx_sequence_number, cp_sequence_number);
+CREATE INDEX tx_recipients_sender ON tx_recipients (sender, recipient, tx_sequence_number);
 
 CREATE TABLE tx_input_objects (
-    cp_sequence_number          BIGINT       NOT NULL,
     tx_sequence_number          BIGINT       NOT NULL,
-    -- Object ID in bytes. 
     object_id                   BYTEA        NOT NULL,
-    PRIMARY KEY(object_id, tx_sequence_number, cp_sequence_number)
+    sender                      BYTEA        NOT NULL,
+    PRIMARY KEY(object_id, tx_sequence_number)
 );
 CREATE INDEX tx_input_objects_tx_sequence_number_index ON tx_input_objects (tx_sequence_number);
+CREATE INDEX tx_input_objects_sender ON tx_input_objects (sender, object_id, tx_sequence_number);
 
 CREATE TABLE tx_changed_objects (
-    cp_sequence_number          BIGINT       NOT NULL,
     tx_sequence_number          BIGINT       NOT NULL,
-    -- Object Id in bytes.
     object_id                   BYTEA        NOT NULL,
-    PRIMARY KEY(object_id, tx_sequence_number, cp_sequence_number)
+    sender                      BYTEA        NOT NULL,
+    PRIMARY KEY(object_id, tx_sequence_number)
 );
 CREATE INDEX tx_changed_objects_tx_sequence_number_index ON tx_changed_objects (tx_sequence_number);
+CREATE INDEX tx_changed_objects_sender ON tx_changed_objects (sender, object_id, tx_sequence_number);
+
+CREATE TABLE tx_calls_pkg (
+    tx_sequence_number          BIGINT       NOT NULL,
+    package                     BYTEA        NOT NULL,
+    sender                      BYTEA        NOT NULL,
+    PRIMARY KEY(package, tx_sequence_number)
+);
+CREATE INDEX tx_calls_pkg_sender ON tx_calls_pkg (sender, package, tx_sequence_number);
+
+CREATE TABLE tx_calls_mod (
+    tx_sequence_number          BIGINT       NOT NULL,
+    package                     BYTEA        NOT NULL,
+    module                      TEXT         NOT NULL,
+    sender                      BYTEA        NOT NULL,
+    PRIMARY KEY(package, module, tx_sequence_number)
+);
+CREATE INDEX tx_calls_mod_sender ON tx_calls_mod (sender, package, module, tx_sequence_number);
 
-CREATE TABLE tx_calls (
-    cp_sequence_number          BIGINT       NOT NULL,
+CREATE TABLE tx_calls_fun (
     tx_sequence_number          BIGINT       NOT NULL,
     package                     BYTEA        NOT NULL,
     module                      TEXT         NOT NULL,
     func                        TEXT         NOT NULL,
-    -- 1. Using Primary Key as a unique index.
-    -- 2. Diesel does not like tables with no primary key.
-    PRIMARY KEY(package, tx_sequence_number, cp_sequence_number)
+    sender                      BYTEA        NOT NULL,
+    PRIMARY KEY(package, module, func, tx_sequence_number)
 );
-CREATE INDEX tx_calls_module ON tx_calls (package, module, tx_sequence_number, cp_sequence_number);
-CREATE INDEX tx_calls_func ON tx_calls (package, module, func, tx_sequence_number, cp_sequence_number);
-CREATE INDEX tx_calls_tx_sequence_number ON tx_calls (tx_sequence_number, cp_sequence_number);
+CREATE INDEX tx_calls_fun_sender ON tx_calls_fun (sender, package, module, func, tx_sequence_number);
 
--- un-partitioned table for tx_digest -> (cp_sequence_number, tx_sequence_number) lookup.
 CREATE TABLE tx_digests (
     tx_digest                   BYTEA        PRIMARY KEY,
-    cp_sequence_number          BIGINT       NOT NULL,
     tx_sequence_number          BIGINT       NOT NULL
 );
 CREATE INDEX tx_digests_tx_sequence_number ON tx_digests (tx_sequence_number);
+
+CREATE TABLE tx_kinds (
+    tx_sequence_number          BIGINT       NOT NULL,
+    tx_kind                     SMALLINT     NOT NULL,
+    PRIMARY KEY(tx_kind, tx_sequence_number)
+);

crates/sui-indexer/migrations/pg/2023-11-29-193859_advance_partition/up.sql

@@ -1,10 +1,10 @@
-CREATE OR REPLACE PROCEDURE advance_partition(table_name TEXT, last_epoch BIGINT, new_epoch BIGINT, last_epoch_start_cp BIGINT, new_epoch_start_cp BIGINT)
+CREATE OR REPLACE PROCEDURE advance_partition(table_name TEXT, last_epoch BIGINT, new_epoch BIGINT, last_epoch_start BIGINT, new_epoch_start BIGINT)
 LANGUAGE plpgsql
 AS $$
 BEGIN
     EXECUTE format('ALTER TABLE %I DETACH PARTITION %I_partition_%s', table_name, table_name, last_epoch);
-    EXECUTE format('ALTER TABLE %I ATTACH PARTITION %I_partition_%s FOR VALUES FROM (%L) TO (%L)', table_name, table_name, last_epoch, last_epoch_start_cp, new_epoch_start_cp);
-    EXECUTE format('CREATE TABLE IF NOT EXISTS %I_partition_%s PARTITION OF %I FOR VALUES FROM (%L) TO (MAXVALUE)', table_name, new_epoch, table_name, new_epoch_start_cp);
+    EXECUTE format('ALTER TABLE %I ATTACH PARTITION %I_partition_%s FOR VALUES FROM (%L) TO (%L)', table_name, table_name, last_epoch, last_epoch_start, new_epoch_start);
+    EXECUTE format('CREATE TABLE IF NOT EXISTS %I_partition_%s PARTITION OF %I FOR VALUES FROM (%L) TO (MAXVALUE)', table_name, new_epoch, table_name, new_epoch_start);
 END;
 $$;
 

crates/sui-indexer/migrations/pg/2024-05-05-155158_obj_indices/down.sql

@@ -0,0 +1 @@
+DROP TABLE IF EXISTS objects_version;

crates/sui-indexer/migrations/pg/2024-05-05-155158_obj_indices/up.sql

@@ -0,0 +1,31 @@
+-- Indexing table mapping an object's ID and version to its checkpoint
+-- sequence number, partitioned by the first byte of its Object ID.
+CREATE TABLE objects_version (
+    object_id           bytea         NOT NULL,
+    object_version      bigint        NOT NULL,
+    cp_sequence_number  bigint        NOT NULL,
+    PRIMARY KEY (object_id, object_version)
+) PARTITION BY RANGE (object_id);
+
+-- Create a partition for each first byte value.
+DO $$
+DECLARE
+    lo text;
+    hi text;
+BEGIN
+    FOR i IN 0..254 LOOP
+        lo := LPAD(TO_HEX(i), 2, '0');
+        hi := LPAD(TO_HEX(i + 1), 2, '0');
+        EXECUTE FORMAT($F$
+            CREATE TABLE objects_version_%1$s PARTITION OF objects_version FOR VALUES
+            FROM (E'\\x%1$s00000000000000000000000000000000000000000000000000000000000000')
+            TO   (E'\\x%2$s00000000000000000000000000000000000000000000000000000000000000');
+        $F$, lo, hi);
+    END LOOP;
+END;
+$$ LANGUAGE plpgsql;
+
+-- Special case for the last partition, because of the upper bound.
+CREATE TABLE objects_version_ff PARTITION OF objects_version FOR VALUES
+FROM (E'\\xff00000000000000000000000000000000000000000000000000000000000000')
+TO   (MAXVALUE);

crates/sui-indexer/migrations/pg/2024-06-14-045801_event_indices/down.sql

@@ -0,0 +1,7 @@
+DROP TABLE IF EXISTS event_emit_package;
+DROP TABLE IF EXISTS event_emit_module;
+DROP TABLE IF EXISTS event_struct_package;
+DROP TABLE IF EXISTS event_struct_module;
+DROP TABLE IF EXISTS event_struct_name;
+DROP TABLE IF EXISTS event_struct_instantiation;
+DROP TABLE IF EXISTS event_senders;