From 19d28788e45f53e6b575cc6266cc782d51b546c8 Mon Sep 17 00:00:00 2001 From: cwastche Date: Sat, 22 Jun 2024 18:23:23 +0200 Subject: [PATCH 1/8] - Changed return statements to array filters - Moved object mutations from map to forEach --- indexer/src/main.ts | 218 ++++++++++++++++++++++++-------------------- 1 file changed, 118 insertions(+), 100 deletions(-) diff --git a/indexer/src/main.ts b/indexer/src/main.ts index 46ab3cbd1..cb48acac0 100644 --- a/indexer/src/main.ts +++ b/indexer/src/main.ts @@ -82,112 +82,126 @@ export default async function transform({ const cumulativeGasUsages: Array = []; const blockNumber = padString(toHexString(header.blockNumber), 8); - const isPendingBlock = padString(header.blockHash, 32) === NULL_BLOCK_HASH; const blockHash = padString(header.blockHash, 32); + const isPendingBlock = blockHash === NULL_BLOCK_HASH; const blockLogsBloom = new Bloom(); const transactionTrie = new Trie(); const receiptTrie = new Trie(); const store: Array = []; - const maybeTrieData: Array = (events ?? []).map( - ({ transaction, receipt, event }) => { - console.log( - "🔍 Processing transaction with Starknet hash: ", - transaction.meta.hash, - ); - // Can be false if the transaction is not related to a specific instance of the Kakarot contract. - // This is typically the case if there are multiple Kakarot contracts on the same chain. - const isKakarotTx = isKakarotTransaction(transaction); - if (!isKakarotTx) { - return null; - } - - // Skip if the transaction_executed event contains "eth validation failed". - if (ethValidationFailed(event)) { - return null; - } - - const typedEthTx = toTypedEthTx({ transaction }); - // Can be null if: - // 1. The transaction is missing calldata. - // 2. The transaction is a multi-call. - // 3. The length of the signature array is different from 5. - // 4. The chain id is not encoded in the v param of the signature for a - // Legacy transaction. - // 5. The deserialization of the transaction fails. - if (!typedEthTx) { - return null; - } + const filteredEvents = (events ?? []) + // Can be false if the transaction is not related to a specific instance of the Kakarot contract. + // This is typically the case if there are multiple Kakarot contracts on the same chain. + .filter((event) => isKakarotTransaction(event.transaction)) + // Skip if the transaction_executed event contains "eth validation failed". + .filter((event) => !ethValidationFailed(event)) + .map((event) => ({ + event: event, + typedEthTx: toTypedEthTx(event.transaction), + })) + // Can be null if: + // 1. The transaction is missing calldata. + // 2. The transaction is a multi-call. + // 3. The length of the signature array is different from 5. + // 4. The chain id is not encoded in the v param of the signature for a + // Legacy transaction. + // 5. The deserialization of the transaction fails. + .filter((eventExtended) => eventExtended.typedEthTx) + .map((eventExtended) => { const ethTx = typedTransactionToEthTx({ - typedTransaction: typedEthTx, - receipt, + typedTransaction: eventExtended.typedEthTx, + receipt: eventExtended.event.receipt, blockNumber, blockHash, isPendingBlock, }); - // Can be null if: - // 1. The typed transaction if missing a signature param (v, r, s). - if (!ethTx) { - return null; - } - // Can be null if: - // 1. The event is part of the defined ignored events (see IGNORED_KEYS). - // 2. The event has an invalid number of keys. - const ethLogs = receipt.events - .map((e) => { - return toEthLog({ - transaction: ethTx, + return { ...eventExtended, ethTx: ethTx }; + }) + // Can be null if: + // 1. The typed transaction if missing a signature param (v, r, s). + .filter((eventExtended) => eventExtended.ethTx) + .map((eventExtended) => { + const ethLogs = eventExtended.event.receipt.events + .map((e) => + toEthLog({ + transaction: eventExtended.ethTx, event: e, blockNumber, blockHash, isPendingBlock, - }); - }) + }) + ) + // Can be null if: + // 1. The event is part of the defined ignored events (see IGNORED_KEYS). + // 2. The event has an invalid number of keys. .filter((e) => e !== null) as JsonRpcLog[]; - const ethLogsIndexed = ethLogs.map((log, index) => { + + return { + ...eventExtended, + ethLogs, + }; + }) + .map((eventExtended) => { + const ethLogsIndexed = eventExtended.ethLogs.map((log, index) => { log.logIndex = index.toString(); return log; }); const ethReceipt = toEthReceipt({ - transaction: ethTx, + transaction: eventExtended.ethTx, logs: ethLogsIndexed, - event, + event: eventExtended.event, cumulativeGasUsed, blockNumber, blockHash, isPendingBlock, }); - cumulativeGasUsed += BigInt(ethReceipt.gasUsed); - // ethTx.transactionIndex can be null (if the block is pending) but - // Number(null) is 0 so this won't panic. - cumulativeGasUsages[Number(ethTx.transactionIndex)] = cumulativeGasUsed; + return { + ...eventExtended, + ethReceipt, + event: eventExtended.event, + }; + }); - // Add all the eth data to the store. - store.push({ collection: "transactions", data: { tx: ethTx } }); - store.push({ collection: "receipts", data: { receipt: ethReceipt } }); - ethLogs.forEach((ethLog) => { - store.push({ collection: "logs", data: { log: ethLog } }); - }); + filteredEvents.forEach((eventExtended) => { + cumulativeGasUsed += BigInt(eventExtended.ethReceipt.gasUsed); + // ethTx.transactionIndex can be null (if the block is pending) but + // Number(null) is 0 so this won't panic. + cumulativeGasUsages[Number(eventExtended.ethTx.transactionIndex)] = + cumulativeGasUsed; - // Add the logs bloom of the receipt to the block logs bloom. - const receiptLogsBloom = new Bloom(hexToBytes(ethReceipt.logsBloom)); - blockLogsBloom.or(receiptLogsBloom); + // Add all the eth data to the store. + store.push({ + collection: "transactions", + data: { tx: eventExtended.ethTx }, + }); + store.push({ + collection: "receipts", + data: { receipt: eventExtended.ethReceipt }, + }); + eventExtended.ethLogs.forEach((ethLog) => + store.push({ collection: "logs", data: { log: ethLog } }) + ); - /// Return the trie data. - return createTrieData({ - transactionIndex: Number(ethTx.transactionIndex), - typedTransaction: typedEthTx, - receipt: ethReceipt, - }); - }, - ); + // Add the logs bloom of the receipt to the block logs bloom. + const receiptLogsBloom = new Bloom( + hexToBytes(eventExtended.ethReceipt.logsBloom), + ); + blockLogsBloom.or(receiptLogsBloom); + }); - // Filter out the null values for the trie data. - const trieData = maybeTrieData.filter((x) => x !== null) as Array; + const trieData: Array = filteredEvents + .map((eventExtended) => + createTrieData({ + transactionIndex: Number(eventExtended.ethTx.transactionIndex), + typedTransaction: eventExtended.typedEthTx, + receipt: eventExtended.ethReceipt, + }) + ) + .filter((x) => x !== null) as Array; // Compute the blooms in an async manner. await Promise.all( @@ -208,55 +222,59 @@ export default async function transform({ // Sort the cumulative gas uses by descending transaction index. cumulativeGasUsages.reverse(); - (transactions ?? []).forEach(({ transaction, receipt }) => { - if (isRevertedWithOutOfResources(receipt)) { - // Can be false if the transaction is not related to a specific instance of the Kakarot contract. - // This is typically the case if there are multiple Kakarot contracts on the same chain. - const isKakarotTx = isKakarotTransaction(transaction); - if (!isKakarotTx) { - return; - } - - const ethTx = toEthTx({ - transaction, - receipt, + const filteredTransactions = (transactions ?? []) + .filter((transactionWithReceipt) => + isRevertedWithOutOfResources(transactionWithReceipt.receipt) + ) + .filter((transactionWithReceipt) => + isKakarotTransaction(transactionWithReceipt.transaction) + ) + .map((transactionWithReceipt) => ({ + transactionWithReceipt: transactionWithReceipt, + ethTx: toEthTx({ + transaction: transactionWithReceipt.transaction, + receipt: transactionWithReceipt.receipt, blockNumber, blockHash, isPendingBlock, - }); - if (!ethTx) { - return; - } - + }), + })) + .map((transactionWithReceiptExtended) => { // Get the cumulative gas used for the reverted transaction. // Example: // const cumulativeGasUsages = [300n, undefined, undefined, 200n, undefined, 100n, undefined, undefined, 10n, undefined]; // const ethTx = { transactionIndex: 5 }; // const revertedTransactionCumulativeGasUsed = 100n; - const len = cumulativeGasUsages.length; const revertedTransactionCumulativeGasUsed = cumulativeGasUsages.find((gas, i) => { return ( - Number(ethTx.transactionIndex) >= len - 1 - i && gas !== undefined + Number(transactionWithReceiptExtended.ethTx.transactionIndex) >= + cumulativeGasUsages.length - 1 - i && gas !== undefined ); }) ?? 0n; const ethReceipt = toRevertedOutOfResourcesReceipt({ - transaction: ethTx, + transaction: transactionWithReceiptExtended.ethTx, blockNumber, blockHash, cumulativeGasUsed: revertedTransactionCumulativeGasUsed, isPendingBlock, }); - store.push({ collection: "transactions", data: { tx: ethTx } }); - store.push({ - collection: "receipts", - data: { - receipt: ethReceipt, - }, - }); - } + return { + ...transactionWithReceiptExtended, + ethReceipt, + }; + }); + + filteredTransactions.forEach((transaction) => { + store.push({ collection: "transactions", data: { tx: transaction.ethTx } }); + store.push({ + collection: "receipts", + data: { + receipt: transaction.ethReceipt, + }, + }); }); const ethHeader = await toEthHeader({ From efff339d6303d2f0bf08f14ed924a34fdc7c9679 Mon Sep 17 00:00:00 2001 From: cwastche Date: Tue, 25 Jun 2024 11:01:04 +0200 Subject: [PATCH 2/8] Address comments: - Add name constants for Collection - cleanup --- indexer/src/main.ts | 21 ++++++++++++--------- indexer/src/types/storeItem.ts | 13 +++++++++---- 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/indexer/src/main.ts b/indexer/src/main.ts index cb48acac0..47840775e 100644 --- a/indexer/src/main.ts +++ b/indexer/src/main.ts @@ -30,7 +30,7 @@ import { } from "./types/receipt.ts"; import { JsonRpcLog, toEthLog } from "./types/log.ts"; import { createTrieData, TrieData } from "./types/tries.ts"; -import { StoreItem } from "./types/storeItem.ts"; +import { Collection, StoreItem } from "./types/storeItem.ts"; // Starknet import { BlockHeader, @@ -117,7 +117,7 @@ export default async function transform({ isPendingBlock, }); - return { ...eventExtended, ethTx: ethTx }; + return { ...eventExtended, ethTx }; }) // Can be null if: // 1. The typed transaction if missing a signature param (v, r, s). @@ -175,15 +175,15 @@ export default async function transform({ // Add all the eth data to the store. store.push({ - collection: "transactions", + collection: Collection.Transactions, data: { tx: eventExtended.ethTx }, }); store.push({ - collection: "receipts", + collection: Collection.Receipts, data: { receipt: eventExtended.ethReceipt }, }); eventExtended.ethLogs.forEach((ethLog) => - store.push({ collection: "logs", data: { log: ethLog } }) + store.push({ collection: Collection.Logs, data: { log: ethLog } }) ); // Add the logs bloom of the receipt to the block logs bloom. @@ -249,7 +249,7 @@ export default async function transform({ cumulativeGasUsages.find((gas, i) => { return ( Number(transactionWithReceiptExtended.ethTx.transactionIndex) >= - cumulativeGasUsages.length - 1 - i && gas !== undefined + cumulativeGasUsages.length - 1 - i && gas ); }) ?? 0n; @@ -268,9 +268,12 @@ export default async function transform({ }); filteredTransactions.forEach((transaction) => { - store.push({ collection: "transactions", data: { tx: transaction.ethTx } }); store.push({ - collection: "receipts", + collection: Collection.Transactions, + data: { tx: transaction.ethTx }, + }); + store.push({ + collection: Collection.Receipts, data: { receipt: transaction.ethReceipt, }, @@ -288,7 +291,7 @@ export default async function transform({ isPendingBlock, }); store.push({ - collection: "headers", + collection: Collection.Headers, data: { header: ethHeader }, }); diff --git a/indexer/src/types/storeItem.ts b/indexer/src/types/storeItem.ts index 891c82ffd..52e75a0cc 100644 --- a/indexer/src/types/storeItem.ts +++ b/indexer/src/types/storeItem.ts @@ -6,12 +6,17 @@ import { JsonRpcReceipt } from "./receipt.ts"; import { JsonRpcTx } from "../deps.ts"; import { JsonRpcBlock } from "./header.ts"; -type Collection = "transactions" | "logs" | "receipts" | "headers"; +export enum Collection { + Transactions = "transactions", + Logs = "logs", + Receipts = "receipts", + Headers = "headers", +} export type StoreItem = { collection: C; - data: C extends "transactions" ? { tx: JsonRpcTx } - : C extends "logs" ? { log: JsonRpcLog } - : C extends "receipts" ? { receipt: JsonRpcReceipt } + data: C extends Collection.Transactions ? { tx: JsonRpcTx } + : C extends Collection.Logs ? { log: JsonRpcLog } + : C extends Collection.Receipts ? { receipt: JsonRpcReceipt } : { header: JsonRpcBlock }; }; From 1e4f39b65c030ff9fdb96e780229aad171440823 Mon Sep 17 00:00:00 2001 From: cwastche Date: Sun, 21 Jul 2024 15:41:38 +0200 Subject: [PATCH 3/8] Fix typing --- indexer/src/main.ts | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/indexer/src/main.ts b/indexer/src/main.ts index 47840775e..d628a1a9a 100644 --- a/indexer/src/main.ts +++ b/indexer/src/main.ts @@ -95,10 +95,10 @@ export default async function transform({ // This is typically the case if there are multiple Kakarot contracts on the same chain. .filter((event) => isKakarotTransaction(event.transaction)) // Skip if the transaction_executed event contains "eth validation failed". - .filter((event) => !ethValidationFailed(event)) + .filter((event) => !ethValidationFailed(event.event)) .map((event) => ({ event: event, - typedEthTx: toTypedEthTx(event.transaction), + typedEthTx: toTypedEthTx({ transaction: event.transaction }), })) // Can be null if: // 1. The transaction is missing calldata. @@ -107,10 +107,10 @@ export default async function transform({ // 4. The chain id is not encoded in the v param of the signature for a // Legacy transaction. // 5. The deserialization of the transaction fails. - .filter((eventExtended) => eventExtended.typedEthTx) + .filter((eventExtended) => eventExtended.typedEthTx !== null) .map((eventExtended) => { const ethTx = typedTransactionToEthTx({ - typedTransaction: eventExtended.typedEthTx, + typedTransaction: eventExtended.typedEthTx!, receipt: eventExtended.event.receipt, blockNumber, blockHash, @@ -121,7 +121,11 @@ export default async function transform({ }) // Can be null if: // 1. The typed transaction if missing a signature param (v, r, s). - .filter((eventExtended) => eventExtended.ethTx) + .filter(( + eventExtended, + ): eventExtended is typeof eventExtended & { + ethTx: NonNullable; + } => eventExtended.ethTx !== null) .map((eventExtended) => { const ethLogs = eventExtended.event.receipt.events .map((e) => @@ -152,7 +156,7 @@ export default async function transform({ const ethReceipt = toEthReceipt({ transaction: eventExtended.ethTx, logs: ethLogsIndexed, - event: eventExtended.event, + event: eventExtended.event.event, cumulativeGasUsed, blockNumber, blockHash, @@ -197,7 +201,7 @@ export default async function transform({ .map((eventExtended) => createTrieData({ transactionIndex: Number(eventExtended.ethTx.transactionIndex), - typedTransaction: eventExtended.typedEthTx, + typedTransaction: eventExtended.typedEthTx!, receipt: eventExtended.ethReceipt, }) ) @@ -248,13 +252,13 @@ export default async function transform({ const revertedTransactionCumulativeGasUsed = cumulativeGasUsages.find((gas, i) => { return ( - Number(transactionWithReceiptExtended.ethTx.transactionIndex) >= + Number(transactionWithReceiptExtended.ethTx!.transactionIndex) >= cumulativeGasUsages.length - 1 - i && gas ); }) ?? 0n; const ethReceipt = toRevertedOutOfResourcesReceipt({ - transaction: transactionWithReceiptExtended.ethTx, + transaction: transactionWithReceiptExtended.ethTx!, blockNumber, blockHash, cumulativeGasUsed: revertedTransactionCumulativeGasUsed, @@ -270,7 +274,7 @@ export default async function transform({ filteredTransactions.forEach((transaction) => { store.push({ collection: Collection.Transactions, - data: { tx: transaction.ethTx }, + data: { tx: transaction.ethTx! }, }); store.push({ collection: Collection.Receipts, From fb5b150eb43d813b4fc36704dc0d95e39597a43a Mon Sep 17 00:00:00 2001 From: cwastche Date: Mon, 5 Aug 2024 05:32:47 +0200 Subject: [PATCH 4/8] Divide transform into helper functions for clarity --- indexer/src/main.ts | 382 +++++++++++++++++--------------- indexer/src/types/interfaces.ts | 27 ++- 2 files changed, 225 insertions(+), 184 deletions(-) diff --git a/indexer/src/main.ts b/indexer/src/main.ts index d628a1a9a..038681220 100644 --- a/indexer/src/main.ts +++ b/indexer/src/main.ts @@ -37,12 +37,18 @@ import { Config, EventWithTransaction, hexToBytes, + JsonRpcTx, NetworkOptions, SinkOptions, TransactionWithReceipt, } from "./deps.ts"; // Eth import { Bloom, Trie } from "./deps.ts"; +import { + BlockInfo, + ProcessedEvent, + ProcessedTransaction, +} from "./types/interfaces.ts"; export const config: Config = { streamUrl: STREAM_URL, @@ -73,33 +79,70 @@ export default async function transform({ events: EventWithTransaction[]; transactions: TransactionWithReceipt[]; }) { - // Accumulate the gas used in the block in order to calculate the cumulative gas used. - // We increment it by the gas used in each transaction in the flatMap iteration. - let cumulativeGasUsed = 0n; - // An array containing the cumulative gas used up to that transaction, indexed by - // transaction index. This is used to later get the cumulative gas used for an out of - // resources transaction. - const cumulativeGasUsages: Array = []; - - const blockNumber = padString(toHexString(header.blockNumber), 8); - const blockHash = padString(header.blockHash, 32); - const isPendingBlock = blockHash === NULL_BLOCK_HASH; - const blockLogsBloom = new Bloom(); - const transactionTrie = new Trie(); - const receiptTrie = new Trie(); - + const blockInfo = createBlockInfo(header); const store: Array = []; + const blockLogsBloom = new Bloom(); - const filteredEvents = (events ?? []) + const processedEvents = events // Can be false if the transaction is not related to a specific instance of the Kakarot contract. // This is typically the case if there are multiple Kakarot contracts on the same chain. .filter((event) => isKakarotTransaction(event.transaction)) // Skip if the transaction_executed event contains "eth validation failed". .filter((event) => !ethValidationFailed(event.event)) - .map((event) => ({ - event: event, - typedEthTx: toTypedEthTx({ transaction: event.transaction }), - })) + .map(processEvent(blockInfo)) + .filter((event) => event !== null); + + const { cumulativeGasUsed, cumulativeGasUsages } = + accumulateGasAndUpdateStore(processedEvents, store, blockLogsBloom); + + const trieData: Array = processedEvents + .map((event) => + createTrieData({ + transactionIndex: Number(event.ethTx.transactionIndex), + typedTransaction: event.typedEthTx, + receipt: event.ethReceipt, + }) + ) + .filter((x) => x !== null); + + const { transactionTrie, receiptTrie } = await computeBlooms(trieData); + + // Sort the cumulative gas uses by descending transaction index. + cumulativeGasUsages.reverse(); + + const processedTransactions = processTransactions( + transactions, + blockInfo, + cumulativeGasUsages + ); + updateStoreWithTransactions(store, processedTransactions); + + const ethHeader = await toEthHeader({ + header: header, + gasUsed: cumulativeGasUsed, + logsBloom: blockLogsBloom, + receiptRoot: receiptTrie.root(), + transactionRoot: transactionTrie.root(), + ...blockInfo, + }); + store.push({ + collection: Collection.Headers, + data: { header: ethHeader }, + }); + + return store; +} + +function createBlockInfo(header: BlockHeader): BlockInfo { + const blockNumber = padString(toHexString(header.blockNumber), 8); + const blockHash = padString(header.blockHash, 32); + const isPendingBlock = blockHash === NULL_BLOCK_HASH; + return { blockNumber, blockHash, isPendingBlock }; +} + +function processEvent(blockInfo: BlockInfo) { + return (event: EventWithTransaction): ProcessedEvent | null => { + const typedEthTx = toTypedEthTx({ transaction: event.transaction }); // Can be null if: // 1. The transaction is missing calldata. // 2. The transaction is a multi-call. @@ -107,105 +150,94 @@ export default async function transform({ // 4. The chain id is not encoded in the v param of the signature for a // Legacy transaction. // 5. The deserialization of the transaction fails. - .filter((eventExtended) => eventExtended.typedEthTx !== null) - .map((eventExtended) => { - const ethTx = typedTransactionToEthTx({ - typedTransaction: eventExtended.typedEthTx!, - receipt: eventExtended.event.receipt, - blockNumber, - blockHash, - isPendingBlock, - }); - - return { ...eventExtended, ethTx }; - }) + if (!typedEthTx) return null; + + const ethTx = typedTransactionToEthTx({ + typedTransaction: typedEthTx!, + receipt: event.receipt, + ...blockInfo, + }); // Can be null if: - // 1. The typed transaction if missing a signature param (v, r, s). - .filter(( - eventExtended, - ): eventExtended is typeof eventExtended & { - ethTx: NonNullable; - } => eventExtended.ethTx !== null) - .map((eventExtended) => { - const ethLogs = eventExtended.event.receipt.events - .map((e) => - toEthLog({ - transaction: eventExtended.ethTx, - event: e, - blockNumber, - blockHash, - isPendingBlock, - }) - ) - // Can be null if: - // 1. The event is part of the defined ignored events (see IGNORED_KEYS). - // 2. The event has an invalid number of keys. - .filter((e) => e !== null) as JsonRpcLog[]; - - return { - ...eventExtended, - ethLogs, - }; - }) - .map((eventExtended) => { - const ethLogsIndexed = eventExtended.ethLogs.map((log, index) => { - log.logIndex = index.toString(); - return log; - }); - - const ethReceipt = toEthReceipt({ - transaction: eventExtended.ethTx, - logs: ethLogsIndexed, - event: eventExtended.event.event, - cumulativeGasUsed, - blockNumber, - blockHash, - isPendingBlock, - }); - - return { - ...eventExtended, - ethReceipt, - event: eventExtended.event, - }; + // The typed transaction is missing a signature param (v, r, s). + if (!ethTx) return null; + + const ethLogs = event.receipt.events + .map((e) => toEthLog({ transaction: ethTx, event: e, ...blockInfo })) + // Can be null if: + // 1. The event is part of the defined ignored events (see IGNORED_KEYS). + // 2. The event has an invalid number of keys. + .filter((e) => e !== null); + + const ethLogsIndexed = ethLogs.map((log, index) => { + // ...log, + // logIndex: index.toString(), + log.logIndex = index.toString(); + return log; }); - filteredEvents.forEach((eventExtended) => { - cumulativeGasUsed += BigInt(eventExtended.ethReceipt.gasUsed); + const ethReceipt = toEthReceipt({ + transaction: ethTx as JsonRpcTx, + logs: ethLogsIndexed, + event: event.event, + cumulativeGasUsed: 0n, // This will be updated later + ...blockInfo, + }); + + return { event, typedEthTx, ethTx, ethLogs, ethReceipt }; + }; +} + +function accumulateGasAndUpdateStore( + processedEvents: ProcessedEvent[], + store: Array, + blockLogsBloom: Bloom +): { cumulativeGasUsed: bigint; cumulativeGasUsages: bigint[] } { + let cumulativeGasUsed = 0n; + const cumulativeGasUsages: bigint[] = []; + + processedEvents.forEach((event) => { + // Accumulate the gas used in the block in order to calculate the cumulative gas used. + // We increment it by the gas used in each transaction. + cumulativeGasUsed += BigInt(event.ethReceipt.gasUsed); // ethTx.transactionIndex can be null (if the block is pending) but // Number(null) is 0 so this won't panic. - cumulativeGasUsages[Number(eventExtended.ethTx.transactionIndex)] = - cumulativeGasUsed; + const transactionIndex = Number(event.ethTx.transactionIndex); + // An array containing the cumulative gas used up to that transaction, indexed by + // transaction index. This is used to later get the cumulative gas used for an out of + // resources transaction. + cumulativeGasUsages[transactionIndex] = cumulativeGasUsed; - // Add all the eth data to the store. - store.push({ - collection: Collection.Transactions, - data: { tx: eventExtended.ethTx }, - }); - store.push({ - collection: Collection.Receipts, - data: { receipt: eventExtended.ethReceipt }, - }); - eventExtended.ethLogs.forEach((ethLog) => - store.push({ collection: Collection.Logs, data: { log: ethLog } }) - ); - - // Add the logs bloom of the receipt to the block logs bloom. - const receiptLogsBloom = new Bloom( - hexToBytes(eventExtended.ethReceipt.logsBloom), - ); - blockLogsBloom.or(receiptLogsBloom); + updateStore(store, event); + updateBlockLogsBloom(blockLogsBloom, event); }); - const trieData: Array = filteredEvents - .map((eventExtended) => - createTrieData({ - transactionIndex: Number(eventExtended.ethTx.transactionIndex), - typedTransaction: eventExtended.typedEthTx!, - receipt: eventExtended.ethReceipt, - }) - ) - .filter((x) => x !== null) as Array; + return { cumulativeGasUsed, cumulativeGasUsages }; +} + +function updateStore(store: Array, event: ProcessedEvent) { + store.push({ + collection: Collection.Transactions, + data: { tx: event.ethTx }, + }); + store.push({ + collection: Collection.Receipts, + data: { receipt: event.ethReceipt }, + }); + event.ethLogs.forEach((log) => + store.push({ collection: Collection.Logs, data: { log } }) + ); +} + +function updateBlockLogsBloom(blockLogsBloom: Bloom, event: ProcessedEvent) { + const receiptLogsBloom = new Bloom(hexToBytes(event.ethReceipt.logsBloom)); + blockLogsBloom.or(receiptLogsBloom); +} + +async function computeBlooms( + trieData: Array +): Promise<{ transactionTrie: Trie; receiptTrie: Trie }> { + const transactionTrie = new Trie(); + const receiptTrie = new Trie(); // Compute the blooms in an async manner. await Promise.all( @@ -219,85 +251,69 @@ export default async function transform({ await transactionTrie.put(encodedTransactionIndex, encodedTransaction); // Add the receipt to the receipt trie. await receiptTrie.put(encodedTransactionIndex, encodedReceipt); - }, - ), + } + ) ); - // Sort the cumulative gas uses by descending transaction index. - cumulativeGasUsages.reverse(); + return { transactionTrie, receiptTrie }; +} - const filteredTransactions = (transactions ?? []) - .filter((transactionWithReceipt) => - isRevertedWithOutOfResources(transactionWithReceipt.receipt) +function processTransactions( + transactions: TransactionWithReceipt[], + blockInfo: BlockInfo, + cumulativeGasUsages: bigint[] +): ProcessedTransaction[] { + return transactions + .filter( + (tx) => + isRevertedWithOutOfResources(tx.receipt) && + isKakarotTransaction(tx.transaction) ) - .filter((transactionWithReceipt) => - isKakarotTransaction(transactionWithReceipt.transaction) - ) - .map((transactionWithReceipt) => ({ - transactionWithReceipt: transactionWithReceipt, - ethTx: toEthTx({ - transaction: transactionWithReceipt.transaction, - receipt: transactionWithReceipt.receipt, - blockNumber, - blockHash, - isPendingBlock, - }), - })) - .map((transactionWithReceiptExtended) => { - // Get the cumulative gas used for the reverted transaction. - // Example: - // const cumulativeGasUsages = [300n, undefined, undefined, 200n, undefined, 100n, undefined, undefined, 10n, undefined]; - // const ethTx = { transactionIndex: 5 }; - // const revertedTransactionCumulativeGasUsed = 100n; - const revertedTransactionCumulativeGasUsed = - cumulativeGasUsages.find((gas, i) => { - return ( - Number(transactionWithReceiptExtended.ethTx!.transactionIndex) >= - cumulativeGasUsages.length - 1 - i && gas - ); - }) ?? 0n; - - const ethReceipt = toRevertedOutOfResourcesReceipt({ - transaction: transactionWithReceiptExtended.ethTx!, - blockNumber, - blockHash, - cumulativeGasUsed: revertedTransactionCumulativeGasUsed, - isPendingBlock, - }); - - return { - ...transactionWithReceiptExtended, - ethReceipt, - }; - }); + .map((tx) => createProcessedTransaction(tx, blockInfo, cumulativeGasUsages)) + .filter((tx) => tx !== null); +} - filteredTransactions.forEach((transaction) => { - store.push({ - collection: Collection.Transactions, - data: { tx: transaction.ethTx! }, - }); - store.push({ - collection: Collection.Receipts, - data: { - receipt: transaction.ethReceipt, - }, - }); +function createProcessedTransaction( + tx: TransactionWithReceipt, + blockInfo: BlockInfo, + cumulativeGasUsages: bigint[] +): ProcessedTransaction | null { + const ethTx = toEthTx({ + transaction: tx.transaction, + receipt: tx.receipt, + ...blockInfo, }); + if (!ethTx) return null; + // Get the cumulative gas used for the reverted transaction. + // Example: + // const cumulativeGasUsages = [300n, undefined, undefined, 200n, undefined, 100n, undefined, undefined, 10n, undefined]; + // const ethTx = { transactionIndex: 5 }; + // const revertedTransactionCumulativeGasUsed = 100n; + const revertedTransactionCumulativeGasUsed = + cumulativeGasUsages.find( + (gas, i) => + Number(ethTx.transactionIndex) >= cumulativeGasUsages.length - 1 - i && + gas + ) ?? 0n; - const ethHeader = await toEthHeader({ - header: header, - gasUsed: cumulativeGasUsed, - logsBloom: blockLogsBloom, - receiptRoot: receiptTrie.root(), - transactionRoot: transactionTrie.root(), - blockNumber, - blockHash, - isPendingBlock, - }); - store.push({ - collection: Collection.Headers, - data: { header: ethHeader }, + const ethReceipt = toRevertedOutOfResourcesReceipt({ + transaction: ethTx as JsonRpcTx, + cumulativeGasUsed: revertedTransactionCumulativeGasUsed, + ...blockInfo, }); - return store; + return { ethTx, ethReceipt }; +} + +function updateStoreWithTransactions( + store: Array, + processedTransactions: ProcessedTransaction[] +) { + processedTransactions.forEach(({ ethTx, ethReceipt }) => { + store.push({ collection: Collection.Transactions, data: { tx: ethTx as JsonRpcTx } }); + store.push({ + collection: Collection.Receipts, + data: { receipt: ethReceipt }, + }); + }); } diff --git a/indexer/src/types/interfaces.ts b/indexer/src/types/interfaces.ts index 4080c9a44..3638d2a94 100644 --- a/indexer/src/types/interfaces.ts +++ b/indexer/src/types/interfaces.ts @@ -1,5 +1,9 @@ // Starknet -import { Transaction, TransactionReceipt } from "../deps.ts"; +import { + EventWithTransaction, + Transaction, + TransactionReceipt, +} from "../deps.ts"; // Eth import { @@ -8,6 +12,8 @@ import { PrefixedHexString, TypedTransaction, } from "../deps.ts"; +import { JsonRpcLog } from "./log.ts"; +import { JsonRpcReceipt } from "./receipt.ts"; /** * Represents a hexadecimal string with a `0x` prefix. @@ -81,3 +87,22 @@ export interface TransactionConversionInput { /** The index of the transaction in the block. */ index: string; } + +export interface BlockInfo { + blockNumber: string; + blockHash: string; + isPendingBlock: boolean; +} + +export interface ProcessedEvent { + event: EventWithTransaction; + typedEthTx: TypedTransaction; + ethTx: JsonRpcTx; + ethLogs: JsonRpcLog[]; + ethReceipt: JsonRpcReceipt; +} + +export interface ProcessedTransaction { + ethTx: JsonRpcTx; + ethReceipt: JsonRpcReceipt; +} From d9d548239a0696bc98103e97842fc0f99a4ffc1a Mon Sep 17 00:00:00 2001 From: Eugenio Paluello Date: Tue, 20 Aug 2024 16:46:43 +0200 Subject: [PATCH 5/8] fix: fmt and type --- indexer/src/main.ts | 58 ++++++++++++++++++++++----------------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/indexer/src/main.ts b/indexer/src/main.ts index 038681220..cf4708f29 100644 --- a/indexer/src/main.ts +++ b/indexer/src/main.ts @@ -90,7 +90,7 @@ export default async function transform({ // Skip if the transaction_executed event contains "eth validation failed". .filter((event) => !ethValidationFailed(event.event)) .map(processEvent(blockInfo)) - .filter((event) => event !== null); + .filter((event): event is ProcessedEvent => event !== null); const { cumulativeGasUsed, cumulativeGasUsages } = accumulateGasAndUpdateStore(processedEvents, store, blockLogsBloom); @@ -98,9 +98,9 @@ export default async function transform({ const trieData: Array = processedEvents .map((event) => createTrieData({ - transactionIndex: Number(event.ethTx.transactionIndex), - typedTransaction: event.typedEthTx, - receipt: event.ethReceipt, + transactionIndex: Number(event!.ethTx.transactionIndex), + typedTransaction: event!.typedEthTx, + receipt: event!.ethReceipt, }) ) .filter((x) => x !== null); @@ -113,7 +113,7 @@ export default async function transform({ const processedTransactions = processTransactions( transactions, blockInfo, - cumulativeGasUsages + cumulativeGasUsages, ); updateStoreWithTransactions(store, processedTransactions); @@ -166,14 +166,12 @@ function processEvent(blockInfo: BlockInfo) { // Can be null if: // 1. The event is part of the defined ignored events (see IGNORED_KEYS). // 2. The event has an invalid number of keys. - .filter((e) => e !== null); + .filter((e): e is JsonRpcLog => e !== null); - const ethLogsIndexed = ethLogs.map((log, index) => { - // ...log, - // logIndex: index.toString(), - log.logIndex = index.toString(); - return log; - }); + const ethLogsIndexed = ethLogs.map((log, index) => ({ + ...log, + logIndex: index.toString(), + })); const ethReceipt = toEthReceipt({ transaction: ethTx as JsonRpcTx, @@ -183,14 +181,14 @@ function processEvent(blockInfo: BlockInfo) { ...blockInfo, }); - return { event, typedEthTx, ethTx, ethLogs, ethReceipt }; + return { event, typedEthTx, ethTx, ethLogs: ethLogsIndexed, ethReceipt }; }; } function accumulateGasAndUpdateStore( processedEvents: ProcessedEvent[], store: Array, - blockLogsBloom: Bloom + blockLogsBloom: Bloom, ): { cumulativeGasUsed: bigint; cumulativeGasUsages: bigint[] } { let cumulativeGasUsed = 0n; const cumulativeGasUsages: bigint[] = []; @@ -234,7 +232,7 @@ function updateBlockLogsBloom(blockLogsBloom: Bloom, event: ProcessedEvent) { } async function computeBlooms( - trieData: Array + trieData: Array, ): Promise<{ transactionTrie: Trie; receiptTrie: Trie }> { const transactionTrie = new Trie(); const receiptTrie = new Trie(); @@ -251,8 +249,8 @@ async function computeBlooms( await transactionTrie.put(encodedTransactionIndex, encodedTransaction); // Add the receipt to the receipt trie. await receiptTrie.put(encodedTransactionIndex, encodedReceipt); - } - ) + }, + ), ); return { transactionTrie, receiptTrie }; @@ -261,22 +259,22 @@ async function computeBlooms( function processTransactions( transactions: TransactionWithReceipt[], blockInfo: BlockInfo, - cumulativeGasUsages: bigint[] + cumulativeGasUsages: bigint[], ): ProcessedTransaction[] { return transactions .filter( (tx) => isRevertedWithOutOfResources(tx.receipt) && - isKakarotTransaction(tx.transaction) + isKakarotTransaction(tx.transaction), ) .map((tx) => createProcessedTransaction(tx, blockInfo, cumulativeGasUsages)) - .filter((tx) => tx !== null); + .filter((tx): tx is ProcessedTransaction => tx !== null); } function createProcessedTransaction( tx: TransactionWithReceipt, blockInfo: BlockInfo, - cumulativeGasUsages: bigint[] + cumulativeGasUsages: bigint[], ): ProcessedTransaction | null { const ethTx = toEthTx({ transaction: tx.transaction, @@ -289,12 +287,11 @@ function createProcessedTransaction( // const cumulativeGasUsages = [300n, undefined, undefined, 200n, undefined, 100n, undefined, undefined, 10n, undefined]; // const ethTx = { transactionIndex: 5 }; // const revertedTransactionCumulativeGasUsed = 100n; - const revertedTransactionCumulativeGasUsed = - cumulativeGasUsages.find( - (gas, i) => - Number(ethTx.transactionIndex) >= cumulativeGasUsages.length - 1 - i && - gas - ) ?? 0n; + const revertedTransactionCumulativeGasUsed = cumulativeGasUsages.find( + (gas, i) => + Number(ethTx.transactionIndex) >= cumulativeGasUsages.length - 1 - i && + gas, + ) ?? 0n; const ethReceipt = toRevertedOutOfResourcesReceipt({ transaction: ethTx as JsonRpcTx, @@ -307,10 +304,13 @@ function createProcessedTransaction( function updateStoreWithTransactions( store: Array, - processedTransactions: ProcessedTransaction[] + processedTransactions: ProcessedTransaction[], ) { processedTransactions.forEach(({ ethTx, ethReceipt }) => { - store.push({ collection: Collection.Transactions, data: { tx: ethTx as JsonRpcTx } }); + store.push({ + collection: Collection.Transactions, + data: { tx: ethTx as JsonRpcTx }, + }); store.push({ collection: Collection.Receipts, data: { receipt: ethReceipt }, From 915b3699baff89161fb6238ab376a64e6f812dc1 Mon Sep 17 00:00:00 2001 From: Eugenio Paluello Date: Tue, 10 Sep 2024 17:51:26 +0200 Subject: [PATCH 6/8] fix: tests --- indexer/src/main.ts | 49 +++++++++++++++++++++------------------------ 1 file changed, 23 insertions(+), 26 deletions(-) diff --git a/indexer/src/main.ts b/indexer/src/main.ts index cf4708f29..61749f5bf 100644 --- a/indexer/src/main.ts +++ b/indexer/src/main.ts @@ -83,7 +83,7 @@ export default async function transform({ const store: Array = []; const blockLogsBloom = new Bloom(); - const processedEvents = events + const processedEvents = (events ?? []) // Can be false if the transaction is not related to a specific instance of the Kakarot contract. // This is typically the case if there are multiple Kakarot contracts on the same chain. .filter((event) => isKakarotTransaction(event.transaction)) @@ -193,17 +193,16 @@ function accumulateGasAndUpdateStore( let cumulativeGasUsed = 0n; const cumulativeGasUsages: bigint[] = []; - processedEvents.forEach((event) => { - // Accumulate the gas used in the block in order to calculate the cumulative gas used. - // We increment it by the gas used in each transaction. + if (!Array.isArray(processedEvents)) { + return { cumulativeGasUsed, cumulativeGasUsages }; + } + + processedEvents?.forEach((event, index) => { cumulativeGasUsed += BigInt(event.ethReceipt.gasUsed); - // ethTx.transactionIndex can be null (if the block is pending) but - // Number(null) is 0 so this won't panic. - const transactionIndex = Number(event.ethTx.transactionIndex); - // An array containing the cumulative gas used up to that transaction, indexed by - // transaction index. This is used to later get the cumulative gas used for an out of - // resources transaction. - cumulativeGasUsages[transactionIndex] = cumulativeGasUsed; + cumulativeGasUsages[index] = cumulativeGasUsed; + + // Update the cumulative gas used in the receipt + event.ethReceipt.cumulativeGasUsed = `0x${cumulativeGasUsed.toString(16)}`; updateStore(store, event); updateBlockLogsBloom(blockLogsBloom, event); @@ -237,22 +236,18 @@ async function computeBlooms( const transactionTrie = new Trie(); const receiptTrie = new Trie(); - // Compute the blooms in an async manner. - await Promise.all( - trieData.map( - async ({ - encodedTransactionIndex, - encodedTransaction, - encodedReceipt, - }) => { - // Add the transaction to the transaction trie. - await transactionTrie.put(encodedTransactionIndex, encodedTransaction); - // Add the receipt to the receipt trie. - await receiptTrie.put(encodedTransactionIndex, encodedReceipt); - }, - ), + trieData.sort((a, b) => + Number(a.encodedTransactionIndex) - Number(b.encodedTransactionIndex) ); + for ( + const { encodedTransactionIndex, encodedTransaction, encodedReceipt } + of trieData + ) { + await transactionTrie.put(encodedTransactionIndex, encodedTransaction); + await receiptTrie.put(encodedTransactionIndex, encodedReceipt); + } + return { transactionTrie, receiptTrie }; } @@ -261,7 +256,7 @@ function processTransactions( blockInfo: BlockInfo, cumulativeGasUsages: bigint[], ): ProcessedTransaction[] { - return transactions + return (transactions ?? []) .filter( (tx) => isRevertedWithOutOfResources(tx.receipt) && @@ -276,6 +271,8 @@ function createProcessedTransaction( blockInfo: BlockInfo, cumulativeGasUsages: bigint[], ): ProcessedTransaction | null { + if (!tx.transaction || !tx.receipt) return null; + const ethTx = toEthTx({ transaction: tx.transaction, receipt: tx.receipt, From 38df95695e44fb36fb13df297ff4da05b706cf7d Mon Sep 17 00:00:00 2001 From: Eugenio Paluello Date: Wed, 11 Sep 2024 13:49:59 +0200 Subject: [PATCH 7/8] fix: refactoring --- indexer/src/main.ts | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/indexer/src/main.ts b/indexer/src/main.ts index 61749f5bf..46173d7ea 100644 --- a/indexer/src/main.ts +++ b/indexer/src/main.ts @@ -86,9 +86,11 @@ export default async function transform({ const processedEvents = (events ?? []) // Can be false if the transaction is not related to a specific instance of the Kakarot contract. // This is typically the case if there are multiple Kakarot contracts on the same chain. - .filter((event) => isKakarotTransaction(event.transaction)) // Skip if the transaction_executed event contains "eth validation failed". - .filter((event) => !ethValidationFailed(event.event)) + .filter((event) => + isKakarotTransaction(event.transaction) && + !ethValidationFailed(event.event) + ) .map(processEvent(blockInfo)) .filter((event): event is ProcessedEvent => event !== null); @@ -193,10 +195,6 @@ function accumulateGasAndUpdateStore( let cumulativeGasUsed = 0n; const cumulativeGasUsages: bigint[] = []; - if (!Array.isArray(processedEvents)) { - return { cumulativeGasUsed, cumulativeGasUsages }; - } - processedEvents?.forEach((event, index) => { cumulativeGasUsed += BigInt(event.ethReceipt.gasUsed); cumulativeGasUsages[index] = cumulativeGasUsed; @@ -204,7 +202,20 @@ function accumulateGasAndUpdateStore( // Update the cumulative gas used in the receipt event.ethReceipt.cumulativeGasUsed = `0x${cumulativeGasUsed.toString(16)}`; - updateStore(store, event); + store.push(...[ + { + collection: Collection.Transactions, + data: { tx: event.ethTx }, + }, + { + collection: Collection.Receipts, + data: { receipt: event.ethReceipt }, + }, + ...event.ethLogs.map((log) => ({ + collection: Collection.Logs, + data: { log }, + })), + ]); updateBlockLogsBloom(blockLogsBloom, event); }); From f267c352760a54049fa158c257dd80a341d19f25 Mon Sep 17 00:00:00 2001 From: Eugenio Paluello Date: Wed, 11 Sep 2024 13:51:17 +0200 Subject: [PATCH 8/8] fix: remove update store --- indexer/src/main.ts | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/indexer/src/main.ts b/indexer/src/main.ts index 46173d7ea..e2c027645 100644 --- a/indexer/src/main.ts +++ b/indexer/src/main.ts @@ -222,20 +222,6 @@ function accumulateGasAndUpdateStore( return { cumulativeGasUsed, cumulativeGasUsages }; } -function updateStore(store: Array, event: ProcessedEvent) { - store.push({ - collection: Collection.Transactions, - data: { tx: event.ethTx }, - }); - store.push({ - collection: Collection.Receipts, - data: { receipt: event.ethReceipt }, - }); - event.ethLogs.forEach((log) => - store.push({ collection: Collection.Logs, data: { log } }) - ); -} - function updateBlockLogsBloom(blockLogsBloom: Bloom, event: ProcessedEvent) { const receiptLogsBloom = new Bloom(hexToBytes(event.ethReceipt.logsBloom)); blockLogsBloom.or(receiptLogsBloom);