Perf/improve ath calculation performance

.env.jungle

@@ -70,7 +70,7 @@ HAPI_EVM_NETWORK_CHAIN_ID=73e4385a2708e6d7048834fbc1079f2fabb17b3c125b146af43897
 HAPI_EVM_EOS_EVM_ACCOUNT=eosio.evm
 HAPI_EVM_BLOCK_INTERVAL_SEC=1
 HAPI_EVM_OLD_BLOCK_INTERVAL_SEC=0.1
-HAPI_EVM_ATH_INTERVAL_SEC=60
+HAPI_EVM_ATH_INTERVAL_SEC=900
 HAPI_EVM_CLEAN_OLD_BLOCK_INTERVAL_SEC=86400
 HAPI_EVM_CLEAN_OLD_TRANSFER_INTERVAL_SEC=86400
 HAPI_EVM_KEEP_HISTORY_FOR_YEARS=1
@@ -110,5 +110,5 @@ REACT_APP_GOOGLE_ANALITIC_PAGE_ID=G-E6Y0EC9FT8
 REACT_APP_PUBLIC_RE_CAPTCHA_KEY=key
 REACT_APP_EOS_INCLUDE_TRANSACTION=
 REACT_APP_EVM_ENDPOINT=https://api.testnet.evm.eosnetwork.com
-REACT_APP_EVM_BLOCK_EXPLORER_URL=https://explorer.testnet.evm.eosnetwork.com/(block)/transactions
+REACT_APP_EVM_BLOCK_EXPLORER_URL=https://explorer.testnet.evm.eosnetwork.com/block/(block)/transactions
 REACT_APP_EVM_ENDPOINTS=["https://api.testnet.evm.eosnetwork.com"]

.env.local

@@ -70,7 +70,7 @@ HAPI_EVM_NETWORK_CHAIN_ID=aca376f206b8fc25a6ed44dbdc66547c36c6c33e3a119ffbeaef94
 HAPI_EVM_EOS_EVM_ACCOUNT=eosio.evm
 HAPI_EVM_BLOCK_INTERVAL_SEC=1
 HAPI_EVM_OLD_BLOCK_INTERVAL_SEC=0.1
-HAPI_EVM_ATH_INTERVAL_SEC=60
+HAPI_EVM_ATH_INTERVAL_SEC=900
 HAPI_EVM_CLEAN_OLD_BLOCK_INTERVAL_SEC=86400
 HAPI_EVM_CLEAN_OLD_TRANSFER_INTERVAL_SEC=86400
 HAPI_EVM_KEEP_HISTORY_FOR_YEARS=1

.env.mainnet

@@ -70,7 +70,7 @@ HAPI_EVM_NETWORK_CHAIN_ID=aca376f206b8fc25a6ed44dbdc66547c36c6c33e3a119ffbeaef94
 HAPI_EVM_EOS_EVM_ACCOUNT=eosio.evm
 HAPI_EVM_BLOCK_INTERVAL_SEC=1
 HAPI_EVM_OLD_BLOCK_INTERVAL_SEC=0.1
-HAPI_EVM_ATH_INTERVAL_SEC=60
+HAPI_EVM_ATH_INTERVAL_SEC=900
 HAPI_EVM_CLEAN_OLD_BLOCK_INTERVAL_SEC=86400
 HAPI_EVM_CLEAN_OLD_TRANSFER_INTERVAL_SEC=86400
 HAPI_EVM_KEEP_HISTORY_FOR_YEARS=1

.env.telos

@@ -69,7 +69,7 @@ HAPI_EVM_NETWORK=TELOS
 HAPI_EVM_EOS_EVM_ACCOUNT=eosio.evm
 HAPI_EVM_BLOCK_INTERVAL_SEC=0.5
 HAPI_EVM_OLD_BLOCK_INTERVAL_SEC=0.1
-HAPI_EVM_ATH_INTERVAL_SEC=60
+HAPI_EVM_ATH_INTERVAL_SEC=900
 HAPI_EVM_CLEAN_OLD_BLOCK_INTERVAL_SEC=86400
 HAPI_EVM_CLEAN_OLD_TRANSFER_INTERVAL_SEC=86400
 HAPI_EVM_KEEP_HISTORY_FOR_YEARS=1

.env.telostestnet

@@ -70,7 +70,7 @@ HAPI_EVM_NETWORK=TELOS
 HAPI_EVM_EOS_EVM_ACCOUNT=eosio.evm
 HAPI_EVM_BLOCK_INTERVAL_SEC=0.5
 HAPI_EVM_OLD_BLOCK_INTERVAL_SEC=0.1
-HAPI_EVM_ATH_INTERVAL_SEC=60
+HAPI_EVM_ATH_INTERVAL_SEC=900
 HAPI_EVM_CLEAN_OLD_BLOCK_INTERVAL_SEC=86400
 HAPI_EVM_CLEAN_OLD_TRANSFER_INTERVAL_SEC=86400
 HAPI_EVM_KEEP_HISTORY_FOR_YEARS=1

.github/workflows/deploy-jungle-testnet.yaml

@@ -122,7 +122,7 @@ jobs:
           HAPI_EVM_EOS_EVM_ACCOUNT: eosio.evm
           HAPI_EVM_BLOCK_INTERVAL_SEC: 1
           HAPI_EVM_OLD_BLOCK_INTERVAL_SEC: 0.1
-          HAPI_EVM_ATH_INTERVAL_SEC: 60
+          HAPI_EVM_ATH_INTERVAL_SEC: 900
           HAPI_EVM_CLEAN_OLD_BLOCK_INTERVAL_SEC: 86400
           HAPI_EVM_CLEAN_OLD_TRANSFER_INTERVAL_SEC: 86400
           HAPI_EVM_KEEP_HISTORY_FOR_YEARS: 1

.github/workflows/deploy-mainnet.yaml

@@ -120,7 +120,7 @@ jobs:
           HAPI_EVM_EOS_EVM_ACCOUNT: eosio.evm
           HAPI_EVM_BLOCK_INTERVAL_SEC: 1
           HAPI_EVM_OLD_BLOCK_INTERVAL_SEC: 0.1
-          HAPI_EVM_ATH_INTERVAL_SEC: 60
+          HAPI_EVM_ATH_INTERVAL_SEC: 900
           HAPI_EVM_CLEAN_OLD_BLOCK_INTERVAL_SEC: 86400
           HAPI_EVM_CLEAN_OLD_TRANSFER_INTERVAL_SEC: 86400
           HAPI_EVM_KEEP_HISTORY_FOR_YEARS: 1

.github/workflows/deploy-telos-testnet.yaml

@@ -121,7 +121,7 @@ jobs:
           HAPI_EVM_EOS_EVM_ACCOUNT: eosio.evm
           HAPI_EVM_BLOCK_INTERVAL_SEC: 0.5
           HAPI_EVM_OLD_BLOCK_INTERVAL_SEC: 0.1
-          HAPI_EVM_ATH_INTERVAL_SEC: 60
+          HAPI_EVM_ATH_INTERVAL_SEC: 900
           HAPI_EVM_CLEAN_OLD_BLOCK_INTERVAL_SEC: 86400
           HAPI_EVM_CLEAN_OLD_TRANSFER_INTERVAL_SEC: 86400
           HAPI_EVM_KEEP_HISTORY_FOR_YEARS: 1

.github/workflows/deploy-telos.yaml

@@ -121,7 +121,7 @@ jobs:
           HAPI_EVM_EOS_EVM_ACCOUNT: eosio.evm
           HAPI_EVM_BLOCK_INTERVAL_SEC: 0.5
           HAPI_EVM_OLD_BLOCK_INTERVAL_SEC: 0.1
-          HAPI_EVM_ATH_INTERVAL_SEC: 60
+          HAPI_EVM_ATH_INTERVAL_SEC: 900
           HAPI_EVM_CLEAN_OLD_BLOCK_INTERVAL_SEC: 86400
           HAPI_EVM_CLEAN_OLD_TRANSFER_INTERVAL_SEC: 86400
           HAPI_EVM_KEEP_HISTORY_FOR_YEARS: 1

hapi-evm/src/models/stats/queries.ts

@@ -1,6 +1,8 @@
 import { gql } from 'graphql-request'
 
 import { coreUtil } from '../../utils'
+import { historicalStatsModel } from '..'
+
 import { Stats } from './interfaces'
 
 interface StatsResponse {
@@ -22,3 +24,25 @@ export const getPartialATH = async () => {
 
   return state
 }
+
+export const updateATH = async (partialATH: Stats) => {
+  if (!partialATH) return
+  const currentState = await historicalStatsModel.queries.getState()
+  if (
+    currentState.tps_all_time_high.transactions_count >
+    partialATH.ath_transactions_count
+  )
+    return
+  if (
+    currentState.tps_all_time_high.transactions_count ||
+    0 < partialATH.ath_transactions_count
+  ) {
+    await historicalStatsModel.queries.saveOrUpdate({
+      tps_all_time_high: {
+        blocks: partialATH.ath_blocks.split(','),
+        transactions_count: partialATH.ath_transactions_count,
+        gas_used: partialATH.ath_gas_used
+      }
+    })
+  }
+}

hapi-evm/src/services/block.service.ts

@@ -5,6 +5,7 @@ import {
   TransactionHash,
   TransactionReceipt
 } from 'web3-types'
+import moment from 'moment'
 
 import {
   defaultModel,
@@ -15,7 +16,9 @@ import {
   StatsModel
 } from '../models'
 import { networkConfig } from '../config'
-import moment from 'moment'
+import { timeUtil } from '../utils'
+
+import { getATHInRange } from './partial-ath.service'
 
 const httpProvider = new Web3.providers.HttpProvider(networkConfig.evmEndpoint)
 const web3 = new Web3(httpProvider)
@@ -135,7 +138,10 @@ const getBlock = async () => {
 const syncOldBlocks = async (): Promise<void> => {
   let blocksInserted = 1
   const paramStats = await paramModel.queries.getState()
-  if (paramStats.isSynced) return
+  if (paramStats.isSynced) {
+    await timeUtil.sleep(86400)
+    return
+  }
   const nextBlock = paramStats.nextBlock
   const nextBlockToNumber =
     paramStats.completeAt ||
@@ -155,6 +161,11 @@ const syncOldBlocks = async (): Promise<void> => {
   } else {
     console.log(`Syncing old blocks complete at ${moment().format()}`)
   }
+  const partialATH = await getATHInRange(
+    nextBlock - 1,
+    nextBlock + blocksInserted - 1
+  )
+  await StatsModel.queries.updateATH(partialATH)
   await paramModel.queries.saveOrUpdate(
     nextBlock + blocksInserted * Number(!isUpToDate),
     !!isUpToDate,
@@ -171,21 +182,8 @@ const cleanOldBlocks = async () => {
 }
 
 const syncATH = async () => {
-  const currentState = await historicalStatsModel.queries.getState()
   const partialATH = await StatsModel.queries.getPartialATH()
-  if (!partialATH) return
-  if (
-    currentState.tps_all_time_high.transactions_count ||
-    0 < partialATH.ath_transactions_count
-  ) {
-    await historicalStatsModel.queries.saveOrUpdate({
-      tps_all_time_high: {
-        blocks: partialATH.ath_blocks.split(','),
-        transactions_count: partialATH.ath_transactions_count,
-        gas_used: partialATH.ath_gas_used
-      }
-    })
-  }
+  await StatsModel.queries.updateATH(partialATH)
 }
 
 const syncBlockWorker = (): defaultModel.Worker => {

hapi-evm/src/services/partial-ath.service.ts

@@ -0,0 +1,59 @@
+import { Stats } from '../models/stats/interfaces'
+import { sequelizeUtil } from '../utils'
+
+export const getATHInRange = async (
+  lowerBlockNumber: number,
+  upperBlockNumber: number
+) => {
+  const [rows] = await sequelizeUtil.sequelize.query(`
+  SELECT
+    ath_in_range.blocks AS ath_blocks,
+    COALESCE(
+        (ath_in_range.max_transaction_sum) :: numeric,
+        (0) :: numeric
+    ) AS ath_transactions_count,
+    COALESCE(ath_in_range.gas_used_sum, (0) :: numeric) AS ath_gas_used
+    FROM
+    (
+      WITH subquery AS (
+        SELECT
+          array_to_string(array_agg(block.number), ',' :: text) AS blocks,
+          sum(jsonb_array_length(block.transactions)) AS total_transaction_count,
+          sum(block.gas_used) AS gas_used_sum
+        FROM
+          evm.block
+        WHERE (block.number >= ${lowerBlockNumber} and block.number <= ${upperBlockNumber})
+        GROUP BY
+          block."timestamp"
+      )
+      SELECT
+        q2.blocks,
+        q1.max_transaction_sum,
+        q2.gas_used_sum
+      FROM
+        (
+          (
+            SELECT
+              max(subquery.total_transaction_count) AS max_transaction_sum
+            FROM
+              subquery
+          ) q1
+          JOIN subquery q2 ON (
+            (
+              q1.max_transaction_sum = q2.total_transaction_count
+            )
+          )
+        )
+      LIMIT
+        1
+    ) ath_in_range
+  `)
+
+  const row = rows[0] as Stats
+
+  return {
+    ath_blocks: row?.ath_blocks,
+    ath_transactions_count: Number(row?.ath_transactions_count) || 0,
+    ath_gas_used: Number(row?.ath_gas_used) || 0
+  } as Stats
+}

hasura/migrations/default/1701189064166_create_evm_stats_view_v5/down.sql

@@ -0,0 +1,32 @@
+-- Could not auto-generate a down migration.
+-- Please write an appropriate down migration for the SQL below:
+-- CREATE OR REPLACE VIEW "evm"."stats" AS
+--  SELECT COALESCE(evm_block.avg_gas_used, (0)::numeric) AS block_gas_avg,
+--     COALESCE(daily_transactions.total_transaction_count, (0)::bigint) AS daily_transaction_count,
+--     partial_ath.blocks AS ath_blocks,
+--     COALESCE((partial_ath.max_transaction_sum)::numeric, (0)::numeric) AS ath_transactions_count,
+--     COALESCE(partial_ath.gas_used_sum, (0)::numeric) AS ath_gas_used
+--    FROM ((( SELECT avg(subquery_alias.gas_used) AS avg_gas_used
+--            FROM ( SELECT block.gas_used,
+--                     block."timestamp"
+--                    FROM evm.block
+--                   ORDER BY block."timestamp" DESC
+--                  LIMIT 100) subquery_alias) evm_block
+--      CROSS JOIN LATERAL ( SELECT sum(jsonb_array_length(block.transactions)) AS total_transaction_count
+--            FROM evm.block
+--           WHERE (block."timestamp" >= (now() - '24:00:00'::interval))) daily_transactions)
+--      CROSS JOIN LATERAL ( WITH subquery AS (
+--                  SELECT array_to_string(array_agg(block.number), ','::text) AS blocks,
+--                     sum(jsonb_array_length(block.transactions)) AS total_transaction_count,
+--                     sum(block.gas_used) AS gas_used_sum
+--                    FROM evm.block
+--                    WHERE (block."timestamp" >= (now() - '00:30:00'::interval))
+--                   GROUP BY block."timestamp"
+--                 )
+--          SELECT q2.blocks,
+--             q1.max_transaction_sum,
+--             q2.gas_used_sum
+--            FROM (( SELECT max(subquery.total_transaction_count) AS max_transaction_sum
+--                    FROM subquery) q1
+--              JOIN subquery q2 ON ((q1.max_transaction_sum = q2.total_transaction_count)))
+--          LIMIT 1) partial_ath);

hasura/migrations/default/1701189064166_create_evm_stats_view_v5/up.sql

@@ -0,0 +1,30 @@
+CREATE OR REPLACE VIEW "evm"."stats" AS 
+ SELECT COALESCE(evm_block.avg_gas_used, (0)::numeric) AS block_gas_avg,
+    COALESCE(daily_transactions.total_transaction_count, (0)::bigint) AS daily_transaction_count,
+    partial_ath.blocks AS ath_blocks,
+    COALESCE((partial_ath.max_transaction_sum)::numeric, (0)::numeric) AS ath_transactions_count,
+    COALESCE(partial_ath.gas_used_sum, (0)::numeric) AS ath_gas_used
+   FROM ((( SELECT avg(subquery_alias.gas_used) AS avg_gas_used
+           FROM ( SELECT block.gas_used,
+                    block."timestamp"
+                   FROM evm.block
+                  ORDER BY block."timestamp" DESC
+                 LIMIT 100) subquery_alias) evm_block
+     CROSS JOIN LATERAL ( SELECT sum(jsonb_array_length(block.transactions)) AS total_transaction_count
+           FROM evm.block
+          WHERE (block."timestamp" >= (now() - '24:00:00'::interval))) daily_transactions)
+     CROSS JOIN LATERAL ( WITH subquery AS (
+                 SELECT array_to_string(array_agg(block.number), ','::text) AS blocks,
+                    sum(jsonb_array_length(block.transactions)) AS total_transaction_count,
+                    sum(block.gas_used) AS gas_used_sum
+                   FROM evm.block
+                   WHERE (block."timestamp" >= (now() - '00:30:00'::interval))
+                  GROUP BY block."timestamp"
+                )
+         SELECT q2.blocks,
+            q1.max_transaction_sum,
+            q2.gas_used_sum
+           FROM (( SELECT max(subquery.total_transaction_count) AS max_transaction_sum
+                   FROM subquery) q1
+             JOIN subquery q2 ON ((q1.max_transaction_sum = q2.total_transaction_count)))
+         LIMIT 1) partial_ath);