Added group batching calls to Por-address-list (#3185)

* add group batching calls * change batchGroupSize input param to config * fix unit tests * change BATCH_GROUP_SIZE to GROUP_SIZE
smartcontractkit · Feb 12, 2024 · 2e46100 · 2e46100
1 parent b8154aa
commit 2e46100
Show file tree

Hide file tree

Showing 7 changed files with 42 additions and 11 deletions.
diff --git a/.changeset/good-fishes-shake.md b/.changeset/good-fishes-shake.md
@@ -0,0 +1,5 @@
+---
+'@chainlink/por-address-list-adapter': patch
+---
+
+Added GROUP_SIZE config
diff --git a/packages/sources/por-address-list/README.md b/packages/sources/por-address-list/README.md
@@ -10,6 +10,7 @@ This document was generated automatically. Please see [README Generator](../../s
 | :-------: | :-------------------: | :---------------------------------------------------------------------------------------: | :----: | :-----: | :-----: |
 |    ✅     |        RPC_URL        |   The RPC URL to connect to the EVM chain the address manager contract is deployed to.    | string |         |         |
 |           |       CHAIN_ID        |                                The chain id to connect to                                 | number |         |   `1`   |
+|           |   BATCH_GROUP_SIZE    |             The number of concurrent batched contract calls to make at a time             | number |         |  `10`   |
 |           | BACKGROUND_EXECUTE_MS | The amount of time the background execute should sleep before performing the next request | number |         | `10000` |
 
 ---

diff --git a/packages/sources/por-address-list/src/config/index.ts b/packages/sources/por-address-list/src/config/index.ts
@@ -12,6 +12,12 @@ export const config = new AdapterConfig({
     type: 'number',
     default: 1,
   },
+  GROUP_SIZE: {
+    description:
+      'The number of concurrent batched contract calls to make at a time. Setting this lower than the default may result in lower performance from the adapter.',
+    type: 'number',
+    default: 10,
+  },
   BACKGROUND_EXECUTE_MS: {
     description:
       'The amount of time the background execute should sleep before performing the next request',

diff --git a/packages/sources/por-address-list/src/transport/address.ts b/packages/sources/por-address-list/src/transport/address.ts
@@ -14,6 +14,7 @@ type RequestParams = typeof inputParameters.validated
 
 export class AddressTransport extends SubscriptionTransport<AddressTransportTypes> {
   provider!: ethers.providers.JsonRpcProvider
+  settings!: AddressTransportTypes['Settings']
 
   async initialize(
     dependencies: TransportDependencies<AddressTransportTypes>,
@@ -26,6 +27,7 @@ export class AddressTransport extends SubscriptionTransport<AddressTransportType
       adapterSettings.RPC_URL,
       adapterSettings.CHAIN_ID,
     )
+    this.settings = adapterSettings
   }
 
   async backgroundHandler(
@@ -69,6 +71,7 @@ export class AddressTransport extends SubscriptionTransport<AddressTransportType
       latestBlockNum,
       confirmations,
       batchSize,
+      this.settings.GROUP_SIZE,
     )
     const addresses: PoRAddress[] = addressList.map((address) => ({
       address,

diff --git a/packages/sources/por-address-list/src/transport/utils.ts b/packages/sources/por-address-list/src/transport/utils.ts
@@ -5,22 +5,34 @@ export const fetchAddressList = async (
   latestBlockNum: number,
   confirmations = 0,
   batchSize = 10,
+  batchGroupSize = 10,
 ): Promise<string[]> => {
   const blockTag = latestBlockNum - confirmations
   const numAddresses = await addressManager.getPoRAddressListLength({
     blockTag,
   })
-  let addresses: string[] = []
+  let totalRequestedAddressesCount = 0
   let startIdx = ethers.BigNumber.from(0)
-  while (ethers.BigNumber.from(addresses.length).lt(numAddresses)) {
+  const addresses: string[] = []
+  let batchRequests: Promise<string[]>[] = []
+
+  while (totalRequestedAddressesCount < numAddresses.toNumber()) {
     const nextEndIdx = startIdx.add(batchSize)
-    const endIdx = nextEndIdx.gt(numAddresses) ? numAddresses.sub(1) : nextEndIdx
+    const endIdx = nextEndIdx.gte(numAddresses) ? numAddresses.sub(1) : nextEndIdx
+    const batchCall = addressManager.getPoRAddressList(startIdx, endIdx, { blockTag })
+    batchRequests.push(batchCall)
     // element at endIdx is included in result
-    const latestAddresses = await addressManager.getPoRAddressList(startIdx, endIdx, {
-      blockTag,
-    })
-    addresses = addresses.concat(latestAddresses)
+    const addressesRequested: number = endIdx.sub(startIdx).add(1).toNumber()
+    totalRequestedAddressesCount += addressesRequested
     startIdx = endIdx.add(1)
+
+    if (
+      batchRequests.length >= batchGroupSize ||
+      totalRequestedAddressesCount >= numAddresses.toNumber()
+    ) {
+      addresses.push(...(await Promise.all(batchRequests)).flat())
+      batchRequests = []
+    }
   }
   return addresses
 }
diff --git a/packages/sources/por-address-list/test/integration/adapter.test.ts b/packages/sources/por-address-list/test/integration/adapter.test.ts
@@ -36,7 +36,11 @@ jest.mock('ethers', () => {
       Contract: function () {
         return {
           getPoRAddressListLength: jest.fn().mockReturnValue(mockAddressListLength),
-          getPoRAddressList: jest.fn().mockReturnValue(mockExpectedAddresses),
+          getPoRAddressList: jest.fn().mockImplementation((startIdx, endIdx) => {
+            const start = startIdx.toNumber()
+            const end = endIdx.toNumber() + 1
+            return mockExpectedAddresses.slice(start, end)
+          }),
         }
       },
     },

diff --git a/packages/sources/por-address-list/test/unit/utils.test.ts b/packages/sources/por-address-list/test/unit/utils.test.ts
@@ -25,7 +25,7 @@ const AddressManagerMock: jest.Mock<ethers.Contract, string[][]> = jest
       getPoRAddressList: jest
         .fn()
         .mockImplementation((startIdx: ethers.BigNumber, endIdx: ethers.BigNumber) => {
-          const lastIdx = endIdx.gt(ethers.BigNumber.from(addresses.length))
+          const lastIdx = endIdx.gte(ethers.BigNumber.from(addresses.length))
             ? addresses.length - 1
             : endIdx.toNumber()
           return addresses.slice(startIdx.toNumber(), lastIdx + 1)
@@ -47,7 +47,7 @@ describe('address endpoint', () => {
         })
         expect(addressManager.getPoRAddressList).toHaveBeenCalledWith(
           ethers.BigNumber.from(0),
-          ethers.BigNumber.from(DEFAULT_EXPECTED_ADDRESSES.length),
+          ethers.BigNumber.from(DEFAULT_EXPECTED_ADDRESSES.length - 1),
           { blockTag: LATEST_BLOCK_NUM },
         )
       })
@@ -62,7 +62,7 @@ describe('address endpoint', () => {
         })
         expect(addressManager.getPoRAddressList).toHaveBeenCalledWith(
           ethers.BigNumber.from(0),
-          ethers.BigNumber.from(DEFAULT_EXPECTED_ADDRESSES.length),
+          ethers.BigNumber.from(DEFAULT_EXPECTED_ADDRESSES.length - 1),
           { blockTag: LATEST_BLOCK_NUM - confirmations },
         )
       })