Odis Combiner 3.0.0 Release (#10547)

* Refactor combiner * Remove combiner timeouts * fix lint * fix combiner e2e tests * point to published dependencies * put auth behind flag * fix config casing * added memory allocation to function * bump version * bump combiner version * switched to dev dependencies * improve signer e2e tests * add mainnet e2e command * add privateKey param to loadTest script * update context name --------- Co-authored-by: Gaston Ponti <ponti@clabs.co> Co-authored-by: soloseng <102702451+soloseng@users.noreply.github.com>
celo-org · Sep 9, 2023 · c4d4791 · c4d4791
1 parent 0d433bc
commit c4d4791
Show file tree

Hide file tree

Showing 10 changed files with 90 additions and 45 deletions.
diff --git a/packages/phone-number-privacy/combiner/package.json b/packages/phone-number-privacy/combiner/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@celo/phone-number-privacy-combiner",
-  "version": "3.0.0-beta.8-dev",
+  "version": "3.0.1-dev",
   "description": "Orchestrates and combines threshold signatures for use in ODIS",
   "author": "Celo",
   "license": "Apache-2.0",
@@ -23,9 +23,10 @@
     "test": "jest --runInBand --testPathIgnorePatterns test/end-to-end",
     "test:coverage": "yarn test --coverage",
     "test:integration": "jest --runInBand test/integration",
-    "test:e2e": "jest test/end-to-end --verbose",
+    "test:e2e": "jest --runInBand test/end-to-end --verbose",
     "test:e2e:staging": "CONTEXT_NAME=staging yarn test:e2e",
-    "test:e2e:alfajores": "CONTEXT_NAME=alfajores yarn test:e2e"
+    "test:e2e:alfajores": "CONTEXT_NAME=alfajores yarn test:e2e",
+    "test:e2e:mainnet": "CONTEXT_NAME=mainnet yarn test:e2e"
   },
   "dependencies": {
     "@celo/contractkit": "^4.1.2-dev",

diff --git a/packages/phone-number-privacy/combiner/src/config.ts b/packages/phone-number-privacy/combiner/src/config.ts
@@ -39,6 +39,7 @@ export interface OdisConfig {
   fullNodeTimeoutMs: number
   fullNodeRetryCount: number
   fullNodeRetryDelayMs: number
+  shouldAuthenticate: boolean
 }
 
 export interface CombinerConfig {
@@ -106,6 +107,7 @@ if (DEV_MODE) {
       fullNodeTimeoutMs: FULL_NODE_TIMEOUT_IN_MS,
       fullNodeRetryCount: RETRY_COUNT,
       fullNodeRetryDelayMs: RETRY_DELAY_IN_MS,
+      shouldAuthenticate: true,
     },
     domains: {
       serviceName: defaultServiceName,
@@ -140,6 +142,7 @@ if (DEV_MODE) {
       fullNodeTimeoutMs: FULL_NODE_TIMEOUT_IN_MS,
       fullNodeRetryCount: RETRY_COUNT,
       fullNodeRetryDelayMs: RETRY_DELAY_IN_MS,
+      shouldAuthenticate: true,
     },
   }
 } else {
@@ -168,6 +171,7 @@ if (DEV_MODE) {
       fullNodeRetryDelayMs: Number(
         functionConfig.pnp.full_node_retry_delay_ms ?? RETRY_DELAY_IN_MS
       ),
+      shouldAuthenticate: toBool(functionConfig.pnp.should_authenticate, true),
     },
     domains: {
       serviceName: functionConfig.domains.service_name ?? defaultServiceName,
@@ -182,11 +186,12 @@ if (DEV_MODE) {
         currentVersion: Number(functionConfig.domains_keys.current_version),
         versions: functionConfig.domains_keys.versions,
       },
-      fullNodeTimeoutMs: Number(functionConfig.pnp.full_node_timeout_ms ?? FULL_NODE_TIMEOUT_IN_MS),
+      fullNodeTimeoutMs: Number(functionConfig.pnp.full_node_timeout_ms ?? FULL_NODE_TIMEOUT_IN_MS), // TODO refactor config - domains endpoints don't use full node
       fullNodeRetryCount: Number(functionConfig.pnp.full_node_retry_count ?? RETRY_COUNT),
       fullNodeRetryDelayMs: Number(
         functionConfig.pnp.full_node_retry_delay_ms ?? RETRY_DELAY_IN_MS
       ),
+      shouldAuthenticate: true,
     },
   }
 }

diff --git a/packages/phone-number-privacy/combiner/src/index.ts b/packages/phone-number-privacy/combiner/src/index.ts
@@ -10,6 +10,7 @@ export const combiner = functions
     // Keep instances warm for mainnet functions
     // Defined check required for running tests vs. deployment
     minInstances: functions.config().service ? Number(functions.config().service.min_instances) : 0,
+    memory: functions.config().service ? functions.config().service.memory : '512MB',
   })
   .https.onRequest(startCombiner(config))
 

diff --git a/packages/phone-number-privacy/combiner/src/pnp/endpoints/quota/action.ts b/packages/phone-number-privacy/combiner/src/pnp/endpoints/quota/action.ts
@@ -31,8 +31,10 @@ export function pnpQuota(
       return errorResult(400, WarningMessage.INVALID_INPUT)
     }
 
-    if (!(await authenticateUser(request, logger, dekFetcher))) {
-      return errorResult(401, WarningMessage.UNAUTHENTICATED_USER)
+    if (config.shouldAuthenticate) {
+      if (!(await authenticateUser(request, logger, dekFetcher))) {
+        return errorResult(401, WarningMessage.UNAUTHENTICATED_USER)
+      }
     }
 
     // TODO remove this, we shouldn't need keyVersionInfo for non-signing endpoints

diff --git a/packages/phone-number-privacy/combiner/src/pnp/endpoints/sign/action.ts b/packages/phone-number-privacy/combiner/src/pnp/endpoints/sign/action.ts
@@ -39,9 +39,12 @@ export function pnpSign(
       return errorResult(400, WarningMessage.INVALID_KEY_VERSION_REQUEST)
     }
 
-    if (!(await authenticateUser(request, logger, dekFetcher))) {
-      return errorResult(401, WarningMessage.UNAUTHENTICATED_USER)
+    if (config.shouldAuthenticate) {
+      if (!(await authenticateUser(request, logger, dekFetcher))) {
+        return errorResult(401, WarningMessage.UNAUTHENTICATED_USER)
+      }
     }
+
     const keyVersionInfo = getKeyVersionInfo(request, config, logger)
     const crypto = new BLSCryptographyClient(keyVersionInfo)
 

diff --git a/packages/phone-number-privacy/combiner/test/end-to-end/pnp.test.ts b/packages/phone-number-privacy/combiner/test/end-to-end/pnp.test.ts
@@ -9,6 +9,7 @@ import {
   SignMessageRequest,
   SignMessageResponseSchema,
 } from '@celo/phone-number-privacy-common'
+import { normalizeAddressWith0x } from '@celo/utils/lib/address'
 import threshold_bls from 'blind-threshold-bls'
 import { randomBytes } from 'crypto'
 import 'isomorphic-fetch'
@@ -19,10 +20,12 @@ import {
   ACCOUNT_ADDRESS_NO_QUOTA,
   BLINDED_PHONE_NUMBER,
   dekAuthSigner,
+  deks,
   getTestContextName,
   PHONE_NUMBER,
   walletAuthSigner,
 } from './resources'
+import { sleep } from '@celo/base'
 
 const { IdentifierPrefix } = OdisUtils.Identifier
 
@@ -36,9 +39,17 @@ const fullNodeUrl = process.env.ODIS_BLOCKCHAIN_PROVIDER
 
 const expectedVersion = getCombinerVersion()
 
-// TODO fix combiner e2e tests
-
 describe(`Running against service deployed at ${combinerUrl} w/ blockchain provider ${fullNodeUrl}`, () => {
+  beforeAll(async () => {
+    const accounts = await walletAuthSigner.contractKit.contracts.getAccounts()
+    const dekPublicKey = normalizeAddressWith0x(deks[0].publicKey)
+    if ((await accounts.getDataEncryptionKey(ACCOUNT_ADDRESS)) !== dekPublicKey) {
+      await accounts
+        .setAccountDataEncryptionKey(dekPublicKey)
+        .sendAndWaitForReceipt({ from: ACCOUNT_ADDRESS })
+    }
+  })
+
   it('Service is deployed at correct version', async () => {
     const response = await fetch(combinerUrl + CombinerEndpoint.STATUS, {
       method: 'GET',
@@ -140,27 +151,30 @@ describe(`Running against service deployed at ${combinerUrl} w/ blockchain provi
   })
 
   describe(`${CombinerEndpoint.PNP_SIGN}`, () => {
-    describe('new requests', () => {
-      beforeAll(async () => {
-        // Replenish quota for ACCOUNT_ADDRESS
-        // If this fails, may be necessary to faucet ACCOUNT_ADDRESS more funds
-        const numQueriesToReplenish = 2
-        const amountInWei = signerConfig.quota.queryPriceInCUSD
-          .times(1e18)
-          .times(numQueriesToReplenish)
-          .toString()
-        const stableToken = await walletAuthSigner.contractKit.contracts.getStableToken(
-          StableToken.cUSD
-        )
-        const odisPayments = await walletAuthSigner.contractKit.contracts.getOdisPayments()
-        await stableToken
-          .approve(odisPayments.address, amountInWei)
-          .sendAndWaitForReceipt({ from: ACCOUNT_ADDRESS })
-        await odisPayments
-          .payInCUSD(ACCOUNT_ADDRESS, amountInWei)
-          .sendAndWaitForReceipt({ from: ACCOUNT_ADDRESS })
-      })
+    beforeAll(async () => {
+      // Replenish quota for ACCOUNT_ADDRESS
+      // If this fails, may be necessary to faucet ACCOUNT_ADDRESS more funds
+      const numQueriesToReplenish = 100
+      const amountInWei = signerConfig.quota.queryPriceInCUSD
+        .times(1e18)
+        .times(numQueriesToReplenish)
+        .toString()
+      const stableToken = await walletAuthSigner.contractKit.contracts.getStableToken(
+        StableToken.cUSD
+      )
+      const odisPayments = await walletAuthSigner.contractKit.contracts.getOdisPayments()
+      await stableToken
+        .approve(odisPayments.address, amountInWei)
+        .sendAndWaitForReceipt({ from: ACCOUNT_ADDRESS })
+      await odisPayments
+        .payInCUSD(ACCOUNT_ADDRESS, amountInWei)
+        .sendAndWaitForReceipt({ from: ACCOUNT_ADDRESS })
+      // wait for cache to expire and then query to refresh
+      await sleep(5 * 1000)
+      await OdisUtils.Quota.getPnpQuotaStatus(ACCOUNT_ADDRESS, dekAuthSigner(0), SERVICE_CONTEXT)
+    })
 
+    describe('new requests', () => {
       // Requests made for PHONE_NUMBER from ACCOUNT_ADDRESS & same blinding factor
       // are replayed from previous test runs (for every run after the very first)
       let startingPerformedQueryCount: number

diff --git a/packages/phone-number-privacy/combiner/test/end-to-end/resources.ts b/packages/phone-number-privacy/combiner/test/end-to-end/resources.ts
@@ -34,8 +34,8 @@ export const getTestContextName = (): OdisContextName => {
 export const DEFAULT_FORNO_URL =
   process.env.ODIS_BLOCKCHAIN_PROVIDER ?? 'https://alfajores-forno.celo-testnet.org'
 
-export const PRIVATE_KEY = '0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef'
-export const ACCOUNT_ADDRESS = normalizeAddressWith0x(privateKeyToAddress(PRIVATE_KEY)) // 0x1be31a94361a391bbafb2a4ccd704f57dc04d4bb
+export const PRIVATE_KEY = '2c63bf6d60b16c8afa13e1069dbe92fef337c23855fff8b27732b3e9c6e7efd4'
+export const ACCOUNT_ADDRESS = normalizeAddressWith0x(privateKeyToAddress(PRIVATE_KEY)) // 0x6037800e91eaa703e38bad40c01410bbdf0fea7e
 
 export const PRIVATE_KEY_NO_QUOTA =
   '0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890000000'

diff --git a/packages/phone-number-privacy/monitor/src/query.ts b/packages/phone-number-privacy/monitor/src/query.ts
@@ -32,7 +32,8 @@ export const queryOdisForSalt = async (
   contextName: OdisContextName,
   timeoutMs: number = 10000,
   bypassQuota: boolean = false,
-  useDEK: boolean = false
+  useDEK: boolean = false,
+  privateKey?: string
 ) => {
   let authSigner: AuthSigner
   let accountAddress: string
@@ -43,12 +44,12 @@ export const queryOdisForSalt = async (
 
   if (useDEK) {
     accountAddress = ACCOUNT_ADDRESS
-    contractKit.connection.addAccount(PRIVATE_KEY)
+    contractKit.connection.addAccount(privateKey ?? PRIVATE_KEY)
     contractKit.defaultAccount = accountAddress
     authSigner = dekAuthSigner(0)
     phoneNumber = generateRandomPhoneNumber()
   } else {
-    const privateKey = await newPrivateKey()
+    privateKey ??= await newPrivateKey()
     accountAddress = normalizeAddressWith0x(privateKeyToAddress(privateKey))
     contractKit.connection.addAccount(privateKey)
     contractKit.defaultAccount = accountAddress
@@ -90,15 +91,16 @@ export const queryOdisForSalt = async (
 export const queryOdisForQuota = async (
   blockchainProvider: string,
   contextName: OdisContextName,
-  timeoutMs: number = 10000
+  timeoutMs: number = 10000,
+  privateKey?: string
 ) => {
   console.log(`contextName: ${contextName}`) // tslint:disable-line:no-console
   console.log(`blockchain provider: ${blockchainProvider}`) // tslint:disable-line:no-console
 
   const serviceContext = getServiceContext(contextName, OdisAPI.PNP)
 
   const contractKit = newKit(blockchainProvider, new LocalWallet())
-  const privateKey = await newPrivateKey()
+  privateKey ??= await newPrivateKey()
   const accountAddress = normalizeAddressWith0x(privateKeyToAddress(privateKey))
   contractKit.connection.addAccount(privateKey)
   contractKit.defaultAccount = accountAddress

diff --git a/packages/phone-number-privacy/monitor/src/scripts/run-load-test.ts b/packages/phone-number-privacy/monitor/src/scripts/run-load-test.ts
@@ -44,6 +44,10 @@ yargs
           type: 'number',
           description: 'number of requests to use when calculating latency moving average',
           default: 50,
+        })
+        .option('privateKey', {
+          type: 'string',
+          description: 'optional private key to send requests from',
         }),
     (args) => {
       if (args.rps == null || args.contextName == null) {
@@ -82,7 +86,8 @@ yargs
         args.duration,
         args.bypassQuota,
         args.useDEK,
-        args.movingAvgRequests
+        args.movingAvgRequests,
+        args.privateKey
       ) // tslint:disable-line:no-floating-promises
     }
   ).argv
diff --git a/packages/phone-number-privacy/monitor/src/test.ts b/packages/phone-number-privacy/monitor/src/test.ts
@@ -15,15 +15,17 @@ export async function testPNPSignQuery(
   contextName: OdisContextName,
   timeoutMs?: number,
   bypassQuota?: boolean,
-  useDEK?: boolean
+  useDEK?: boolean,
+  privateKey?: string
 ) {
   try {
     const odisResponse: IdentifierHashDetails = await queryOdisForSalt(
       blockchainProvider,
       contextName,
       timeoutMs,
       bypassQuota,
-      useDEK
+      useDEK,
+      privateKey
     )
     logger.debug({ odisResponse }, 'ODIS salt request successful. System is healthy.')
   } catch (err) {
@@ -43,14 +45,16 @@ export async function testPNPSignQuery(
 export async function testPNPQuotaQuery(
   blockchainProvider: string,
   contextName: OdisContextName,
-  timeoutMs?: number
+  timeoutMs?: number,
+  privateKey?: string
 ) {
   logger.info(`Performing test PNP query for ${CombinerEndpointPNP.PNP_QUOTA}`)
   try {
     const odisResponse: PnpClientQuotaStatus = await queryOdisForQuota(
       blockchainProvider,
       contextName,
-      timeoutMs
+      timeoutMs,
+      privateKey
     )
     logger.info({ odisResponse }, 'ODIS quota request successful. System is healthy.')
   } catch (err) {
@@ -88,7 +92,8 @@ export async function concurrentRPSLoadTest(
   duration: number = 0,
   bypassQuota: boolean = false,
   useDEK: boolean = false,
-  movingAverageRequests: number = 50
+  movingAverageRequests: number = 50,
+  privateKey?: string
 ) {
   const latencyQueue: number[] = []
   let movingAvgLatencySum = 0
@@ -128,8 +133,15 @@ export async function concurrentRPSLoadTest(
   const testFn = async () => {
     try {
       await (endpoint === CombinerEndpointPNP.PNP_SIGN
-        ? testPNPSignQuery(blockchainProvider, contextName, undefined, bypassQuota, useDEK)
-        : testPNPQuotaQuery(blockchainProvider, contextName))
+        ? testPNPSignQuery(
+            blockchainProvider,
+            contextName,
+            undefined,
+            bypassQuota,
+            useDEK,
+            privateKey
+          )
+        : testPNPQuotaQuery(blockchainProvider, contextName, undefined, privateKey))
     } catch (_) {
       logger.error('load test request failed')
     }