update pnpcommon README, fix release sdks script, combiner deps (#10463)

update pnpcommon README, release sdks script, combiner deps

packages/phone-number-privacy/combiner/package.json

@@ -32,14 +32,13 @@
     "@celo/phone-number-privacy-common": "^3.0.0-dev",
     "@celo/identity": "^4.1.1-dev",
     "@celo/encrypted-backup": "^4.1.1-dev",
-    "@celo/identity-prev": "npm:@celo/identity@1.2.0",
     "@celo/poprf": "^0.1.9",
     "@types/bunyan": "^1.8.8",
     "blind-threshold-bls": "https://github.com/celo-org/blind-threshold-bls-wasm#e1e2f8a",
     "dotenv": "^8.2.0",
     "express": "^4.17.1",
-    "firebase-admin": "^9.12.0",
-    "firebase-functions": "^3.15.7",
+    "firebase-admin": "^11.10.1",
+    "firebase-functions": "^4.4.1",
     "knex": "^2.1.0",
     "node-fetch": "^2.6.9",
     "pg": "^8.2.1",
@@ -51,13 +50,13 @@
     "@types/express": "^4.17.6",
     "@types/supertest": "^2.0.12",
     "@types/uuid": "^7.0.3",
-    "firebase-functions-test": "^0.3.3",
-    "firebase-tools": "9.20.0"
+    "firebase-functions-test": "^3.1.0",
+    "firebase-tools": "12.4.7"
   },
   "peerDependencies": {
-    "@celo/phone-number-privacy-signer": "^2.0.2"
+    "@celo/phone-number-privacy-signer": "^3.0.0-dev"
   },
   "engines": {
-    "node": ">=14"
+    "node": ">=18"
   }
 }

packages/phone-number-privacy/combiner/src/common/combine.ts

@@ -157,12 +157,12 @@ export abstract class CombineAction<R extends OdisRequest> implements Action<R>
   }
 
   private addFailureToSession(signer: Signer, errorCode: number | undefined, session: Session<R>) {
-    session.logger.warn(
-      `Received failure from ${session.failedSigners.size}/${this.signers.length} signers`
-    )
     // Tracking failed request count via signer url prevents
     // double counting the same failed request by mistake
     session.failedSigners.add(signer.url)
+    session.logger.warn(
+      `Received failure from ${session.failedSigners.size}/${this.signers.length} signers`
+    )
     if (errorCode) {
       session.incrementErrorCodeCount(errorCode)
     }

packages/phone-number-privacy/combiner/src/common/crypto-clients/crypto-client.ts

@@ -1,5 +1,6 @@
 import { ErrorMessage, KeyVersionInfo } from '@celo/phone-number-privacy-common'
 import Logger from 'bunyan'
+import { performance } from 'perf_hooks'
 
 export interface ServicePartialSignature {
   url: string
@@ -38,7 +39,17 @@ export abstract class CryptoClient {
         `${ErrorMessage.NOT_ENOUGH_PARTIAL_SIGNATURES} ${this.allSignaturesLength}/${threshold}`
       )
     }
-    return this._combineBlindedSignatureShares(blindedMessage, logger)
+
+    const start = `Start combineBlindedSignatureShares`
+    const end = `End combineBlindedSignatureShares`
+    performance.mark(start)
+
+    const combinedSignature = this._combineBlindedSignatureShares(blindedMessage, logger)
+
+    performance.mark(end)
+    performance.measure('combineBlindedSignatureShares', start, end)
+
+    return combinedSignature
   }
 
   /*

packages/phone-number-privacy/combiner/src/index.ts

@@ -10,7 +10,7 @@ export const combiner = functions
   .runWith({
     // Keep instances warm for mainnet functions
     // Defined check required for running tests vs. deployment
-    minInstances: functions.config().service ? functions.config().service.min_instances : undefined,
+    minInstances: functions.config().service ? Number(functions.config().service.min_instances) : 0,
   })
   .https.onRequest(startCombiner(config, getContractKit(config.blockchain)))
 export * from './config'

packages/phone-number-privacy/common/README.md

@@ -27,7 +27,8 @@ These instructions assume the following scenario for readability:
 4. Same idea as above -- ensure the version of the `@celo/phone-number-privacy-common` package is set to the version you are trying to release (i.e. `2.0.3-beta.1`) and that all other packages are importing this version.
 5. From the monorepo root directory, run `yarn reset && yarn && yarn build` (expect this to take at least 10 mins)
 6. Commit your changes with the message `3.2.0-beta.1`
-7. Publish the ODIS common package by navigating to the `phone-number-privacy/common` directory and running `npm publish —-tag beta --version 3.2.0-beta.1 —-otp <OTP>`
+7. Publish the ODIS common package by navigating to the `phone-number-privacy/common` directory and running `npm publish —-tag beta`
+   - You will be prompted to enter your OTP
    - When publishing as `latest`, omit the `--tag beta`
 8. Publish the sdks by running `npm run deploy-sdks` from the monorepo root directory
     - You will be prompted to enter a version number that you wish to publish. i.e. `3.2.0-beta.1`

packages/phone-number-privacy/common/package.json

@@ -45,8 +45,7 @@
     "@types/elliptic": "^6.4.12",
     "@types/express": "^4.17.6",
     "@types/is-base64": "^1.1.0",
-    "@types/node-fetch": "^2.5.7",
-    "web3": "1.10.0"
+    "@types/node-fetch": "^2.5.7"
   },
   "engines": {
     "node": ">=10"

packages/phone-number-privacy/common/src/test/utils.ts

@@ -1,7 +1,6 @@
 import { privateKeyToAddress } from '@celo/utils/lib/address'
 import { serializeSignature, Signature, signMessage } from '@celo/utils/lib/signatureUtils'
 import BigNumber from 'bignumber.js'
-import Web3 from 'web3'
 import {
   AuthenticationMethod,
   PhoneNumberPrivacyRequest,
@@ -50,7 +49,7 @@ export function createMockOdisPayments(totalPaidCUSDFunc: jest.Mock<BigNumber, [
 
 export function createMockContractKit(
   c: { [contractName in ContractRetrieval]?: any },
-  mockWeb3?: any
+  mockWeb3: any
 ) {
   const contracts: any = {}
   for (const t of Object.keys(c)) {
@@ -66,8 +65,7 @@ export function createMockContractKit(
   }
 }
 
-export function createMockConnection(mockWeb3?: any) {
-  mockWeb3 = mockWeb3 ?? new Web3()
+export function createMockConnection(mockWeb3: any) {
   return {
     web3: mockWeb3,
     getTransactionCount: jest.fn(() => mockWeb3.eth.getTransactionCount()),

packages/phone-number-privacy/monitor/src/query.ts

@@ -18,8 +18,9 @@ import { genSessionID } from '@celo/phone-number-privacy-common/lib/utils/logger
 import { normalizeAddressWith0x, privateKeyToAddress } from '@celo/utils/lib/address'
 import { defined } from '@celo/utils/lib/sign-typed-data-utils'
 import { LocalWallet } from '@celo/wallet-local'
+import { ACCOUNT_ADDRESS, dekAuthSigner, generateRandomPhoneNumber, PRIVATE_KEY } from './resources'
 
-const phoneNumber = fetchEnv('PHONE_NUMBER')
+let phoneNumber = fetchEnv('PHONE_NUMBER')
 
 const newPrivateKey = async () => {
   const mnemonic = await generateMnemonic(MnemonicStrength.s256_24words)
@@ -29,21 +30,35 @@ const newPrivateKey = async () => {
 export const queryOdisForSalt = async (
   blockchainProvider: string,
   contextName: OdisContextName,
-  timeoutMs: number = 10000
+  timeoutMs: number = 10000,
+  bypassQuota: boolean = false,
+  useDEK: boolean = false
 ) => {
+  let authSigner: AuthSigner
+  let accountAddress: string
   console.log(`contextName: ${contextName}`) // tslint:disable-line:no-console
   console.log(`blockchain provider: ${blockchainProvider}`) // tslint:disable-line:no-console
+  console.log(`using DEK: ${useDEK}`) // tslint:disable-line:no-console
 
   const serviceContext = getServiceContext(contextName, OdisAPI.PNP)
 
   const contractKit = newKit(blockchainProvider, new LocalWallet())
-  const privateKey = await newPrivateKey()
-  const accountAddress = normalizeAddressWith0x(privateKeyToAddress(privateKey))
-  contractKit.connection.addAccount(privateKey)
-  contractKit.defaultAccount = accountAddress
-  const authSigner: AuthSigner = {
-    authenticationMethod: OdisUtils.Query.AuthenticationMethod.WALLET_KEY,
-    contractKit,
+
+  if (useDEK) {
+    accountAddress = ACCOUNT_ADDRESS
+    contractKit.connection.addAccount(PRIVATE_KEY)
+    contractKit.defaultAccount = accountAddress
+    authSigner = dekAuthSigner(0)
+    phoneNumber = generateRandomPhoneNumber()
+  } else {
+    const privateKey = await newPrivateKey()
+    accountAddress = normalizeAddressWith0x(privateKeyToAddress(privateKey))
+    contractKit.connection.addAccount(privateKey)
+    contractKit.defaultAccount = accountAddress
+    authSigner = {
+      authenticationMethod: OdisUtils.Query.AuthenticationMethod.WALLET_KEY,
+      contractKit,
+    }
   }
 
   const abortController = new AbortController()
@@ -52,6 +67,7 @@ export const queryOdisForSalt = async (
     console.log(`ODIS salt request timed out after ${timeoutMs} ms`) // tslint:disable-line:no-console
   }, timeoutMs)
   try {
+    const testSessionId = Math.floor(Math.random() * 100000).toString()
     const res = await OdisUtils.Identifier.getObfuscatedIdentifier(
       phoneNumber,
       OdisUtils.Identifier.IdentifierPrefix.PHONE_NUMBER,
@@ -61,7 +77,7 @@ export const queryOdisForSalt = async (
       undefined,
       undefined,
       undefined,
-      genSessionID(),
+      bypassQuota ? testSessionId : genSessionID(),
       undefined,
       abortController
     )

packages/phone-number-privacy/monitor/src/resources.ts

@@ -0,0 +1,48 @@
+import { EncryptionKeySigner } from '@celo/identity/lib/odis/query'
+import { AuthenticationMethod } from '@celo/phone-number-privacy-common'
+import {
+  ensureLeading0x,
+  normalizeAddressWith0x,
+  privateKeyToAddress,
+} from '@celo/utils/lib/address'
+
+export const PRIVATE_KEY = '0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef'
+export const ACCOUNT_ADDRESS = normalizeAddressWith0x(privateKeyToAddress(PRIVATE_KEY)) // 0x1be31a94361a391bbafb2a4ccd704f57dc04d4bb
+
+interface DEK {
+  privateKey: string
+  publicKey: string
+  address: string
+}
+
+export const deks: DEK[] = [
+  {
+    privateKey: 'bf8a2b73baf8402f8fe906ad3f42b560bf14b39f7df7797ece9e293d6f162188',
+    publicKey: '034846bc781cacdafc66f3a77aa9fc3c56a9dadcd683c72be3c446fee8da041070',
+    address: '0x7b33dF2607b85e3211738a49A6Ad6E8Ed4d13F6E',
+  },
+  {
+    privateKey: '0975b0c565abc75b6638a749ea3008cb52676af3eabe4b80e19c516d82330364',
+    publicKey: '03b1ac8c445f0796978018c087b97e8213b32c39e6a8642ae63dce71da33a19f65',
+    address: '0x34332049B07Fab9a2e843A7C8991469d93cF6Ae6',
+  },
+]
+
+// The following code can be used to generate more test DEKs
+// const generateDEKs = (n: number): Promise<DEK[]> => Promise.all([...Array(n).keys()].map(
+//   async () => await deriveDek(await generateMnemonic())
+// ))
+
+export const dekAuthSigner = (index: number): EncryptionKeySigner => {
+  return {
+    authenticationMethod: AuthenticationMethod.ENCRYPTION_KEY,
+    rawKey: ensureLeading0x(deks[index].privateKey),
+  }
+}
+
+export function generateRandomPhoneNumber() {
+  const min = 1000000000 // Smallest 10-digit number
+  const max = 9999999999 // Largest 10-digit number
+  const randomNumber = Math.floor(Math.random() * (max - min + 1)) + min
+  return '+1' + randomNumber.toString()
+}

packages/phone-number-privacy/monitor/src/scripts/run-load-test.ts

@@ -1,10 +1,19 @@
 import { OdisContextName } from '@celo/identity/lib/odis/query'
+import { CombinerEndpointPNP } from '@celo/phone-number-privacy-common'
 import yargs from 'yargs'
 import { concurrentLoadTest, serialLoadTest } from '../test'
 
 /* tslint:disable:no-console */
 
-const runLoadTest = (contextName: string, numWorker: number, isSerial: boolean) => {
+const runLoadTest = (
+  contextName: string,
+  numWorker: number,
+  isSerial: boolean,
+  pnpQuotaEndpoint: boolean,
+  timeoutMs: number,
+  bypassQuota: boolean,
+  useDEK: boolean
+) => {
   let blockchainProvider: string
   switch (contextName) {
     case 'alfajoresstaging':
@@ -25,11 +34,30 @@ const runLoadTest = (contextName: string, numWorker: number, isSerial: boolean)
     process.exit(1)
   }
   if (isSerial) {
-    serialLoadTest(numWorker, blockchainProvider!, contextName as OdisContextName) // tslint:disable-line:no-floating-promises
+    // tslint:disable-next-line: no-floating-promises
+    serialLoadTest(
+      numWorker,
+      blockchainProvider!,
+      contextName as OdisContextName,
+      pnpQuotaEndpoint ? CombinerEndpointPNP.PNP_QUOTA : CombinerEndpointPNP.PNP_SIGN,
+      timeoutMs,
+      bypassQuota,
+      useDEK
+    )
   } else {
-    concurrentLoadTest(numWorker, blockchainProvider!, contextName as OdisContextName) // tslint:disable-line:no-floating-promises
+    // tslint:disable-next-line: no-floating-promises
+    concurrentLoadTest(
+      numWorker,
+      blockchainProvider!,
+      contextName as OdisContextName,
+      pnpQuotaEndpoint ? CombinerEndpointPNP.PNP_QUOTA : CombinerEndpointPNP.PNP_SIGN,
+      timeoutMs,
+      bypassQuota,
+      useDEK
+    )
   }
 }
+
 // tslint:disable-next-line: no-unused-expression
 yargs
   .scriptName('ODIS-load-test')
@@ -52,8 +80,38 @@ yargs
         })
         .option('isSerial', {
           type: 'boolean',
-          description: 'run test workers in series.',
+          description: 'Run test workers in series.',
+          default: false,
+        })
+        .option('timeoutMs', {
+          type: 'number',
+          description: 'Timout in ms.',
+          default: 10000,
+        })
+        .option('bypassQuota', {
+          type: 'boolean',
+          description: 'Bypass Signer quota check.',
+          default: false,
+        })
+        .option('useDEK', {
+          type: 'boolean',
+          description: 'Use Data Encryption Key (DEK) to authenticate.',
+          default: false,
+        })
+        .option('pnpQuotaEndpoint', {
+          type: 'boolean',
+          description:
+            'Use this flag to load test PNP_QUOTA endpoint instead of PNP_SIGN endpoint.',
           default: false,
         }),
-    (args) => runLoadTest(args.contextName!, args.numWorkers!, args.isSerial)
+    (args) =>
+      runLoadTest(
+        args.contextName!,
+        args.numWorkers!,
+        args.isSerial,
+        args.pnpQuotaEndpoint,
+        args.timeoutMs,
+        args.bypassQuota,
+        args.useDEK
+      )
   ).argv