ODIS 3.0.0 Release Branch (#10462)

* sdks 4.1.1-beta.1, pnpcommon 3.0.0-beta.1

* fix sdk release script and formatting

* remove unneccessary combiner dependency

* get rid of trx

* update signer version used in combiner integration tests

* add metering

* get rid of trx

* highlighting other relevant code for reviewers

* highlighting other relevant code for reviewers again

* Soloseng/fix-release (#10466)

* update packages

* change format to number

* add back trx

* rename tableWithLockForTrx to queryWithOptionalTrx

* drive-by remove attestations from config

* remove comment

* add back trx

* add back trx comment

* add back rollack

* bump version numbers to beta.3

* mock combiner authentication

* Odis signer refactor  (#10464)

Refactors Signer to be more performant

* Combiner tracing - staging (#10496)

Adds tracing to the combiner

* removed blocknumber (#10507)

* Signer mock interfaces (#10510)

add mock signer interfaces

* add moving avg to load test (#10509)

* Big Refactor of the combiner (#10513)

* first pass into transforming into handlers

* Extract methods out of IO class (story: IO class destruction)

* io: getSignerEndpoint lifted up to base class

* Remove inputChecks and authenticate from IO

* wip: kill Combiner and Signer actions

* use handler function (remove IO and Action)

* kill ThresholdService and ResponseLogger classes

* Removes Session and CryptoSession

* Fix typing errors

* fix tests

* Uses httpAgent for http/s connections

* use shouldCheckKeyVerion boolean argument

* fix some tests

* cosmetic cleanup from PR review

* Fixes some tests

* fix error reporting

* fix pnp integration test

* bump combiner version

* fix integrations test

---------

Co-authored-by: alecps <alec@cLabs.co>

* fix lint errors

* fix unit tests

* fix issue on domainQuota signer endpoint

* publish common package for combiner deploy

* implement AbortSignal any

* add back log based metrics

* add back trx, fix signer tests

* prevent loadtest from crashing on error response (#10517)

* Set user agent to the kit (#10518)

* Add Remove database error (#10519)

* fix log based metric

* store signature in requests table (#10520)

* store signature in requests table

* remove it.only

* fix keyVersion parsing

* fix fallback url bug

* lint fix + bump combiner version

* make signature optional

* Signer Database requests prunning (#10521)

* Signer Database requests prunning

* Fix tests

* Refactor code

* Update packages/phone-number-privacy/signer/src/pnp/services/request-service.ts

Co-authored-by: soloseng <102702451+soloseng@users.noreply.github.com>

* Adding tests

* Update tests

* Remove unnecessary transaction wrapper

---------

Co-authored-by: soloseng <102702451+soloseng@users.noreply.github.com>

* Fixed linter

* decrease coverage threshold 76% -> 68%

* Add index in timestamp (#10525)

* Update packages/phone-number-privacy/signer/src/pnp/services/request-service.ts

* fix signer e2e tests

* add TODO

* fix response metrics

* misc cosmetic changes from review

* remove unused code from common pkg

* misc edits, remove unused test

* remove blockNumber from response type schemas

* publish common package and bump odis versions

* use 4.1.1-dev everywhere except for ODIS packages

* revert change to identity sdk

* add back change to identity sdk

* remove duplicate countAndThrowDBError in domains request table wrapper

* misc fixes from reviewing metrics

* add TODO statements and remove unused QService

* misc cleanup

* cleanup combiner comments

* remove combiner comment

---------

Co-authored-by: soloseng <102702451+soloseng@users.noreply.github.com>
Co-authored-by: Mariano Cortesi <mariano@clabs.co>
Co-authored-by: Álvaro Fernández <35505302+alvarof2@users.noreply.github.com>
Co-authored-by: Gaston Ponti <ponti@clabs.co>

.vscode/settings.json

@@ -43,5 +43,6 @@
   "javascript.format.enable": false,
   "editor.tabSize": 2,
   "editor.detectIndentation": false,
-  "tslint.jsEnable": true
+  "tslint.jsEnable": true,
+  "typescript.tsdk": "node_modules/typescript/lib"
 }

dependency-graph.json

@@ -78,7 +78,6 @@
       "@celo/encrypted-backup",
       "@celo/identity",
       "@celo/phone-number-privacy-common",
-      "@celo/phone-number-privacy-signer",
       "@celo/phone-utils",
       "@celo/utils"
     ]

packages/celotool/package.json

@@ -17,7 +17,7 @@
     "@celo/network-utils": "4.1.1-dev",
     "@celo/utils": "4.1.1-dev",
     "@ethereumjs/util": "8.0.5",
-    "@ethereumjs/rlp":"4.0.1",
+    "@ethereumjs/rlp": "4.0.1",
     "@google-cloud/monitoring": "0.7.1",
     "@google-cloud/pubsub": "^0.28.1",
     "@google-cloud/secret-manager": "3.0.0",
@@ -66,4 +66,4 @@
     "build": "tsc -b ."
   },
   "private": true
-}
+}

packages/metadata-crawler/package.json

@@ -34,4 +34,4 @@
     "clean:all": "yarn clean && rm -rf lib"
   },
   "private": true
-}
+}

packages/phone-number-privacy/TODO.md

@@ -0,0 +1,9 @@
+# TODO
+
+- Add caching to Combiner 
+- Fix Combiner e2e tests
+- Fix types in errorResult and sendFailure so we don't have to use ANY in Signer
+- Refactor domain sign handler to use db transactions properly
+- refactor authorization function with the new account model
+- Make caching config parameters configurable by environment
+- TODO comments

packages/phone-number-privacy/combiner/.env

@@ -6,3 +6,7 @@ SERVICE_NAME='odis-combiner'
 # For e2e Tests
 ODIS_BLOCKCHAIN_PROVIDER=https://alfajores-forno.celo-testnet.org
 CONTEXT_NAME='alfajores'
+# TODO investigate why these are defined here 
+NODE_OPTIONS='--require ./dist/tracing.js'
+TRACER_ENDPOINT='https://grafana-agent.staging-odis2-centralus.celo-networks-dev.org/api/traces'
+TRACING_SERVICE_NAME='odis-combiner-staging'

packages/phone-number-privacy/combiner/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@celo/phone-number-privacy-combiner",
-  "version": "3.0.0-dev",
+  "version": "3.0.0-beta.7",
   "description": "Orchestrates and combines threshold signatures for use in ODIS",
   "author": "Celo",
   "license": "Apache-2.0",
@@ -28,12 +28,20 @@
     "test:e2e:alfajores": "CONTEXT_NAME=alfajores yarn test:e2e"
   },
   "dependencies": {
-    "@celo/contractkit": "^4.1.1-dev",
-    "@celo/phone-number-privacy-common": "^3.0.0-dev",
-    "@celo/identity": "^4.1.1-dev",
-    "@celo/encrypted-backup": "^4.1.1-dev",
+    "@celo/contractkit": "^4.1.1-beta.1",
+    "@celo/phone-number-privacy-common": "^3.0.0-beta.3",
+    "@celo/identity": "^4.1.1-beta.1",
+    "@celo/encrypted-backup": "^4.1.1-beta.1",
     "@celo/poprf": "^0.1.9",
     "@types/bunyan": "^1.8.8",
+    "@opentelemetry/api": "^1.4.1",
+    "@opentelemetry/auto-instrumentations-node": "^0.38.0",
+    "@opentelemetry/exporter-jaeger": "^1.15.2",
+    "@opentelemetry/propagator-ot-trace": "^0.27.0",
+    "@opentelemetry/sdk-metrics": "^1.15.2",
+    "@opentelemetry/sdk-node": "^0.41.2",
+    "@opentelemetry/sdk-trace-web": "^1.15.2",
+    "@opentelemetry/semantic-conventions": "^1.15.2",
     "blind-threshold-bls": "https://github.com/celo-org/blind-threshold-bls-wasm#e1e2f8a",
     "dotenv": "^8.2.0",
     "express": "^4.17.1",
@@ -45,18 +53,15 @@
     "uuid": "^7.0.3"
   },
   "devDependencies": {
-    "@celo/utils": "^4.1.1-dev",
-    "@celo/phone-utils": "^4.1.1-dev",
+    "@celo/utils": "^4.1.1-beta.1",
+    "@celo/phone-utils": "^4.1.1-beta.1",
     "@types/express": "^4.17.6",
     "@types/supertest": "^2.0.12",
     "@types/uuid": "^7.0.3",
     "firebase-functions-test": "^3.1.0",
     "firebase-tools": "12.4.7"
   },
-  "peerDependencies": {
-    "@celo/phone-number-privacy-signer": "^3.0.0-dev"
-  },
   "engines": {
-    "node": ">=18"
+    "node": "18"
   }
-}
+}