Reloading api (#982)

* Reloading API

* Rename env files

* undo changes to test docker file

* reloader fiel

* fix import

* Make tests work with most recent substrate contracts node. Fix CaptchaEvent model creation

* Update package-lock.json

* Use new error type

* Set default provider port to 9229

.github/workflows/provider_image.yml

@@ -100,8 +100,8 @@ jobs:
             # build the cli production bundle
             - name: Build cli production bundle
               run: |
-                  cp ./dev/scripts/env.rococo ./packages/cli/.env.production
-                  cp ./dev/scripts/env.rococo ./.env.production
+                  cp ./dev/scripts/env.production ./packages/cli/.env.production
+                  cp ./dev/scripts/env.production ./.env.production
                   NODE_ENV=production npm run -w @prosopo/cli bundle:prod
 
             # create the provider image

.github/workflows/publish.yml

@@ -358,7 +358,7 @@ jobs:
               run: |
                   # Copy the rococo env file to production env file
                   echo "Copying the rococo env to production env file in procaptcha-bundle"
-                  cp ./dev/scripts/env.rococo ./packages/procaptcha-bundle/.env.production
+                  cp ./dev/scripts/env.production ./packages/procaptcha-bundle/.env.production
 
                   # Navigate to the JS bundle directory and build
                   echo "Navigating to 'packages/procaptcha-bundle' and building JS bundle..."
@@ -497,7 +497,7 @@ jobs:
               run: |
                   # Copy the rococo env file to production env file
                   echo "Copying the rococo env to production env file in cli package"
-                  cp ./dev/scripts/env.rococo ./packages/cli/.env.production
+                  cp ./dev/scripts/env.production ./packages/cli/.env.production
 
                   # Navigate to the provider CLI directory and build
                   echo "Navigating to 'packages/cli' and bundling..."

dev/scripts/env.development

@@ -1,5 +1,3 @@
-PROSOPO_DAPP_WASM_PATH=./src/contract/sources/dapp/411053b7ec79cc77f5ec9f5eb18610b24daaaaf0/dapp.wasm
-PROSOPO_DAPP_ABI_PATH=./src/contract/sources/dapp/411053b7ec79cc77f5ec9f5eb18610b24daaaaf0/dapp.json
 PROSOPO_SITE_KEY=5HGjWAeFDfFCWPsjFQdVV2Msvz2XtMktvgocEZcCj68kUMaw
 PROSOPO_CONTRACT_ADDRESS=
 PROSOPO_DATABASE_PASSWORD=root
@@ -8,8 +6,6 @@ PROSOPO_DATABASE_NAME=prosopo
 PROSOPO_DATABASE_HOST=127.0.0.1
 PROSOPO_DATABASE_PORT=27017
 PROSOPO_SUBSTRATE_ENDPOINT=ws://localhost:9944
-PROSOPO_API_BASE_URL=http://localhost
-PROSOPO_API_PORT=9229
 PROSOPO_PROVIDER_ADDRESS=5EjTA28bKSbFPPyMbUjNtArxyqjwq38r1BapVmLZShaqEedV
 PROSOPO_PROVIDER_MNEMONIC=puppy cream effort carbon despair leg pyramid cotton endorse immense drill peasant
 NODE_ENV=development

dev/scripts/env.test

@@ -1,5 +1,3 @@
-PROSOPO_DAPP_WASM_PATH=./src/contract/sources/dapp/411053b7ec79cc77f5ec9f5eb18610b24daaaaf0/dapp.wasm
-PROSOPO_DAPP_ABI_PATH=./src/contract/sources/dapp/411053b7ec79cc77f5ec9f5eb18610b24daaaaf0/dapp.json
 PROSOPO_SITE_KEY=5HGjWAeFDfFCWPsjFQdVV2Msvz2XtMktvgocEZcCj68kUMaw
 PROSOPO_CONTRACT_ADDRESS=
 PROSOPO_DATABASE_PASSWORD=root
@@ -8,10 +6,6 @@ PROSOPO_DATABASE_NAME=prosopo_test
 PROSOPO_DATABASE_HOST=127.0.0.1
 PROSOPO_DATABASE_PORT=27017
 PROSOPO_SUBSTRATE_ENDPOINT=ws://localhost:9944
-PROSOPO_API_BASE_URL=http://localhost
-PROSOPO_API_PORT=9229
-PROSOPO_PAIR_TYPE=sr25519
-PROSOPO_SS58_FORMAT=42
 PROSOPO_PROVIDER_ADDRESS=5EjTA28bKSbFPPyMbUjNtArxyqjwq38r1BapVmLZShaqEedV
 PROSOPO_PROVIDER_MNEMONIC=puppy cream effort carbon despair leg pyramid cotton endorse immense drill peasant
 NODE_ENV=test

dev/scripts/src/setup/funds.ts

@@ -73,7 +73,7 @@ export async function sendFunds(
     // eslint-disable-next-line no-async-promise-executor
     const result = new Promise<ISubmittableResult>(async (resolve, reject) => {
         const unsub = await api.tx.balances
-            .transfer(address, amount)
+            .transferAllowDeath(address, amount)
             .signAndSend(pair, { nonce }, (result: ISubmittableResult) => {
                 if (result.status.isInBlock || result.status.isFinalized) {
                     result.events

dev/scripts/src/setup/setup.ts

@@ -47,7 +47,7 @@ function getDefaultProvider(): IProviderAccount {
     return {
         url: process.env.PROSOPO_API_PORT
             ? `http://localhost:${process.env.PROSOPO_API_PORT}`
-            : 'http://localhost:3000',
+            : 'http://localhost:9229',
         fee: 10,
         payee: Payee.dapp,
         stake: Math.pow(10, 13),

docker/docker-compose.test.yml

@@ -1,7 +1,7 @@
 version: '3.8'
 services:
   substrate:
-    image: prosopo/substrate-contracts-node:v0.25
+    image: prosopo/substrate-contracts-node:v0.35
     ports:
       - "9615:9615"
       - "9944:9944"

packages/cli/src/argv.ts

@@ -33,6 +33,8 @@ import {
 import { hideBin } from 'yargs/helpers'
 import yargs from 'yargs'
 
+export type AwaitedProcessedArgs = { [x: string]: unknown; api: boolean; _: (string | number)[]; $0: string }
+
 export function processArgs(args: string[], pair: KeyringPair, config: ProsopoConfigOutput) {
     const logger = getLogger(LogLevel.enum.info, 'CLI')
     return yargs(hideBin(args))

packages/cli/src/cli.ts

@@ -13,20 +13,19 @@
 // limitations under the License.
 import { LogLevel, getLogger } from '@prosopo/common'
 import { ProsopoConfigOutput } from '@prosopo/types'
-import { ProviderEnvironment } from '@prosopo/env'
 import { getPairAsync } from '@prosopo/contract'
 import { getSecret } from './process.env.js'
 import { isMain } from '@prosopo/util'
 import { loadEnv } from './env.js'
 import { processArgs } from './argv.js'
-import { start } from './start.js'
+import ReloadingAPI from './reloader.js'
 import getConfig from './prosopo.config.js'
 import process from 'process'
 
 const log = getLogger(LogLevel.enum.info, 'CLI')
 
 async function main() {
-    loadEnv()
+    const envPath = loadEnv()
 
     const secret = getSecret()
 
@@ -43,11 +42,10 @@ async function main() {
     log.info(`Contract address: ${process.env.PROSOPO_CONTRACT_ADDRESS}`)
 
     const processedArgs = await processArgs(process.argv, pair, config)
+
+    log.info(`Processsed args: ${JSON.stringify(processedArgs, null, 4)}`)
     if (processedArgs.api) {
-        const env = new ProviderEnvironment(config, pair)
-        await env.isReady()
-        log.info('Starting API')
-        await start(env)
+        await new ReloadingAPI(envPath, config, pair, processedArgs).start()
     } else {
         process.exit(0)
     }

packages/cli/src/env.ts

@@ -27,11 +27,12 @@ export function getEnv() {
     return 'development'
 }
 
-export function loadEnv(rootDir?: string, filename?: string, filePath?: string) {
+export function loadEnv(rootDir?: string, filename?: string, filePath?: string): string {
     const envPath = getEnvFile(path.resolve(rootDir || '.'), filename, filePath)
     const args = { path: envPath }
     logger.info(`Loading env from ${envPath}`)
     dotenv.config(args)
+    return envPath
 }
 
 export function getEnvFile(rootDir?: string, filename = '.env', filepath = path.join(__dirname, '../..')) {

packages/cli/src/reloader.ts

@@ -0,0 +1,63 @@
+// Copyright 2021-2024 Prosopo (UK) Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+import { AwaitedProcessedArgs } from './argv.js'
+import { KeyringPair } from '@polkadot/keyring/types'
+import { LogLevel, getLogger } from '@prosopo/common'
+import { ProsopoConfigOutput } from '@prosopo/types'
+import { ProviderEnvironment } from '@prosopo/env'
+import { Server } from 'node:net'
+import { loadEnv } from './env.js'
+import { start } from './start.js'
+import fs from 'fs'
+
+const log = getLogger(LogLevel.enum.info, 'CLI')
+
+export default class ReloadingAPI {
+    private _envWatcher: any
+    private _envPath: string
+    private _config: ProsopoConfigOutput
+    private _pair: KeyringPair
+    private _processedArgs: AwaitedProcessedArgs
+    private api: Server | undefined
+
+    constructor(envPath: string, config: ProsopoConfigOutput, pair: KeyringPair, processedArgs: AwaitedProcessedArgs) {
+        this._envPath = envPath
+        this._config = config
+        this._pair = pair
+        this._processedArgs = processedArgs
+    }
+
+    public async start() {
+        log.info('Starting API')
+        this._envWatcher = await this._watchEnv()
+        const env = new ProviderEnvironment(this._config, this._pair)
+        await env.isReady()
+        this.api = await start(env)
+    }
+
+    public async stop() {
+        log.info('Stopping API')
+        if (this.api) {
+            this.api.close()
+        }
+    }
+
+    private async _watchEnv() {
+        return fs.watchFile(this._envPath, async () => {
+            await this.stop()
+            loadEnv()
+            await this.start()
+        })
+    }
+}

packages/cli/src/start.ts

@@ -13,6 +13,7 @@
 // limitations under the License.
 import { ProsopoApiError, i18nMiddleware } from '@prosopo/common'
 import { ProviderEnvironment } from '@prosopo/env'
+import { Server } from 'node:net'
 import { getDB, getSecret } from './process.env.js'
 import { getPairAsync } from '@prosopo/contract'
 import { loadEnv } from './env.js'
@@ -41,7 +42,7 @@ export const handleErrors = (
     })
 }
 
-function startApi(env: ProviderEnvironment) {
+function startApi(env: ProviderEnvironment): Server {
     env.logger.info(`Starting Prosopo API`)
     const apiApp = express()
     const apiPort = env.config.server.port
@@ -52,7 +53,7 @@ function startApi(env: ProviderEnvironment) {
     apiApp.use(prosopoRouter(env))
 
     apiApp.use(handleErrors)
-    apiApp.listen(apiPort, () => {
+    return apiApp.listen(apiPort, () => {
         env.logger.info(`Prosopo app listening at http://localhost:${apiPort}`)
     })
 }
@@ -73,5 +74,5 @@ export async function start(env?: ProviderEnvironment) {
         env = new ProviderEnvironment(config, pair)
     }
     await env.isReady()
-    startApi(env)
+    return startApi(env)
 }

packages/database/src/eventsDatabase/eventsDatabase.ts

@@ -1,5 +1,8 @@
-import { StoredEvents } from '@prosopo/types'
-import mongoose from 'mongoose'
+import { ProsopoDBError, getLoggerDefault } from '@prosopo/common'
+import { StoredEventRecord, StoredEvents } from '@prosopo/types'
+import mongoose, { Model } from 'mongoose'
+const logger = getLoggerDefault()
+const MAX_RETRIES = 3
 
 const captchaEventSchema = new mongoose.Schema({
     touchEvents: [
@@ -20,42 +23,60 @@ const captchaEventSchema = new mongoose.Schema({
         {
             key: String,
             timestamp: Number,
-            isShiftKey: Boolean,
-            isCtrlKey: Boolean,
+            isShiftKey: { type: Boolean, required: false },
+            isCtrlKey: { type: Boolean, required: false },
         },
     ],
     accountId: String,
 })
+let CaptchaEvent: typeof Model<StoredEventRecord>
+try {
+    CaptchaEvent = mongoose.model('CaptchaEvent')
+} catch (error) {
+    CaptchaEvent = mongoose.model('CaptchaEvent', captchaEventSchema)
+}
 
-const CaptchaEvent = mongoose.model('CaptchaEvent', captchaEventSchema)
-
-const addCaptchaEventRecord = async (record: {
-    touchEvents?: { x: number; y: number; timestamp: number }[]
-    mouseEvents?: { x: number; y: number; timestamp: number }[]
-    keyboardEvents?: { key: string; timestamp: number; isShiftKey?: boolean; isCtrlKey?: boolean }[]
-    accountId: string
-}): Promise<void> => {
+const addCaptchaEventRecord = async (record: StoredEventRecord): Promise<void> => {
     try {
         const newRecord = new CaptchaEvent(record)
         await newRecord.save()
-        console.log('Record added successfully')
+        logger.info('Record added successfully')
     } catch (error) {
-        console.error('Error adding record to the database:', error)
+        logger.error('Error adding record to the database:', error)
     }
 }
 
 export const saveCaptchaEvent = async (events: StoredEvents, accountId: string, atlasUri: string) => {
-    await mongoose
-        .connect(atlasUri)
-        .then(() => console.log('Connected to MongoDB Atlas'))
-        .catch((err) => console.error('Error connecting to MongoDB:', err))
+    return new Promise((resolve, reject) => {
+        const connection = mongoose.createConnection(atlasUri)
+        for (let attempt = 1; attempt <= MAX_RETRIES; attempt++) {
+            connection.once('open', resolve).on('error', (e) => {
+                logger.warn(`Mongoose connection error`)
+                logger.error(e)
 
-    const captchaEventData = {
-        ...events,
-        accountId,
-    }
+                // Only reject on the last attempt, otherwise handle the retry logic
+                if (attempt === MAX_RETRIES) {
+                    reject(
+                        new ProsopoDBError('DATABASE.CONNECT_ERROR', {
+                            context: { failedFuncName: saveCaptchaEvent.name, db: 'events database' },
+                            logger: logger,
+                        })
+                    )
+                } else {
+                    // Remove the error listener to avoid accumulated listeners on retries
+                    connection?.removeAllListeners('error')
+                }
+            })
+        }
+
+        const captchaEventData = {
+            ...events,
+            accountId,
+        }
 
-    addCaptchaEventRecord(captchaEventData)
-        .then(() => console.log('Captcha event data saved'))
-        .catch((error) => console.error('Error saving captcha event data:', error))
+        addCaptchaEventRecord(captchaEventData)
+            .then(() => logger.info('Captcha event data saved'))
+            .catch((error) => logger.error('Error saving captcha event data:', error))
+            .finally(() => connection.close())
+    })
 }

packages/provider/src/tasks/tasks.ts

@@ -555,7 +555,7 @@ export class Tasks {
         return { provider, dbConnectionOk }
     }
 
-    /** Get the dataset from the databse */
+    /** Get the dataset from the database */
     async getProviderDataset(datasetId: string): Promise<DatasetWithIds> {
         return await this.db.getDataset(datasetId)
     }

packages/provider/src/tests/dataUtils/funds.ts

@@ -69,7 +69,7 @@ export async function sendFunds(
     // eslint-disable-next-line no-async-promise-executor
     const result = new Promise<ISubmittableResult>(async (resolve, reject) => {
         const unsub = await api.tx.balances
-            .transfer(address, amount)
+            .transferAllowDeath(address, amount)
             .signAndSend(pair, { nonce }, (result: ISubmittableResult) => {
                 if (result.status.isInBlock || result.status.isFinalized) {
                     result.events

packages/types/src/procaptcha/collector.ts

@@ -22,3 +22,7 @@ export type StoredEvents = {
     touchEvents?: ProsopoTouchEvent[]
     keyboardEvents?: ProsopoKeyboardEvent[]
 }
+
+export interface StoredEventRecord extends StoredEvents {
+    accountId: string
+}