Skip to content

Commit

Permalink
Provider rate limiting (#1289)
Browse files Browse the repository at this point in the history
* Provider rate limiting setup

* Fixing defaults for api rate limits

* zod tidy up

* Removing redundant readme changes

* Removing unused code

* Unused import

* Fixing linting issues

* Fixing circular deps

* Linting command fix

* Circular dep import resolve

* deps fixes

* Linting

* Sneaky uncaught linting error

* Removing console log
  • Loading branch information
HughParry authored Jul 3, 2024
1 parent e196bb2 commit 02887ee
Show file tree
Hide file tree
Showing 11 changed files with 216 additions and 31 deletions.
16 changes: 15 additions & 1 deletion package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@
"lint:cmd": "f() { npm run -w @prosopo/scripts license; npm run eslint:cmd -- $@; npm run prettier:cmd -- $@; }; f",
"lint:workspace": "npm run eslint:workspace && npm run prettier:workspace",
"lint:contracts": "npm -w @prosopo/captcha-contract -w @prosopo/common-contract -w @prosopo/proxy-contract run lint",
"lint:fix": "FILES=$(git diff --name-status main | sed '/^[M]/!D' | awk -F ' ' '{print $2}'); echo $FILES; f() { npm run -w @prosopo/scripts license:fix; npm run eslint:fix -- $FILES; npm run prettier:fix -- $FILES; }; f",
"lint:fix": "FILES=$(git diff --name-status main | sed '/^[M|A]/!D' | awk -F ' ' '{print $2}'); echo $FILES; f() { npm run -w @prosopo/scripts license:fix; npm run eslint:fix -- $FILES; npm run prettier:fix -- $FILES; }; f",
"lint:fix:contracts": "npm run -w @prosopo/scripts license:fix && npm -w @prosopo/captcha-contract -w @prosopo/common-contract -w @prosopo/proxy-contract run lint:fix",
"lint:fix:workspace": "npm run eslint:fix:workspace && npm run prettier:fix:workspace",
"removePolkadotJSWarnings": "sed -i 's/console.warn\\(.*\\);//g' ./node_modules/@polkadot/util/versionDetect.js && sed -i 's/console.warn\\(.*\\);//g' ./node_modules/@polkadot/util/cjs/versionDetect.js || true",
Expand Down
12 changes: 6 additions & 6 deletions packages/cli/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@
},
"dependencies": {
"@polkadot/keyring": "12.6.2",
"@polkadot/types": "10.13.1",
"@polkadot/util": "12.6.2",
"@polkadot/util-crypto": "12.6.2",
"@prosopo/captcha-contract": "1.0.2",
Expand All @@ -45,18 +44,19 @@
"cors": "^2.8.5",
"cron-parser": "^4.9.0",
"dotenv": "^16.0.1",
"express-rate-limit": "^7.3.1",
"yargs": "^17.7.2",
"zod": "^3.22.4"
},
"devDependencies": {
"es-main": "^1.2.0",
"express": "^4.18.2",
"vite": "^5.1.7",
"vitest": "^1.3.1",
"@prosopo/config": "1.0.2",
"@types/cors": "^2.8.14",
"es-main": "^1.2.0",
"express": "^4.18.2",
"tslib": "2.6.2",
"typescript": "5.1.6"
"typescript": "5.1.6",
"vite": "^5.1.7",
"vitest": "^1.3.1"
},
"author": "Prosopo",
"license": "Apache-2.0",
Expand Down
75 changes: 75 additions & 0 deletions packages/cli/src/RateLimiter.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
// Copyright 2021-2024 Prosopo (UK) Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
import { AdminApiPaths, ApiPaths } from '@prosopo/types'

export const getRateLimitConfig = () => {
return {
[ApiPaths.GetImageCaptchaChallenge]: {
windowMs: process.env.PROSOPO_GET_IMAGE_CAPTCHA_CHALLENGE_WINDOW,
limit: process.env.PROSOPO_GET_IMAGE_CAPTCHA_CHALLENGE_LIMIT,
},
[ApiPaths.GetPowCaptchaChallenge]: {
windowMs: process.env.PROSOPO_GET_POW_CAPTCHA_CHALLENGE_WINDOW,
limit: process.env.PROSOPO_GET_POW_CAPTCHA_CHALLENGE_LIMIT,
},
[ApiPaths.SubmitImageCaptchaSolution]: {
windowMs: process.env.PROSOPO_SUBMIT_IMAGE_CAPTCHA_SOLUTION_WINDOW,
limit: process.env.PROSOPO_SUBMIT_IMAGE_CAPTCHA_SOLUTION_LIMIT,
},
[ApiPaths.SubmitPowCaptchaSolution]: {
windowMs: process.env.PROSOPO_SUBMIT_POW_CAPTCHA_SOLUTION_WINDOW,
limit: process.env.PROSOPO_SUBMIT_POW_CAPTCHA_SOLUTION_LIMIT,
},
[ApiPaths.VerifyPowCaptchaSolution]: {
windowMs: process.env.PROSOPO_VERIFY_POW_CAPTCHA_SOLUTION_WINDOW,
limit: process.env.PROSOPO_VERIFY_POW_CAPTCHA_SOLUTION_LIMIT,
},
[ApiPaths.VerifyImageCaptchaSolutionDapp]: {
windowMs: process.env.PROSOPO_VERIFY_IMAGE_CAPTCHA_SOLUTION_DAPP_WINDOW,
limit: process.env.PROSOPO_VERIFY_IMAGE_CAPTCHA_SOLUTION_DAPP_LIMIT,
},
[ApiPaths.VerifyImageCaptchaSolutionUser]: {
windowMs: process.env.PROSOPO_VERIFY_IMAGE_CAPTCHA_SOLUTION_USER_WINDOW,
limit: process.env.PROSOPO_VERIFY_IMAGE_CAPTCHA_SOLUTION_USER_LIMIT,
},
[ApiPaths.GetProviderStatus]: {
windowMs: process.env.PROSOPO_GET_PROVIDER_STATUS_WINDOW,
limit: process.env.PROSOPO_GET_PROVIDER_STATUS_LIMIT,
},
[ApiPaths.GetProviderDetails]: {
windowMs: process.env.PROSOPO_GET_PROVIDER_DETAILS_WINDOW,
limit: process.env.PROSOPO_GET_PROVIDER_DETAILS_LIMIT,
},
[ApiPaths.SubmitUserEvents]: {
windowMs: process.env.PROSOPO_SUBMIT_USER_EVENTS_WINDOW,
limit: process.env.PROSOPO_SUBMIT_USER_EVENTS_LIMIT,
},
[AdminApiPaths.BatchCommit]: {
windowMs: process.env.PROSOPO_BATCH_COMMIT_WINDOW,
limit: process.env.PROSOPO_BATCH_COMMIT_LIMIT,
},
[AdminApiPaths.UpdateDataset]: {
windowMs: process.env.PROSOPO_UPDATE_DATASET_WINDOW,
limit: process.env.PROSOPO_UPDATE_DATASET_LIMIT,
},
[AdminApiPaths.ProviderDeregister]: {
windowMs: process.env.PROSOPO_PROVIDER_DEREGISTER_WINDOW,
limit: process.env.PROSOPO_PROVIDER_DEREGISTER_LIMIT,
},
[AdminApiPaths.ProviderUpdate]: {
windowMs: process.env.PROSOPO_PROVIDER_UPDATE_WINDOW,
limit: process.env.PROSOPO_PROVIDER_UPDATE_LIMIT,
},
}
}
2 changes: 2 additions & 0 deletions packages/cli/src/prosopo.config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import {
} from '@prosopo/types'
import { getAddress, getPassword, getSecret } from './process.env.js'
import { getLogLevel } from '@prosopo/common'
import { getRateLimitConfig } from './RateLimiter.js'

function getMongoURI(): string {
const protocol = process.env.PROSOPO_DATABASE_PROTOCOL || 'mongodb'
Expand Down Expand Up @@ -83,5 +84,6 @@ export default function getConfig(
devOnlyWatchEvents: process.env._DEV_ONLY_WATCH_EVENTS === 'true',
mongoEventsUri: process.env.PROSOPO_MONGO_EVENTS_URI || '',
mongoCaptchaUri: process.env.PROSOPO_MONGO_CAPTCHA_URI || '',
rateLimits: getRateLimitConfig(),
} as ProsopoConfigInput)
}
9 changes: 9 additions & 0 deletions packages/cli/src/start.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
import { CombinedApiPaths } from '@prosopo/types'
import { ProviderEnvironment } from '@prosopo/env'
import { Server } from 'node:net'
import { getDB, getSecret } from './process.env.js'
Expand All @@ -21,6 +22,7 @@ import { prosopoAdminRouter, prosopoRouter, prosopoVerifyRouter, storeCaptchasEx
import cors from 'cors'
import express from 'express'
import getConfig from './prosopo.config.js'
import rateLimit from 'express-rate-limit'

function startApi(env: ProviderEnvironment, admin = false): Server {
env.logger.info(`Starting Prosopo API`)
Expand All @@ -37,6 +39,13 @@ function startApi(env: ProviderEnvironment, admin = false): Server {
apiApp.use(prosopoAdminRouter(env))
}

// Rate limiting
const rateLimits = env.config.rateLimits
for (const [path, limit] of Object.entries(rateLimits)) {
const enumPath = path as CombinedApiPaths
apiApp.use(enumPath, rateLimit(limit))
}

return apiApp.listen(apiPort, () => {
env.logger.info(`Prosopo app listening at http://localhost:${apiPort}`)
})
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,12 +11,12 @@
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
import { BotDetectionFunction, ProcaptchaFrictionlessProps } from '@prosopo/types'
import { Procaptcha } from '@prosopo/procaptcha-react'
import { ProcaptchaPlaceholder } from '@prosopo/web-components'
import { ProcaptchaPow } from '@prosopo/procaptcha-pow'
import { useEffect, useState } from 'react'
import { BotDetectionFunction, ProcaptchaFrictionlessProps } from '@prosopo/types'
import { isBot } from '@prosopo/detector'
import { useEffect, useState } from 'react'

const customDetectBot: BotDetectionFunction = async () => {
return await isBot().then((result) => {
Expand Down
31 changes: 12 additions & 19 deletions packages/types/src/config/config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,17 @@
// See the License for the specific language governing permissions and
// limitations under the License.

import { ApiPathRateLimits, ProviderDefaultRateLimits } from '../provider/index.js'
import {
DEFAULT_IMAGE_CAPTCHA_SOLUTION_TIMEOUT,
DEFAULT_IMAGE_CAPTCHA_TIMEOUT,
DEFAULT_IMAGE_CAPTCHA_VERIFIED_TIMEOUT,
DEFAULT_IMAGE_MAX_VERIFIED_TIME_CACHED,
DEFAULT_MAX_VERIFIED_TIME_CONTRACT,
DEFAULT_POW_CAPTCHA_CACHED_TIMEOUT,
DEFAULT_POW_CAPTCHA_SOLUTION_TIMEOUT,
DEFAULT_POW_CAPTCHA_VERIFIED_TIMEOUT,
} from './timeouts.js'
import { NetworkNamesSchema, ProsopoNetworkSchema } from './network.js'
import { input } from 'zod'
import { literal } from 'zod'
Expand All @@ -32,25 +43,6 @@ export const EnvironmentTypesSchema = zEnum(['development', 'staging', 'producti

export type EnvironmentTypes = zInfer<typeof EnvironmentTypesSchema>

const ONE_MINUTE = 60 * 1000
// The timeframe in which a user must complete an image captcha (1 minute)
export const DEFAULT_IMAGE_CAPTCHA_TIMEOUT = ONE_MINUTE
// The timeframe in which an image captcha solution remains valid on the page before timing out (2 minutes)
export const DEFAULT_IMAGE_CAPTCHA_SOLUTION_TIMEOUT = DEFAULT_IMAGE_CAPTCHA_TIMEOUT * 2
// The timeframe in which an image captcha solution must be verified within (3 minutes)
export const DEFAULT_IMAGE_CAPTCHA_VERIFIED_TIMEOUT = DEFAULT_IMAGE_CAPTCHA_TIMEOUT * 3
// The time in milliseconds that a cached, verified, image captcha solution is valid for (15 minutes)
export const DEFAULT_IMAGE_MAX_VERIFIED_TIME_CACHED = DEFAULT_IMAGE_CAPTCHA_TIMEOUT * 15
// The timeframe in which a pow captcha solution remains valid on the page before timing out (1 minute)
export const DEFAULT_POW_CAPTCHA_SOLUTION_TIMEOUT = ONE_MINUTE
// The timeframe in which a pow captcha must be completed and verified (2 minutes)
export const DEFAULT_POW_CAPTCHA_VERIFIED_TIMEOUT = DEFAULT_POW_CAPTCHA_SOLUTION_TIMEOUT * 2
// The time in milliseconds that a Provider cached, verified, pow captcha solution is valid for (3 minutes)
export const DEFAULT_POW_CAPTCHA_CACHED_TIMEOUT = DEFAULT_POW_CAPTCHA_SOLUTION_TIMEOUT * 3
// The time in milliseconds since the last correct captcha recorded in the contract (15 minutes), after which point, the
// user will be required to complete another captcha
export const DEFAULT_MAX_VERIFIED_TIME_CONTRACT = ONE_MINUTE * 15

export const DatabaseConfigSchema = record(
EnvironmentTypesSchema,
object({
Expand Down Expand Up @@ -252,6 +244,7 @@ export const ProsopoConfigSchema = ProsopoBasicConfigSchema.merge(
server: ProsopoImageServerConfigSchema,
mongoEventsUri: string().optional(),
mongoCaptchaUri: string().optional(),
rateLimits: ApiPathRateLimits.default(ProviderDefaultRateLimits),
})
)

Expand Down
1 change: 1 addition & 0 deletions packages/types/src/config/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,3 +13,4 @@
// limitations under the License.
export * from './config.js'
export * from './network.js'
export * from './timeouts.js'
31 changes: 31 additions & 0 deletions packages/types/src/config/timeouts.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
// Copyright 2021-2024 Prosopo (UK) Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
const ONE_MINUTE = 60 * 1000
// The timeframe in which a user must complete an image captcha (1 minute)
export const DEFAULT_IMAGE_CAPTCHA_TIMEOUT = ONE_MINUTE
// The timeframe in which an image captcha solution remains valid on the page before timing out (2 minutes)
export const DEFAULT_IMAGE_CAPTCHA_SOLUTION_TIMEOUT = DEFAULT_IMAGE_CAPTCHA_TIMEOUT * 2
// The timeframe in which an image captcha solution must be verified within (3 minutes)
export const DEFAULT_IMAGE_CAPTCHA_VERIFIED_TIMEOUT = DEFAULT_IMAGE_CAPTCHA_TIMEOUT * 3
// The time in milliseconds that a cached, verified, image captcha solution is valid for (15 minutes)
export const DEFAULT_IMAGE_MAX_VERIFIED_TIME_CACHED = DEFAULT_IMAGE_CAPTCHA_TIMEOUT * 15
// The timeframe in which a pow captcha solution remains valid on the page before timing out (1 minute)
export const DEFAULT_POW_CAPTCHA_SOLUTION_TIMEOUT = ONE_MINUTE
// The timeframe in which a pow captcha must be completed and verified (2 minutes)
export const DEFAULT_POW_CAPTCHA_VERIFIED_TIMEOUT = DEFAULT_POW_CAPTCHA_SOLUTION_TIMEOUT * 2
// The time in milliseconds that a Provider cached, verified, pow captcha solution is valid for (3 minutes)
export const DEFAULT_POW_CAPTCHA_CACHED_TIMEOUT = DEFAULT_POW_CAPTCHA_SOLUTION_TIMEOUT * 3
// The time in milliseconds since the last correct captcha recorded in the contract (15 minutes), after which point, the
// user will be required to complete another captcha
export const DEFAULT_MAX_VERIFIED_TIME_CONTRACT = ONE_MINUTE * 15
64 changes: 62 additions & 2 deletions packages/types/src/provider/api.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,22 @@
// limitations under the License.
import { ApiParams } from '../api/params.js'
import { CaptchaSolutionSchema, CaptchaWithProof } from '../datasets/index.js'
import { DEFAULT_IMAGE_MAX_VERIFIED_TIME_CACHED, DEFAULT_POW_CAPTCHA_VERIFIED_TIMEOUT } from '../config/index.js'
import { DEFAULT_IMAGE_MAX_VERIFIED_TIME_CACHED, DEFAULT_POW_CAPTCHA_VERIFIED_TIMEOUT } from '../config/timeouts.js'
import { Hash, Provider } from '@prosopo/captcha-contract/types-returns'
import { ProcaptchaTokenSpec } from '../procaptcha/index.js'
import { array, input, number, object, output, string, infer as zInfer } from 'zod'
import {
ZodDefault,
ZodNumber,
ZodObject,
ZodOptional,
array,
input,
number,
object,
output,
string,
infer as zInfer,
} from 'zod'

export enum ApiPaths {
GetImageCaptchaChallenge = '/v1/prosopo/provider/captcha/image',
Expand All @@ -38,6 +50,54 @@ export enum AdminApiPaths {
ProviderUpdate = '/v1/prosopo/provider/admin/update',
}

export type CombinedApiPaths = ApiPaths | AdminApiPaths

export const ProviderDefaultRateLimits = {
[ApiPaths.GetImageCaptchaChallenge]: { windowMs: 60000, limit: 30 },
[ApiPaths.GetPowCaptchaChallenge]: { windowMs: 60000, limit: 60 },
[ApiPaths.SubmitImageCaptchaSolution]: { windowMs: 60000, limit: 60 },
[ApiPaths.SubmitPowCaptchaSolution]: { windowMs: 60000, limit: 60 },
[ApiPaths.VerifyPowCaptchaSolution]: { windowMs: 60000, limit: 60 },
[ApiPaths.VerifyImageCaptchaSolutionDapp]: { windowMs: 60000, limit: 60 },
[ApiPaths.VerifyImageCaptchaSolutionUser]: { windowMs: 60000, limit: 60 },
[ApiPaths.GetProviderStatus]: { windowMs: 60000, limit: 60 },
[ApiPaths.GetProviderDetails]: { windowMs: 60000, limit: 60 },
[ApiPaths.SubmitUserEvents]: { windowMs: 60000, limit: 60 },
[AdminApiPaths.BatchCommit]: { windowMs: 60000, limit: 5 },
[AdminApiPaths.UpdateDataset]: { windowMs: 60000, limit: 5 },
[AdminApiPaths.ProviderDeregister]: { windowMs: 60000, limit: 1 },
[AdminApiPaths.ProviderUpdate]: { windowMs: 60000, limit: 5 },
}

type RateLimit = {
windowMs: number
limit: number
}

type RateLimitSchemaType = ZodObject<{
windowMs: ZodDefault<ZodOptional<ZodNumber>>
limit: ZodDefault<ZodOptional<ZodNumber>>
}>

// Utility function to create Zod schemas with defaults
const createRateLimitSchemaWithDefaults = (paths: Record<CombinedApiPaths, RateLimit>) =>
object(
Object.entries(paths).reduce(
(schemas, [path, defaults]) => {
const enumPath = path as CombinedApiPaths
schemas[enumPath] = object({
windowMs: number().optional().default(defaults.windowMs),
limit: number().optional().default(defaults.limit),
})

return schemas
},
{} as Record<CombinedApiPaths, RateLimitSchemaType>
)
)

export const ApiPathRateLimits = createRateLimitSchemaWithDefaults(ProviderDefaultRateLimits)

export interface DappUserSolutionResult {
[ApiParams.captchas]: CaptchaIdAndProof[]
partialFee?: string
Expand Down

0 comments on commit 02887ee

Please sign in to comment.