From 9fd8f464d207aa0d61cab9a8f3b38ddb80c4c7a2 Mon Sep 17 00:00:00 2001 From: dblythy Date: Thu, 19 Jan 2023 12:18:50 +1100 Subject: [PATCH 1/8] feat: Add redis store for rate limiter --- package-lock.json | 18 ++++++++++++++++++ package.json | 1 + spec/RateLimit.spec.js | 30 ++++++++++++++++++++++++++++++ src/Options/Definitions.js | 4 ++++ src/Options/docs.js | 1 + src/Options/index.js | 3 +++ src/middlewares.js | 32 ++++++++++++++++++++++++++++++++ 7 files changed, 89 insertions(+) diff --git a/package-lock.json b/package-lock.json index f92bd73dbd..dd0ab11c11 100644 --- a/package-lock.json +++ b/package-lock.json @@ -44,6 +44,7 @@ "pg-monitor": "1.5.0", "pg-promise": "10.12.1", "pluralize": "8.0.0", + "rate-limit-redis": "3.0.1", "redis": "4.0.6", "semver": "7.3.8", "subscriptions-transport-ws": "0.11.0", @@ -16994,6 +16995,17 @@ "node": ">= 0.6" } }, + "node_modules/rate-limit-redis": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/rate-limit-redis/-/rate-limit-redis-3.0.1.tgz", + "integrity": "sha512-L6yhOUBrAZ8VEMX9DwlM3X6hfm8yq+gBO4LoOW7+JgmNq59zE7QmLz4v5VnwYPvLeSh/e7PDcrzUI3UumJw1iw==", + "engines": { + "node": ">= 14.5.0" + }, + "peerDependencies": { + "express-rate-limit": "^6" + } + }, "node_modules/raw-body": { "version": "2.5.1", "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.1.tgz", @@ -33525,6 +33537,12 @@ "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==" }, + "rate-limit-redis": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/rate-limit-redis/-/rate-limit-redis-3.0.1.tgz", + "integrity": "sha512-L6yhOUBrAZ8VEMX9DwlM3X6hfm8yq+gBO4LoOW7+JgmNq59zE7QmLz4v5VnwYPvLeSh/e7PDcrzUI3UumJw1iw==", + "requires": {} + }, "raw-body": { "version": "2.5.1", "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.1.tgz", diff --git a/package.json b/package.json index 5da1f96d39..005ab004d8 100644 --- a/package.json +++ b/package.json @@ -53,6 +53,7 @@ "pg-monitor": "1.5.0", "pg-promise": "10.12.1", "pluralize": "8.0.0", + "rate-limit-redis": "3.0.1", "redis": "4.0.6", "semver": "7.3.8", "subscriptions-transport-ws": "0.11.0", diff --git a/spec/RateLimit.spec.js b/spec/RateLimit.spec.js index 60aff61381..f5f03cda51 100644 --- a/spec/RateLimit.spec.js +++ b/spec/RateLimit.spec.js @@ -1,3 +1,4 @@ +const RedisCacheAdapter = require('../lib/Adapters/Cache/RedisCacheAdapter').default; describe('rate limit', () => { it('can limit cloud functions', async () => { Parse.Cloud.define('test', () => 'Abc'); @@ -367,4 +368,33 @@ describe('rate limit', () => { }) ).toBeRejectedWith(`Invalid rate limit option "path"`); }); + describe_only(() => { + return process.env.PARSE_SERVER_TEST_CACHE === 'redis'; + })('with RedisCache', function () { + fit('does work with cache', async () => { + await reconfigureServer({ + rateLimit: [ + { + requestPath: '/classes/*', + requestTimeWindow: 10000, + requestCount: 1, + errorResponseMessage: 'Too many requests', + includeInternalRequests: true, + redisURL: 'redis://localhost:6379', + }, + ], + }); + const obj = new Parse.Object('Test'); + await obj.save(); + await expectAsync(obj.save()).toBeRejectedWith( + new Parse.Error(Parse.Error.CONNECTION_FAILED, 'Too many requests') + ); + const cache = new RedisCacheAdapter(); + await cache.connect(); + const value = await cache.get('rl:127.0.0.1'); + expect(value).toEqual(2); + const ttl = await cache.client.ttl('rl:127.0.0.1'); + expect(ttl).toEqual(10); + }); + }); }); diff --git a/src/Options/Definitions.js b/src/Options/Definitions.js index f7a4f822d7..ce5c19b2fe 100644 --- a/src/Options/Definitions.js +++ b/src/Options/Definitions.js @@ -557,6 +557,10 @@ module.exports.RateLimitOptions = { action: parsers.booleanParser, default: false, }, + redisURL: { + env: 'PARSE_SERVER_RATE_LIMIT_REDIS_URL', + help: 'Optional, a RedisURL used to store requests across multiple servers or clusters', + }, requestCount: { env: 'PARSE_SERVER_RATE_LIMIT_REQUEST_COUNT', help: diff --git a/src/Options/docs.js b/src/Options/docs.js index b0378d327e..aba9dceb1e 100644 --- a/src/Options/docs.js +++ b/src/Options/docs.js @@ -104,6 +104,7 @@ * @property {String} errorResponseMessage The error message that should be returned in the body of the HTTP 429 response when the rate limit is hit. Default is `Too many requests.`. * @property {Boolean} includeInternalRequests Optional, if `true` the rate limit will also apply to requests that are made in by Cloud Code, default is `false`. Note that a public Cloud Code function that triggers internal requests may circumvent rate limiting and be vulnerable to attacks. * @property {Boolean} includeMasterKey Optional, if `true` the rate limit will also apply to requests using the `masterKey`, default is `false`. Note that a public Cloud Code function that triggers internal requests using the `masterKey` may circumvent rate limiting and be vulnerable to attacks. + * @property {String} redisURL Optional, a RedisURL used to store requests across multiple servers or clusters * @property {Number} requestCount The number of requests that can be made per IP address within the time window set in `requestTimeWindow` before the rate limit is applied. * @property {String[]} requestMethods Optional, the HTTP request methods to which the rate limit should be applied, default is all methods. * @property {String} requestPath The path of the API route to be rate limited. Route paths, in combination with a request method, define the endpoints at which requests can be made. Route paths can be strings, string patterns, or regular expression. See: https://expressjs.com/en/guide/routing.html diff --git a/src/Options/index.js b/src/Options/index.js index 661d062de6..eebcae6e0a 100644 --- a/src/Options/index.js +++ b/src/Options/index.js @@ -320,6 +320,9 @@ export interface RateLimitOptions { /* Optional, if `true` the rate limit will also apply to requests that are made in by Cloud Code, default is `false`. Note that a public Cloud Code function that triggers internal requests may circumvent rate limiting and be vulnerable to attacks. :DEFAULT: false */ includeInternalRequests: ?boolean; + /* Optional, a RedisURL used to store requests across multiple servers or clusters + */ + redisURL: ?string; } export interface SecurityOptions { diff --git a/src/middlewares.js b/src/middlewares.js index 46c4a32a43..97bd4b2ce2 100644 --- a/src/middlewares.js +++ b/src/middlewares.js @@ -11,6 +11,8 @@ import rateLimit from 'express-rate-limit'; import { RateLimitOptions } from './Options/Definitions'; import pathToRegexp from 'path-to-regexp'; import ipRangeCheck from 'ip-range-check'; +import RedisStore from 'rate-limit-redis'; +import { createClient } from 'redis'; export const DEFAULT_ALLOWED_HEADERS = 'X-Parse-Master-Key, X-Parse-REST-API-Key, X-Parse-Javascript-Key, X-Parse-Application-Id, X-Parse-Client-Version, X-Parse-Session-Token, X-Requested-With, X-Parse-Revocable-Session, X-Parse-Request-Id, Content-Type, Pragma, Cache-Control'; @@ -470,6 +472,35 @@ export const addRateLimit = (route, config) => { if (!config.rateLimits) { config.rateLimits = []; } + const redisStore = { + connectionPromise: Promise.resolve(), + store: null, + connected: false, + }; + if (route.redisURL) { + const client = createClient({ + url: route.redisURL, + }); + redisStore.connectionPromise = async () => { + if (redisStore.connected) { + return; + } + try { + await client.connect(); + redisStore.connected = true; + } catch (e) { + const log = config?.loggerController || defaultLogger; + log.error(`Could not connect to redisURL in rate limit: ${e}`); + } + }; + redisStore.connectionPromise(); + redisStore.store = new RedisStore({ + sendCommand: async (...args) => { + await redisStore.connectionPromise(); + return client.sendCommand(args); + }, + }); + } config.rateLimits.push({ path: pathToRegexp(route.requestPath), handler: rateLimit({ @@ -503,6 +534,7 @@ export const addRateLimit = (route, config) => { keyGenerator: request => { return request.config.ip; }, + store: redisStore.store, }), }); Config.put(config); From c2ab8e4934d8950fdd1174371b74bc9c38884371 Mon Sep 17 00:00:00 2001 From: dblythy Date: Thu, 19 Jan 2023 12:21:56 +1100 Subject: [PATCH 2/8] Update RateLimit.spec.js --- spec/RateLimit.spec.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/RateLimit.spec.js b/spec/RateLimit.spec.js index f5f03cda51..5e7861a540 100644 --- a/spec/RateLimit.spec.js +++ b/spec/RateLimit.spec.js @@ -371,7 +371,7 @@ describe('rate limit', () => { describe_only(() => { return process.env.PARSE_SERVER_TEST_CACHE === 'redis'; })('with RedisCache', function () { - fit('does work with cache', async () => { + it('does work with cache', async () => { await reconfigureServer({ rateLimit: [ { From 8b4583c3f3c180c012db52f88ea572aba2ff7833 Mon Sep 17 00:00:00 2001 From: Daniel Date: Sat, 21 Jan 2023 11:38:26 +1100 Subject: [PATCH 3/8] Update src/Options/index.js Co-authored-by: Manuel <5673677+mtrezza@users.noreply.github.com> --- src/Options/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Options/index.js b/src/Options/index.js index eebcae6e0a..d05351d0ae 100644 --- a/src/Options/index.js +++ b/src/Options/index.js @@ -320,7 +320,7 @@ export interface RateLimitOptions { /* Optional, if `true` the rate limit will also apply to requests that are made in by Cloud Code, default is `false`. Note that a public Cloud Code function that triggers internal requests may circumvent rate limiting and be vulnerable to attacks. :DEFAULT: false */ includeInternalRequests: ?boolean; - /* Optional, a RedisURL used to store requests across multiple servers or clusters + /* Optional, the URL of the Redis server to store rate limit data. This allows to rate limit requests for multiple server instances by calculating the sum of all requests across all servers. This is useful if a multiple servers are processing requests behind a load balancer. For example, the limit of 10 requests is reached if each of 2 servers processed 5 requests. */ redisURL: ?string; } From ce9603c298aadddfb73a19761513c6af4c0e91f7 Mon Sep 17 00:00:00 2001 From: dblythy Date: Sat, 21 Jan 2023 16:00:52 +1100 Subject: [PATCH 4/8] defintions --- src/Options/Definitions.js | 3 ++- src/Options/docs.js | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Options/Definitions.js b/src/Options/Definitions.js index ce5c19b2fe..a442a53c24 100644 --- a/src/Options/Definitions.js +++ b/src/Options/Definitions.js @@ -559,7 +559,8 @@ module.exports.RateLimitOptions = { }, redisURL: { env: 'PARSE_SERVER_RATE_LIMIT_REDIS_URL', - help: 'Optional, a RedisURL used to store requests across multiple servers or clusters', + help: + 'Optional, the URL of the Redis server to store rate limit data. This allows to rate limit requests for multiple server instances by calculating the sum of all requests across all servers. This is useful if a multiple servers are processing requests behind a load balancer. For example, the limit of 10 requests is reached if each of 2 servers processed 5 requests.', }, requestCount: { env: 'PARSE_SERVER_RATE_LIMIT_REQUEST_COUNT', diff --git a/src/Options/docs.js b/src/Options/docs.js index aba9dceb1e..399a2aade1 100644 --- a/src/Options/docs.js +++ b/src/Options/docs.js @@ -104,7 +104,7 @@ * @property {String} errorResponseMessage The error message that should be returned in the body of the HTTP 429 response when the rate limit is hit. Default is `Too many requests.`. * @property {Boolean} includeInternalRequests Optional, if `true` the rate limit will also apply to requests that are made in by Cloud Code, default is `false`. Note that a public Cloud Code function that triggers internal requests may circumvent rate limiting and be vulnerable to attacks. * @property {Boolean} includeMasterKey Optional, if `true` the rate limit will also apply to requests using the `masterKey`, default is `false`. Note that a public Cloud Code function that triggers internal requests using the `masterKey` may circumvent rate limiting and be vulnerable to attacks. - * @property {String} redisURL Optional, a RedisURL used to store requests across multiple servers or clusters + * @property {String} redisURL Optional, the URL of the Redis server to store rate limit data. This allows to rate limit requests for multiple server instances by calculating the sum of all requests across all servers. This is useful if a multiple servers are processing requests behind a load balancer. For example, the limit of 10 requests is reached if each of 2 servers processed 5 requests. * @property {Number} requestCount The number of requests that can be made per IP address within the time window set in `requestTimeWindow` before the rate limit is applied. * @property {String[]} requestMethods Optional, the HTTP request methods to which the rate limit should be applied, default is all methods. * @property {String} requestPath The path of the API route to be rate limited. Route paths, in combination with a request method, define the endpoints at which requests can be made. Route paths can be strings, string patterns, or regular expression. See: https://expressjs.com/en/guide/routing.html From 5fa0557f85708a3b718f92046d9204e7f779b56f Mon Sep 17 00:00:00 2001 From: Daniel Date: Sun, 22 Jan 2023 10:12:14 +1100 Subject: [PATCH 5/8] Update src/Options/index.js Co-authored-by: Manuel <5673677+mtrezza@users.noreply.github.com> --- src/Options/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Options/index.js b/src/Options/index.js index d05351d0ae..fc868d5c10 100644 --- a/src/Options/index.js +++ b/src/Options/index.js @@ -320,7 +320,7 @@ export interface RateLimitOptions { /* Optional, if `true` the rate limit will also apply to requests that are made in by Cloud Code, default is `false`. Note that a public Cloud Code function that triggers internal requests may circumvent rate limiting and be vulnerable to attacks. :DEFAULT: false */ includeInternalRequests: ?boolean; - /* Optional, the URL of the Redis server to store rate limit data. This allows to rate limit requests for multiple server instances by calculating the sum of all requests across all servers. This is useful if a multiple servers are processing requests behind a load balancer. For example, the limit of 10 requests is reached if each of 2 servers processed 5 requests. + /* Optional, the URL of the Redis server to store rate limit data. This allows to rate limit requests for multiple servers by calculating the sum of all requests across all servers. This is useful if multiple servers are processing requests behind a load balancer. For example, the limit of 10 requests is reached if each of 2 servers processed 5 requests. */ redisURL: ?string; } From 4a766f4b01cd38b757aef69269ca1d1b9bff46fe Mon Sep 17 00:00:00 2001 From: dblythy Date: Mon, 23 Jan 2023 10:32:17 +1100 Subject: [PATCH 6/8] definitions --- src/Options/Definitions.js | 2 +- src/Options/docs.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Options/Definitions.js b/src/Options/Definitions.js index a442a53c24..86e89ca051 100644 --- a/src/Options/Definitions.js +++ b/src/Options/Definitions.js @@ -560,7 +560,7 @@ module.exports.RateLimitOptions = { redisURL: { env: 'PARSE_SERVER_RATE_LIMIT_REDIS_URL', help: - 'Optional, the URL of the Redis server to store rate limit data. This allows to rate limit requests for multiple server instances by calculating the sum of all requests across all servers. This is useful if a multiple servers are processing requests behind a load balancer. For example, the limit of 10 requests is reached if each of 2 servers processed 5 requests.', + 'Optional, the URL of the Redis server to store rate limit data. This allows to rate limit requests for multiple servers by calculating the sum of all requests across all servers. This is useful if multiple servers are processing requests behind a load balancer. For example, the limit of 10 requests is reached if each of 2 servers processed 5 requests.', }, requestCount: { env: 'PARSE_SERVER_RATE_LIMIT_REQUEST_COUNT', diff --git a/src/Options/docs.js b/src/Options/docs.js index 399a2aade1..1c89217655 100644 --- a/src/Options/docs.js +++ b/src/Options/docs.js @@ -104,7 +104,7 @@ * @property {String} errorResponseMessage The error message that should be returned in the body of the HTTP 429 response when the rate limit is hit. Default is `Too many requests.`. * @property {Boolean} includeInternalRequests Optional, if `true` the rate limit will also apply to requests that are made in by Cloud Code, default is `false`. Note that a public Cloud Code function that triggers internal requests may circumvent rate limiting and be vulnerable to attacks. * @property {Boolean} includeMasterKey Optional, if `true` the rate limit will also apply to requests using the `masterKey`, default is `false`. Note that a public Cloud Code function that triggers internal requests using the `masterKey` may circumvent rate limiting and be vulnerable to attacks. - * @property {String} redisURL Optional, the URL of the Redis server to store rate limit data. This allows to rate limit requests for multiple server instances by calculating the sum of all requests across all servers. This is useful if a multiple servers are processing requests behind a load balancer. For example, the limit of 10 requests is reached if each of 2 servers processed 5 requests. + * @property {String} redisURL Optional, the URL of the Redis server to store rate limit data. This allows to rate limit requests for multiple servers by calculating the sum of all requests across all servers. This is useful if multiple servers are processing requests behind a load balancer. For example, the limit of 10 requests is reached if each of 2 servers processed 5 requests. * @property {Number} requestCount The number of requests that can be made per IP address within the time window set in `requestTimeWindow` before the rate limit is applied. * @property {String[]} requestMethods Optional, the HTTP request methods to which the rate limit should be applied, default is all methods. * @property {String} requestPath The path of the API route to be rate limited. Route paths, in combination with a request method, define the endpoints at which requests can be made. Route paths can be strings, string patterns, or regular expression. See: https://expressjs.com/en/guide/routing.html From 2148ec3c1cf997d8948b3da2110046c5047d7d4a Mon Sep 17 00:00:00 2001 From: dblythy Date: Wed, 1 Feb 2023 13:15:25 +1100 Subject: [PATCH 7/8] change to redisUrl --- spec/RateLimit.spec.js | 2 +- src/Options/Definitions.js | 3 ++- src/Options/docs.js | 2 +- src/Options/index.js | 2 +- src/middlewares.js | 4 ++-- 5 files changed, 7 insertions(+), 6 deletions(-) diff --git a/spec/RateLimit.spec.js b/spec/RateLimit.spec.js index 6164e95552..894c8fcf82 100644 --- a/spec/RateLimit.spec.js +++ b/spec/RateLimit.spec.js @@ -401,7 +401,7 @@ describe('rate limit', () => { requestCount: 1, errorResponseMessage: 'Too many requests', includeInternalRequests: true, - redisURL: 'redis://localhost:6379', + redisUrl: 'redis://localhost:6379', }, ], }); diff --git a/src/Options/Definitions.js b/src/Options/Definitions.js index 86e89ca051..788f97f7c0 100644 --- a/src/Options/Definitions.js +++ b/src/Options/Definitions.js @@ -4,6 +4,7 @@ This code has been generated by resources/buildConfigDefinitions.js Do not edit manually, but update Options/index.js */ var parsers = require('./parsers'); + module.exports.SchemaOptions = { afterMigration: { env: 'PARSE_SERVER_SCHEMA_AFTER_MIGRATION', @@ -557,7 +558,7 @@ module.exports.RateLimitOptions = { action: parsers.booleanParser, default: false, }, - redisURL: { + redisUrl: { env: 'PARSE_SERVER_RATE_LIMIT_REDIS_URL', help: 'Optional, the URL of the Redis server to store rate limit data. This allows to rate limit requests for multiple servers by calculating the sum of all requests across all servers. This is useful if multiple servers are processing requests behind a load balancer. For example, the limit of 10 requests is reached if each of 2 servers processed 5 requests.', diff --git a/src/Options/docs.js b/src/Options/docs.js index 1c89217655..3d99dd4904 100644 --- a/src/Options/docs.js +++ b/src/Options/docs.js @@ -104,7 +104,7 @@ * @property {String} errorResponseMessage The error message that should be returned in the body of the HTTP 429 response when the rate limit is hit. Default is `Too many requests.`. * @property {Boolean} includeInternalRequests Optional, if `true` the rate limit will also apply to requests that are made in by Cloud Code, default is `false`. Note that a public Cloud Code function that triggers internal requests may circumvent rate limiting and be vulnerable to attacks. * @property {Boolean} includeMasterKey Optional, if `true` the rate limit will also apply to requests using the `masterKey`, default is `false`. Note that a public Cloud Code function that triggers internal requests using the `masterKey` may circumvent rate limiting and be vulnerable to attacks. - * @property {String} redisURL Optional, the URL of the Redis server to store rate limit data. This allows to rate limit requests for multiple servers by calculating the sum of all requests across all servers. This is useful if multiple servers are processing requests behind a load balancer. For example, the limit of 10 requests is reached if each of 2 servers processed 5 requests. + * @property {String} redisUrl Optional, the URL of the Redis server to store rate limit data. This allows to rate limit requests for multiple servers by calculating the sum of all requests across all servers. This is useful if multiple servers are processing requests behind a load balancer. For example, the limit of 10 requests is reached if each of 2 servers processed 5 requests. * @property {Number} requestCount The number of requests that can be made per IP address within the time window set in `requestTimeWindow` before the rate limit is applied. * @property {String[]} requestMethods Optional, the HTTP request methods to which the rate limit should be applied, default is all methods. * @property {String} requestPath The path of the API route to be rate limited. Route paths, in combination with a request method, define the endpoints at which requests can be made. Route paths can be strings, string patterns, or regular expression. See: https://expressjs.com/en/guide/routing.html diff --git a/src/Options/index.js b/src/Options/index.js index fc868d5c10..cbe79fe043 100644 --- a/src/Options/index.js +++ b/src/Options/index.js @@ -322,7 +322,7 @@ export interface RateLimitOptions { includeInternalRequests: ?boolean; /* Optional, the URL of the Redis server to store rate limit data. This allows to rate limit requests for multiple servers by calculating the sum of all requests across all servers. This is useful if multiple servers are processing requests behind a load balancer. For example, the limit of 10 requests is reached if each of 2 servers processed 5 requests. */ - redisURL: ?string; + redisUrl: ?string; } export interface SecurityOptions { diff --git a/src/middlewares.js b/src/middlewares.js index 08296a050b..8f3915e4b6 100644 --- a/src/middlewares.js +++ b/src/middlewares.js @@ -483,9 +483,9 @@ export const addRateLimit = (route, config) => { store: null, connected: false, }; - if (route.redisURL) { + if (route.redisrUrl) { const client = createClient({ - url: route.redisURL, + url: route.redisrUrl, }); redisStore.connectionPromise = async () => { if (redisStore.connected) { From d41db3037f2d19d23ae9e419c4731a64ad5ae96e Mon Sep 17 00:00:00 2001 From: dblythy Date: Tue, 21 Feb 2023 17:23:50 +1100 Subject: [PATCH 8/8] remove --- src/Options/Definitions.js | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Options/Definitions.js b/src/Options/Definitions.js index 788f97f7c0..a0f111cf37 100644 --- a/src/Options/Definitions.js +++ b/src/Options/Definitions.js @@ -4,7 +4,6 @@ This code has been generated by resources/buildConfigDefinitions.js Do not edit manually, but update Options/index.js */ var parsers = require('./parsers'); - module.exports.SchemaOptions = { afterMigration: { env: 'PARSE_SERVER_SCHEMA_AFTER_MIGRATION',