feat: rate limits for GET

config/default.cjs

@@ -61,14 +61,22 @@ module.exports = {
 		syncInterval: 60000,
 	},
 	measurement: {
-		anonymousRateLimit: 100000,
-		authenticatedRateLimit: 250,
-		rateLimitReset: 3600,
-		maxInProgressProbes: 5,
+		maxInProgressTests: 5,
 		// Timeout after which measurement will be marked as finished even if not all probes respond
 		timeout: 30, // 30 seconds
 		// measurement result TTL in redis
 		resultTTL: 7 * 24 * 60 * 60, // 7 days
+		rateLimit: {
+			post: {
+				anonymousLimit: 100000,
+				authenticatedLimit: 250,
+				reset: 3600,
+			},
+			getPerMeasurement: {
+				limit: 5,
+				reset: 2,
+			},
+		},
 		limits: {
 			anonymousTestsPerLocation: 200,
 			anonymousTestsPerMeasurement: 500,

config/test.cjs

@@ -29,7 +29,12 @@ module.exports = {
 		},
 	},
 	measurement: {
-		maxInProgressProbes: 2,
+		maxInProgressTests: 2,
+		rateLimit: {
+			getPerMeasurement: {
+				limit: 1000,
+			},
+		},
 	},
 	sigtermDelay: 0,
 };

public/v1/components/headers.yaml

@@ -41,3 +41,8 @@ components:
       required: false
       schema:
         type: integer
+    RetryAfter:
+      description: The number of seconds to wait before retrying this request.
+      required: true
+      schema:
+        type: integer

public/v1/components/responses.yaml

@@ -169,6 +169,35 @@ components:
               $ref: 'examples.yaml#/components/examples/getMtrMeasurementResponse'
             httpMeasurement:
               $ref: 'examples.yaml#/components/examples/getHttpMeasurementResponse'
+    measurement429:
+      description: |
+        If you've exceeded the API rate limit, you'll receive status `429 Too Many Requests` and a body containing more information about the error.
+      headers:
+        Retry-After:
+          $ref: 'headers.yaml#/components/headers/RetryAfter'
+      content:
+        application/json:
+          schema:
+            type: object
+            required:
+              - error
+            properties:
+              error:
+                type: object
+                required:
+                  - type
+                  - message
+                properties:
+                  type:
+                    type: string
+                  message:
+                    type: string
+          examples:
+            json:
+              value:
+                error:
+                  type: too_many_requests
+                  message: Too many requests. Please retry in 4 seconds.
     probes200:
       description: |
         A successful request returns status `200 OK` and a body containing a list of all probes currently online and their metadata.

public/v1/spec.yaml

@@ -100,11 +100,16 @@ paths:
           1. Request the measurement to retrieve its status.
           2. If the status is `in-progress`, wait 500 milliseconds and start again at step 1. Note that it's important to wait 500 ms *after* receiving the response rather than using an "every 500ms" interval as for large measurements, the request itself may take a few hundred milliseconds to complete.
           3. If the status is anything **other** than `in-progress`, stop. The measurement is no longer running, and its results are final.
+
+        > **Important**: Do not query the results of a single measurement more often than every 500 milliseconds. Sending more than two
+        requests per second may trigger a rate limit and prevent you from accessing the results for a few seconds.
       responses:
         '200':
           $ref: 'components/responses.yaml#/components/responses/measurement200'
         '404':
           $ref: 'components/responses.yaml#/components/responses/404'
+        '429':
+          $ref: 'components/responses.yaml#/components/responses/measurement429'
       tags:
         - Measurements
   /v1/probes:

src/lib/rate-limiter/rate-limiter-get.ts

@@ -0,0 +1,47 @@
+import config from 'config';
+import { RateLimiterRedis, RateLimiterRes } from 'rate-limiter-flexible';
+import requestIp from 'request-ip';
+import { getPersistentRedisClient } from '../redis/persistent-client.js';
+import createHttpError from 'http-errors';
+import type { ExtendedContext } from '../../types.js';
+import type { Next } from 'koa';
+
+const redisClient = getPersistentRedisClient();
+
+export const rateLimiter = new RateLimiterRedis({
+	storeClient: redisClient,
+	keyPrefix: 'rate:get',
+	points: config.get<number>('measurement.rateLimit.getPerMeasurement.limit'),
+	duration: config.get<number>('measurement.rateLimit.getPerMeasurement.reset'),
+	blockDuration: 5,
+});
+
+export const getMeasurementRateLimit = async (ctx: ExtendedContext, next: Next) => {
+	if (ctx['isAdmin']) {
+		return next();
+	}
+
+	const ip = requestIp.getClientIp(ctx.req) ?? '';
+	const measurementId = ctx.params['id'] ?? '';
+	const id = `${ip}:${measurementId}`;
+
+	try {
+		await rateLimiter.consume(id);
+	} catch (error) {
+		if (error instanceof RateLimiterRes) {
+			const retryAfter = Math.ceil(error.msBeforeNext / 1000);
+			const units = retryAfter === 1 ? 'second' : 'seconds';
+
+			setRateLimitHeaders(ctx, error);
+			throw createHttpError(429, `Too many requests. Please retry in ${retryAfter} ${units}.`, { type: 'too_many_requests' });
+		}
+
+		throw createHttpError(500);
+	}
+
+	await next();
+};
+
+const setRateLimitHeaders = (ctx: ExtendedContext, error: RateLimiterRes) => {
+	ctx.set('Retry-After', `${Math.ceil(error.msBeforeNext / 1000)}`);
+};

src/lib/rate-limiter.ts → src/lib/rate-limiter/rate-limiter-post.ts

@@ -1,25 +1,25 @@
 import config from 'config';
 import { RateLimiterRedis, RateLimiterRes } from 'rate-limiter-flexible';
 import requestIp from 'request-ip';
-import { getPersistentRedisClient } from './redis/persistent-client.js';
+import { getPersistentRedisClient } from '../redis/persistent-client.js';
 import createHttpError from 'http-errors';
-import type { ExtendedContext } from '../types.js';
-import { credits } from './credits.js';
+import type { ExtendedContext } from '../../types.js';
+import { credits } from '../credits.js';
 
 const redisClient = getPersistentRedisClient();
 
 export const anonymousRateLimiter = new RateLimiterRedis({
 	storeClient: redisClient,
-	keyPrefix: 'rate:anon',
-	points: config.get<number>('measurement.anonymousRateLimit'),
-	duration: config.get<number>('measurement.rateLimitReset'),
+	keyPrefix: 'rate:post:anon',
+	points: config.get<number>('measurement.rateLimit.post.anonymousLimit'),
+	duration: config.get<number>('measurement.rateLimit.post.reset'),
 });
 
 export const authenticatedRateLimiter = new RateLimiterRedis({
 	storeClient: redisClient,
-	keyPrefix: 'rate:auth',
-	points: config.get<number>('measurement.authenticatedRateLimit'),
-	duration: config.get<number>('measurement.rateLimitReset'),
+	keyPrefix: 'rate:post:auth',
+	points: config.get<number>('measurement.rateLimit.post.authenticatedLimit'),
+	duration: config.get<number>('measurement.rateLimit.post.reset'),
 });
 
 const getRateLimiter = (ctx: ExtendedContext): {
@@ -91,22 +91,22 @@ export const getRateLimitState = async (ctx: ExtendedContext) => {
 	} else if (type === 'user') {
 		return {
 			type,
-			limit: config.get<number>('measurement.authenticatedRateLimit'),
-			remaining: config.get<number>('measurement.authenticatedRateLimit'),
+			limit: config.get<number>('measurement.rateLimit.post.authenticatedLimit'),
+			remaining: config.get<number>('measurement.rateLimit.post.authenticatedLimit'),
 			reset: 0,
 		};
 	}
 
 	return {
 		type,
-		limit: config.get<number>('measurement.anonymousRateLimit'),
-		remaining: config.get<number>('measurement.anonymousRateLimit'),
+		limit: config.get<number>('measurement.rateLimit.post.anonymousLimit'),
+		remaining: config.get<number>('measurement.rateLimit.post.anonymousLimit'),
 		reset: 0,
 	};
 };
 
 const consumeCredits = async (userId: string, rateLimiterRes: RateLimiterRes, numberOfProbes: number) => {
-	const freeCredits = config.get<number>('measurement.authenticatedRateLimit');
+	const freeCredits = config.get<number>('measurement.rateLimit.post.authenticatedLimit');
 	const requiredCredits = Math.min(rateLimiterRes.consumedPoints - freeCredits, numberOfProbes);
 	const { isConsumed, remainingCredits } = await credits.consume(userId, requiredCredits);
 

src/limits/route/get-limits.ts

@@ -1,5 +1,5 @@
 import type Router from '@koa/router';
-import { getRateLimitState } from '../../lib/rate-limiter.js';
+import { getRateLimitState } from '../../lib/rate-limiter/rate-limiter-post.js';
 import type { ExtendedContext } from '../../types.js';
 import { credits } from '../../lib/credits.js';
 import { authenticate } from '../../lib/http/middleware/authenticate.js';

src/measurement/route/get-measurement.ts

@@ -1,6 +1,8 @@
 import type { DefaultContext, DefaultState, ParameterizedContext } from 'koa';
 import type Router from '@koa/router';
 import { getMeasurementStore } from '../store.js';
+import { getMeasurementRateLimit } from '../../lib/rate-limiter/rate-limiter-get.js';
+import createHttpError from 'http-errors';
 
 const store = getMeasurementStore();
 
@@ -15,14 +17,13 @@ const handle = async (ctx: ParameterizedContext<DefaultState, DefaultContext & R
 	const result = await store.getMeasurementString(id);
 
 	if (!result) {
-		ctx.status = 404;
-		return;
+		throw createHttpError(404, `Couldn't find the requested measurement.`, { type: 'not_found' });
 	}
 
 	ctx.type = 'application/json';
 	ctx.body = result;
 };
 
 export const registerGetMeasurementRoute = (router: Router): void => {
-	router.get('/measurements/:id', '/measurements/:id([a-zA-Z0-9]+)', handle);
+	router.get('/measurements/:id', '/measurements/:id([a-zA-Z0-9]+)', getMeasurementRateLimit, handle);
 };

src/measurement/runner.ts

@@ -8,7 +8,7 @@ import { getMetricsAgent, type MetricsAgent } from '../lib/metrics.js';
 import type { MeasurementStore } from './store.js';
 import { getMeasurementStore } from './store.js';
 import type { MeasurementRequest, MeasurementResultMessage, MeasurementProgressMessage, UserRequest } from './types.js';
-import { rateLimit } from '../lib/rate-limiter.js';
+import { rateLimit } from '../lib/rate-limiter/rate-limiter-post.js';
 import type { ExtendedContext } from '../types.js';
 
 export class MeasurementRunner {
@@ -58,10 +58,10 @@ export class MeasurementRunner {
 	}
 
 	private sendToProbes (measurementId: string, onlineProbesMap: Map<number, Probe>, request: MeasurementRequest) {
-		let inProgressProbes = 0;
-		const maxInProgressProbes = config.get<number>('measurement.maxInProgressProbes');
+		let inProgressTests = 0;
+		const maxInProgressTests = config.get<number>('measurement.maxInProgressTests');
 		onlineProbesMap.forEach((probe, index) => {
-			const inProgressUpdates = request.inProgressUpdates && inProgressProbes++ < maxInProgressProbes;
+			const inProgressUpdates = request.inProgressUpdates && inProgressTests++ < maxInProgressTests;
 			this.io.of(PROBES_NAMESPACE).to(probe.client).emit('probe:measurement:request', {
 				measurementId,
 				testId: index.toString(),

test/e2e/cases/limits.test.ts

@@ -6,7 +6,7 @@ describe('/limits endpoint', () => {
 	const redis =	getPersistentRedisClient();
 
 	before(async () => {
-		const keys = await redis.keys('rate:anon:*');
+		const keys = await redis.keys('rate:post:anon:*');
 		await redis.del(keys);
 	});
 

test/tests/integration/limits.test.ts

@@ -4,7 +4,7 @@ import requestIp from 'request-ip';
 import { expect } from 'chai';
 import { getTestServer, addFakeProbe, deleteFakeProbes, waitForProbesUpdate } from '../../utils/server.js';
 import nockGeoIpProviders from '../../utils/nock-geo-ip.js';
-import { anonymousRateLimiter, authenticatedRateLimiter } from '../../../src/lib/rate-limiter.js';
+import { anonymousRateLimiter, authenticatedRateLimiter } from '../../../src/lib/rate-limiter/rate-limiter-post.js';
 import { client } from '../../../src/lib/sql/client.js';
 import { GP_TOKENS_TABLE } from '../../../src/lib/http/auth.js';
 import { CREDITS_TABLE } from '../../../src/lib/credits.js';