From 0f0a4a31078fd1ed9b6364b86a229c451229a27b Mon Sep 17 00:00:00 2001 From: Oli Evans Date: Wed, 5 Jul 2023 15:57:30 +0100 Subject: [PATCH] feat: check cids in denylist before providing (#215) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - cap the max number of cids we'll accept in a single message. - We're seeing 20k spikes in our `bitswap-pending-entries` metrics per node, so I'm putting in a hard cap of 500 wanted cids per message that we'll process. The caller can ask again if they need more. This also means i can put a sensible cap on how many cids the denylist service should handle in a batch. - check batches of cids against our denylist api. - cache entries that are on the denylist; they are rarely removed. - use cache to avoid asking about items we already know, and as a fallback if denylist service cannot be reached. - add `bitswap-denied` counter metric to see how many CIDs we skip due to being on the denylist see: batch endpoint for denylist api – https://github.com/web3-storage/reads/pull/166 see: set DENYLIST_URL in env - https://github.com/elastic-ipfs/bitswap-peer-deployment/pull/99 License: MIT --------- Signed-off-by: Oli Evans --- .env.sample | 1 + README.md | 1 + metrics.yml | 8 ++- package-lock.json | 146 ++++++++++++++++++++++---------------------- package.json | 2 + src/config.js | 4 +- src/deny.js | 72 ++++++++++++++++++++++ src/index.js | 3 +- src/limit.js | 40 ++++++++++++ src/service.js | 23 ++++++- test/config.test.js | 7 ++- test/deny.test.js | 39 ++++++++++++ test/limit.test.js | 17 ++++++ 13 files changed, 281 insertions(+), 82 deletions(-) create mode 100644 src/deny.js create mode 100644 src/limit.js create mode 100644 test/deny.test.js create mode 100644 test/limit.test.js diff --git a/.env.sample b/.env.sample index 8a4a81d..45f0e5b 100644 --- a/.env.sample +++ b/.env.sample @@ -12,3 +12,4 @@ DYNAMO_LINK_TABLE_V1= CACHE_BLOCK_INFO=true CACHE_BLOCK_SIZE=128 CONCURRENCY=4 +DENYLIST_URL=https://denylist.dag.haus diff --git a/README.md b/README.md index b194a22..be5e8e0 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,7 @@ _Variables in bold are required._ | NODE_DEBUG | | If it contains `aws-ipfs`, debug mode is enabled. | | LOG_LEVEL | `info` | Logging level. | | LOG_PRETTY | `false` | Enable pretty logging. | +| DENYLIST_URL | `https://denylist.dag.haus` | URL for cid checking api. | Also check [AWS specifics configuration](https://github.com/elastic-ipfs/elastic-ipfs/blob/main/aws.md). diff --git a/metrics.yml b/metrics.yml index 8487d9a..c776e2b 100644 --- a/metrics.yml +++ b/metrics.yml @@ -60,12 +60,14 @@ metrics: description: BitSwap Total Entries served bitswap-block-error: description: Block error (on parsing) - + bitswap-denied: + description: Count of CIDs found on denylist + process: elu: name: bitswap-elu description: Bitswap Event Loop Utilization interval: 500 -version: 0.3.0 -buildDate: "20230426.1252" +version: 0.4.0 +buildDate: "20230706.1402" diff --git a/package-lock.json b/package-lock.json index dd87514..8dc21de 100644 --- a/package-lock.json +++ b/package-lock.json @@ -24,6 +24,8 @@ "libp2p": "0.42.2", "lru-cache": "7.14.1", "mnemonist": "0.39.5", + "multiformats": "^10.0.3", + "p-retry": "^5.1.2", "pino": "8.8.0", "piscina": "3.2.0", "protobufjs": "7.2.0", @@ -1580,9 +1582,9 @@ } }, "node_modules/@chainsafe/libp2p-noise/node_modules/multiformats": { - "version": "11.0.1", - "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-11.0.1.tgz", - "integrity": "sha512-atWruyH34YiknSdL5yeIir00EDlJRpHzELYQxG7Iy29eCyL+VrZHpPrX5yqlik3jnuqpLpRKVZ0SGVb9UzKaSA==", + "version": "11.0.2", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-11.0.2.tgz", + "integrity": "sha512-b5mYMkOkARIuVZCpvijFj9a6m5wMVLC7cf/jIPd5D/ARDOfLC5+IFkbgDXQgcU2goIsTD/O9NY4DI/Mt4OGvlg==", "engines": { "node": ">=16.0.0", "npm": ">=7.0.0" @@ -2012,9 +2014,9 @@ } }, "node_modules/@ipld/car/node_modules/multiformats": { - "version": "11.0.1", - "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-11.0.1.tgz", - "integrity": "sha512-atWruyH34YiknSdL5yeIir00EDlJRpHzELYQxG7Iy29eCyL+VrZHpPrX5yqlik3jnuqpLpRKVZ0SGVb9UzKaSA==", + "version": "11.0.2", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-11.0.2.tgz", + "integrity": "sha512-b5mYMkOkARIuVZCpvijFj9a6m5wMVLC7cf/jIPd5D/ARDOfLC5+IFkbgDXQgcU2goIsTD/O9NY4DI/Mt4OGvlg==", "dev": true, "engines": { "node": ">=16.0.0", @@ -2036,9 +2038,9 @@ } }, "node_modules/@ipld/dag-cbor/node_modules/multiformats": { - "version": "11.0.1", - "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-11.0.1.tgz", - "integrity": "sha512-atWruyH34YiknSdL5yeIir00EDlJRpHzELYQxG7Iy29eCyL+VrZHpPrX5yqlik3jnuqpLpRKVZ0SGVb9UzKaSA==", + "version": "11.0.2", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-11.0.2.tgz", + "integrity": "sha512-b5mYMkOkARIuVZCpvijFj9a6m5wMVLC7cf/jIPd5D/ARDOfLC5+IFkbgDXQgcU2goIsTD/O9NY4DI/Mt4OGvlg==", "dev": true, "engines": { "node": ">=16.0.0", @@ -2059,9 +2061,9 @@ } }, "node_modules/@ipld/dag-pb/node_modules/multiformats": { - "version": "11.0.1", - "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-11.0.1.tgz", - "integrity": "sha512-atWruyH34YiknSdL5yeIir00EDlJRpHzELYQxG7Iy29eCyL+VrZHpPrX5yqlik3jnuqpLpRKVZ0SGVb9UzKaSA==", + "version": "11.0.2", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-11.0.2.tgz", + "integrity": "sha512-b5mYMkOkARIuVZCpvijFj9a6m5wMVLC7cf/jIPd5D/ARDOfLC5+IFkbgDXQgcU2goIsTD/O9NY4DI/Mt4OGvlg==", "dev": true, "engines": { "node": ">=16.0.0", @@ -2257,9 +2259,9 @@ } }, "node_modules/@libp2p/crypto/node_modules/multiformats": { - "version": "11.0.1", - "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-11.0.1.tgz", - "integrity": "sha512-atWruyH34YiknSdL5yeIir00EDlJRpHzELYQxG7Iy29eCyL+VrZHpPrX5yqlik3jnuqpLpRKVZ0SGVb9UzKaSA==", + "version": "11.0.2", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-11.0.2.tgz", + "integrity": "sha512-b5mYMkOkARIuVZCpvijFj9a6m5wMVLC7cf/jIPd5D/ARDOfLC5+IFkbgDXQgcU2goIsTD/O9NY4DI/Mt4OGvlg==", "engines": { "node": ">=16.0.0", "npm": ">=7.0.0" @@ -2379,9 +2381,9 @@ } }, "node_modules/@libp2p/interface-keychain/node_modules/multiformats": { - "version": "11.0.1", - "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-11.0.1.tgz", - "integrity": "sha512-atWruyH34YiknSdL5yeIir00EDlJRpHzELYQxG7Iy29eCyL+VrZHpPrX5yqlik3jnuqpLpRKVZ0SGVb9UzKaSA==", + "version": "11.0.2", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-11.0.2.tgz", + "integrity": "sha512-b5mYMkOkARIuVZCpvijFj9a6m5wMVLC7cf/jIPd5D/ARDOfLC5+IFkbgDXQgcU2goIsTD/O9NY4DI/Mt4OGvlg==", "engines": { "node": ">=16.0.0", "npm": ">=7.0.0" @@ -2463,9 +2465,9 @@ } }, "node_modules/@libp2p/interface-libp2p/node_modules/multiformats": { - "version": "11.0.1", - "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-11.0.1.tgz", - "integrity": "sha512-atWruyH34YiknSdL5yeIir00EDlJRpHzELYQxG7Iy29eCyL+VrZHpPrX5yqlik3jnuqpLpRKVZ0SGVb9UzKaSA==", + "version": "11.0.2", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-11.0.2.tgz", + "integrity": "sha512-b5mYMkOkARIuVZCpvijFj9a6m5wMVLC7cf/jIPd5D/ARDOfLC5+IFkbgDXQgcU2goIsTD/O9NY4DI/Mt4OGvlg==", "engines": { "node": ">=16.0.0", "npm": ">=7.0.0" @@ -2660,9 +2662,9 @@ } }, "node_modules/@libp2p/logger/node_modules/multiformats": { - "version": "11.0.1", - "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-11.0.1.tgz", - "integrity": "sha512-atWruyH34YiknSdL5yeIir00EDlJRpHzELYQxG7Iy29eCyL+VrZHpPrX5yqlik3jnuqpLpRKVZ0SGVb9UzKaSA==", + "version": "11.0.2", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-11.0.2.tgz", + "integrity": "sha512-b5mYMkOkARIuVZCpvijFj9a6m5wMVLC7cf/jIPd5D/ARDOfLC5+IFkbgDXQgcU2goIsTD/O9NY4DI/Mt4OGvlg==", "engines": { "node": ">=16.0.0", "npm": ">=7.0.0" @@ -2794,9 +2796,9 @@ } }, "node_modules/@libp2p/peer-id-factory/node_modules/multiformats": { - "version": "11.0.1", - "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-11.0.1.tgz", - "integrity": "sha512-atWruyH34YiknSdL5yeIir00EDlJRpHzELYQxG7Iy29eCyL+VrZHpPrX5yqlik3jnuqpLpRKVZ0SGVb9UzKaSA==", + "version": "11.0.2", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-11.0.2.tgz", + "integrity": "sha512-b5mYMkOkARIuVZCpvijFj9a6m5wMVLC7cf/jIPd5D/ARDOfLC5+IFkbgDXQgcU2goIsTD/O9NY4DI/Mt4OGvlg==", "engines": { "node": ">=16.0.0", "npm": ">=7.0.0" @@ -2815,9 +2817,9 @@ } }, "node_modules/@libp2p/peer-id/node_modules/multiformats": { - "version": "11.0.1", - "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-11.0.1.tgz", - "integrity": "sha512-atWruyH34YiknSdL5yeIir00EDlJRpHzELYQxG7Iy29eCyL+VrZHpPrX5yqlik3jnuqpLpRKVZ0SGVb9UzKaSA==", + "version": "11.0.2", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-11.0.2.tgz", + "integrity": "sha512-b5mYMkOkARIuVZCpvijFj9a6m5wMVLC7cf/jIPd5D/ARDOfLC5+IFkbgDXQgcU2goIsTD/O9NY4DI/Mt4OGvlg==", "engines": { "node": ">=16.0.0", "npm": ">=7.0.0" @@ -8552,9 +8554,9 @@ } }, "node_modules/libp2p/node_modules/multiformats": { - "version": "11.0.1", - "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-11.0.1.tgz", - "integrity": "sha512-atWruyH34YiknSdL5yeIir00EDlJRpHzELYQxG7Iy29eCyL+VrZHpPrX5yqlik3jnuqpLpRKVZ0SGVb9UzKaSA==", + "version": "11.0.2", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-11.0.2.tgz", + "integrity": "sha512-b5mYMkOkARIuVZCpvijFj9a6m5wMVLC7cf/jIPd5D/ARDOfLC5+IFkbgDXQgcU2goIsTD/O9NY4DI/Mt4OGvlg==", "engines": { "node": ">=16.0.0", "npm": ">=7.0.0" @@ -9353,9 +9355,9 @@ } }, "node_modules/multiformats": { - "version": "10.0.2", - "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-10.0.2.tgz", - "integrity": "sha512-nJEHLFOYhO4L+aNApHhCnWqa31FyqAHv9Q77AhmwU3KsM2f1j7tuJpCk5ByZ33smzycNCpSG5klNIejIyfFx2A==", + "version": "10.0.3", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-10.0.3.tgz", + "integrity": "sha512-K2yGSmstS/oEmYiEIieHb53jJCaqp4ERPDQAYrm5sV3UUrVDZeshJQCK6GHAKyIGufU1vAcbS0PdAAZmC7Tzcw==", "engines": { "node": ">=16.0.0", "npm": ">=7.0.0" @@ -15554,9 +15556,9 @@ } }, "multiformats": { - "version": "11.0.1", - "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-11.0.1.tgz", - "integrity": "sha512-atWruyH34YiknSdL5yeIir00EDlJRpHzELYQxG7Iy29eCyL+VrZHpPrX5yqlik3jnuqpLpRKVZ0SGVb9UzKaSA==" + "version": "11.0.2", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-11.0.2.tgz", + "integrity": "sha512-b5mYMkOkARIuVZCpvijFj9a6m5wMVLC7cf/jIPd5D/ARDOfLC5+IFkbgDXQgcU2goIsTD/O9NY4DI/Mt4OGvlg==" } } }, @@ -15896,9 +15898,9 @@ }, "dependencies": { "multiformats": { - "version": "11.0.1", - "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-11.0.1.tgz", - "integrity": "sha512-atWruyH34YiknSdL5yeIir00EDlJRpHzELYQxG7Iy29eCyL+VrZHpPrX5yqlik3jnuqpLpRKVZ0SGVb9UzKaSA==", + "version": "11.0.2", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-11.0.2.tgz", + "integrity": "sha512-b5mYMkOkARIuVZCpvijFj9a6m5wMVLC7cf/jIPd5D/ARDOfLC5+IFkbgDXQgcU2goIsTD/O9NY4DI/Mt4OGvlg==", "dev": true } } @@ -15914,9 +15916,9 @@ }, "dependencies": { "multiformats": { - "version": "11.0.1", - "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-11.0.1.tgz", - "integrity": "sha512-atWruyH34YiknSdL5yeIir00EDlJRpHzELYQxG7Iy29eCyL+VrZHpPrX5yqlik3jnuqpLpRKVZ0SGVb9UzKaSA==", + "version": "11.0.2", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-11.0.2.tgz", + "integrity": "sha512-b5mYMkOkARIuVZCpvijFj9a6m5wMVLC7cf/jIPd5D/ARDOfLC5+IFkbgDXQgcU2goIsTD/O9NY4DI/Mt4OGvlg==", "dev": true } } @@ -15931,9 +15933,9 @@ }, "dependencies": { "multiformats": { - "version": "11.0.1", - "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-11.0.1.tgz", - "integrity": "sha512-atWruyH34YiknSdL5yeIir00EDlJRpHzELYQxG7Iy29eCyL+VrZHpPrX5yqlik3jnuqpLpRKVZ0SGVb9UzKaSA==", + "version": "11.0.2", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-11.0.2.tgz", + "integrity": "sha512-b5mYMkOkARIuVZCpvijFj9a6m5wMVLC7cf/jIPd5D/ARDOfLC5+IFkbgDXQgcU2goIsTD/O9NY4DI/Mt4OGvlg==", "dev": true } } @@ -16088,9 +16090,9 @@ }, "dependencies": { "multiformats": { - "version": "11.0.1", - "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-11.0.1.tgz", - "integrity": "sha512-atWruyH34YiknSdL5yeIir00EDlJRpHzELYQxG7Iy29eCyL+VrZHpPrX5yqlik3jnuqpLpRKVZ0SGVb9UzKaSA==" + "version": "11.0.2", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-11.0.2.tgz", + "integrity": "sha512-b5mYMkOkARIuVZCpvijFj9a6m5wMVLC7cf/jIPd5D/ARDOfLC5+IFkbgDXQgcU2goIsTD/O9NY4DI/Mt4OGvlg==" } } }, @@ -16176,9 +16178,9 @@ } }, "multiformats": { - "version": "11.0.1", - "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-11.0.1.tgz", - "integrity": "sha512-atWruyH34YiknSdL5yeIir00EDlJRpHzELYQxG7Iy29eCyL+VrZHpPrX5yqlik3jnuqpLpRKVZ0SGVb9UzKaSA==" + "version": "11.0.2", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-11.0.2.tgz", + "integrity": "sha512-b5mYMkOkARIuVZCpvijFj9a6m5wMVLC7cf/jIPd5D/ARDOfLC5+IFkbgDXQgcU2goIsTD/O9NY4DI/Mt4OGvlg==" } } }, @@ -16238,9 +16240,9 @@ } }, "multiformats": { - "version": "11.0.1", - "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-11.0.1.tgz", - "integrity": "sha512-atWruyH34YiknSdL5yeIir00EDlJRpHzELYQxG7Iy29eCyL+VrZHpPrX5yqlik3jnuqpLpRKVZ0SGVb9UzKaSA==" + "version": "11.0.2", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-11.0.2.tgz", + "integrity": "sha512-b5mYMkOkARIuVZCpvijFj9a6m5wMVLC7cf/jIPd5D/ARDOfLC5+IFkbgDXQgcU2goIsTD/O9NY4DI/Mt4OGvlg==" } } }, @@ -16377,9 +16379,9 @@ } }, "multiformats": { - "version": "11.0.1", - "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-11.0.1.tgz", - "integrity": "sha512-atWruyH34YiknSdL5yeIir00EDlJRpHzELYQxG7Iy29eCyL+VrZHpPrX5yqlik3jnuqpLpRKVZ0SGVb9UzKaSA==" + "version": "11.0.2", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-11.0.2.tgz", + "integrity": "sha512-b5mYMkOkARIuVZCpvijFj9a6m5wMVLC7cf/jIPd5D/ARDOfLC5+IFkbgDXQgcU2goIsTD/O9NY4DI/Mt4OGvlg==" } } }, @@ -16468,9 +16470,9 @@ } }, "multiformats": { - "version": "11.0.1", - "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-11.0.1.tgz", - "integrity": "sha512-atWruyH34YiknSdL5yeIir00EDlJRpHzELYQxG7Iy29eCyL+VrZHpPrX5yqlik3jnuqpLpRKVZ0SGVb9UzKaSA==" + "version": "11.0.2", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-11.0.2.tgz", + "integrity": "sha512-b5mYMkOkARIuVZCpvijFj9a6m5wMVLC7cf/jIPd5D/ARDOfLC5+IFkbgDXQgcU2goIsTD/O9NY4DI/Mt4OGvlg==" } } }, @@ -16498,9 +16500,9 @@ } }, "multiformats": { - "version": "11.0.1", - "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-11.0.1.tgz", - "integrity": "sha512-atWruyH34YiknSdL5yeIir00EDlJRpHzELYQxG7Iy29eCyL+VrZHpPrX5yqlik3jnuqpLpRKVZ0SGVb9UzKaSA==" + "version": "11.0.2", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-11.0.2.tgz", + "integrity": "sha512-b5mYMkOkARIuVZCpvijFj9a6m5wMVLC7cf/jIPd5D/ARDOfLC5+IFkbgDXQgcU2goIsTD/O9NY4DI/Mt4OGvlg==" } } }, @@ -20710,9 +20712,9 @@ } }, "multiformats": { - "version": "11.0.1", - "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-11.0.1.tgz", - "integrity": "sha512-atWruyH34YiknSdL5yeIir00EDlJRpHzELYQxG7Iy29eCyL+VrZHpPrX5yqlik3jnuqpLpRKVZ0SGVb9UzKaSA==" + "version": "11.0.2", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-11.0.2.tgz", + "integrity": "sha512-b5mYMkOkARIuVZCpvijFj9a6m5wMVLC7cf/jIPd5D/ARDOfLC5+IFkbgDXQgcU2goIsTD/O9NY4DI/Mt4OGvlg==" }, "private-ip": { "version": "3.0.0", @@ -21643,9 +21645,9 @@ } }, "multiformats": { - "version": "10.0.2", - "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-10.0.2.tgz", - "integrity": "sha512-nJEHLFOYhO4L+aNApHhCnWqa31FyqAHv9Q77AhmwU3KsM2f1j7tuJpCk5ByZ33smzycNCpSG5klNIejIyfFx2A==" + "version": "10.0.3", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-10.0.3.tgz", + "integrity": "sha512-K2yGSmstS/oEmYiEIieHb53jJCaqp4ERPDQAYrm5sV3UUrVDZeshJQCK6GHAKyIGufU1vAcbS0PdAAZmC7Tzcw==" }, "murmurhash3js-revisited": { "version": "3.0.0", diff --git a/package.json b/package.json index d6239e5..aa598f1 100644 --- a/package.json +++ b/package.json @@ -28,6 +28,8 @@ "libp2p": "0.42.2", "lru-cache": "7.14.1", "mnemonist": "0.39.5", + "multiformats": "^10.0.3", + "p-retry": "^5.1.2", "pino": "8.8.0", "piscina": "3.2.0", "protobufjs": "7.2.0", diff --git a/src/config.js b/src/config.js index f811722..b53d329 100644 --- a/src/config.js +++ b/src/config.js @@ -74,7 +74,9 @@ export function makeConfig () { s3MaxRetries: process.env.S3_MAX_RETRIES ? parseInt(process.env.S3_MAX_RETRIES) : 3, s3RetryDelay: process.env.S3_RETRY_DELAY ? parseInt(process.env.S3_RETRY_DELAY) : 100, // ms - allowReadinessTweak: process.env.ALLOW_READINESS_TWEAK === 'true' + allowReadinessTweak: process.env.ALLOW_READINESS_TWEAK === 'true', + + denylistUrl: process.env.DENYLIST_URL ? new URL(process.env.DENYLIST_URL) : undefined } } diff --git a/src/deny.js b/src/deny.js new file mode 100644 index 0000000..1cf63f3 --- /dev/null +++ b/src/deny.js @@ -0,0 +1,72 @@ +import retry from 'p-retry' +import { fetch } from 'undici' +import LRUCache from 'mnemonist/lru-cache.js' + +/** + * 46 bytes per multihash * 200k = ~9.2MB (nodes have 2-4GB available) + * @type {LRUCache} + */ +const denyCache = new LRUCache(200_000) + +/** + * Check if we already know a CID to be on the denylist + * TODO: purge cached items if removed from denylist. + * @param {import('multiformats/cid').CID} cid + **/ +function inDenyCache (cid) { + // use .get here to update the last used time in the LRU cache + return !!denyCache.get(cid.toString()) +} + +/** + * Add a cid to the denylist cache + * NOTE: we could set the reason as the cache value if we need it. + * @param {string} cidStr + */ +function cacheDeniedCid (cidStr) { + denyCache.set(cidStr, true) +} + +/** + * Remove items that should not be served, determined by the denylist api + * @param {{cancel: boolean, cid: CID}[]} entries + * @param {{error: () => void}} logger + * @param {URL?} denylistUrl - https://denylist.dag.haus or undefined to disable denylist filtering + */ +export async function denylistFilter (entries, logger, denylistUrl) { + if (!denylistUrl) { + return entries + } + + // skip cancels and things we already know are denied. + const filtered = entries.filter(entry => entry.cancel || !inDenyCache(entry.cid)) + + try { + const res = await retry(() => fetch(denylistUrl, { + body: JSON.stringify(filtered.map(e => e.cid.toString())), + method: 'POST', + headers: { + 'content-type': 'application/json' + } + }), { + retries: 5, // try once then 5 more times + factor: 1.2, // scaling factor so total wait is 10s see: https://www.wolframalpha.com/input?i=Sum%5B1000*x%5Ek,+%7Bk,+0,+5%7D%5D+%3D+10+*+1000 + minTimeout: 1000 // ms to wait before first retry + }) + + if (res.ok) { + const denySet = new Set(await res.json()) + for (const cidStr of denySet.values()) { + cacheDeniedCid(cidStr) + } + // filter again to remove any items the api says are denied + return filtered.filter(entry => !denySet.has(entry.cid.toString())) + } + } catch (err) { + logger.error({ err }, 'denylist check failed') + } + + // we know the denylist api can hit rate limits so we 'fail open' here, + // with a fallback to our local cache. + return filtered +} diff --git a/src/index.js b/src/index.js index f15dfdd..4d39321 100644 --- a/src/index.js +++ b/src/index.js @@ -57,7 +57,8 @@ async function boot () { peerId, peerAnnounceAddr: config.peerAnnounceAddr, connectionConfig: createConnectionConfig(config), - taggedPeers: JSON.parse(taggedPeers.value) + taggedPeers: JSON.parse(taggedPeers.value), + denylistUrl: config.denylistUrl })) } catch (err) { logger.fatal({ err }, 'Cannot start the service') diff --git a/src/limit.js b/src/limit.js new file mode 100644 index 0000000..3755550 --- /dev/null +++ b/src/limit.js @@ -0,0 +1,40 @@ +/** @typedef {{priority: number, cancel: boolean}} Entry */ + +/** + * @param {Entry[]} entries + * @param {number} max + */ +export function truncateWantlist (entries = [], max = 500) { + // boxo aims to send less than 16KiB messages... + if (entries.length <= max) { return entries } + + // Prefer high priority and cancel messages. Copy before mutating. + const sorted = [...entries].sort(entryPrioritySort) + return sorted.slice(0, max) +} + +/** + * Compare function to sort highest priority messages first + * @param {Entry} a + * @param {Entry} b + */ +export function entryPrioritySort (a, b) { + const score = priority(b) - priority(a) + if (score === 0) { + if (a.cancel && !b.cancel) { return -1 } + if (!a.cancel && b.cancel) { return 1 } + return 0 + } + return score +} + +/** + * Safely get priority + * @param {Entry} entry + */ +function priority (entry = {}) { + if (Number.isSafeInteger(entry.priority)) { + return entry.priority + } + return 0 +} diff --git a/src/service.js b/src/service.js index 5d2a98a..13cdcf4 100644 --- a/src/service.js +++ b/src/service.js @@ -13,6 +13,8 @@ import { handle, createContext } from './handler.js' import { telemetry } from './telemetry.js' import { logger as defaultLogger } from './logging.js' import { createPeerIdFromMultihash } from './peer-id.js' +import { denylistFilter } from './deny.js' +import { truncateWantlist } from './limit.js' // TODO validate all the params function validateParams ({ taggedPeers, logger }) { @@ -52,7 +54,7 @@ function validateParams ({ taggedPeers, logger }) { return { taggedPeers: peers } } -async function startService ({ peerId, port, peerAnnounceAddr, awsClient, connectionConfig, logger = defaultLogger, taggedPeers } = {}) { +async function startService ({ peerId, port, peerAnnounceAddr, awsClient, connectionConfig, logger = defaultLogger, taggedPeers, denylistUrl } = {}) { try { const validatedParams = validateParams({ taggedPeers, logger }) const service = await createLibp2p({ @@ -120,7 +122,7 @@ async function startService ({ peerId, port, peerAnnounceAddr, awsClient, connec const connectionId = hrTime[0] * 1000000000 + hrTime[1] // Open a send connection to the peer - connection.on('data', data => { + connection.on('data', async data => { let message try { @@ -130,12 +132,27 @@ async function startService ({ peerId, port, peerAnnounceAddr, awsClient, connec return } + // limit the number of cids we'll process from a single message. they can ask again. + const { wantlist } = message + wantlist.entries = truncateWantlist(wantlist.entries, 500) + + try { + const count = wantlist.entries.length + wantlist.entries = await denylistFilter(wantlist.entries, logger, denylistUrl) + const diff = count - wantlist.entries.length + if (diff > 0) { + telemetry.increaseCount('bitswap-denied', diff) + } + } catch (err) { + logger.error({ err }, 'Error filtering by denylist') + } + try { const context = createContext({ service, peerId: dial.remotePeer, protocol, - wantlist: message.wantlist, + wantlist, awsClient, connectionId, canceled diff --git a/test/config.test.js b/test/config.test.js index ac8abfc..9413974 100644 --- a/test/config.test.js +++ b/test/config.test.js @@ -57,7 +57,8 @@ t.test('config - defaults', async t => { dynamoRetryDelay: 50, s3MaxRetries: 3, s3RetryDelay: 50, - allowReadinessTweak: false + allowReadinessTweak: false, + denylistUrl: undefined }) }) @@ -112,6 +113,7 @@ t.test('config - all by env vars', async t => { process.env.S3_MAX_RETRIES = '7' process.env.S3_RETRY_DELAY = '600' process.env.ALLOW_READINESS_TWEAK = 'true' + process.env.DENYLIST_URL = 'http://example.org' t.same(makeConfig(), { maxBlockDataSize: 987, @@ -167,6 +169,7 @@ t.test('config - all by env vars', async t => { dynamoRetryDelay: 500, s3MaxRetries: 7, s3RetryDelay: 600, - allowReadinessTweak: true + allowReadinessTweak: true, + denylistUrl: new URL('http://example.org') }) }) diff --git a/test/deny.test.js b/test/deny.test.js new file mode 100644 index 0000000..4b60222 --- /dev/null +++ b/test/deny.test.js @@ -0,0 +1,39 @@ +import t from 'tap' +import { CID } from 'multiformats/cid' +import { denylistFilter } from '../src/deny.js' +import { createMockAgent } from './utils/mock.js' +import { setGlobalDispatcher } from 'undici' + +t.test('denylistFilter', async () => { + const denylistUrl = 'http://example.org' + const denylist = ['QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn'] + + const mock = createMockAgent() + setGlobalDispatcher(mock) + + mock.get(denylistUrl).intercept({ + method: 'POST', + path: '/' + }).reply(200, JSON.stringify(denylist)) + + const okList = ['QmSzQpWhK1jbLofRWBoWr1VUsKhYU9GeHWCPf31Hb653XM'] + + const entries = denylist.concat(okList).map(x => ({ + cid: CID.parse(x), + priority: 1, + cancel: false + })) + + const filteredList = await denylistFilter(entries, { error: console.error }, new URL(denylistUrl)) + const expected = entries.slice(1) + t.same(filteredList, expected, 'should filter out items on denylist') + + // denylist api request fails + mock.get(denylistUrl).intercept({ + method: 'POST', + path: '/' + }).reply(429, 'TOO MUCH REQ!') + + const filteredFromCache = await denylistFilter(entries, { error: console.error }, new URL(denylistUrl)) + t.same(filteredFromCache, entries.slice(1), 'should filter out items from denylist cache') +}) diff --git a/test/limit.test.js b/test/limit.test.js new file mode 100644 index 0000000..176b62f --- /dev/null +++ b/test/limit.test.js @@ -0,0 +1,17 @@ +import t from 'tap' +import { CID } from 'multiformats/cid' +import { truncateWantlist } from '../src/limit.js' + +t.test('truncateWantlist', async () => { + // should end up with last one first. Should sort by priority then cancel: true + const entries = [ + { priority: 1, cancel: true, cid: CID.parse('Qmd66enYAFadcPRraKbbWsXNN8oENBFo4UanvCEuVhENLU') }, + { priority: 1, cancel: false, cid: CID.parse('Qmc89yFTx1pEgWnPaUhiJGLxChCpvjkmy1HfRnSzhg61Nv') }, + { priority: 2, cancel: false, cid: CID.parse('QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn') } + ] + const max = 2 + const res = truncateWantlist(entries, max) + t.same(res.length, max, 'should truncate to max length') + t.same(res[0].cid.toString(), entries[2].cid.toString(), 'should sort by priority') + t.same(res[1], entries[0], 'should prefer cancel: true') +})