From 5f8bc22c57a150dec99fed73097a4931873cd9f1 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Mon, 13 Feb 2023 17:26:43 +0100 Subject: [PATCH] fix: stop writing to blocks v1 table (#72) --- README.md | 1 - src/config.js | 3 - src/lib/block.js | 51 +----------- tap-snapshots/test/index.test.js.test.cjs | 96 ----------------------- 4 files changed, 1 insertion(+), 150 deletions(-) diff --git a/README.md b/README.md index fd7fd3d..1005e0f 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,6 @@ _Variables in bold are required._ | --------------------------- | ------------------ | ------------------------------------------------------------------------------ | | CONCURRENCY | `32` | Concurrent batch inserts of blocks. | | BLOCKS_BATCH_SIZE | `10` | Batch size for blocks ops (insert, publish). 10 is max for SQS, 25 is max for Dynamo | -| DYNAMO_BLOCKS_TABLE | `v1-blocks` | The DynamoDB table where store CIDs informations to. | | DYNAMO_CARS_TABLE | `v1-cars` | The DynamoDB table where store CAR files informations to. | | DYNAMO_LINK_TABLE | `v1-blocks-cars-position` | The DynamoDB table with CARs-blocks links. | | DYNAMO_MAX_RETRIES | `3` | DynamoDB max attempts in case of query failure. | diff --git a/src/config.js b/src/config.js index 425a11b..636730c 100644 --- a/src/config.js +++ b/src/config.js @@ -8,7 +8,6 @@ require('dotenv').config({ path: process.env.ENV_FILE_PATH || resolve(process.cw const { CONCURRENCY: rawConcurrency, - DYNAMO_BLOCKS_TABLE: blocksTable, DYNAMO_CARS_TABLE: carsTable, DYNAMO_LINK_TABLE: linkTable, @@ -49,11 +48,9 @@ const concurrency = parseInt(rawConcurrency) module.exports = { RAW_BLOCK_CODEC, - blocksTable: blocksTable ?? 'v1-blocks', carsTable: carsTable ?? 'v1-cars', linkTable: linkTable ?? 'v1-blocks-cars-position', - blocksTablePrimaryKey: 'multihash', carsTablePrimaryKey: 'path', linkTableBlockKey: 'blockmultihash', linkTableCarKey: 'carpath', diff --git a/src/lib/block.js b/src/lib/block.js index af8b315..83f4d72 100644 --- a/src/lib/block.js +++ b/src/lib/block.js @@ -1,51 +1,11 @@ 'use strict' -const { - UnixFS: { unmarshal: decodeUnixFs } -} = require('ipfs-unixfs') const config = require('../config') -const { now, queuedTasks } = require('./util') +const { queuedTasks } = require('./util') const { serializeError } = require('./logging') const { cidToKey, batchWriteDynamoItems } = require('./storage') const { publishBatch } = require('./publish') -function blockType(blockId, codecs = config.codecs) { - return codecs[blockId.code]?.label ?? 'unsupported' -} - -function blockData(block) { - if (!block.data) { - return - } - - const data = decodeBlock(block) - if (data.Links) { - for (const link of data.Links) { - link.Hash = link.Hash?.toString() - } - } - - return data -} - -function decodeBlock(block, codecs = config.codecs) { - const codec = codecs[block.cid.code] - - if (!codec) { - throw new Error(`Unsupported codec ${block.cid.code} in the block at offset ${block.blockOffset}`) - } - - const data = codec.decode(block.data) - - if (codec.label.startsWith('dag')) { - const { type, blocks } = decodeUnixFs(data.Data) - - data.Data = { type, blocks } - } - - return data -} - async function publishBlocks({ blocks, logger, queue = config.publishingQueue }) { await publishBatch({ queue, messages: blocks.map(b => b.key), logger }) } @@ -98,7 +58,6 @@ async function storeBlocks({ car, source, logger, batchSize = config.blocksBatch * @see https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Programming.Errors.html#Programming.Errors.BatchOperations */ async function writeBlocksBatch({ blocks, car, logger }) { - const blockItems = [] const linkItems = [] const keys = [] for (let i = 0; i < blocks.length; i++) { @@ -112,13 +71,6 @@ async function writeBlocksBatch({ blocks, car, logger }) { continue } - blockItems.push({ - [config.blocksTablePrimaryKey]: block.key, - type: blockType(block.cid), - createdAt: now(), - data: blockData(block) - }) - linkItems.push({ [config.linkTableBlockKey]: block.key, [config.linkTableCarKey]: car.id, @@ -131,7 +83,6 @@ async function writeBlocksBatch({ blocks, car, logger }) { } const batch = [ - { table: config.blocksTable, items: blockItems }, { table: config.linkTable, items: linkItems } ] diff --git a/tap-snapshots/test/index.test.js.test.cjs b/tap-snapshots/test/index.test.js.test.cjs index b45b6a5..038c581 100644 --- a/tap-snapshots/test/index.test.js.test.cjs +++ b/tap-snapshots/test/index.test.js.test.cjs @@ -11,68 +11,6 @@ Object { "batchCreates": Array [ Object { "RequestItems": Object { - "v1-blocks": Array [ - Object { - "PutRequest": Object { - "Item": Object { - "createdAt": Object { - "S": "2022-06-24T15:15:17.401Z", - }, - "multihash": Object { - "S": "zQmY13QWtykrcwmQmLVdxAQnJsRq7xBs5FAqH5zpG9ZvJpC", - }, - "type": Object { - "S": "raw", - }, - }, - }, - }, - Object { - "PutRequest": Object { - "Item": Object { - "createdAt": Object { - "S": "2022-06-24T15:15:17.401Z", - }, - "multihash": Object { - "S": "zQmSGtsqx7aYH8gP21AgidxXuX5vsseFJgHKa75kg8HepXL", - }, - "type": Object { - "S": "dag-pb", - }, - }, - }, - }, - Object { - "PutRequest": Object { - "Item": Object { - "createdAt": Object { - "S": "2022-06-24T15:15:17.401Z", - }, - "multihash": Object { - "S": "zQmSHc8o3PxQgMccYgGtuStaNQKXTBX1rTHN5W9cUCwrcHX", - }, - "type": Object { - "S": "dag-pb", - }, - }, - }, - }, - Object { - "PutRequest": Object { - "Item": Object { - "createdAt": Object { - "S": "2022-06-24T15:15:17.401Z", - }, - "multihash": Object { - "S": "zQmTgGQZ3ZcbcHxZiFNHs76Y7Ca8DfFGjdsxXDVnr41h339", - }, - "type": Object { - "S": "dag-pb", - }, - }, - }, - }, - ], "v1-blocks-cars-position": Array [ Object { "PutRequest": Object { @@ -151,23 +89,6 @@ Object { }, Object { "RequestItems": Object { - "v1-blocks": Array [ - Object { - "PutRequest": Object { - "Item": Object { - "createdAt": Object { - "S": "2022-06-24T15:15:17.401Z", - }, - "multihash": Object { - "S": "zQmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn", - }, - "type": Object { - "S": "dag-pb", - }, - }, - }, - }, - ], "v1-blocks-cars-position": Array [ Object { "PutRequest": Object { @@ -271,23 +192,6 @@ Object { "batchCreates": Array [ Object { "RequestItems": Object { - "v1-blocks": Array [ - Object { - "PutRequest": Object { - "Item": Object { - "createdAt": Object { - "S": "2022-06-24T15:15:17.401Z", - }, - "multihash": Object { - "S": "zQmPH3Su9xAqw4WRbXT6DvwNpmaXYvTKKAY2hBKJsC7j2b4", - }, - "type": Object { - "S": "unsupported", - }, - }, - }, - }, - ], "v1-blocks-cars-position": Array [ Object { "PutRequest": Object {