fix: stop writing to blocks v1 table (#72)

elastic-ipfs · Feb 13, 2023 · 5f8bc22 · 5f8bc22
1 parent 34a303f
commit 5f8bc22
Show file tree

Hide file tree

Showing 4 changed files with 1 addition and 150 deletions.
diff --git a/README.md b/README.md
@@ -8,7 +8,6 @@ _Variables in bold are required._
 | --------------------------- | ------------------ | ------------------------------------------------------------------------------ |
 | CONCURRENCY                 | `32`               | Concurrent batch inserts of blocks.                                            |
 | BLOCKS_BATCH_SIZE           | `10`               | Batch size for blocks ops (insert, publish). 10 is max for SQS, 25 is max for Dynamo |
-| DYNAMO_BLOCKS_TABLE         | `v1-blocks`        | The DynamoDB table where store CIDs informations to.                           |
 | DYNAMO_CARS_TABLE           | `v1-cars`          | The DynamoDB table where store CAR files informations to.                      |
 | DYNAMO_LINK_TABLE           | `v1-blocks-cars-position`   | The DynamoDB table with CARs-blocks links.                                     |
 | DYNAMO_MAX_RETRIES          | `3`                | DynamoDB max attempts in case of query failure.                                |

diff --git a/src/config.js b/src/config.js
@@ -8,7 +8,6 @@ require('dotenv').config({ path: process.env.ENV_FILE_PATH || resolve(process.cw
 const {
   CONCURRENCY: rawConcurrency,
 
-  DYNAMO_BLOCKS_TABLE: blocksTable,
   DYNAMO_CARS_TABLE: carsTable,
   DYNAMO_LINK_TABLE: linkTable,
 
@@ -49,11 +48,9 @@ const concurrency = parseInt(rawConcurrency)
 
 module.exports = {
   RAW_BLOCK_CODEC,
-  blocksTable: blocksTable ?? 'v1-blocks',
   carsTable: carsTable ?? 'v1-cars',
   linkTable: linkTable ?? 'v1-blocks-cars-position',
 
-  blocksTablePrimaryKey: 'multihash',
   carsTablePrimaryKey: 'path',
   linkTableBlockKey: 'blockmultihash',
   linkTableCarKey: 'carpath',

diff --git a/src/lib/block.js b/src/lib/block.js
@@ -1,51 +1,11 @@
 'use strict'
 
-const {
-  UnixFS: { unmarshal: decodeUnixFs }
-} = require('ipfs-unixfs')
 const config = require('../config')
-const { now, queuedTasks } = require('./util')
+const { queuedTasks } = require('./util')
 const { serializeError } = require('./logging')
 const { cidToKey, batchWriteDynamoItems } = require('./storage')
 const { publishBatch } = require('./publish')
 
-function blockType(blockId, codecs = config.codecs) {
-  return codecs[blockId.code]?.label ?? 'unsupported'
-}
-
-function blockData(block) {
-  if (!block.data) {
-    return
-  }
-
-  const data = decodeBlock(block)
-  if (data.Links) {
-    for (const link of data.Links) {
-      link.Hash = link.Hash?.toString()
-    }
-  }
-
-  return data
-}
-
-function decodeBlock(block, codecs = config.codecs) {
-  const codec = codecs[block.cid.code]
-
-  if (!codec) {
-    throw new Error(`Unsupported codec ${block.cid.code} in the block at offset ${block.blockOffset}`)
-  }
-
-  const data = codec.decode(block.data)
-
-  if (codec.label.startsWith('dag')) {
-    const { type, blocks } = decodeUnixFs(data.Data)
-
-    data.Data = { type, blocks }
-  }
-
-  return data
-}
-
 async function publishBlocks({ blocks, logger, queue = config.publishingQueue }) {
   await publishBatch({ queue, messages: blocks.map(b => b.key), logger })
 }
@@ -98,7 +58,6 @@ async function storeBlocks({ car, source, logger, batchSize = config.blocksBatch
  * @see https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Programming.Errors.html#Programming.Errors.BatchOperations
  */
 async function writeBlocksBatch({ blocks, car, logger }) {
-  const blockItems = []
   const linkItems = []
   const keys = []
   for (let i = 0; i < blocks.length; i++) {
@@ -112,13 +71,6 @@ async function writeBlocksBatch({ blocks, car, logger }) {
       continue
     }
 
-    blockItems.push({
-      [config.blocksTablePrimaryKey]: block.key,
-      type: blockType(block.cid),
-      createdAt: now(),
-      data: blockData(block)
-    })
-
     linkItems.push({
       [config.linkTableBlockKey]: block.key,
       [config.linkTableCarKey]: car.id,
@@ -131,7 +83,6 @@ async function writeBlocksBatch({ blocks, car, logger }) {
   }
 
   const batch = [
-    { table: config.blocksTable, items: blockItems },
     { table: config.linkTable, items: linkItems }
   ]
 

diff --git a/tap-snapshots/test/index.test.js.test.cjs b/tap-snapshots/test/index.test.js.test.cjs
@@ -11,68 +11,6 @@ Object {
     "batchCreates": Array [
       Object {
         "RequestItems": Object {
-          "v1-blocks": Array [
-            Object {
-              "PutRequest": Object {
-                "Item": Object {
-                  "createdAt": Object {
-                    "S": "2022-06-24T15:15:17.401Z",
-                  },
-                  "multihash": Object {
-                    "S": "zQmY13QWtykrcwmQmLVdxAQnJsRq7xBs5FAqH5zpG9ZvJpC",
-                  },
-                  "type": Object {
-                    "S": "raw",
-                  },
-                },
-              },
-            },
-            Object {
-              "PutRequest": Object {
-                "Item": Object {
-                  "createdAt": Object {
-                    "S": "2022-06-24T15:15:17.401Z",
-                  },
-                  "multihash": Object {
-                    "S": "zQmSGtsqx7aYH8gP21AgidxXuX5vsseFJgHKa75kg8HepXL",
-                  },
-                  "type": Object {
-                    "S": "dag-pb",
-                  },
-                },
-              },
-            },
-            Object {
-              "PutRequest": Object {
-                "Item": Object {
-                  "createdAt": Object {
-                    "S": "2022-06-24T15:15:17.401Z",
-                  },
-                  "multihash": Object {
-                    "S": "zQmSHc8o3PxQgMccYgGtuStaNQKXTBX1rTHN5W9cUCwrcHX",
-                  },
-                  "type": Object {
-                    "S": "dag-pb",
-                  },
-                },
-              },
-            },
-            Object {
-              "PutRequest": Object {
-                "Item": Object {
-                  "createdAt": Object {
-                    "S": "2022-06-24T15:15:17.401Z",
-                  },
-                  "multihash": Object {
-                    "S": "zQmTgGQZ3ZcbcHxZiFNHs76Y7Ca8DfFGjdsxXDVnr41h339",
-                  },
-                  "type": Object {
-                    "S": "dag-pb",
-                  },
-                },
-              },
-            },
-          ],
           "v1-blocks-cars-position": Array [
             Object {
               "PutRequest": Object {
@@ -151,23 +89,6 @@ Object {
       },
       Object {
         "RequestItems": Object {
-          "v1-blocks": Array [
-            Object {
-              "PutRequest": Object {
-                "Item": Object {
-                  "createdAt": Object {
-                    "S": "2022-06-24T15:15:17.401Z",
-                  },
-                  "multihash": Object {
-                    "S": "zQmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn",
-                  },
-                  "type": Object {
-                    "S": "dag-pb",
-                  },
-                },
-              },
-            },
-          ],
           "v1-blocks-cars-position": Array [
             Object {
               "PutRequest": Object {
@@ -271,23 +192,6 @@ Object {
     "batchCreates": Array [
       Object {
         "RequestItems": Object {
-          "v1-blocks": Array [
-            Object {
-              "PutRequest": Object {
-                "Item": Object {
-                  "createdAt": Object {
-                    "S": "2022-06-24T15:15:17.401Z",
-                  },
-                  "multihash": Object {
-                    "S": "zQmPH3Su9xAqw4WRbXT6DvwNpmaXYvTKKAY2hBKJsC7j2b4",
-                  },
-                  "type": Object {
-                    "S": "unsupported",
-                  },
-                },
-              },
-            },
-          ],
           "v1-blocks-cars-position": Array [
             Object {
               "PutRequest": Object {