feat: add HAMT sharded directories support

packages/upload-client/package.json

@@ -65,7 +65,7 @@
   "dependencies": {
     "@ipld/car": "^5.0.3",
     "@ipld/dag-ucan": "^3.2.0",
-    "@ipld/unixfs": "^2.0.1",
+    "@ipld/unixfs": "^2.1.0",
     "@ucanto/client": "^5.1.0",
     "@ucanto/interface": "^6.0.0",
     "@ucanto/transport": "^5.1.0",

packages/upload-client/src/unixfs.js

@@ -1,6 +1,7 @@
 import * as UnixFS from '@ipld/unixfs'
 import * as raw from 'multiformats/codecs/raw'
 
+const SHARD_THRESHOLD = 1000 // shard directory after > 1,000 items
 const queuingStrategy = UnixFS.withCapacity()
 
 // TODO: configure chunk size and max children https://github.com/ipld/js-unixfs/issues/36
@@ -64,7 +65,10 @@ class UnixFSDirectoryBuilder {
 
   /** @param {import('@ipld/unixfs').View} writer */
   async finalize(writer) {
-    const dirWriter = UnixFS.createDirectoryWriter(writer)
+    const dirWriter =
+      this.entries.size <= SHARD_THRESHOLD
+        ? UnixFS.createDirectoryWriter(writer)
+        : UnixFS.createShardedDirectoryWriter(writer)
     for (const [name, entry] of this.entries) {
       const link = await entry.finalize(writer)
       dirWriter.set(name, link)

packages/upload-client/test/unixfs.test.js

@@ -1,4 +1,5 @@
 import assert from 'assert'
+import { decode, NodeType } from '@ipld/unixfs'
 import { exporter } from 'ipfs-unixfs-exporter'
 import { MemoryBlockstore } from 'blockstore-core/memory'
 import * as raw from 'multiformats/codecs/raw'
@@ -67,6 +68,31 @@ describe('UnixFS', () => {
     expectedPaths.forEach((p) => assert(actualPaths.includes(p)))
   })
 
+  it('encodes a sharded directory', async () => {
+    const files = []
+    for (let i = 0; i < 1001; i++) {
+      files.push(new File([`data${i}`], `file${i}.txt`))
+    }
+
+    const { cid, blocks } = await encodeDirectory(files)
+    const blockstore = await blocksToBlockstore(blocks)
+    const dirEntry = await exporter(cid.toString(), blockstore)
+    assert.equal(dirEntry.type, 'directory')
+
+    const expectedPaths = files.map((f) => path.join(cid.toString(), f.name))
+    // @ts-expect-error
+    const entries = await collectDir(dirEntry)
+    const actualPaths = entries.map((e) => e.path)
+
+    expectedPaths.forEach((p) => assert(actualPaths.includes(p)))
+
+    // check root node is a HAMT sharded directory
+    // @ts-expect-error
+    const bytes = await blockstore.get(cid)
+    const node = decode(bytes)
+    assert.equal(node.type, NodeType.HAMTShard)
+  })
+
   it('throws then treating a file as a directory', () =>
     assert.rejects(
       encodeDirectory([