diff --git a/.github/dependabot.yml b/.github/dependabot.yml
new file mode 100644
index 0000000..0bc3b42
--- /dev/null
+++ b/.github/dependabot.yml
@@ -0,0 +1,11 @@
+version: 2
+updates:
+- package-ecosystem: npm
+ directory: "/"
+ schedule:
+ interval: daily
+ time: "10:00"
+ open-pull-requests-limit: 10
+ commit-message:
+ prefix: "deps"
+ prefix-development: "deps(dev)"
diff --git a/.github/workflows/automerge.yml b/.github/workflows/automerge.yml
new file mode 100644
index 0000000..d57c2a0
--- /dev/null
+++ b/.github/workflows/automerge.yml
@@ -0,0 +1,8 @@
+name: Automerge
+on: [ pull_request ]
+
+jobs:
+ automerge:
+ uses: protocol/.github/.github/workflows/automerge.yml@master
+ with:
+ job: 'automerge'
diff --git a/.github/workflows/js-test-and-release.yml b/.github/workflows/js-test-and-release.yml
new file mode 100644
index 0000000..081b272
--- /dev/null
+++ b/.github/workflows/js-test-and-release.yml
@@ -0,0 +1,143 @@
+name: test & maybe release
+on:
+ push:
+ branches:
+ - main
+ pull_request:
+
+jobs:
+
+ check:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v3
+ - uses: actions/setup-node@v3
+ with:
+ node-version: lts/*
+ - uses: ipfs/aegir/actions/cache-node-modules@master
+ - run: npm run --if-present lint
+ - run: npm run --if-present dep-check
+
+ test-node:
+ needs: check
+ runs-on: ${{ matrix.os }}
+ strategy:
+ matrix:
+ os: [windows-latest, ubuntu-latest, macos-latest]
+ node: [lts/*]
+ fail-fast: true
+ steps:
+ - uses: actions/checkout@v3
+ - uses: actions/setup-node@v3
+ with:
+ node-version: ${{ matrix.node }}
+ - uses: ipfs/aegir/actions/cache-node-modules@master
+ - run: npm run --if-present test:node
+ - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1
+ with:
+ flags: node
+
+ test-chrome:
+ needs: check
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v3
+ - uses: actions/setup-node@v3
+ with:
+ node-version: lts/*
+ - uses: ipfs/aegir/actions/cache-node-modules@master
+ - run: npm run --if-present test:chrome
+ - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1
+ with:
+ flags: chrome
+
+ test-chrome-webworker:
+ needs: check
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v3
+ - uses: actions/setup-node@v3
+ with:
+ node-version: lts/*
+ - uses: ipfs/aegir/actions/cache-node-modules@master
+ - run: npm run --if-present test:chrome-webworker
+ - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1
+ with:
+ flags: chrome-webworker
+
+ test-firefox:
+ needs: check
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v3
+ - uses: actions/setup-node@v3
+ with:
+ node-version: lts/*
+ - uses: ipfs/aegir/actions/cache-node-modules@master
+ - run: npm run --if-present test:firefox
+ - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1
+ with:
+ flags: firefox
+
+ test-firefox-webworker:
+ needs: check
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v3
+ - uses: actions/setup-node@v3
+ with:
+ node-version: lts/*
+ - uses: ipfs/aegir/actions/cache-node-modules@master
+ - run: npm run --if-present test:firefox-webworker
+ - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1
+ with:
+ flags: firefox-webworker
+
+ test-electron-main:
+ needs: check
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v3
+ - uses: actions/setup-node@v3
+ with:
+ node-version: lts/*
+ - uses: ipfs/aegir/actions/cache-node-modules@master
+ - run: npx xvfb-maybe npm run --if-present test:electron-main
+ - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1
+ with:
+ flags: electron-main
+
+ test-electron-renderer:
+ needs: check
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v3
+ - uses: actions/setup-node@v3
+ with:
+ node-version: lts/*
+ - uses: ipfs/aegir/actions/cache-node-modules@master
+ - run: npx xvfb-maybe npm run --if-present test:electron-renderer
+ - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1
+ with:
+ flags: electron-renderer
+
+ release:
+ needs: [test-node, test-chrome, test-chrome-webworker, test-firefox, test-firefox-webworker, test-electron-main, test-electron-renderer]
+ runs-on: ubuntu-latest
+ if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+ steps:
+ - uses: actions/checkout@v3
+ with:
+ fetch-depth: 0
+ - uses: actions/setup-node@v3
+ with:
+ node-version: lts/*
+ - uses: ipfs/aegir/actions/cache-node-modules@master
+ - uses: ipfs/aegir/actions/docker-login@master
+ with:
+ docker-token: ${{ secrets.DOCKER_TOKEN }}
+ docker-username: ${{ secrets.DOCKER_USERNAME }}
+ - run: npm run --if-present release
+ env:
+ GITHUB_TOKEN: ${{ github.token }}
+ NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..7ad9e67
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,9 @@
+node_modules
+build
+dist
+.docs
+.coverage
+node_modules
+package-lock.json
+yarn.lock
+.vscode
diff --git a/LICENSE b/LICENSE
new file mode 100644
index 0000000..20ce483
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,4 @@
+This project is dual licensed under MIT and Apache-2.0.
+
+MIT: https://www.opensource.org/licenses/mit
+Apache-2.0: https://www.apache.org/licenses/license-2.0
diff --git a/LICENSE-APACHE b/LICENSE-APACHE
new file mode 100644
index 0000000..14478a3
--- /dev/null
+++ b/LICENSE-APACHE
@@ -0,0 +1,5 @@
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
diff --git a/LICENSE-MIT b/LICENSE-MIT
new file mode 100644
index 0000000..72dc60d
--- /dev/null
+++ b/LICENSE-MIT
@@ -0,0 +1,19 @@
+The MIT License (MIT)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
diff --git a/README.md b/README.md
index ffff3cb..dc448fc 100644
--- a/README.md
+++ b/README.md
@@ -1,19 +1,26 @@
+
+
+ 
+
+
+
# @helia/unixfs
[](https://ipfs.tech)
[](https://discuss.ipfs.tech)
-[](https://codecov.io/gh/ipfs/helia)
-[](https://github.com/ipfs/helia/actions/workflows/js-test-and-release.yml?query=branch%3Amain)
+[](https://codecov.io/gh/ipfs/helia-unixfs)
+[](https://github.com/ipfs/helia-unixfs/actions/workflows/js-test-and-release.yml?query=branch%3Amain)
> A Helia-compatible wrapper for UnixFS
## Table of contents
-- [Install](#install)
- - [Browser `
```
+
+
+
+
+
+
+# @helia/unixfs
+
+[](https://ipfs.tech)
+[](https://discuss.ipfs.tech)
+[](https://codecov.io/gh/ipfs/helia-unixfs)
+[](https://github.com/ipfs/helia-unixfs/actions/workflows/js-test-and-release.yml?query=branch%3Amaster)
+
## API Docs
--
+-
## License
@@ -42,7 +62,7 @@ Licensed under either of
## Contribute
-Contributions welcome! Please check out [the issues](https://github.com/ipfs/helia/issues).
+Contributions welcome! Please check out [the issues](https://github.com/ipfs/helia-unixfs/issues).
Also see our [contributing document](https://github.com/ipfs/community/blob/master/CONTRIBUTING_JS.md) for more information on how we work, and about contributing in general.
diff --git a/package.json b/package.json
new file mode 100644
index 0000000..eb97459
--- /dev/null
+++ b/package.json
@@ -0,0 +1,167 @@
+{
+ "name": "@helia/unixfs",
+ "version": "0.0.0",
+ "description": "A Helia-compatible wrapper for UnixFS",
+ "license": "Apache-2.0 OR MIT",
+ "homepage": "https://github.com/ipfs/helia-unixfs#readme",
+ "repository": {
+ "type": "git",
+ "url": "git+https://github.com/ipfs/helia-unixfs.git"
+ },
+ "bugs": {
+ "url": "https://github.com/ipfs/helia-unixfs/issues"
+ },
+ "keywords": [
+ "IPFS"
+ ],
+ "engines": {
+ "node": ">=16.0.0",
+ "npm": ">=7.0.0"
+ },
+ "type": "module",
+ "types": "./dist/src/index.d.ts",
+ "files": [
+ "src",
+ "dist",
+ "!dist/test",
+ "!**/*.tsbuildinfo"
+ ],
+ "exports": {
+ ".": {
+ "types": "./dist/src/index.d.ts",
+ "import": "./dist/src/index.js"
+ }
+ },
+ "eslintConfig": {
+ "extends": "ipfs",
+ "parserOptions": {
+ "sourceType": "module"
+ }
+ },
+ "release": {
+ "branches": [
+ "main"
+ ],
+ "plugins": [
+ [
+ "@semantic-release/commit-analyzer",
+ {
+ "preset": "conventionalcommits",
+ "releaseRules": [
+ {
+ "breaking": true,
+ "release": "major"
+ },
+ {
+ "revert": true,
+ "release": "patch"
+ },
+ {
+ "type": "feat",
+ "release": "minor"
+ },
+ {
+ "type": "fix",
+ "release": "patch"
+ },
+ {
+ "type": "docs",
+ "release": "patch"
+ },
+ {
+ "type": "test",
+ "release": "patch"
+ },
+ {
+ "type": "deps",
+ "release": "patch"
+ },
+ {
+ "scope": "no-release",
+ "release": false
+ }
+ ]
+ }
+ ],
+ [
+ "@semantic-release/release-notes-generator",
+ {
+ "preset": "conventionalcommits",
+ "presetConfig": {
+ "types": [
+ {
+ "type": "feat",
+ "section": "Features"
+ },
+ {
+ "type": "fix",
+ "section": "Bug Fixes"
+ },
+ {
+ "type": "chore",
+ "section": "Trivial Changes"
+ },
+ {
+ "type": "docs",
+ "section": "Documentation"
+ },
+ {
+ "type": "deps",
+ "section": "Dependencies"
+ },
+ {
+ "type": "test",
+ "section": "Tests"
+ }
+ ]
+ }
+ }
+ ],
+ "@semantic-release/changelog",
+ "@semantic-release/npm",
+ "@semantic-release/github",
+ "@semantic-release/git"
+ ]
+ },
+ "scripts": {
+ "clean": "aegir clean",
+ "lint": "aegir lint",
+ "dep-check": "aegir dep-check",
+ "build": "aegir build",
+ "docs": "aegir docs",
+ "test": "aegir test",
+ "test:chrome": "aegir test -t browser --cov",
+ "test:chrome-webworker": "aegir test -t webworker",
+ "test:firefox": "aegir test -t browser -- --browser firefox",
+ "test:firefox-webworker": "aegir test -t webworker -- --browser firefox",
+ "test:node": "aegir test -t node --cov",
+ "test:electron-main": "aegir test -t electron-main",
+ "release": "aegir release"
+ },
+ "dependencies": {
+ "@helia/interface": "next",
+ "@ipld/dag-pb": "^4.0.0",
+ "@libp2p/interfaces": "^3.3.1",
+ "@libp2p/logger": "^2.0.5",
+ "@multiformats/murmur3": "^2.1.2",
+ "hamt-sharding": "^3.0.2",
+ "interface-blockstore": "^4.0.1",
+ "ipfs-unixfs": "^11.0.0",
+ "ipfs-unixfs-exporter": "^12.0.0",
+ "ipfs-unixfs-importer": "^14.0.0",
+ "it-last": "^2.0.0",
+ "it-pipe": "^2.0.5",
+ "merge-options": "^3.0.4",
+ "multiformats": "^11.0.1"
+ },
+ "devDependencies": {
+ "aegir": "^38.1.0",
+ "blockstore-core": "^3.0.0",
+ "delay": "^5.0.0",
+ "it-all": "^2.0.0",
+ "it-drain": "^2.0.0",
+ "it-first": "^2.0.0",
+ "it-to-buffer": "^3.0.0",
+ "uint8arrays": "^4.0.3"
+ }
+}
diff --git a/src/commands/cat.ts b/src/commands/cat.ts
new file mode 100644
index 0000000..2c6d1bd
--- /dev/null
+++ b/src/commands/cat.ts
@@ -0,0 +1,32 @@
+import { exporter } from 'ipfs-unixfs-exporter'
+import type { CID } from 'multiformats/cid'
+import type { CatOptions } from '../index.js'
+import { resolve } from './utils/resolve.js'
+import mergeOpts from 'merge-options'
+import type { Blockstore } from 'interface-blockstore'
+import { NoContentError, NotAFileError } from './utils/errors.js'
+
+const mergeOptions = mergeOpts.bind({ ignoreUndefined: true })
+
+const defaultOptions: CatOptions = {
+
+}
+
+export async function * cat (cid: CID, blockstore: Blockstore, options: Partial = {}): AsyncIterable {
+ const opts: CatOptions = mergeOptions(defaultOptions, options)
+ const resolved = await resolve(cid, opts.path, blockstore, opts)
+ const result = await exporter(resolved.cid, blockstore, opts)
+
+ if (result.type !== 'file' && result.type !== 'raw') {
+ throw new NotAFileError()
+ }
+
+ if (result.content == null) {
+ throw new NoContentError()
+ }
+
+ yield * result.content({
+ offset: opts.offset,
+ length: opts.length
+ })
+}
diff --git a/src/commands/chmod.ts b/src/commands/chmod.ts
new file mode 100644
index 0000000..2f3f610
--- /dev/null
+++ b/src/commands/chmod.ts
@@ -0,0 +1,138 @@
+import { recursive } from 'ipfs-unixfs-exporter'
+import { CID } from 'multiformats/cid'
+import type { ChmodOptions } from '../index.js'
+import mergeOpts from 'merge-options'
+import { logger } from '@libp2p/logger'
+import { UnixFS } from 'ipfs-unixfs'
+import { pipe } from 'it-pipe'
+import { InvalidPBNodeError, NotUnixFSError, UnknownError } from './utils/errors.js'
+import * as dagPB from '@ipld/dag-pb'
+import type { PBNode, PBLink } from '@ipld/dag-pb'
+import { importer } from 'ipfs-unixfs-importer'
+import { persist } from './utils/persist.js'
+import type { Blockstore } from 'interface-blockstore'
+import last from 'it-last'
+import { sha256 } from 'multiformats/hashes/sha2'
+import { resolve, updatePathCids } from './utils/resolve.js'
+import * as raw from 'multiformats/codecs/raw'
+import { SHARD_SPLIT_THRESHOLD_BYTES } from './utils/constants.js'
+
+const mergeOptions = mergeOpts.bind({ ignoreUndefined: true })
+const log = logger('helia:unixfs:chmod')
+
+const defaultOptions: ChmodOptions = {
+ recursive: false,
+ shardSplitThresholdBytes: SHARD_SPLIT_THRESHOLD_BYTES
+}
+
+export async function chmod (cid: CID, mode: number, blockstore: Blockstore, options: Partial = {}): Promise {
+ const opts: ChmodOptions = mergeOptions(defaultOptions, options)
+ const resolved = await resolve(cid, opts.path, blockstore, options)
+
+ log('chmod %c %d', resolved.cid, mode)
+
+ if (opts.recursive) {
+ // recursively export from root CID, change perms of each entry then reimport
+ // but do not reimport files, only manipulate dag-pb nodes
+ const root = await pipe(
+ async function * () {
+ for await (const entry of recursive(resolved.cid, blockstore)) {
+ let metadata: UnixFS
+ let links: PBLink[] = []
+
+ if (entry.type === 'raw') {
+ // convert to UnixFS
+ metadata = new UnixFS({ type: 'file', data: entry.node })
+ } else if (entry.type === 'file' || entry.type === 'directory') {
+ metadata = entry.unixfs
+ links = entry.node.Links
+ } else {
+ throw new NotUnixFSError()
+ }
+
+ metadata.mode = mode
+
+ const node = {
+ Data: metadata.marshal(),
+ Links: links
+ }
+
+ yield {
+ path: entry.path,
+ content: node
+ }
+ }
+ },
+ // @ts-expect-error we account for the incompatible source type with our custom dag builder below
+ (source) => importer(source, blockstore, {
+ ...opts,
+ pin: false,
+ dagBuilder: async function * (source, block) {
+ for await (const entry of source) {
+ yield async function () {
+ // @ts-expect-error cannot derive type
+ const node: PBNode = entry.content
+
+ const buf = dagPB.encode(node)
+ const updatedCid = await persist(buf, block, {
+ ...opts,
+ cidVersion: cid.version
+ })
+
+ if (node.Data == null) {
+ throw new InvalidPBNodeError(`${updatedCid} had no data`)
+ }
+
+ const unixfs = UnixFS.unmarshal(node.Data)
+
+ return {
+ cid: updatedCid,
+ size: buf.length,
+ path: entry.path,
+ unixfs
+ }
+ }
+ }
+ }
+ }),
+ async (nodes) => await last(nodes)
+ )
+
+ if (root == null) {
+ throw new UnknownError(`Could not chmod ${resolved.cid.toString()}`)
+ }
+
+ return await updatePathCids(root.cid, resolved, blockstore, opts)
+ }
+
+ const block = await blockstore.get(resolved.cid)
+ let metadata: UnixFS
+ let links: PBLink[] = []
+
+ if (resolved.cid.code === raw.code) {
+ // convert to UnixFS
+ metadata = new UnixFS({ type: 'file', data: block })
+ } else {
+ const node = dagPB.decode(block)
+
+ if (node.Data == null) {
+ throw new InvalidPBNodeError(`${resolved.cid.toString()} had no data`)
+ }
+
+ links = node.Links
+ metadata = UnixFS.unmarshal(node.Data)
+ }
+
+ metadata.mode = mode
+ const updatedBlock = dagPB.encode({
+ Data: metadata.marshal(),
+ Links: links
+ })
+
+ const hash = await sha256.digest(updatedBlock)
+ const updatedCid = CID.create(resolved.cid.version, dagPB.code, hash)
+
+ await blockstore.put(updatedCid, updatedBlock)
+
+ return await updatePathCids(updatedCid, resolved, blockstore, opts)
+}
diff --git a/src/commands/cp.ts b/src/commands/cp.ts
new file mode 100644
index 0000000..4ca837f
--- /dev/null
+++ b/src/commands/cp.ts
@@ -0,0 +1,44 @@
+import type { Blockstore } from 'interface-blockstore'
+import type { CID } from 'multiformats/cid'
+import type { CpOptions } from '../index.js'
+import mergeOpts from 'merge-options'
+import { logger } from '@libp2p/logger'
+import { addLink } from './utils/add-link.js'
+import { cidToPBLink } from './utils/cid-to-pblink.js'
+import { cidToDirectory } from './utils/cid-to-directory.js'
+import { SHARD_SPLIT_THRESHOLD_BYTES } from './utils/constants.js'
+import { InvalidParametersError } from './utils/errors.js'
+
+const mergeOptions = mergeOpts.bind({ ignoreUndefined: true })
+const log = logger('helia:unixfs:cp')
+
+const defaultOptions: CpOptions = {
+ force: false,
+ shardSplitThresholdBytes: SHARD_SPLIT_THRESHOLD_BYTES
+}
+
+export async function cp (source: CID, target: CID, name: string, blockstore: Blockstore, options: Partial = {}): Promise {
+ const opts: CpOptions = mergeOptions(defaultOptions, options)
+
+ if (name.includes('/')) {
+ throw new InvalidParametersError('Name must not have slashes')
+ }
+
+ const [
+ directory,
+ pblink
+ ] = await Promise.all([
+ cidToDirectory(target, blockstore, opts),
+ cidToPBLink(source, name, blockstore, opts)
+ ])
+
+ log('Adding %c as "%s" to %c', source, name, target)
+
+ const result = await addLink(directory, pblink, blockstore, {
+ allowOverwriting: opts.force,
+ cidVersion: target.version,
+ ...opts
+ })
+
+ return result.cid
+}
diff --git a/src/commands/ls.ts b/src/commands/ls.ts
new file mode 100644
index 0000000..ca52acf
--- /dev/null
+++ b/src/commands/ls.ts
@@ -0,0 +1,37 @@
+import { exporter, UnixFSEntry } from 'ipfs-unixfs-exporter'
+import type { CID } from 'multiformats/cid'
+import type { LsOptions } from '../index.js'
+import { resolve } from './utils/resolve.js'
+import mergeOpts from 'merge-options'
+import type { Blockstore } from 'interface-blockstore'
+import { NoContentError, NotADirectoryError } from './utils/errors.js'
+
+const mergeOptions = mergeOpts.bind({ ignoreUndefined: true })
+
+const defaultOptions: LsOptions = {
+
+}
+
+export async function * ls (cid: CID, blockstore: Blockstore, options: Partial = {}): AsyncIterable {
+ const opts: LsOptions = mergeOptions(defaultOptions, options)
+ const resolved = await resolve(cid, opts.path, blockstore, opts)
+ const result = await exporter(resolved.cid, blockstore)
+
+ if (result.type === 'file' || result.type === 'raw') {
+ yield result
+ return
+ }
+
+ if (result.content == null) {
+ throw new NoContentError()
+ }
+
+ if (result.type !== 'directory') {
+ throw new NotADirectoryError()
+ }
+
+ yield * result.content({
+ offset: options.offset,
+ length: options.length
+ })
+}
diff --git a/src/commands/mkdir.ts b/src/commands/mkdir.ts
new file mode 100644
index 0000000..6b96905
--- /dev/null
+++ b/src/commands/mkdir.ts
@@ -0,0 +1,73 @@
+import { CID } from 'multiformats/cid'
+import mergeOpts from 'merge-options'
+import { logger } from '@libp2p/logger'
+import type { MkdirOptions } from '../index.js'
+import * as dagPB from '@ipld/dag-pb'
+import { addLink } from './utils/add-link.js'
+import type { Blockstore } from 'interface-blockstore'
+import { UnixFS } from 'ipfs-unixfs'
+import { sha256 } from 'multiformats/hashes/sha2'
+import { exporter } from 'ipfs-unixfs-exporter'
+import { cidToDirectory } from './utils/cid-to-directory.js'
+import { cidToPBLink } from './utils/cid-to-pblink.js'
+import { SHARD_SPLIT_THRESHOLD_BYTES } from './utils/constants.js'
+import { InvalidParametersError, NotADirectoryError } from './utils/errors.js'
+
+const mergeOptions = mergeOpts.bind({ ignoreUndefined: true })
+const log = logger('helia:unixfs:mkdir')
+
+const defaultOptions: MkdirOptions = {
+ cidVersion: 1,
+ force: false,
+ shardSplitThresholdBytes: SHARD_SPLIT_THRESHOLD_BYTES
+}
+
+export async function mkdir (parentCid: CID, dirname: string, blockstore: Blockstore, options: Partial = {}): Promise {
+ const opts: MkdirOptions = mergeOptions(defaultOptions, options)
+
+ if (dirname.includes('/')) {
+ throw new InvalidParametersError('Path must not have slashes')
+ }
+
+ const entry = await exporter(parentCid, blockstore, options)
+
+ if (entry.type !== 'directory') {
+ throw new NotADirectoryError(`${parentCid.toString()} was not a UnixFS directory`)
+ }
+
+ log('creating %s', dirname)
+
+ const metadata = new UnixFS({
+ type: 'directory',
+ mode: opts.mode,
+ mtime: opts.mtime
+ })
+
+ // Persist the new parent PBNode
+ const node = {
+ Data: metadata.marshal(),
+ Links: []
+ }
+ const buf = dagPB.encode(node)
+ const hash = await sha256.digest(buf)
+ const emptyDirCid = CID.create(opts.cidVersion, dagPB.code, hash)
+
+ await blockstore.put(emptyDirCid, buf)
+
+ const [
+ directory,
+ pblink
+ ] = await Promise.all([
+ cidToDirectory(parentCid, blockstore, opts),
+ cidToPBLink(emptyDirCid, dirname, blockstore, opts)
+ ])
+
+ log('adding empty dir called %s to %c', dirname, parentCid)
+
+ const result = await addLink(directory, pblink, blockstore, {
+ ...opts,
+ allowOverwriting: opts.force
+ })
+
+ return result.cid
+}
diff --git a/src/commands/rm.ts b/src/commands/rm.ts
new file mode 100644
index 0000000..28f7532
--- /dev/null
+++ b/src/commands/rm.ts
@@ -0,0 +1,35 @@
+import type { Blockstore } from 'interface-blockstore'
+import type { CID } from 'multiformats/cid'
+import type { RmOptions } from '../index.js'
+import mergeOpts from 'merge-options'
+import { logger } from '@libp2p/logger'
+import { removeLink } from './utils/remove-link.js'
+import { cidToDirectory } from './utils/cid-to-directory.js'
+import { SHARD_SPLIT_THRESHOLD_BYTES } from './utils/constants.js'
+import { InvalidParametersError } from './utils/errors.js'
+
+const mergeOptions = mergeOpts.bind({ ignoreUndefined: true })
+const log = logger('helia:unixfs:rm')
+
+const defaultOptions: RmOptions = {
+ shardSplitThresholdBytes: SHARD_SPLIT_THRESHOLD_BYTES
+}
+
+export async function rm (target: CID, name: string, blockstore: Blockstore, options: Partial = {}): Promise {
+ const opts: RmOptions = mergeOptions(defaultOptions, options)
+
+ if (name.includes('/')) {
+ throw new InvalidParametersError('Name must not have slashes')
+ }
+
+ const directory = await cidToDirectory(target, blockstore, opts)
+
+ log('Removing %s from %c', name, target)
+
+ const result = await removeLink(directory, name, blockstore, {
+ ...opts,
+ cidVersion: target.version
+ })
+
+ return result.cid
+}
diff --git a/src/commands/stat.ts b/src/commands/stat.ts
new file mode 100644
index 0000000..9564eca
--- /dev/null
+++ b/src/commands/stat.ts
@@ -0,0 +1,142 @@
+import { exporter } from 'ipfs-unixfs-exporter'
+import type { CID } from 'multiformats/cid'
+import type { StatOptions, UnixFSStats } from '../index.js'
+import mergeOpts from 'merge-options'
+import { logger } from '@libp2p/logger'
+import { UnixFS } from 'ipfs-unixfs'
+import { InvalidPBNodeError, NotUnixFSError, UnknownError } from './utils/errors.js'
+import * as dagPb from '@ipld/dag-pb'
+import type { AbortOptions } from '@libp2p/interfaces'
+import type { Mtime } from 'ipfs-unixfs'
+import { resolve } from './utils/resolve.js'
+import * as raw from 'multiformats/codecs/raw'
+import type { Blockstore } from 'interface-blockstore'
+
+const mergeOptions = mergeOpts.bind({ ignoreUndefined: true })
+const log = logger('helia:unixfs:stat')
+
+const defaultOptions: StatOptions = {
+
+}
+
+export async function stat (cid: CID, blockstore: Blockstore, options: Partial = {}): Promise {
+ const opts: StatOptions = mergeOptions(defaultOptions, options)
+ const resolved = await resolve(cid, options.path, blockstore, opts)
+
+ log('stat %c', resolved.cid)
+
+ const result = await exporter(resolved.cid, blockstore, opts)
+
+ if (result.type !== 'file' && result.type !== 'directory' && result.type !== 'raw') {
+ throw new NotUnixFSError()
+ }
+
+ let fileSize: bigint = 0n
+ let dagSize: bigint = 0n
+ let localFileSize: bigint = 0n
+ let localDagSize: bigint = 0n
+ let blocks: number = 0
+ let mode: number | undefined
+ let mtime: Mtime | undefined
+ const type = result.type
+ let unixfs: UnixFS | undefined
+
+ if (result.type === 'raw') {
+ fileSize = BigInt(result.node.byteLength)
+ dagSize = BigInt(result.node.byteLength)
+ localFileSize = BigInt(result.node.byteLength)
+ localDagSize = BigInt(result.node.byteLength)
+ blocks = 1
+ }
+
+ if (result.type === 'directory') {
+ fileSize = 0n
+ dagSize = BigInt(result.unixfs.marshal().byteLength)
+ localFileSize = 0n
+ localDagSize = dagSize
+ blocks = 1
+ mode = result.unixfs.mode
+ mtime = result.unixfs.mtime
+ unixfs = result.unixfs
+ }
+
+ if (result.type === 'file') {
+ const results = await inspectDag(resolved.cid, blockstore, opts)
+
+ fileSize = result.unixfs.fileSize()
+ dagSize = BigInt((result.node.Data?.byteLength ?? 0) + result.node.Links.reduce((acc, curr) => acc + (curr.Tsize ?? 0), 0))
+ localFileSize = BigInt(results.localFileSize)
+ localDagSize = BigInt(results.localDagSize)
+ blocks = results.blocks
+ mode = result.unixfs.mode
+ mtime = result.unixfs.mtime
+ unixfs = result.unixfs
+ }
+
+ return {
+ cid: resolved.cid,
+ mode,
+ mtime,
+ fileSize,
+ dagSize,
+ localFileSize,
+ localDagSize,
+ blocks,
+ type,
+ unixfs
+ }
+}
+
+interface InspectDagResults {
+ localFileSize: number
+ localDagSize: number
+ blocks: number
+}
+
+async function inspectDag (cid: CID, blockstore: Blockstore, options: AbortOptions): Promise {
+ const results = {
+ localFileSize: 0,
+ localDagSize: 0,
+ blocks: 0
+ }
+
+ if (await blockstore.has(cid, options)) {
+ const block = await blockstore.get(cid, options)
+ results.blocks++
+ results.localDagSize += block.byteLength
+
+ if (cid.code === raw.code) {
+ results.localFileSize += block.byteLength
+ } else if (cid.code === dagPb.code) {
+ const pbNode = dagPb.decode(block)
+
+ if (pbNode.Links.length > 0) {
+ // intermediate node
+ for (const link of pbNode.Links) {
+ const linkResult = await inspectDag(link.Hash, blockstore, options)
+
+ results.localFileSize += linkResult.localFileSize
+ results.localDagSize += linkResult.localDagSize
+ results.blocks += linkResult.blocks
+ }
+ } else {
+ // leaf node
+ if (pbNode.Data == null) {
+ throw new InvalidPBNodeError(`PBNode ${cid.toString()} had no data`)
+ }
+
+ const unixfs = UnixFS.unmarshal(pbNode.Data)
+
+ if (unixfs.data == null) {
+ throw new InvalidPBNodeError(`UnixFS node ${cid.toString()} had no data`)
+ }
+
+ results.localFileSize += unixfs.data.byteLength ?? 0
+ }
+ } else {
+ throw new UnknownError(`${cid.toString()} was neither DAG_PB nor RAW`)
+ }
+ }
+
+ return results
+}
diff --git a/src/commands/touch.ts b/src/commands/touch.ts
new file mode 100644
index 0000000..379778f
--- /dev/null
+++ b/src/commands/touch.ts
@@ -0,0 +1,141 @@
+import { recursive } from 'ipfs-unixfs-exporter'
+import { CID } from 'multiformats/cid'
+import type { TouchOptions } from '../index.js'
+import mergeOpts from 'merge-options'
+import { logger } from '@libp2p/logger'
+import { UnixFS } from 'ipfs-unixfs'
+import { pipe } from 'it-pipe'
+import { InvalidPBNodeError, NotUnixFSError, UnknownError } from './utils/errors.js'
+import * as dagPB from '@ipld/dag-pb'
+import type { PBNode, PBLink } from '@ipld/dag-pb'
+import { importer } from 'ipfs-unixfs-importer'
+import { persist } from './utils/persist.js'
+import type { Blockstore } from 'interface-blockstore'
+import last from 'it-last'
+import { sha256 } from 'multiformats/hashes/sha2'
+import { resolve, updatePathCids } from './utils/resolve.js'
+import * as raw from 'multiformats/codecs/raw'
+import { SHARD_SPLIT_THRESHOLD_BYTES } from './utils/constants.js'
+
+const mergeOptions = mergeOpts.bind({ ignoreUndefined: true })
+const log = logger('helia:unixfs:touch')
+
+const defaultOptions: TouchOptions = {
+ recursive: false,
+ shardSplitThresholdBytes: SHARD_SPLIT_THRESHOLD_BYTES
+}
+
+export async function touch (cid: CID, blockstore: Blockstore, options: Partial = {}): Promise {
+ const opts: TouchOptions = mergeOptions(defaultOptions, options)
+ const resolved = await resolve(cid, opts.path, blockstore, opts)
+ const mtime = opts.mtime ?? {
+ secs: BigInt(Math.round(Date.now() / 1000)),
+ nsecs: 0
+ }
+
+ log('touch %c %o', resolved.cid, mtime)
+
+ if (opts.recursive) {
+ // recursively export from root CID, change perms of each entry then reimport
+ // but do not reimport files, only manipulate dag-pb nodes
+ const root = await pipe(
+ async function * () {
+ for await (const entry of recursive(resolved.cid, blockstore)) {
+ let metadata: UnixFS
+ let links: PBLink[]
+
+ if (entry.type === 'raw') {
+ metadata = new UnixFS({ data: entry.node })
+ links = []
+ } else if (entry.type === 'file' || entry.type === 'directory') {
+ metadata = entry.unixfs
+ links = entry.node.Links
+ } else {
+ throw new NotUnixFSError()
+ }
+
+ metadata.mtime = mtime
+
+ const node = {
+ Data: metadata.marshal(),
+ Links: links
+ }
+
+ yield {
+ path: entry.path,
+ content: node
+ }
+ }
+ },
+ // @ts-expect-error we account for the incompatible source type with our custom dag builder below
+ (source) => importer(source, blockstore, {
+ ...opts,
+ pin: false,
+ dagBuilder: async function * (source, block) {
+ for await (const entry of source) {
+ yield async function () {
+ // @ts-expect-error cannot derive type
+ const node: PBNode = entry.content
+
+ const buf = dagPB.encode(node)
+ const updatedCid = await persist(buf, block, {
+ ...opts,
+ cidVersion: cid.version
+ })
+
+ if (node.Data == null) {
+ throw new InvalidPBNodeError(`${updatedCid} had no data`)
+ }
+
+ const unixfs = UnixFS.unmarshal(node.Data)
+
+ return {
+ cid: updatedCid,
+ size: buf.length,
+ path: entry.path,
+ unixfs
+ }
+ }
+ }
+ }
+ }),
+ async (nodes) => await last(nodes)
+ )
+
+ if (root == null) {
+ throw new UnknownError(`Could not chmod ${resolved.cid.toString()}`)
+ }
+
+ return await updatePathCids(root.cid, resolved, blockstore, opts)
+ }
+
+ const block = await blockstore.get(resolved.cid)
+ let metadata: UnixFS
+ let links: PBLink[] = []
+
+ if (resolved.cid.code === raw.code) {
+ metadata = new UnixFS({ data: block })
+ } else {
+ const node = dagPB.decode(block)
+ links = node.Links
+
+ if (node.Data == null) {
+ throw new InvalidPBNodeError(`${resolved.cid.toString()} had no data`)
+ }
+
+ metadata = UnixFS.unmarshal(node.Data)
+ }
+
+ metadata.mtime = mtime
+ const updatedBlock = dagPB.encode({
+ Data: metadata.marshal(),
+ Links: links
+ })
+
+ const hash = await sha256.digest(updatedBlock)
+ const updatedCid = CID.create(resolved.cid.version, dagPB.code, hash)
+
+ await blockstore.put(updatedCid, updatedBlock)
+
+ return await updatePathCids(updatedCid, resolved, blockstore, opts)
+}
diff --git a/src/commands/utils/add-link.ts b/src/commands/utils/add-link.ts
new file mode 100644
index 0000000..b8f1510
--- /dev/null
+++ b/src/commands/utils/add-link.ts
@@ -0,0 +1,320 @@
+import * as dagPB from '@ipld/dag-pb'
+import { CID, Version } from 'multiformats/cid'
+import { logger } from '@libp2p/logger'
+import { UnixFS } from 'ipfs-unixfs'
+import { DirSharded } from './dir-sharded.js'
+import {
+ updateHamtDirectory,
+ recreateHamtLevel,
+ recreateInitialHamtLevel,
+ createShard,
+ toPrefix,
+ addLinksToHamtBucket
+} from './hamt-utils.js'
+import last from 'it-last'
+import type { PBNode, PBLink } from '@ipld/dag-pb/interface'
+import { sha256 } from 'multiformats/hashes/sha2'
+import type { Bucket } from 'hamt-sharding'
+import { AlreadyExistsError, InvalidParametersError, InvalidPBNodeError } from './errors.js'
+import type { ImportResult } from 'ipfs-unixfs-importer'
+import type { AbortOptions } from '@libp2p/interfaces'
+import type { Directory } from './cid-to-directory.js'
+import type { Blockstore } from 'interface-blockstore'
+import { isOverShardThreshold } from './is-over-shard-threshold.js'
+
+const log = logger('helia:unixfs:components:utils:add-link')
+
+export interface AddLinkResult {
+ node: PBNode
+ cid: CID
+ size: number
+}
+
+export interface AddLinkOptions extends AbortOptions {
+ allowOverwriting: boolean
+ shardSplitThresholdBytes: number
+ cidVersion: Version
+}
+
+export async function addLink (parent: Directory, child: Required, blockstore: Blockstore, options: AddLinkOptions): Promise {
+ if (parent.node.Data == null) {
+ throw new InvalidParametersError('Invalid parent passed to addLink')
+ }
+
+ const meta = UnixFS.unmarshal(parent.node.Data)
+
+ if (meta.type === 'hamt-sharded-directory') {
+ log('adding link to sharded directory')
+
+ return await addToShardedDirectory(parent, child, blockstore, options)
+ }
+
+ log(`adding ${child.Name} (${child.Hash}) to regular directory`)
+
+ const result = await addToDirectory(parent, child, blockstore, options)
+
+ if (await isOverShardThreshold(result.node, blockstore, options.shardSplitThresholdBytes)) {
+ log('converting directory to sharded directory')
+
+ const converted = await convertToShardedDirectory(result, blockstore)
+ result.cid = converted.cid
+ result.node = dagPB.decode(await blockstore.get(converted.cid))
+ }
+
+ return result
+}
+
+const convertToShardedDirectory = async (parent: Directory, blockstore: Blockstore): Promise => {
+ if (parent.node.Data == null) {
+ throw new InvalidParametersError('Invalid parent passed to convertToShardedDirectory')
+ }
+
+ const unixfs = UnixFS.unmarshal(parent.node.Data)
+
+ const result = await createShard(blockstore, parent.node.Links.map(link => ({
+ name: (link.Name ?? ''),
+ size: BigInt(link.Tsize ?? 0),
+ cid: link.Hash
+ })), {
+ mode: unixfs.mode,
+ mtime: unixfs.mtime,
+ cidVersion: parent.cid.version
+ })
+
+ log(`Converted directory to sharded directory ${result.cid}`)
+
+ return result
+}
+
+const addToDirectory = async (parent: Directory, child: PBLink, blockstore: Blockstore, options: AddLinkOptions): Promise => {
+ // Remove existing link if it exists
+ const parentLinks = parent.node.Links.filter((link) => {
+ const matches = link.Name === child.Name
+
+ if (matches && !options.allowOverwriting) {
+ throw new AlreadyExistsError()
+ }
+
+ return !matches
+ })
+ parentLinks.push(child)
+
+ if (parent.node.Data == null) {
+ throw new InvalidPBNodeError('Parent node with no data passed to addToDirectory')
+ }
+
+ const node = UnixFS.unmarshal(parent.node.Data)
+
+ let data
+ if (node.mtime != null) {
+ // Update mtime if previously set
+ const ms = Date.now()
+ const secs = Math.floor(ms / 1000)
+
+ node.mtime = {
+ secs: BigInt(secs),
+ nsecs: (ms - (secs * 1000)) * 1000
+ }
+
+ data = node.marshal()
+ } else {
+ data = parent.node.Data
+ }
+ parent.node = dagPB.prepare({
+ Data: data,
+ Links: parentLinks
+ })
+
+ // Persist the new parent PbNode
+ const buf = dagPB.encode(parent.node)
+ const hash = await sha256.digest(buf)
+ const cid = CID.create(parent.cid.version, dagPB.code, hash)
+
+ await blockstore.put(cid, buf)
+
+ return {
+ node: parent.node,
+ cid,
+ size: buf.length
+ }
+}
+
+const addToShardedDirectory = async (parent: Directory, child: Required, blockstore: Blockstore, options: AddLinkOptions): Promise => {
+ const {
+ shard, path
+ } = await addFileToShardedDirectory(parent, child, blockstore, options)
+ const result = await last(shard.flush(blockstore))
+
+ if (result == null) {
+ throw new Error('No result from flushing shard')
+ }
+
+ const block = await blockstore.get(result.cid)
+ const node = dagPB.decode(block)
+
+ // we have written out the shard, but only one sub-shard will have been written so replace it in the original shard
+ const parentLinks = parent.node.Links.filter((link) => {
+ return (link.Name ?? '').substring(0, 2) !== path[0].prefix
+ })
+
+ const newLink = node.Links
+ .find(link => (link.Name ?? '').substring(0, 2) === path[0].prefix)
+
+ if (newLink == null) {
+ throw new Error(`No link found with prefix ${path[0].prefix}`)
+ }
+
+ parentLinks.push(newLink)
+
+ return await updateHamtDirectory({
+ Data: parent.node.Data,
+ Links: parentLinks
+ }, blockstore, path[0].bucket, options)
+}
+
+const addFileToShardedDirectory = async (parent: Directory, child: Required, blockstore: Blockstore, options: AddLinkOptions): Promise<{ shard: DirSharded, path: BucketPath[] }> => {
+ if (parent.node.Data == null) {
+ throw new InvalidPBNodeError('Parent node with no data passed to addFileToShardedDirectory')
+ }
+
+ // start at the root bucket and descend, loading nodes as we go
+ const rootBucket = await recreateInitialHamtLevel(parent.node.Links)
+ const node = UnixFS.unmarshal(parent.node.Data)
+
+ const shard = new DirSharded({
+ root: true,
+ dir: true,
+ parent: undefined,
+ parentKey: undefined,
+ path: '',
+ dirty: true,
+ flat: false,
+ mode: node.mode
+ }, {
+ ...options,
+ cidVersion: parent.cid.version
+ })
+ shard._bucket = rootBucket
+
+ if (node.mtime != null) {
+ // update mtime if previously set
+ shard.mtime = {
+ secs: BigInt(Math.round(Date.now() / 1000))
+ }
+ }
+
+ // load subshards until the bucket & position no longer changes
+ const position = await rootBucket._findNewBucketAndPos(child.Name)
+ const path = toBucketPath(position)
+ path[0].node = parent.node
+ let index = 0
+
+ while (index < path.length) {
+ const segment = path[index]
+ index++
+ const node = segment.node
+
+ if (node == null) {
+ throw new Error('Segment had no node')
+ }
+
+ const link = node.Links
+ .find(link => (link.Name ?? '').substring(0, 2) === segment.prefix)
+
+ if (link == null) {
+ // prefix is new, file will be added to the current bucket
+ log(`Link ${segment.prefix}${child.Name} will be added`)
+ index = path.length
+
+ break
+ }
+
+ if (link.Name === `${segment.prefix}${child.Name}`) {
+ if (!options.allowOverwriting) {
+ throw new AlreadyExistsError()
+ }
+
+ // file already existed, file will be added to the current bucket
+ log(`Link ${segment.prefix}${child.Name} will be replaced`)
+ index = path.length
+
+ break
+ }
+
+ if ((link.Name ?? '').length > 2) {
+ // another file had the same prefix, will be replaced with a subshard
+ log(`Link ${link.Name} ${link.Hash} will be replaced with a subshard`)
+ index = path.length
+
+ break
+ }
+
+ // load sub-shard
+ log(`Found subshard ${segment.prefix}`)
+ const block = await blockstore.get(link.Hash)
+ const subShard = dagPB.decode(block)
+
+ // subshard hasn't been loaded, descend to the next level of the HAMT
+ if (path[index] == null) {
+ log(`Loaded new subshard ${segment.prefix}`)
+ await recreateHamtLevel(blockstore, subShard.Links, rootBucket, segment.bucket, parseInt(segment.prefix, 16), options)
+
+ const position = await rootBucket._findNewBucketAndPos(child.Name)
+
+ path.push({
+ bucket: position.bucket,
+ prefix: toPrefix(position.pos),
+ node: subShard
+ })
+
+ break
+ }
+
+ const nextSegment = path[index]
+
+ // add next levels worth of links to bucket
+ await addLinksToHamtBucket(blockstore, subShard.Links, nextSegment.bucket, rootBucket, options)
+
+ nextSegment.node = subShard
+ }
+
+ // finally add the new file into the shard
+ await shard._bucket.put(child.Name, {
+ size: BigInt(child.Tsize),
+ cid: child.Hash
+ })
+
+ return {
+ shard, path
+ }
+}
+
+export interface BucketPath {
+ bucket: Bucket
+ prefix: string
+ node?: PBNode
+}
+
+const toBucketPath = (position: { pos: number, bucket: Bucket }): BucketPath[] => {
+ const path = [{
+ bucket: position.bucket,
+ prefix: toPrefix(position.pos)
+ }]
+
+ let bucket = position.bucket._parent
+ let positionInBucket = position.bucket._posAtParent
+
+ while (bucket != null) {
+ path.push({
+ bucket,
+ prefix: toPrefix(positionInBucket)
+ })
+
+ positionInBucket = bucket._posAtParent
+ bucket = bucket._parent
+ }
+
+ path.reverse()
+
+ return path
+}
diff --git a/src/commands/utils/cid-to-directory.ts b/src/commands/utils/cid-to-directory.ts
new file mode 100644
index 0000000..8d01526
--- /dev/null
+++ b/src/commands/utils/cid-to-directory.ts
@@ -0,0 +1,23 @@
+import { exporter, ExporterOptions } from 'ipfs-unixfs-exporter'
+import type { CID } from 'multiformats/cid'
+import type { PBNode } from '@ipld/dag-pb'
+import type { Blockstore } from 'interface-blockstore'
+import { NotADirectoryError } from './errors.js'
+
+export interface Directory {
+ cid: CID
+ node: PBNode
+}
+
+export async function cidToDirectory (cid: CID, blockstore: Blockstore, options: ExporterOptions = {}): Promise {
+ const entry = await exporter(cid, blockstore, options)
+
+ if (entry.type !== 'directory') {
+ throw new NotADirectoryError(`${cid.toString()} was not a UnixFS directory`)
+ }
+
+ return {
+ cid,
+ node: entry.node
+ }
+}
diff --git a/src/commands/utils/cid-to-pblink.ts b/src/commands/utils/cid-to-pblink.ts
new file mode 100644
index 0000000..e819e92
--- /dev/null
+++ b/src/commands/utils/cid-to-pblink.ts
@@ -0,0 +1,26 @@
+import { exporter, ExporterOptions } from 'ipfs-unixfs-exporter'
+import type { CID } from 'multiformats/cid'
+import { NotUnixFSError } from './errors.js'
+import * as dagPb from '@ipld/dag-pb'
+import type { PBNode, PBLink } from '@ipld/dag-pb'
+import type { Blockstore } from 'interface-blockstore'
+
+export async function cidToPBLink (cid: CID, name: string, blockstore: Blockstore, options?: ExporterOptions): Promise> {
+ const sourceEntry = await exporter(cid, blockstore, options)
+
+ if (sourceEntry.type !== 'directory' && sourceEntry.type !== 'file' && sourceEntry.type !== 'raw') {
+ throw new NotUnixFSError(`${cid.toString()} was not a UnixFS node`)
+ }
+
+ return {
+ Name: name,
+ Tsize: sourceEntry.node instanceof Uint8Array ? sourceEntry.node.byteLength : dagNodeTsize(sourceEntry.node),
+ Hash: cid
+ }
+}
+
+function dagNodeTsize (node: PBNode): number {
+ const linkSizes = node.Links.reduce((acc, curr) => acc + (curr.Tsize ?? 0), 0)
+
+ return dagPb.encode(node).byteLength + linkSizes
+}
diff --git a/src/commands/utils/constants.ts b/src/commands/utils/constants.ts
new file mode 100644
index 0000000..919f10d
--- /dev/null
+++ b/src/commands/utils/constants.ts
@@ -0,0 +1,2 @@
+
+export const SHARD_SPLIT_THRESHOLD_BYTES = 262144
diff --git a/src/commands/utils/dir-sharded.ts b/src/commands/utils/dir-sharded.ts
new file mode 100644
index 0000000..2f89677
--- /dev/null
+++ b/src/commands/utils/dir-sharded.ts
@@ -0,0 +1,318 @@
+import { encode, PBLink, prepare } from '@ipld/dag-pb'
+import { UnixFS } from 'ipfs-unixfs'
+import { persist, PersistOptions } from './persist.js'
+import { createHAMT, Bucket, BucketChild } from 'hamt-sharding'
+import {
+ hamtHashCode,
+ hamtHashFn
+} from './hamt-constants.js'
+import { CID } from 'multiformats/cid'
+import type { Mtime } from 'ipfs-unixfs'
+import type { Blockstore } from 'interface-blockstore'
+
+interface InProgressImportResult extends ImportResult {
+ single?: boolean
+ originalPath?: string
+}
+
+interface ImportResult {
+ cid: CID
+ size: bigint
+ path?: string
+ unixfs?: UnixFS
+}
+
+interface DirProps {
+ root: boolean
+ dir: boolean
+ path: string
+ dirty: boolean
+ flat: boolean
+ parent?: Dir
+ parentKey?: string
+ unixfs?: UnixFS
+ mode?: number
+ mtime?: Mtime
+}
+
+abstract class Dir {
+ public options: PersistOptions
+ public root: boolean
+ public dir: boolean
+ public path: string
+ public dirty: boolean
+ public flat: boolean
+ public parent?: Dir
+ public parentKey?: string
+ public unixfs?: UnixFS
+ public mode?: number
+ public mtime?: Mtime
+ public cid?: CID
+ public size?: number
+ public nodeSize?: number
+
+ constructor (props: DirProps, options: PersistOptions) {
+ this.options = options ?? {}
+
+ this.root = props.root
+ this.dir = props.dir
+ this.path = props.path
+ this.dirty = props.dirty
+ this.flat = props.flat
+ this.parent = props.parent
+ this.parentKey = props.parentKey
+ this.unixfs = props.unixfs
+ this.mode = props.mode
+ this.mtime = props.mtime
+ }
+
+ abstract put (name: string, value: InProgressImportResult | Dir): Promise
+ abstract get (name: string): Promise
+ abstract eachChildSeries (): AsyncIterable<{ key: string, child: InProgressImportResult | Dir }>
+ abstract flush (blockstore: Blockstore): AsyncGenerator
+ abstract estimateNodeSize (): number
+ abstract childCount (): number
+}
+
+export class DirSharded extends Dir {
+ public _bucket: Bucket
+
+ constructor (props: DirProps, options: PersistOptions) {
+ super(props, options)
+
+ this._bucket = createHAMT({
+ hashFn: hamtHashFn,
+ bits: 8
+ })
+ }
+
+ async put (name: string, value: InProgressImportResult | Dir): Promise {
+ this.cid = undefined
+ this.size = undefined
+ this.nodeSize = undefined
+
+ await this._bucket.put(name, value)
+ }
+
+ async get (name: string): Promise {
+ return await this._bucket.get(name)
+ }
+
+ childCount (): number {
+ return this._bucket.leafCount()
+ }
+
+ directChildrenCount (): number {
+ return this._bucket.childrenCount()
+ }
+
+ onlyChild (): Bucket | BucketChild {
+ return this._bucket.onlyChild()
+ }
+
+ async * eachChildSeries (): AsyncGenerator<{ key: string, child: InProgressImportResult | Dir }> {
+ for await (const { key, value } of this._bucket.eachLeafSeries()) {
+ yield {
+ key,
+ child: value
+ }
+ }
+ }
+
+ estimateNodeSize (): number {
+ if (this.nodeSize !== undefined) {
+ return this.nodeSize
+ }
+
+ this.nodeSize = calculateSize(this._bucket, this, this.options)
+
+ return this.nodeSize
+ }
+
+ async * flush (blockstore: Blockstore): AsyncGenerator {
+ for await (const entry of flush(this._bucket, blockstore, this, this.options)) {
+ yield {
+ ...entry,
+ path: this.path
+ }
+ }
+ }
+}
+
+async function * flush (bucket: Bucket, blockstore: Blockstore, shardRoot: DirSharded | null, options: PersistOptions): AsyncIterable {
+ const children = bucket._children
+ const links: PBLink[] = []
+ let childrenSize = 0n
+
+ for (let i = 0; i < children.length; i++) {
+ const child = children.get(i)
+
+ if (child == null) {
+ continue
+ }
+
+ const labelPrefix = i.toString(16).toUpperCase().padStart(2, '0')
+
+ if (child instanceof Bucket) {
+ let shard
+
+ for await (const subShard of flush(child, blockstore, null, options)) {
+ shard = subShard
+ }
+
+ if (shard == null) {
+ throw new Error('Could not flush sharded directory, no subshard found')
+ }
+
+ links.push({
+ Name: labelPrefix,
+ Tsize: Number(shard.size),
+ Hash: shard.cid
+ })
+ childrenSize += shard.size
+ } else if (isDir(child.value)) {
+ const dir = child.value
+ let flushedDir: ImportResult | undefined
+
+ for await (const entry of dir.flush(blockstore)) {
+ flushedDir = entry
+
+ yield flushedDir
+ }
+
+ if (flushedDir == null) {
+ throw new Error('Did not flush dir')
+ }
+
+ const label = labelPrefix + child.key
+ links.push({
+ Name: label,
+ Tsize: Number(flushedDir.size),
+ Hash: flushedDir.cid
+ })
+
+ childrenSize += flushedDir.size
+ } else {
+ const value = child.value
+
+ if (value.cid == null) {
+ continue
+ }
+
+ const label = labelPrefix + child.key
+ const size = value.size
+
+ links.push({
+ Name: label,
+ Tsize: Number(size),
+ Hash: value.cid
+ })
+ childrenSize += BigInt(size ?? 0)
+ }
+ }
+
+ // go-ipfs uses little endian, that's why we have to
+ // reverse the bit field before storing it
+ const data = Uint8Array.from(children.bitField().reverse())
+ const dir = new UnixFS({
+ type: 'hamt-sharded-directory',
+ data,
+ fanout: BigInt(bucket.tableSize()),
+ hashType: hamtHashCode,
+ mtime: shardRoot?.mtime,
+ mode: shardRoot?.mode
+ })
+
+ const node = {
+ Data: dir.marshal(),
+ Links: links
+ }
+ const buffer = encode(prepare(node))
+ const cid = await persist(buffer, blockstore, options)
+ const size = BigInt(buffer.byteLength) + childrenSize
+
+ yield {
+ cid,
+ unixfs: dir,
+ size
+ }
+}
+
+function isDir (obj: any): obj is Dir {
+ return typeof obj.flush === 'function'
+}
+
+function calculateSize (bucket: Bucket, shardRoot: DirSharded | null, options: PersistOptions): number {
+ const children = bucket._children
+ const links: PBLink[] = []
+
+ for (let i = 0; i < children.length; i++) {
+ const child = children.get(i)
+
+ if (child == null) {
+ continue
+ }
+
+ const labelPrefix = i.toString(16).toUpperCase().padStart(2, '0')
+
+ if (child instanceof Bucket) {
+ const size = calculateSize(child, null, options)
+
+ links.push({
+ Name: labelPrefix,
+ Tsize: Number(size),
+ Hash: options.cidVersion === 0 ? CID_V0 : CID_V1
+ })
+ } else if (typeof child.value.flush === 'function') {
+ const dir = child.value
+ const size = dir.nodeSize()
+
+ links.push({
+ Name: labelPrefix + child.key,
+ Tsize: Number(size),
+ Hash: options.cidVersion === 0 ? CID_V0 : CID_V1
+ })
+ } else {
+ const value = child.value
+
+ if (value.cid == null) {
+ continue
+ }
+
+ const label = labelPrefix + child.key
+ const size = value.size
+
+ links.push({
+ Name: label,
+ Tsize: Number(size),
+ Hash: value.cid
+ })
+ }
+ }
+
+ // go-ipfs uses little endian, that's why we have to
+ // reverse the bit field before storing it
+ const data = Uint8Array.from(children.bitField().reverse())
+ const dir = new UnixFS({
+ type: 'hamt-sharded-directory',
+ data,
+ fanout: BigInt(bucket.tableSize()),
+ hashType: hamtHashCode,
+ mtime: shardRoot?.mtime,
+ mode: shardRoot?.mode
+ })
+
+ const buffer = encode(prepare({
+ Data: dir.marshal(),
+ Links: links
+ }))
+
+ return buffer.length
+}
+
+// we use these to calculate the node size to use as a check for whether a directory
+// should be sharded or not. Since CIDs have a constant length and We're only
+// interested in the data length and not the actual content identifier we can use
+// any old CID instead of having to hash the data which is expensive.
+export const CID_V0 = CID.parse('QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn')
+export const CID_V1 = CID.parse('zdj7WbTaiJT1fgatdet9Ei9iDB5hdCxkbVyhyh8YTUnXMiwYi')
diff --git a/src/commands/utils/errors.ts b/src/commands/utils/errors.ts
new file mode 100644
index 0000000..22614c6
--- /dev/null
+++ b/src/commands/utils/errors.ts
@@ -0,0 +1,65 @@
+export abstract class UnixFSError extends Error {
+ public readonly name: string
+ public readonly code: string
+
+ constructor (message: string, name: string, code: string) {
+ super(message)
+
+ this.name = name
+ this.code = code
+ }
+}
+
+export class NotUnixFSError extends UnixFSError {
+ constructor (message = 'not a Unixfs node') {
+ super(message, 'NotUnixFSError', 'ERR_NOT_UNIXFS')
+ }
+}
+
+export class InvalidPBNodeError extends UnixFSError {
+ constructor (message = 'invalid PBNode') {
+ super(message, 'InvalidPBNodeError', 'ERR_INVALID_PBNODE')
+ }
+}
+
+export class UnknownError extends UnixFSError {
+ constructor (message = 'unknown error') {
+ super(message, 'InvalidPBNodeError', 'ERR_UNKNOWN_ERROR')
+ }
+}
+
+export class AlreadyExistsError extends UnixFSError {
+ constructor (message = 'path already exists') {
+ super(message, 'AlreadyExistsError', 'ERR_ALREADY_EXISTS')
+ }
+}
+
+export class DoesNotExistError extends UnixFSError {
+ constructor (message = 'path does not exist') {
+ super(message, 'DoesNotExistError', 'ERR_DOES_NOT_EXIST')
+ }
+}
+
+export class NoContentError extends UnixFSError {
+ constructor (message = 'no content') {
+ super(message, 'NoContentError', 'ERR_NO_CONTENT')
+ }
+}
+
+export class NotAFileError extends UnixFSError {
+ constructor (message = 'not a file') {
+ super(message, 'NotAFileError', 'ERR_NOT_A_FILE')
+ }
+}
+
+export class NotADirectoryError extends UnixFSError {
+ constructor (message = 'not a directory') {
+ super(message, 'NotADirectoryError', 'ERR_NOT_A_DIRECTORY')
+ }
+}
+
+export class InvalidParametersError extends UnixFSError {
+ constructor (message = 'invalid parameters') {
+ super(message, 'InvalidParametersError', 'ERR_INVALID_PARAMETERS')
+ }
+}
diff --git a/src/commands/utils/hamt-constants.ts b/src/commands/utils/hamt-constants.ts
new file mode 100644
index 0000000..66d9dcf
--- /dev/null
+++ b/src/commands/utils/hamt-constants.ts
@@ -0,0 +1,14 @@
+import { murmur3128 } from '@multiformats/murmur3'
+
+export const hamtHashCode = BigInt(murmur3128.code)
+export const hamtBucketBits = 8
+
+export async function hamtHashFn (buf: Uint8Array): Promise {
+ return (await murmur3128.encode(buf))
+ // Murmur3 outputs 128 bit but, accidentally, IPFS Go's
+ // implementation only uses the first 64, so we must do the same
+ // for parity..
+ .subarray(0, 8)
+ // Invert buffer because that's how Go impl does it
+ .reverse()
+}
diff --git a/src/commands/utils/hamt-utils.ts b/src/commands/utils/hamt-utils.ts
new file mode 100644
index 0000000..90147ba
--- /dev/null
+++ b/src/commands/utils/hamt-utils.ts
@@ -0,0 +1,294 @@
+import * as dagPB from '@ipld/dag-pb'
+import {
+ Bucket,
+ createHAMT
+} from 'hamt-sharding'
+import { DirSharded } from './dir-sharded.js'
+import { logger } from '@libp2p/logger'
+import { UnixFS } from 'ipfs-unixfs'
+import last from 'it-last'
+import type { CID, Version } from 'multiformats/cid'
+import {
+ hamtHashCode,
+ hamtHashFn,
+ hamtBucketBits
+} from './hamt-constants.js'
+import type { PBLink, PBNode } from '@ipld/dag-pb/interface'
+import type { Blockstore } from 'interface-blockstore'
+import type { Mtime } from 'ipfs-unixfs'
+import type { Directory } from './cid-to-directory.js'
+import type { AbortOptions } from '@libp2p/interfaces'
+import type { ImportResult } from 'ipfs-unixfs-importer'
+import { persist } from './persist.js'
+
+const log = logger('helia:unixfs:commands:utils:hamt-utils')
+
+export interface UpdateHamtResult {
+ node: PBNode
+ cid: CID
+ size: number
+}
+
+export interface UpdateHamtDirectoryOptions extends AbortOptions {
+ cidVersion: Version
+}
+
+export const updateHamtDirectory = async (pbNode: PBNode, blockstore: Blockstore, bucket: Bucket, options: UpdateHamtDirectoryOptions): Promise => {
+ if (pbNode.Data == null) {
+ throw new Error('Could not update HAMT directory because parent had no data')
+ }
+
+ // update parent with new bit field
+ const node = UnixFS.unmarshal(pbNode.Data)
+ const dir = new UnixFS({
+ type: 'hamt-sharded-directory',
+ data: Uint8Array.from(bucket._children.bitField().reverse()),
+ fanout: BigInt(bucket.tableSize()),
+ hashType: hamtHashCode,
+ mode: node.mode,
+ mtime: node.mtime
+ })
+
+ const updatedPbNode = {
+ Data: dir.marshal(),
+ Links: pbNode.Links
+ }
+
+ const buf = dagPB.encode(dagPB.prepare(updatedPbNode))
+ const cid = await persist(buf, blockstore, options)
+
+ return {
+ node: updatedPbNode,
+ cid,
+ size: pbNode.Links.reduce((sum, link) => sum + (link.Tsize ?? 0), buf.byteLength)
+ }
+}
+
+export const recreateHamtLevel = async (blockstore: Blockstore, links: PBLink[], rootBucket: Bucket, parentBucket: Bucket, positionAtParent: number, options: AbortOptions): Promise> => {
+ // recreate this level of the HAMT
+ const bucket = new Bucket({
+ hash: rootBucket._options.hash,
+ bits: rootBucket._options.bits
+ }, parentBucket, positionAtParent)
+ parentBucket._putObjectAt(positionAtParent, bucket)
+
+ await addLinksToHamtBucket(blockstore, links, bucket, rootBucket, options)
+
+ return bucket
+}
+
+export const recreateInitialHamtLevel = async (links: PBLink[]): Promise> => {
+ const bucket = createHAMT({
+ hashFn: hamtHashFn,
+ bits: hamtBucketBits
+ })
+
+ // populate sub bucket but do not recurse as we do not want to load the whole shard
+ await Promise.all(
+ links.map(async link => {
+ const linkName = (link.Name ?? '')
+
+ if (linkName.length === 2) {
+ const pos = parseInt(linkName, 16)
+ const subBucket = new Bucket({
+ hash: bucket._options.hash,
+ bits: bucket._options.bits
+ }, bucket, pos)
+
+ bucket._putObjectAt(pos, subBucket)
+ }
+
+ await bucket.put(linkName.substring(2), {
+ size: link.Tsize,
+ cid: link.Hash
+ })
+ })
+ )
+
+ return bucket
+}
+
+export const addLinksToHamtBucket = async (blockstore: Blockstore, links: PBLink[], bucket: Bucket, rootBucket: Bucket, options: AbortOptions): Promise => {
+ await Promise.all(
+ links.map(async link => {
+ const linkName = (link.Name ?? '')
+
+ if (linkName.length === 2) {
+ log('Populating sub bucket', linkName)
+ const pos = parseInt(linkName, 16)
+ const block = await blockstore.get(link.Hash, options)
+ const node = dagPB.decode(block)
+
+ const subBucket = new Bucket({
+ hash: rootBucket._options.hash,
+ bits: rootBucket._options.bits
+ }, bucket, pos)
+ bucket._putObjectAt(pos, subBucket)
+
+ await addLinksToHamtBucket(blockstore, node.Links, subBucket, rootBucket, options)
+ }
+
+ await rootBucket.put(linkName.substring(2), {
+ size: link.Tsize,
+ cid: link.Hash
+ })
+ })
+ )
+}
+
+export const toPrefix = (position: number): string => {
+ return position
+ .toString(16)
+ .toUpperCase()
+ .padStart(2, '0')
+ .substring(0, 2)
+}
+
+export interface HamtPathSegment {
+ bucket?: Bucket
+ prefix?: string
+ node?: PBNode
+ cid?: CID
+ size?: number
+}
+
+export const generatePath = async (root: Directory, name: string, blockstore: Blockstore, options: AbortOptions): Promise => {
+ // start at the root bucket and descend, loading nodes as we go
+ const rootBucket = await recreateInitialHamtLevel(root.node.Links)
+ const position = await rootBucket._findNewBucketAndPos(name)
+ const path: HamtPathSegment[] = [{
+ bucket: position.bucket,
+ prefix: toPrefix(position.pos)
+ }]
+ let currentBucket = position.bucket
+
+ while (currentBucket !== rootBucket) {
+ path.push({
+ bucket: currentBucket,
+ prefix: toPrefix(currentBucket._posAtParent)
+ })
+
+ if (currentBucket._parent == null) {
+ break
+ }
+
+ currentBucket = currentBucket._parent
+ }
+
+ // add the root bucket to the path
+ path.push({
+ bucket: rootBucket,
+ node: root.node
+ })
+
+ path.reverse()
+
+ // load PbNode for each path segment
+ for (let i = 1; i < path.length; i++) {
+ const segment = path[i]
+ const previousSegment = path[i - 1]
+
+ if (previousSegment.node == null) {
+ throw new Error('Could not generate HAMT path')
+ }
+
+ // find prefix in links
+ const link = previousSegment.node.Links
+ .filter(link => (link.Name ?? '').substring(0, 2) === segment.prefix)
+ .pop()
+
+ // entry was not in shard
+ if (link == null) {
+ // reached bottom of tree, file will be added to the current bucket
+ log(`Link ${segment.prefix}${name} will be added`)
+ // return path
+ continue
+ }
+
+ const linkName = link.Name ?? ''
+
+ // found entry
+ if (linkName === `${segment.prefix}${name}`) {
+ log(`Link ${segment.prefix}${name} will be replaced`)
+ // file already existed, file will be added to the current bucket
+ // return path
+ continue
+ }
+
+ // found subshard
+ log(`Found subshard ${segment.prefix}`)
+ const block = await blockstore.get(link.Hash)
+ const node = dagPB.decode(block)
+
+ // subshard hasn't been loaded, descend to the next level of the HAMT
+ if (path[i + 1] == null) {
+ log(`Loaded new subshard ${segment.prefix}`)
+
+ if (segment.bucket == null || segment.prefix == null) {
+ throw new Error('Shard was invalid')
+ }
+
+ await recreateHamtLevel(blockstore, node.Links, rootBucket, segment.bucket, parseInt(segment.prefix, 16), options)
+ const position = await rootBucket._findNewBucketAndPos(name)
+
+ // i--
+ path.push({
+ bucket: position.bucket,
+ prefix: toPrefix(position.pos),
+ node
+ })
+
+ continue
+ }
+
+ if (segment.bucket == null) {
+ throw new Error('Shard was invalid')
+ }
+
+ // add intermediate links to bucket
+ await addLinksToHamtBucket(blockstore, node.Links, segment.bucket, rootBucket, options)
+
+ segment.node = node
+ }
+
+ await rootBucket.put(name, true)
+
+ path.reverse()
+
+ return path
+}
+
+export interface CreateShardOptions {
+ mtime?: Mtime
+ mode?: number
+ cidVersion: Version
+}
+
+export const createShard = async (blockstore: Blockstore, contents: Array<{ name: string, size: bigint, cid: CID }>, options: CreateShardOptions): Promise => {
+ const shard = new DirSharded({
+ root: true,
+ dir: true,
+ parent: undefined,
+ parentKey: undefined,
+ path: '',
+ dirty: true,
+ flat: false,
+ mtime: options.mtime,
+ mode: options.mode
+ }, options)
+
+ for (let i = 0; i < contents.length; i++) {
+ await shard._bucket.put(contents[i].name, {
+ size: contents[i].size,
+ cid: contents[i].cid
+ })
+ }
+
+ const res = await last(shard.flush(blockstore))
+
+ if (res == null) {
+ throw new Error('Flushing shard yielded no result')
+ }
+
+ return res
+}
diff --git a/src/commands/utils/is-over-shard-threshold.ts b/src/commands/utils/is-over-shard-threshold.ts
new file mode 100644
index 0000000..9902eb9
--- /dev/null
+++ b/src/commands/utils/is-over-shard-threshold.ts
@@ -0,0 +1,78 @@
+import type { PBNode } from '@ipld/dag-pb'
+import type { Blockstore } from 'interface-blockstore'
+import { UnixFS } from 'ipfs-unixfs'
+import * as dagPb from '@ipld/dag-pb'
+import { CID_V0, CID_V1 } from './dir-sharded.js'
+
+/**
+ * Estimate node size only based on DAGLink name and CID byte lengths
+ * https://github.com/ipfs/go-unixfsnode/blob/37b47f1f917f1b2f54c207682f38886e49896ef9/data/builder/directory.go#L81-L96
+ *
+ * If the node is a hamt sharded directory the calculation is based on if it was a regular directory.
+ */
+export async function isOverShardThreshold (node: PBNode, blockstore: Blockstore, threshold: number): Promise {
+ if (node.Data == null) {
+ throw new Error('DagPB node had no data')
+ }
+
+ const unixfs = UnixFS.unmarshal(node.Data)
+ let size: number
+
+ if (unixfs.type === 'directory') {
+ size = estimateNodeSize(node)
+ } else if (unixfs.type === 'hamt-sharded-directory') {
+ size = await estimateShardSize(node, 0, threshold, blockstore)
+ } else {
+ throw new Error('Can only estimate the size of directories or shards')
+ }
+
+ return size > threshold
+}
+
+function estimateNodeSize (node: PBNode): number {
+ let size = 0
+
+ // estimate size only based on DAGLink name and CID byte lengths
+ // https://github.com/ipfs/go-unixfsnode/blob/37b47f1f917f1b2f54c207682f38886e49896ef9/data/builder/directory.go#L81-L96
+ for (const link of node.Links) {
+ size += (link.Name ?? '').length
+ size += link.Hash.version === 1 ? CID_V1.bytes.byteLength : CID_V0.bytes.byteLength
+ }
+
+ return size
+}
+
+async function estimateShardSize (node: PBNode, current: number, max: number, blockstore: Blockstore): Promise {
+ if (current > max) {
+ return max
+ }
+
+ if (node.Data == null) {
+ return current
+ }
+
+ const unixfs = UnixFS.unmarshal(node.Data)
+
+ if (!unixfs.isDirectory()) {
+ return current
+ }
+
+ for (const link of node.Links) {
+ let name = link.Name ?? ''
+
+ // remove hamt hash prefix from name
+ name = name.substring(2)
+
+ current += name.length
+ current += link.Hash.bytes.byteLength
+
+ if (link.Hash.code === dagPb.code) {
+ const block = await blockstore.get(link.Hash)
+ const node = dagPb.decode(block)
+
+ current += await estimateShardSize(node, current, max, blockstore)
+ }
+ }
+
+ return current
+}
diff --git a/src/commands/utils/persist.ts b/src/commands/utils/persist.ts
new file mode 100644
index 0000000..15f1275
--- /dev/null
+++ b/src/commands/utils/persist.ts
@@ -0,0 +1,27 @@
+import { CID } from 'multiformats/cid'
+import * as dagPb from '@ipld/dag-pb'
+import { sha256 } from 'multiformats/hashes/sha2'
+import type { Blockstore } from 'interface-blockstore'
+import type { BlockCodec } from 'multiformats/codecs/interface'
+import type { Version as CIDVersion } from 'multiformats/cid'
+
+export interface PersistOptions {
+ codec?: BlockCodec
+ cidVersion: CIDVersion
+ signal?: AbortSignal
+}
+
+export const persist = async (buffer: Uint8Array, blockstore: Blockstore, options: PersistOptions): Promise => {
+ if (options.codec == null) {
+ options.codec = dagPb
+ }
+
+ const multihash = await sha256.digest(buffer)
+ const cid = CID.create(options.cidVersion, options.codec.code, multihash)
+
+ await blockstore.put(cid, buffer, {
+ signal: options.signal
+ })
+
+ return cid
+}
diff --git a/src/commands/utils/remove-link.ts b/src/commands/utils/remove-link.ts
new file mode 100644
index 0000000..e9765d3
--- /dev/null
+++ b/src/commands/utils/remove-link.ts
@@ -0,0 +1,252 @@
+
+import * as dagPB from '@ipld/dag-pb'
+import type { CID, Version } from 'multiformats/cid'
+import { logger } from '@libp2p/logger'
+import { UnixFS } from 'ipfs-unixfs'
+import {
+ generatePath,
+ HamtPathSegment,
+ updateHamtDirectory,
+ UpdateHamtDirectoryOptions
+} from './hamt-utils.js'
+import type { PBNode } from '@ipld/dag-pb'
+import type { Blockstore } from 'interface-blockstore'
+import type { Directory } from './cid-to-directory.js'
+import type { AbortOptions } from '@libp2p/interfaces'
+import { InvalidParametersError, InvalidPBNodeError } from './errors.js'
+import { exporter } from 'ipfs-unixfs-exporter'
+import { persist } from './persist.js'
+import { isOverShardThreshold } from './is-over-shard-threshold.js'
+
+const log = logger('helia:unixfs:utils:remove-link')
+
+export interface RmLinkOptions extends AbortOptions {
+ shardSplitThresholdBytes: number
+ cidVersion: Version
+}
+
+export interface RemoveLinkResult {
+ node: PBNode
+ cid: CID
+}
+
+export async function removeLink (parent: Directory, name: string, blockstore: Blockstore, options: RmLinkOptions): Promise {
+ if (parent.node.Data == null) {
+ throw new InvalidPBNodeError('Parent node had no data')
+ }
+
+ const meta = UnixFS.unmarshal(parent.node.Data)
+
+ if (meta.type === 'hamt-sharded-directory') {
+ log(`removing ${name} from sharded directory`)
+
+ const result = await removeFromShardedDirectory(parent, name, blockstore, options)
+
+ if (!(await isOverShardThreshold(result.node, blockstore, options.shardSplitThresholdBytes))) {
+ log('converting shard to flat directory %c', parent.cid)
+
+ return await convertToFlatDirectory(result, blockstore, options)
+ }
+
+ return result
+ }
+
+ log(`removing link ${name} regular directory`)
+
+ return await removeFromDirectory(parent, name, blockstore, options)
+}
+
+const removeFromDirectory = async (parent: Directory, name: string, blockstore: Blockstore, options: AbortOptions): Promise => {
+ // Remove existing link if it exists
+ parent.node.Links = parent.node.Links.filter((link) => {
+ return link.Name !== name
+ })
+
+ const parentBlock = dagPB.encode(parent.node)
+ const parentCid = await persist(parentBlock, blockstore, {
+ ...options,
+ cidVersion: parent.cid.version
+ })
+
+ log(`Updated regular directory ${parentCid}`)
+
+ return {
+ node: parent.node,
+ cid: parentCid
+ }
+}
+
+const removeFromShardedDirectory = async (parent: Directory, name: string, blockstore: Blockstore, options: UpdateHamtDirectoryOptions): Promise<{ cid: CID, node: PBNode }> => {
+ const path = await generatePath(parent, name, blockstore, options)
+
+ // remove file from root bucket
+ const rootBucket = path[path.length - 1].bucket
+
+ if (rootBucket == null) {
+ throw new Error('Could not generate HAMT path')
+ }
+
+ await rootBucket.del(name)
+
+ // update all nodes in the shard path
+ return await updateShard(path, name, blockstore, options)
+}
+
+/**
+ * The `path` param is a list of HAMT path segments starting with th
+ */
+const updateShard = async (path: HamtPathSegment[], name: string, blockstore: Blockstore, options: UpdateHamtDirectoryOptions): Promise<{ node: PBNode, cid: CID }> => {
+ const fileName = `${path[0].prefix}${name}`
+
+ // skip first path segment as it is the file to remove
+ for (let i = 1; i < path.length; i++) {
+ const lastPrefix = path[i - 1].prefix
+ const segment = path[i]
+
+ if (segment.node == null) {
+ throw new InvalidParametersError('Path segment had no associated PBNode')
+ }
+
+ const link = segment.node.Links
+ .find(link => (link.Name ?? '').substring(0, 2) === lastPrefix)
+
+ if (link == null) {
+ throw new InvalidParametersError(`No link found with prefix ${lastPrefix} for file ${name}`)
+ }
+
+ if (link.Name == null) {
+ throw new InvalidParametersError(`${lastPrefix} link had no name`)
+ }
+
+ if (link.Name === fileName) {
+ log(`removing existing link ${link.Name}`)
+
+ const links = segment.node.Links.filter((nodeLink) => {
+ return nodeLink.Name !== link.Name
+ })
+
+ if (segment.bucket == null) {
+ throw new Error('Segment bucket was missing')
+ }
+
+ await segment.bucket.del(name)
+
+ const result = await updateHamtDirectory({
+ Data: segment.node.Data,
+ Links: links
+ }, blockstore, segment.bucket, options)
+
+ segment.node = result.node
+ segment.cid = result.cid
+ segment.size = result.size
+ }
+
+ if (link.Name === lastPrefix) {
+ log(`updating subshard with prefix ${lastPrefix}`)
+
+ const lastSegment = path[i - 1]
+
+ if (lastSegment.node?.Links.length === 1) {
+ log(`removing subshard for ${lastPrefix}`)
+
+ // convert subshard back to normal file entry
+ const link = lastSegment.node.Links[0]
+ link.Name = `${lastPrefix}${(link.Name ?? '').substring(2)}`
+
+ // remove existing prefix
+ segment.node.Links = segment.node.Links.filter((link) => {
+ return link.Name !== lastPrefix
+ })
+
+ // add new child
+ segment.node.Links.push(link)
+ } else {
+ // replace subshard entry
+ log(`replacing subshard for ${lastPrefix}`)
+
+ // remove existing prefix
+ segment.node.Links = segment.node.Links.filter((link) => {
+ return link.Name !== lastPrefix
+ })
+
+ if (lastSegment.cid == null) {
+ throw new Error('Did not persist previous segment')
+ }
+
+ // add new child
+ segment.node.Links.push({
+ Name: lastPrefix,
+ Hash: lastSegment.cid,
+ Tsize: lastSegment.size
+ })
+ }
+
+ if (segment.bucket == null) {
+ throw new Error('Segment bucket was missing')
+ }
+
+ const result = await updateHamtDirectory(segment.node, blockstore, segment.bucket, options)
+ segment.node = result.node
+ segment.cid = result.cid
+ segment.size = result.size
+ }
+ }
+
+ const rootSegment = path[path.length - 1]
+
+ if (rootSegment == null || rootSegment.cid == null || rootSegment.node == null) {
+ throw new InvalidParametersError('Failed to update shard')
+ }
+
+ return {
+ cid: rootSegment.cid,
+ node: rootSegment.node
+ }
+}
+
+const convertToFlatDirectory = async (parent: Directory, blockstore: Blockstore, options: RmLinkOptions): Promise => {
+ if (parent.node.Data == null) {
+ throw new InvalidParametersError('Invalid parent passed to convertToFlatDirectory')
+ }
+
+ const rootNode: PBNode = {
+ Links: []
+ }
+ const dir = await exporter(parent.cid, blockstore)
+
+ if (dir.type !== 'directory') {
+ throw new Error('Unexpected node type')
+ }
+
+ for await (const entry of dir.content()) {
+ let tsize = 0
+
+ if (entry.node instanceof Uint8Array) {
+ tsize = entry.node.byteLength
+ } else {
+ tsize = dagPB.encode(entry.node).length
+ }
+
+ rootNode.Links.push({
+ Hash: entry.cid,
+ Name: entry.name,
+ Tsize: tsize
+ })
+ }
+
+ // copy mode/mtime over if set
+ const oldUnixfs = UnixFS.unmarshal(parent.node.Data)
+ rootNode.Data = new UnixFS({ type: 'directory', mode: oldUnixfs.mode, mtime: oldUnixfs.mtime }).marshal()
+ const block = dagPB.encode(dagPB.prepare(rootNode))
+
+ const cid = await persist(block, blockstore, {
+ codec: dagPB,
+ cidVersion: parent.cid.version,
+ signal: options.signal
+ })
+
+ return {
+ cid,
+ node: rootNode
+ }
+}
diff --git a/src/commands/utils/resolve.ts b/src/commands/utils/resolve.ts
new file mode 100644
index 0000000..33530d8
--- /dev/null
+++ b/src/commands/utils/resolve.ts
@@ -0,0 +1,135 @@
+import type { CID } from 'multiformats/cid'
+import { exporter } from 'ipfs-unixfs-exporter'
+import type { AbortOptions } from '@libp2p/interfaces'
+import { logger } from '@libp2p/logger'
+import { DoesNotExistError, InvalidParametersError } from './errors.js'
+import { addLink } from './add-link.js'
+import { cidToDirectory } from './cid-to-directory.js'
+import { cidToPBLink } from './cid-to-pblink.js'
+import type { Blockstore } from 'interface-blockstore'
+
+const log = logger('helia:unixfs:components:utils:add-link')
+
+export interface Segment {
+ name: string
+ cid: CID
+ size: bigint
+}
+
+export interface ResolveResult {
+ /**
+ * The CID at the end of the path
+ */
+ cid: CID
+
+ path?: string
+
+ /**
+ * If present, these are the CIDs and path segments that were traversed through to reach the final CID
+ *
+ * If not present, there was no path passed or the path was an empty string
+ */
+ segments?: Segment[]
+}
+
+export async function resolve (cid: CID, path: string | undefined, blockstore: Blockstore, options: AbortOptions): Promise {
+ log('resolve "%s" under %c', path, cid)
+
+ if (path == null || path === '') {
+ return { cid }
+ }
+
+ const parts = path.split('/').filter(Boolean)
+ const segments: Segment[] = [{
+ name: '',
+ cid,
+ size: 0n
+ }]
+
+ for (let i = 0; i < parts.length; i++) {
+ const part = parts[i]
+ const result = await exporter(cid, blockstore, options)
+
+ if (result.type === 'file') {
+ if (i < parts.length - 1) {
+ throw new InvalidParametersError('Path was invalid')
+ }
+
+ cid = result.cid
+ } else if (result.type === 'directory') {
+ let dirCid: CID | undefined
+
+ for await (const entry of result.content()) {
+ if (entry.name === part) {
+ dirCid = entry.cid
+ }
+ }
+
+ if (dirCid == null) {
+ throw new DoesNotExistError('Could not find path in directory')
+ }
+
+ cid = dirCid
+
+ segments.push({
+ name: part,
+ cid,
+ size: result.size
+ })
+ } else {
+ throw new InvalidParametersError('Could not resolve path')
+ }
+ }
+
+ return {
+ cid,
+ path,
+ segments
+ }
+}
+
+export interface UpdatePathCidsOptions extends AbortOptions {
+ shardSplitThresholdBytes: number
+}
+
+/**
+ * Where we have descended into a DAG to update a child node, ascend up the DAG creating
+ * new hashes and blocks for the changed content
+ */
+export async function updatePathCids (cid: CID, result: ResolveResult, blockstore: Blockstore, options: UpdatePathCidsOptions): Promise {
+ if (result.segments == null || result.segments.length === 0) {
+ return cid
+ }
+
+ let child = result.segments.pop()
+
+ if (child == null) {
+ throw new Error('Insufficient segments')
+ }
+
+ child.cid = cid
+
+ result.segments.reverse()
+
+ for (const parent of result.segments) {
+ const [
+ directory,
+ pblink
+ ] = await Promise.all([
+ cidToDirectory(parent.cid, blockstore, options),
+ cidToPBLink(child.cid, child.name, blockstore, options)
+ ])
+
+ const result = await addLink(directory, pblink, blockstore, {
+ ...options,
+ allowOverwriting: true,
+ cidVersion: cid.version
+ })
+
+ cid = result.cid
+ parent.cid = cid
+ child = parent
+ }
+
+ return cid
+}
diff --git a/src/index.ts b/src/index.ts
new file mode 100644
index 0000000..a2c6d73
--- /dev/null
+++ b/src/index.ts
@@ -0,0 +1,172 @@
+import type { CID, Version } from 'multiformats/cid'
+import type { Blockstore } from 'interface-blockstore'
+import type { AbortOptions } from '@libp2p/interfaces'
+import { cat } from './commands/cat.js'
+import { mkdir } from './commands/mkdir.js'
+import type { Mtime } from 'ipfs-unixfs'
+import { cp } from './commands/cp.js'
+import { rm } from './commands/rm.js'
+import { stat } from './commands/stat.js'
+import { touch } from './commands/touch.js'
+import { chmod } from './commands/chmod.js'
+import type { UnixFSEntry } from 'ipfs-unixfs-exporter'
+import { ls } from './commands/ls.js'
+
+export interface UnixFSComponents {
+ blockstore: Blockstore
+}
+
+export interface CatOptions extends AbortOptions {
+ offset?: number
+ length?: number
+ path?: string
+}
+
+export interface ChmodOptions extends AbortOptions {
+ recursive: boolean
+ path?: string
+ shardSplitThresholdBytes: number
+}
+
+export interface CpOptions extends AbortOptions {
+ force: boolean
+ shardSplitThresholdBytes: number
+}
+
+export interface LsOptions extends AbortOptions {
+ path?: string
+ offset?: number
+ length?: number
+}
+
+export interface MkdirOptions extends AbortOptions {
+ cidVersion: Version
+ force: boolean
+ mode?: number
+ mtime?: Mtime
+ shardSplitThresholdBytes: number
+}
+
+export interface RmOptions extends AbortOptions {
+ shardSplitThresholdBytes: number
+}
+
+export interface StatOptions extends AbortOptions {
+ path?: string
+}
+
+export interface UnixFSStats {
+ /**
+ * The file or directory CID
+ */
+ cid: CID
+
+ /**
+ * The file or directory mode
+ */
+ mode?: number
+
+ /**
+ * The file or directory mtime
+ */
+ mtime?: Mtime
+
+ /**
+ * The size of the file in bytes
+ */
+ fileSize: bigint
+
+ /**
+ * The size of the DAG that holds the file in bytes
+ */
+ dagSize: bigint
+
+ /**
+ * How much of the file is in the local block store
+ */
+ localFileSize: bigint
+
+ /**
+ * How much of the DAG that holds the file is in the local blockstore
+ */
+ localDagSize: bigint
+
+ /**
+ * How many blocks make up the DAG - nb. this will only be accurate
+ * if all blocks are present in the local blockstore
+ */
+ blocks: number
+
+ /**
+ * The type of file
+ */
+ type: 'file' | 'directory' | 'raw'
+
+ /**
+ * UnixFS metadata about this file or directory. Will not be present
+ * if the node is a `raw` type.
+ */
+ unixfs?: import('ipfs-unixfs').UnixFS
+}
+
+export interface TouchOptions extends AbortOptions {
+ mtime?: Mtime
+ path?: string
+ recursive: boolean
+ shardSplitThresholdBytes: number
+}
+
+export interface UnixFS {
+ cat: (cid: CID, options?: Partial) => AsyncIterable
+ chmod: (source: CID, mode: number, options?: Partial) => Promise
+ cp: (source: CID, target: CID, name: string, options?: Partial) => Promise
+ ls: (cid: CID, options?: Partial) => AsyncIterable
+ mkdir: (cid: CID, dirname: string, options?: Partial) => Promise
+ rm: (cid: CID, path: string, options?: Partial) => Promise
+ stat: (cid: CID, options?: Partial) => Promise
+ touch: (cid: CID, options?: Partial) => Promise
+}
+
+class DefaultUnixFS implements UnixFS {
+ private readonly components: UnixFSComponents
+
+ constructor (components: UnixFSComponents) {
+ this.components = components
+ }
+
+ async * cat (cid: CID, options: Partial = {}): AsyncIterable {
+ yield * cat(cid, this.components.blockstore, options)
+ }
+
+ async chmod (source: CID, mode: number, options: Partial = {}): Promise {
+ return await chmod(source, mode, this.components.blockstore, options)
+ }
+
+ async cp (source: CID, target: CID, name: string, options: Partial = {}): Promise {
+ return await cp(source, target, name, this.components.blockstore, options)
+ }
+
+ async * ls (cid: CID, options: Partial = {}): AsyncIterable {
+ yield * ls(cid, this.components.blockstore, options)
+ }
+
+ async mkdir (cid: CID, dirname: string, options: Partial = {}): Promise {
+ return await mkdir(cid, dirname, this.components.blockstore, options)
+ }
+
+ async rm (cid: CID, path: string, options: Partial = {}): Promise {
+ return await rm(cid, path, this.components.blockstore, options)
+ }
+
+ async stat (cid: CID, options: Partial = {}): Promise {
+ return await stat(cid, this.components.blockstore, options)
+ }
+
+ async touch (cid: CID, options: Partial = {}): Promise {
+ return await touch(cid, this.components.blockstore, options)
+ }
+}
+
+export function unixfs (helia: { blockstore: Blockstore }): UnixFS {
+ return new DefaultUnixFS(helia)
+}
diff --git a/test/cat.spec.ts b/test/cat.spec.ts
new file mode 100644
index 0000000..3f67001
--- /dev/null
+++ b/test/cat.spec.ts
@@ -0,0 +1,87 @@
+/* eslint-env mocha */
+
+import { expect } from 'aegir/chai'
+import type { CID } from 'multiformats/cid'
+import type { Blockstore } from 'interface-blockstore'
+import { unixfs, UnixFS } from '../src/index.js'
+import { MemoryBlockstore } from 'blockstore-core'
+import toBuffer from 'it-to-buffer'
+import drain from 'it-drain'
+import { importContent, importBytes } from 'ipfs-unixfs-importer'
+import { createShardedDirectory } from './fixtures/create-sharded-directory.js'
+
+const smallFile = Uint8Array.from(new Array(13).fill(0).map(() => Math.random() * 100))
+
+describe('cat', () => {
+ let blockstore: Blockstore
+ let fs: UnixFS
+ let emptyDirCid: CID
+
+ beforeEach(async () => {
+ blockstore = new MemoryBlockstore()
+
+ fs = unixfs({ blockstore })
+
+ const imported = await importContent({ path: 'empty' }, blockstore)
+ emptyDirCid = imported.cid
+ })
+
+ it('reads a small file', async () => {
+ const { cid } = await importBytes(smallFile, blockstore)
+ const bytes = await toBuffer(fs.cat(cid))
+
+ expect(bytes).to.equalBytes(smallFile)
+ })
+
+ it('reads a file with an offset', async () => {
+ const offset = 10
+ const { cid } = await importBytes(smallFile, blockstore)
+ const bytes = await toBuffer(fs.cat(cid, {
+ offset
+ }))
+
+ expect(bytes).to.equalBytes(smallFile.subarray(offset))
+ })
+
+ it('reads a file with a length', async () => {
+ const length = 10
+ const { cid } = await importBytes(smallFile, blockstore)
+ const bytes = await toBuffer(fs.cat(cid, {
+ length
+ }))
+
+ expect(bytes).to.equalBytes(smallFile.subarray(0, length))
+ })
+
+ it('reads a file with an offset and a length', async () => {
+ const offset = 2
+ const length = 5
+ const { cid } = await importBytes(smallFile, blockstore)
+ const bytes = await toBuffer(fs.cat(cid, {
+ offset,
+ length
+ }))
+
+ expect(bytes).to.equalBytes(smallFile.subarray(offset, offset + length))
+ })
+
+ it('refuses to read a directory', async () => {
+ await expect(drain(fs.cat(emptyDirCid))).to.eventually.be.rejected
+ .with.property('code', 'ERR_NOT_A_FILE')
+ })
+
+ it('reads file from inside a sharded directory', async () => {
+ const content = Uint8Array.from([0, 1, 2, 3, 4])
+ const dirCid = await createShardedDirectory(blockstore)
+ const { cid: fileCid } = await importBytes(content, blockstore)
+ const path = 'new-file.txt'
+
+ const updatedCid = await fs.cp(fileCid, dirCid, path)
+
+ const bytes = await toBuffer(fs.cat(updatedCid, {
+ path
+ }))
+
+ expect(bytes).to.deep.equal(content)
+ })
+})
diff --git a/test/chmod.spec.ts b/test/chmod.spec.ts
new file mode 100644
index 0000000..e9ce3cf
--- /dev/null
+++ b/test/chmod.spec.ts
@@ -0,0 +1,95 @@
+/* eslint-env mocha */
+
+import { expect } from 'aegir/chai'
+import type { Blockstore } from 'interface-blockstore'
+import { MemoryBlockstore } from 'blockstore-core'
+import { UnixFS, unixfs } from '../src/index.js'
+import type { CID } from 'multiformats/cid'
+import { importContent, importBytes } from 'ipfs-unixfs-importer'
+import { createShardedDirectory } from './fixtures/create-sharded-directory.js'
+
+const smallFile = Uint8Array.from(new Array(13).fill(0).map(() => Math.random() * 100))
+
+describe('chmod', () => {
+ let blockstore: Blockstore
+ let fs: UnixFS
+ let emptyDirCid: CID
+
+ beforeEach(async () => {
+ blockstore = new MemoryBlockstore()
+
+ fs = unixfs({ blockstore })
+
+ const imported = await importContent({ path: 'empty' }, blockstore)
+ emptyDirCid = imported.cid
+ })
+
+ it('should update the mode for a raw node', async () => {
+ const { cid } = await importBytes(smallFile, blockstore)
+ const originalMode = (await fs.stat(cid)).mode
+ const updatedCid = await fs.chmod(cid, 0o777)
+
+ const updatedMode = (await fs.stat(updatedCid)).mode
+ expect(updatedMode).to.not.equal(originalMode)
+ expect(updatedMode).to.equal(0o777)
+ })
+
+ it('should update the mode for a file', async () => {
+ const { cid } = await importBytes(smallFile, blockstore, {
+ rawLeaves: false
+ })
+ const originalMode = (await fs.stat(cid)).mode
+ const updatedCid = await fs.chmod(cid, 0o777)
+
+ const updatedMode = (await fs.stat(updatedCid)).mode
+ expect(updatedMode).to.not.equal(originalMode)
+ expect(updatedMode).to.equal(0o777)
+ })
+
+ it('should update the mode for a directory', async () => {
+ const path = `foo-${Math.random()}`
+
+ const dirCid = await fs.mkdir(emptyDirCid, path)
+ const originalMode = (await fs.stat(dirCid, {
+ path
+ })).mode
+ const updatedCid = await fs.chmod(dirCid, 0o777, {
+ path
+ })
+
+ const updatedMode = (await fs.stat(updatedCid, {
+ path
+ })).mode
+ expect(updatedMode).to.not.equal(originalMode)
+ expect(updatedMode).to.equal(0o777)
+ })
+
+ it('should update mode recursively', async () => {
+ const path = 'path'
+ const { cid } = await importBytes(smallFile, blockstore)
+ const dirCid = await fs.cp(cid, emptyDirCid, path)
+ const originalMode = (await fs.stat(dirCid, {
+ path
+ })).mode
+ const updatedCid = await fs.chmod(dirCid, 0o777, {
+ recursive: true
+ })
+
+ const updatedMode = (await fs.stat(updatedCid, {
+ path
+ })).mode
+ expect(updatedMode).to.not.equal(originalMode)
+ expect(updatedMode).to.equal(0o777)
+ })
+
+ it('should update the mode for a hamt-sharded-directory', async () => {
+ const shardedDirCid = await createShardedDirectory(blockstore)
+
+ const originalMode = (await fs.stat(shardedDirCid)).mode
+ const updatedShardCid = await fs.chmod(shardedDirCid, 0o777)
+
+ const updatedMode = (await fs.stat(updatedShardCid)).mode
+ expect(updatedMode).to.not.equal(originalMode)
+ expect(updatedMode).to.equal(0o777)
+ })
+})
diff --git a/test/cp.spec.ts b/test/cp.spec.ts
new file mode 100644
index 0000000..b849325
--- /dev/null
+++ b/test/cp.spec.ts
@@ -0,0 +1,169 @@
+/* eslint-env mocha */
+
+import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string'
+import { expect } from 'aegir/chai'
+import { identity } from 'multiformats/hashes/identity'
+import { CID } from 'multiformats/cid'
+import type { Blockstore } from 'interface-blockstore'
+import { unixfs, UnixFS } from '../src/index.js'
+import { MemoryBlockstore } from 'blockstore-core'
+import toBuffer from 'it-to-buffer'
+import { importContent, importBytes } from 'ipfs-unixfs-importer'
+import { createShardedDirectory } from './fixtures/create-sharded-directory.js'
+import first from 'it-first'
+
+describe('cp', () => {
+ let blockstore: Blockstore
+ let fs: UnixFS
+ let emptyDirCid: CID
+
+ beforeEach(async () => {
+ blockstore = new MemoryBlockstore()
+
+ fs = unixfs({ blockstore })
+
+ const imported = await importContent({ path: 'empty' }, blockstore)
+ emptyDirCid = imported.cid
+ })
+
+ it('refuses to copy files without a source', async () => {
+ // @ts-expect-error invalid args
+ await expect(fs.cp()).to.eventually.be.rejected.with('Please supply at least one source')
+ })
+
+ it('refuses to copy files without a source, even with options', async () => {
+ // @ts-expect-error invalid args
+ await expect(fs.cp({})).to.eventually.be.rejected.with('Please supply at least one source')
+ })
+
+ it('refuses to copy files without a destination', async () => {
+ // @ts-expect-error invalid args
+ await expect(fs.cp('/source')).to.eventually.be.rejected.with('Please supply at least one source')
+ })
+
+ it('refuses to copy files without a destination, even with options', async () => {
+ // @ts-expect-error invalid args
+ await expect(fs.cp('/source', {})).to.eventually.be.rejected.with('Please supply at least one source')
+ })
+
+ it('refuses to copy files to an unreadable node', async () => {
+ const hash = identity.digest(uint8ArrayFromString('derp'))
+ const { cid: source } = await importBytes(Uint8Array.from([0, 1, 3, 4]), blockstore)
+ const target = CID.createV1(identity.code, hash)
+
+ await expect(fs.cp(source, target, 'foo')).to.eventually.be.rejected
+ .with.property('code', 'ERR_NOT_A_DIRECTORY')
+ })
+
+ it('refuses to copy files from an unreadable node', async () => {
+ const hash = identity.digest(uint8ArrayFromString('derp'))
+ const source = CID.createV1(identity.code, hash)
+
+ await expect(fs.cp(source, emptyDirCid, 'foo')).to.eventually.be.rejected
+ .with.property('code', 'ERR_NOT_UNIXFS')
+ })
+
+ it('refuses to copy files to an existing file', async () => {
+ const path = 'path'
+ const { cid: source } = await importBytes(Uint8Array.from([0, 1, 3, 4]), blockstore)
+ const target = await fs.cp(source, emptyDirCid, path)
+
+ await expect(fs.cp(source, target, path)).to.eventually.be.rejected
+ .with.property('code', 'ERR_ALREADY_EXISTS')
+
+ // should succeed with force option
+ await expect(fs.cp(source, target, path, {
+ force: true
+ })).to.eventually.be.ok()
+ })
+
+ it('copies a file to new location', async () => {
+ const data = Uint8Array.from([0, 1, 3, 4])
+ const path = 'path'
+ const { cid: source } = await importBytes(data, blockstore)
+ const dirCid = await fs.cp(source, emptyDirCid, path)
+
+ const bytes = await toBuffer(fs.cat(dirCid, {
+ path
+ }))
+
+ expect(bytes).to.deep.equal(data)
+ })
+
+ it('copies directories', async () => {
+ const path = 'path'
+ const dirCid = await fs.cp(emptyDirCid, emptyDirCid, path)
+
+ await expect(fs.stat(dirCid, {
+ path
+ })).to.eventually.include({
+ type: 'directory'
+ })
+ })
+
+ it('copies a sharded directory to a normal directory', async () => {
+ const shardedDirCid = await createShardedDirectory(blockstore)
+ const path = 'sharded-dir'
+ const containingDirCid = await fs.cp(shardedDirCid, emptyDirCid, path)
+
+ // should still be a regular directory
+ await expect(fs.stat(containingDirCid)).to.eventually.have.nested.property('unixfs.type', 'directory')
+
+ const subDirStats = await fs.stat(containingDirCid, {
+ path
+ })
+ expect(subDirStats).to.have.nested.property('unixfs.type', 'hamt-sharded-directory')
+ expect(subDirStats.cid).to.eql(shardedDirCid)
+ })
+
+ it('copies a normal directory to a sharded directory', async () => {
+ const shardedDirCid = await createShardedDirectory(blockstore)
+ const path = 'normal-dir'
+ const containingDirCid = await fs.cp(emptyDirCid, shardedDirCid, path)
+
+ // should still be a sharded directory
+ await expect(fs.stat(containingDirCid)).to.eventually.have.nested.property('unixfs.type', 'hamt-sharded-directory')
+
+ const subDirStats = await fs.stat(containingDirCid, {
+ path
+ })
+ expect(subDirStats).to.have.nested.property('unixfs.type', 'directory')
+ expect(subDirStats.cid).to.eql(emptyDirCid)
+ })
+
+ it('copies a file from a normal directory to a sharded directory', async () => {
+ const shardedDirCid = await createShardedDirectory(blockstore)
+ const path = `file-${Math.random()}.txt`
+ const { cid: fileCid } = await importBytes(Uint8Array.from([0, 1, 2, 3]), blockstore, {
+ rawLeaves: false
+ })
+
+ const containingDirCid = await fs.cp(fileCid, shardedDirCid, path)
+
+ // should still be a sharded directory
+ await expect(fs.stat(containingDirCid)).to.eventually.have.nested.property('unixfs.type', 'hamt-sharded-directory')
+
+ const fileInDirStats = await fs.stat(containingDirCid, {
+ path
+ })
+ expect(fileInDirStats).to.have.nested.property('unixfs.type', 'file')
+ expect(fileInDirStats.cid).to.eql(fileCid)
+ })
+
+ it('refuses to copy files to an existing file in a sharded directory', async () => {
+ const shardedDirCid = await createShardedDirectory(blockstore)
+ const file = await first(fs.ls(shardedDirCid))
+
+ if (file == null) {
+ throw new Error('No files listed')
+ }
+
+ await expect(fs.cp(file.cid, shardedDirCid, file.name)).to.eventually.be.rejected
+ .with.property('code', 'ERR_ALREADY_EXISTS')
+
+ // should succeed with force option
+ await expect(fs.cp(file.cid, shardedDirCid, file.name, {
+ force: true
+ })).to.eventually.be.ok()
+ })
+})
diff --git a/test/fixtures/create-sharded-directory.ts b/test/fixtures/create-sharded-directory.ts
new file mode 100644
index 0000000..ae5f5e4
--- /dev/null
+++ b/test/fixtures/create-sharded-directory.ts
@@ -0,0 +1,27 @@
+import { expect } from 'aegir/chai'
+import last from 'it-last'
+import { importer } from 'ipfs-unixfs-importer'
+import type { CID } from 'multiformats/cid'
+import type { Blockstore } from 'interface-blockstore'
+
+export async function createShardedDirectory (blockstore: Blockstore, files = 1001): Promise {
+ const result = await last(importer((function * () {
+ for (let i = 0; i < files; i++) {
+ yield {
+ path: `./file-${i}`,
+ content: Uint8Array.from([0, 1, 2, 3, 4, 5, i])
+ }
+ }
+ }()), blockstore, {
+ shardSplitThresholdBytes: 1,
+ wrapWithDirectory: true
+ }))
+
+ if (result == null) {
+ throw new Error('No result received from ipfs.addAll')
+ }
+
+ expect(result).to.have.nested.property('unixfs.type', 'hamt-sharded-directory', 'tried to create a shared directory but the result was not a shard')
+
+ return result.cid
+}
diff --git a/test/fixtures/create-subsharded-directory.ts b/test/fixtures/create-subsharded-directory.ts
new file mode 100644
index 0000000..64bcd96
--- /dev/null
+++ b/test/fixtures/create-subsharded-directory.ts
@@ -0,0 +1,93 @@
+import type { Blockstore } from 'interface-blockstore'
+import { importBytes, importer } from 'ipfs-unixfs-importer'
+import { CID } from 'multiformats/cid'
+import { unixfs } from '../../src/index.js'
+import * as dagPb from '@ipld/dag-pb'
+import last from 'it-last'
+
+export async function createSubshardedDirectory (blockstore: Blockstore, depth: number = 1, files: number = 5000): Promise<{
+ importerCid: CID
+ containingDirCid: CID
+ fileName: string
+}> {
+ const fs = unixfs({ blockstore })
+
+ const { cid: fileCid } = await importBytes(Uint8Array.from([0, 1, 2, 3, 4]), blockstore)
+ let containingDirCid = CID.parse('bafybeiczsscdsbs7ffqz55asqdf3smv6klcw3gofszvwlyarci47bgf354')
+ let fileName: string | undefined
+ let count = 0
+
+ for (let i = 0; i < files; i++) {
+ fileName = `file-${i}-${new Array(512).fill('0').join('')}.txt`
+
+ containingDirCid = await fs.cp(fileCid, containingDirCid, fileName, {
+ shardSplitThresholdBytes: 1
+ })
+
+ if (await searchCIDForSubshards(containingDirCid, blockstore, depth)) {
+ count = i
+
+ break
+ }
+ }
+
+ if (fileName == null) {
+ throw new Error('could not find file that would create a subshard')
+ }
+
+ // create a shard with the importer that is the same as the directory after we delete the file that causes a sub-shard to be created
+ const importResult = await last(importer(
+ new Array(count).fill(0).map((_, i) => {
+ return {
+ path: `file-${i}-${new Array(512).fill('0').join('')}.txt`,
+ content: Uint8Array.from([0, 1, 2, 3, 4])
+ }
+ }), blockstore, {
+ wrapWithDirectory: true,
+ shardSplitThresholdBytes: 1
+ }))
+
+ if (importResult == null) {
+ throw new Error('Nothing imported')
+ }
+
+ const { cid: importerCid } = importResult
+
+ return {
+ importerCid,
+ containingDirCid,
+ fileName
+ }
+}
+
+async function searchCIDForSubshards (cid: CID, blockstore: Blockstore, depth: number = 1): Promise {
+ const block = await blockstore.get(cid)
+ const node = dagPb.decode(block)
+
+ // search links for subshard
+ for (const link of node.Links) {
+ if (link.Name?.length === 2) {
+ const block = await blockstore.get(link.Hash)
+ const node = dagPb.decode(block)
+ const firstLink = node.Links[1]
+
+ if (firstLink == null) {
+ throw new Error('Subshard had no child links')
+ }
+
+ if (firstLink.Name == null) {
+ throw new Error('Subshard child had no name')
+ }
+
+ if (depth === 1) {
+ return true
+ }
+
+ if (await searchCIDForSubshards(link.Hash, blockstore, depth - 1)) {
+ return true
+ }
+ }
+ }
+
+ return false
+}
diff --git a/test/fixtures/print-tree.ts b/test/fixtures/print-tree.ts
new file mode 100644
index 0000000..e2cb75a
--- /dev/null
+++ b/test/fixtures/print-tree.ts
@@ -0,0 +1,33 @@
+import type { Blockstore } from 'interface-blockstore'
+import { UnixFS } from 'ipfs-unixfs'
+import type { CID } from 'multiformats/cid'
+import * as dagPb from '@ipld/dag-pb'
+import * as raw from 'multiformats/codecs/raw'
+
+export async function printTree (cid: CID, blockstore: Blockstore, name: string = '', indent: string = ''): Promise {
+ const block = await blockstore.get(cid)
+
+ if (cid.code === dagPb.code) {
+ const node = dagPb.decode(block)
+
+ if (node.Data == null) {
+ return
+ }
+
+ const unixfs = UnixFS.unmarshal(node.Data)
+
+ console.info(indent, cid.toString(), name, unixfs.type) // eslint-disable-line no-console
+
+ for (const link of node.Links) {
+ let name = link.Name ?? ''
+
+ if (name.length > 12) {
+ name = name.substring(0, 12) + '...'
+ }
+
+ await printTree(link.Hash, blockstore, name, `${indent} `)
+ }
+ } else if (cid.code === raw.code) {
+ console.info(indent, cid.toString(), name, 'dag-raw') // eslint-disable-line no-console
+ }
+}
diff --git a/test/ls.spec.ts b/test/ls.spec.ts
new file mode 100644
index 0000000..6c316ae
--- /dev/null
+++ b/test/ls.spec.ts
@@ -0,0 +1,123 @@
+/* eslint-env mocha */
+
+import { expect } from 'aegir/chai'
+import type { CID } from 'multiformats/cid'
+import all from 'it-all'
+import type { Blockstore } from 'interface-blockstore'
+import { unixfs, UnixFS } from '../src/index.js'
+import { MemoryBlockstore } from 'blockstore-core'
+import { importContent, importBytes } from 'ipfs-unixfs-importer'
+import { createShardedDirectory } from './fixtures/create-sharded-directory.js'
+
+describe('ls', () => {
+ let blockstore: Blockstore
+ let fs: UnixFS
+ let emptyDirCid: CID
+
+ beforeEach(async () => {
+ blockstore = new MemoryBlockstore()
+
+ fs = unixfs({ blockstore })
+
+ const imported = await importContent({ path: 'empty' }, blockstore)
+ emptyDirCid = imported.cid
+ })
+
+ it('should require a path', async () => {
+ // @ts-expect-error invalid args
+ await expect(all(fs.ls())).to.eventually.be.rejected()
+ })
+
+ it('lists files in a directory', async () => {
+ const path = 'path'
+ const data = Uint8Array.from([0, 1, 2, 3])
+ const { cid: fileCid } = await importBytes(data, blockstore)
+ const dirCid = await fs.cp(fileCid, emptyDirCid, path)
+ const files = await all(fs.ls(dirCid))
+
+ expect(files).to.have.lengthOf(1).and.to.containSubset([{
+ cid: fileCid,
+ name: path,
+ size: BigInt(data.byteLength),
+ type: 'raw'
+ }])
+ })
+
+ it('lists a file', async () => {
+ const path = 'path'
+ const data = Uint8Array.from([0, 1, 2, 3])
+ const { cid: fileCid } = await importBytes(data, blockstore, {
+ rawLeaves: false
+ })
+ const dirCid = await fs.cp(fileCid, emptyDirCid, path)
+ const files = await all(fs.ls(dirCid, {
+ path
+ }))
+
+ expect(files).to.have.lengthOf(1).and.to.containSubset([{
+ cid: fileCid,
+ size: BigInt(data.byteLength),
+ type: 'file'
+ }])
+ })
+
+ it('lists a raw node', async () => {
+ const path = 'path'
+ const data = Uint8Array.from([0, 1, 2, 3])
+ const { cid: fileCid } = await importBytes(data, blockstore)
+ const dirCid = await fs.cp(fileCid, emptyDirCid, path)
+ const files = await all(fs.ls(dirCid, {
+ path
+ }))
+
+ expect(files).to.have.lengthOf(1).and.to.containSubset([{
+ cid: fileCid,
+ size: BigInt(data.byteLength),
+ type: 'raw'
+ }])
+ })
+
+ it('lists a sharded directory contents', async () => {
+ const fileCount = 1001
+ const shardedDirCid = await createShardedDirectory(blockstore, fileCount)
+ const files = await all(fs.ls(shardedDirCid))
+
+ expect(files.length).to.equal(fileCount)
+
+ files.forEach(file => {
+ // should be a file
+ expect(file.type).to.equal('raw')
+ })
+ })
+
+ it('lists a file inside a sharded directory directly', async () => {
+ const shardedDirCid = await createShardedDirectory(blockstore)
+ const files = await all(fs.ls(shardedDirCid))
+ const fileName = files[0].name
+
+ // should be able to ls new file directly
+ const directFiles = await all(fs.ls(shardedDirCid, {
+ path: fileName
+ }))
+
+ expect(directFiles.length).to.equal(1)
+ expect(directFiles.filter(file => file.name === fileName)).to.be.ok()
+ })
+
+ it('lists the contents of a directory inside a sharded directory', async () => {
+ const shardedDirCid = await createShardedDirectory(blockstore)
+ const dirName = `subdir-${Math.random()}`
+ const fileName = `small-file-${Math.random()}.txt`
+
+ const { cid: fileCid } = await importBytes(Uint8Array.from([0, 1, 2, 3]), blockstore)
+ const containingDirectoryCid = await fs.cp(fileCid, emptyDirCid, fileName)
+ const updatedShardCid = await fs.cp(containingDirectoryCid, shardedDirCid, dirName)
+
+ const files = await all(fs.ls(updatedShardCid, {
+ path: dirName
+ }))
+
+ expect(files.length).to.equal(1)
+ expect(files.filter(file => file.name === fileName)).to.be.ok()
+ })
+})
diff --git a/test/mkdir.spec.ts b/test/mkdir.spec.ts
new file mode 100644
index 0000000..eb94437
--- /dev/null
+++ b/test/mkdir.spec.ts
@@ -0,0 +1,123 @@
+/* eslint-env mocha */
+
+import { expect } from 'aegir/chai'
+import all from 'it-all'
+import type { Blockstore } from 'interface-blockstore'
+import { unixfs, UnixFS } from '../src/index.js'
+import { MemoryBlockstore } from 'blockstore-core'
+import type { CID } from 'multiformats/cid'
+import type { Mtime } from 'ipfs-unixfs'
+import { importContent } from 'ipfs-unixfs-importer'
+import { createShardedDirectory } from './fixtures/create-sharded-directory.js'
+
+describe('mkdir', () => {
+ let blockstore: Blockstore
+ let fs: UnixFS
+ let emptyDirCid: CID
+ let emptyDirCidV0: CID
+
+ beforeEach(async () => {
+ blockstore = new MemoryBlockstore()
+
+ fs = unixfs({ blockstore })
+
+ const imported = await importContent({ path: 'empty' }, blockstore)
+ emptyDirCid = imported.cid
+
+ const importedV0 = await importContent({ path: 'empty' }, blockstore, {
+ cidVersion: 0
+ })
+ emptyDirCidV0 = importedV0.cid
+ })
+
+ async function testMode (mode: number | undefined, expectedMode: number): Promise {
+ const path = 'sub-directory'
+ const dirCid = await fs.mkdir(emptyDirCid, path, {
+ mode
+ })
+
+ await expect(fs.stat(dirCid, {
+ path
+ })).to.eventually.have.property('mode', expectedMode)
+ }
+
+ async function testMtime (mtime: Mtime, expectedMtime: Mtime): Promise {
+ const path = 'sub-directory'
+ const dirCid = await fs.mkdir(emptyDirCid, path, {
+ mtime
+ })
+
+ await expect(fs.stat(dirCid, {
+ path
+ })).to.eventually.have.deep.property('mtime', expectedMtime)
+ }
+
+ it('requires a directory', async () => {
+ // @ts-expect-error not enough arguments
+ await expect(fs.mkdir(emptyDirCid)).to.eventually.be.rejected()
+ })
+
+ it('creates a directory', async () => {
+ const path = 'foo'
+ const dirCid = await fs.mkdir(emptyDirCid, path)
+
+ const stats = await fs.stat(dirCid)
+ expect(stats.type).to.equal('directory')
+
+ const files = await all(fs.ls(dirCid))
+
+ expect(files.length).to.equal(1)
+ expect(files).to.have.nested.property('[0].name', path)
+ })
+
+ it('refuses to create a directory that already exists', async () => {
+ const path = 'qux'
+ const dirCid = await fs.mkdir(emptyDirCid, path)
+
+ await expect(fs.mkdir(dirCid, path)).to.eventually.be.rejected()
+ .with.property('code', 'ERR_ALREADY_EXISTS')
+ })
+
+ it('creates a nested directory with a different CID version to the parent', async () => {
+ const subDirectory = 'sub-dir'
+
+ expect(emptyDirCidV0).to.have.property('version', 0)
+
+ const dirCid = await fs.mkdir(emptyDirCidV0, subDirectory, {
+ cidVersion: 1
+ })
+
+ await expect(fs.stat(dirCid)).to.eventually.have.nested.property('cid.version', 0)
+ await expect(fs.stat(dirCid, {
+ path: subDirectory
+ })).to.eventually.have.nested.property('cid.version', 1)
+ })
+
+ it('should make directory and have default mode', async function () {
+ await testMode(undefined, parseInt('0755', 8))
+ })
+
+ it('should make directory and specify mtime as { nsecs, secs }', async function () {
+ const mtime = {
+ secs: 5n,
+ nsecs: 0
+ }
+
+ await testMtime(mtime, mtime)
+ })
+
+ it('makes a directory inside a sharded directory', async () => {
+ const shardedDirCid = await createShardedDirectory(blockstore)
+ const dirName = `subdir-${Math.random()}`
+
+ const updatedShardCid = await fs.mkdir(shardedDirCid, dirName)
+
+ // should still be a sharded directory
+ await expect(fs.stat(updatedShardCid)).to.eventually.have.nested.property('unixfs.type', 'hamt-sharded-directory')
+
+ // subdir should be a regular directory
+ await expect(fs.stat(updatedShardCid, {
+ path: dirName
+ })).to.eventually.have.nested.property('unixfs.type', 'directory')
+ })
+})
diff --git a/test/rm.spec.ts b/test/rm.spec.ts
new file mode 100644
index 0000000..5b14c35
--- /dev/null
+++ b/test/rm.spec.ts
@@ -0,0 +1,219 @@
+/* eslint-env mocha */
+
+import { expect } from 'aegir/chai'
+import type { Blockstore } from 'interface-blockstore'
+import { unixfs, UnixFS } from '../src/index.js'
+import { MemoryBlockstore } from 'blockstore-core'
+import type { CID } from 'multiformats/cid'
+import { importContent, importBytes, importer } from 'ipfs-unixfs-importer'
+import { createShardedDirectory } from './fixtures/create-sharded-directory.js'
+import last from 'it-last'
+import { createSubshardedDirectory } from './fixtures/create-subsharded-directory.js'
+
+const smallFile = Uint8Array.from(new Array(13).fill(0).map(() => Math.random() * 100))
+
+describe('rm', () => {
+ let blockstore: Blockstore
+ let fs: UnixFS
+ let emptyDirCid: CID
+
+ beforeEach(async () => {
+ blockstore = new MemoryBlockstore()
+
+ fs = unixfs({ blockstore })
+
+ const imported = await importContent({ path: 'empty' }, blockstore)
+ emptyDirCid = imported.cid
+ })
+
+ it('refuses to remove files without arguments', async () => {
+ // @ts-expect-error invalid args
+ await expect(fs.rm()).to.eventually.be.rejected()
+ })
+
+ it('removes a file', async () => {
+ const path = 'foo'
+ const { cid: fileCid } = await importBytes(smallFile, blockstore)
+ const dirCid = await fs.cp(fileCid, emptyDirCid, path)
+ const updatedDirCid = await fs.rm(dirCid, path)
+
+ await expect(fs.stat(updatedDirCid, {
+ path
+ })).to.eventually.be.rejected
+ .with.property('code', 'ERR_DOES_NOT_EXIST')
+ })
+
+ it('removes a directory', async () => {
+ const path = 'foo'
+ const dirCid = await fs.cp(emptyDirCid, emptyDirCid, path)
+ const updatedDirCid = await fs.rm(dirCid, path)
+
+ await expect(fs.stat(updatedDirCid, {
+ path
+ })).to.eventually.be.rejected
+ .with.property('code', 'ERR_DOES_NOT_EXIST')
+ })
+
+ it('removes a sharded directory inside a normal directory', async () => {
+ const shardedDirCid = await createShardedDirectory(blockstore)
+ const dirName = `subdir-${Math.random()}`
+ const containingDirCid = await fs.cp(shardedDirCid, emptyDirCid, dirName)
+
+ await expect(fs.stat(containingDirCid)).to.eventually.have.nested.property('unixfs.type', 'directory')
+ await expect(fs.stat(containingDirCid, {
+ path: dirName
+ })).to.eventually.have.nested.property('unixfs.type', 'hamt-sharded-directory')
+
+ const updatedContainingDirCid = await fs.rm(containingDirCid, dirName)
+
+ await expect(fs.stat(updatedContainingDirCid, {
+ path: dirName
+ })).to.eventually.be.rejected
+ .with.property('code', 'ERR_DOES_NOT_EXIST')
+ })
+
+ it('removes a sharded directory inside a sharded directory', async () => {
+ const shardedDirCid = await createShardedDirectory(blockstore)
+ const otherShardedDirCid = await createShardedDirectory(blockstore)
+ const dirName = `subdir-${Math.random()}`
+ const containingDirCid = await fs.cp(shardedDirCid, otherShardedDirCid, dirName)
+
+ await expect(fs.stat(containingDirCid)).to.eventually.have.nested.property('unixfs.type', 'hamt-sharded-directory')
+ await expect(fs.stat(containingDirCid, {
+ path: dirName
+ })).to.eventually.have.nested.property('unixfs.type', 'hamt-sharded-directory')
+
+ const updatedContainingDirCid = await fs.rm(containingDirCid, dirName)
+
+ await expect(fs.stat(updatedContainingDirCid, {
+ path: dirName
+ })).to.eventually.be.rejected
+ .with.property('code', 'ERR_DOES_NOT_EXIST')
+ })
+
+ it('results in the same hash as a sharded directory created by the importer when removing a file and removing the file means the root node is below the shard threshold', async function () {
+ const shardSplitThresholdBytes = 55
+
+ // create a shard with the importer
+ const importResult = await last(importer([{
+ path: 'file-1.txt',
+ content: Uint8Array.from([0, 1, 2, 3, 4])
+ }], blockstore, {
+ wrapWithDirectory: true,
+ shardSplitThresholdBytes
+ }))
+
+ if (importResult == null) {
+ throw new Error('Nothing imported')
+ }
+
+ const { cid: importerCid } = importResult
+ await expect(fs.stat(importerCid)).to.eventually.have.nested.property('unixfs.type', 'directory')
+
+ // create the same shard with unixfs command
+ const { cid: fileCid } = await importBytes(Uint8Array.from([0, 1, 2, 3, 4]), blockstore)
+ let containingDirCid = await fs.cp(fileCid, emptyDirCid, 'file-1.txt', {
+ shardSplitThresholdBytes
+ })
+
+ await expect(fs.stat(containingDirCid)).to.eventually.have.nested.property('unixfs.type', 'directory')
+
+ containingDirCid = await fs.cp(fileCid, containingDirCid, 'file-2.txt', {
+ shardSplitThresholdBytes
+ })
+
+ await expect(fs.stat(containingDirCid)).to.eventually.have.nested.property('unixfs.type', 'hamt-sharded-directory')
+
+ containingDirCid = await fs.rm(containingDirCid, 'file-2.txt', {
+ shardSplitThresholdBytes
+ })
+
+ await expect(fs.stat(containingDirCid)).to.eventually.have.nested.property('unixfs.type', 'directory')
+
+ expect(containingDirCid).to.eql(importerCid)
+ })
+
+ it('results in the same hash as a sharded directory created by the importer when removing a file', async function () {
+ const shardSplitThresholdBytes = 1
+
+ // create a shard with the importer
+ const importResult = await last(importer([{
+ path: 'file-1.txt',
+ content: Uint8Array.from([0, 1, 2, 3, 4])
+ }], blockstore, {
+ wrapWithDirectory: true,
+ shardSplitThresholdBytes
+ }))
+
+ if (importResult == null) {
+ throw new Error('Nothing imported')
+ }
+
+ const { cid: importerCid } = importResult
+ await expect(fs.stat(importerCid)).to.eventually.have.nested.property('unixfs.type', 'hamt-sharded-directory')
+
+ // create the same shard with unixfs command
+ const { cid: fileCid } = await importBytes(Uint8Array.from([0, 1, 2, 3, 4]), blockstore)
+ let containingDirCid = await fs.cp(fileCid, emptyDirCid, 'file-1.txt', {
+ shardSplitThresholdBytes
+ })
+
+ await expect(fs.stat(containingDirCid)).to.eventually.have.nested.property('unixfs.type', 'hamt-sharded-directory')
+
+ containingDirCid = await fs.cp(fileCid, containingDirCid, 'file-2.txt', {
+ shardSplitThresholdBytes
+ })
+
+ await expect(fs.stat(containingDirCid)).to.eventually.have.nested.property('unixfs.type', 'hamt-sharded-directory')
+
+ containingDirCid = await fs.rm(containingDirCid, 'file-2.txt', {
+ shardSplitThresholdBytes
+ })
+
+ await expect(fs.stat(containingDirCid)).to.eventually.have.nested.property('unixfs.type', 'hamt-sharded-directory')
+
+ expect(containingDirCid).to.eql(importerCid)
+ })
+
+ it.skip('results in the same hash as a sharded directory created by the importer when removing a subshard', async function () {
+ let {
+ containingDirCid,
+ fileName,
+ importerCid
+ } = await createSubshardedDirectory(blockstore)
+
+ await expect(fs.stat(importerCid))
+ .to.eventually.have.nested.property('unixfs.type', 'hamt-sharded-directory')
+
+ // remove the file that caused the subshard to be created and the CID should be the same as the importer
+ containingDirCid = await fs.rm(containingDirCid, fileName, {
+ shardSplitThresholdBytes: 1
+ })
+
+ await expect(fs.stat(containingDirCid))
+ .to.eventually.have.nested.property('unixfs.type', 'hamt-sharded-directory')
+
+ expect(containingDirCid).to.eql(importerCid)
+ })
+
+ it.skip('results in the same hash as a sharded directory created by the importer when removing a subshard of a subshard', async function () {
+ let {
+ containingDirCid,
+ fileName,
+ importerCid
+ } = await createSubshardedDirectory(blockstore, 2)
+
+ await expect(fs.stat(importerCid))
+ .to.eventually.have.nested.property('unixfs.type', 'hamt-sharded-directory')
+
+ // remove the file that caused the subshard to be created and the CID should be the same as the importer
+ containingDirCid = await fs.rm(containingDirCid, fileName, {
+ shardSplitThresholdBytes: 1
+ })
+
+ await expect(fs.stat(containingDirCid))
+ .to.eventually.have.nested.property('unixfs.type', 'hamt-sharded-directory')
+
+ expect(containingDirCid).to.eql(importerCid)
+ })
+})
diff --git a/test/stat.spec.ts b/test/stat.spec.ts
new file mode 100644
index 0000000..041768b
--- /dev/null
+++ b/test/stat.spec.ts
@@ -0,0 +1,204 @@
+/* eslint-env mocha */
+
+import { expect } from 'aegir/chai'
+import type { Blockstore } from 'interface-blockstore'
+import { unixfs, UnixFS } from '../src/index.js'
+import { MemoryBlockstore } from 'blockstore-core'
+import type { CID } from 'multiformats/cid'
+import * as dagPb from '@ipld/dag-pb'
+import { importContent, importBytes } from 'ipfs-unixfs-importer'
+import { createShardedDirectory } from './fixtures/create-sharded-directory.js'
+
+const smallFile = Uint8Array.from(new Array(13).fill(0).map(() => Math.random() * 100))
+const largeFile = Uint8Array.from(new Array(490668).fill(0).map(() => Math.random() * 100))
+
+describe('stat', function () {
+ this.timeout(120 * 1000)
+
+ let blockstore: Blockstore
+ let fs: UnixFS
+ let emptyDirCid: CID
+
+ beforeEach(async () => {
+ blockstore = new MemoryBlockstore()
+
+ fs = unixfs({ blockstore })
+
+ const imported = await importContent({ path: 'empty' }, blockstore)
+ emptyDirCid = imported.cid
+ })
+
+ it('stats an empty directory', async () => {
+ await expect(fs.stat(emptyDirCid)).to.eventually.include({
+ fileSize: 0n,
+ dagSize: 2n,
+ blocks: 1,
+ type: 'directory'
+ })
+ })
+
+ it('computes how much of the DAG is local', async () => {
+ const { cid: largeFileCid } = await importBytes(largeFile, blockstore)
+ const block = await blockstore.get(largeFileCid)
+ const node = dagPb.decode(block)
+
+ expect(node.Links).to.have.lengthOf(2)
+
+ await expect(fs.stat(largeFileCid)).to.eventually.include({
+ fileSize: 490668n,
+ blocks: 3,
+ localDagSize: 490776n
+ })
+
+ // remove one of the blocks so we now have an incomplete DAG
+ await blockstore.delete(node.Links[0].Hash)
+
+ // block count and local file/dag sizes should be smaller
+ await expect(fs.stat(largeFileCid)).to.eventually.include({
+ fileSize: 490668n,
+ blocks: 2,
+ localFileSize: 228524n,
+ localDagSize: 228632n
+ })
+ })
+
+ it('stats a raw node', async () => {
+ const { cid: fileCid } = await importBytes(smallFile, blockstore)
+
+ await expect(fs.stat(fileCid)).to.eventually.include({
+ fileSize: BigInt(smallFile.length),
+ dagSize: 13n,
+ blocks: 1,
+ type: 'raw'
+ })
+ })
+
+ it('stats a small file', async () => {
+ const { cid: fileCid } = await importBytes(smallFile, blockstore, {
+ cidVersion: 0,
+ rawLeaves: false
+ })
+
+ await expect(fs.stat(fileCid)).to.eventually.include({
+ fileSize: BigInt(smallFile.length),
+ dagSize: 19n,
+ blocks: 1,
+ type: 'file'
+ })
+ })
+
+ it('stats a large file', async () => {
+ const { cid } = await importBytes(largeFile, blockstore)
+
+ await expect(fs.stat(cid)).to.eventually.include({
+ fileSize: BigInt(largeFile.length),
+ dagSize: 490682n,
+ blocks: 3,
+ type: 'file'
+ })
+ })
+
+ it('should stat file with mode', async () => {
+ const mode = 0o644
+ const { cid } = await importContent({
+ content: smallFile,
+ mode
+ }, blockstore)
+
+ await expect(fs.stat(cid)).to.eventually.include({
+ mode
+ })
+ })
+
+ it('should stat file with mtime', async function () {
+ const mtime = {
+ secs: 5n,
+ nsecs: 0
+ }
+ const { cid } = await importContent({
+ content: smallFile,
+ mtime
+ }, blockstore)
+
+ await expect(fs.stat(cid)).to.eventually.deep.include({
+ mtime
+ })
+ })
+
+ it('should stat a directory', async function () {
+ await expect(fs.stat(emptyDirCid)).to.eventually.include({
+ type: 'directory',
+ blocks: 1,
+ fileSize: 0n
+ })
+ })
+
+ it('should stat dir with mode', async function () {
+ const mode = 0o755
+ const path = 'test-dir'
+ const dirCid = await fs.mkdir(emptyDirCid, path, {
+ mode
+ })
+
+ await expect(fs.stat(dirCid, {
+ path
+ })).to.eventually.include({
+ mode
+ })
+ })
+
+ it('should stat dir with mtime', async function () {
+ const mtime = {
+ secs: 5n,
+ nsecs: 0
+ }
+
+ const path = 'test-dir'
+ const dirCid = await fs.mkdir(emptyDirCid, path, {
+ mtime
+ })
+
+ await expect(fs.stat(dirCid, {
+ path
+ })).to.eventually.deep.include({
+ mtime
+ })
+ })
+
+ it('sstats a sharded directory', async function () {
+ const mtime = {
+ secs: 5n,
+ nsecs: 0
+ }
+ const shardedDirCid = await createShardedDirectory(blockstore)
+ const updatedShardCid = await fs.touch(shardedDirCid, {
+ mtime
+ })
+
+ const stat = await fs.stat(updatedShardCid)
+ expect(stat).to.have.property('type', 'directory')
+ expect(stat).to.have.nested.property('unixfs.type', 'hamt-sharded-directory')
+ expect(stat).to.include({
+ mode: 0o755
+ })
+ expect(stat).to.deep.include({
+ mtime
+ })
+ })
+
+ it('stats a file inside a sharded directory', async () => {
+ const shardedDirCid = await createShardedDirectory(blockstore)
+ const { cid: fileCid } = await importBytes(Uint8Array.from([0, 1, 2, 3]), blockstore, {
+ rawLeaves: false
+ })
+ const fileName = `small-file-${Math.random()}.txt`
+ const updatedShardCid = await fs.cp(fileCid, shardedDirCid, fileName)
+
+ const stats = await fs.stat(updatedShardCid, {
+ path: fileName
+ })
+
+ expect(stats.type).to.equal('file')
+ expect(stats.fileSize).to.equal(4n)
+ })
+})
diff --git a/test/touch.spec.ts b/test/touch.spec.ts
new file mode 100644
index 0000000..52a75dc
--- /dev/null
+++ b/test/touch.spec.ts
@@ -0,0 +1,150 @@
+/* eslint-env mocha */
+
+import { expect } from 'aegir/chai'
+import type { Blockstore } from 'interface-blockstore'
+import { unixfs, UnixFS } from '../src/index.js'
+import { MemoryBlockstore } from 'blockstore-core'
+import type { CID } from 'multiformats/cid'
+import delay from 'delay'
+import { importContent, importBytes } from 'ipfs-unixfs-importer'
+import { createShardedDirectory } from './fixtures/create-sharded-directory.js'
+
+describe('.files.touch', () => {
+ let blockstore: Blockstore
+ let fs: UnixFS
+ let emptyDirCid: CID
+
+ beforeEach(async () => {
+ blockstore = new MemoryBlockstore()
+
+ fs = unixfs({ blockstore })
+
+ const imported = await importContent({ path: 'empty' }, blockstore)
+ emptyDirCid = imported.cid
+ })
+
+ it('should have default mtime', async () => {
+ const { cid } = await importBytes(Uint8Array.from([0, 1, 2, 3, 4]), blockstore)
+
+ await expect(fs.stat(cid)).to.eventually.have.property('mtime')
+ .that.is.undefined()
+
+ const updatedCid = await fs.touch(cid)
+
+ await expect(fs.stat(updatedCid)).to.eventually.have.property('mtime')
+ .that.is.not.undefined().and.does.not.deep.equal({
+ secs: 0,
+ nsecs: 0
+ })
+ })
+
+ it('should update file mtime', async function () {
+ this.slow(5 * 1000)
+ const mtime = new Date()
+ const seconds = BigInt(Math.floor(mtime.getTime() / 1000))
+
+ const { cid } = await importContent({
+ content: Uint8Array.from([0, 1, 2, 3, 4]),
+ mtime: {
+ secs: seconds
+ }
+ }, blockstore)
+
+ await delay(2000)
+ const updatedCid = await fs.touch(cid)
+
+ await expect(fs.stat(updatedCid)).to.eventually.have.nested.property('mtime.secs')
+ // no bigint support
+ .that.satisfies((s: bigint) => s > seconds)
+ })
+
+ it('should update directory mtime', async function () {
+ this.slow(5 * 1000)
+ const path = 'path'
+ const mtime = new Date()
+ const seconds = BigInt(Math.floor(mtime.getTime() / 1000))
+
+ const cid = await fs.mkdir(emptyDirCid, path, {
+ mtime: {
+ secs: seconds
+ }
+ })
+ await delay(2000)
+ const updateCid = await fs.touch(cid)
+
+ await expect(fs.stat(updateCid)).to.eventually.have.nested.property('mtime.secs')
+ // no bigint support
+ .that.satisfies((s: bigint) => s > seconds)
+ })
+
+ it('should update mtime recursively', async function () {
+ this.slow(5 * 1000)
+ const path = 'path'
+ const mtime = new Date()
+ const seconds = Math.floor(mtime.getTime() / 1000)
+
+ const { cid } = await importBytes(Uint8Array.from([0, 1, 2, 3, 4]), blockstore)
+ const dirCid = await fs.cp(cid, emptyDirCid, path)
+
+ await delay(2000)
+
+ const updatedCid = await fs.touch(dirCid, {
+ recursive: true
+ })
+
+ await expect(fs.stat(updatedCid)).to.eventually.have.nested.property('mtime.secs')
+ // no bigint support
+ .that.satisfies((s: bigint) => s > seconds)
+
+ await expect(fs.stat(updatedCid, {
+ path
+ })).to.eventually.have.nested.property('mtime.secs')
+ // no bigint support
+ .that.satisfies((s: bigint) => s > seconds)
+ })
+
+ it('should update the mtime for a hamt-sharded-directory', async function () {
+ this.slow(5 * 1000)
+ const shardedDirCid = await createShardedDirectory(blockstore)
+ let updatedShardCid = await fs.touch(shardedDirCid)
+ const originalMtime = (await fs.stat(updatedShardCid)).mtime
+
+ if (originalMtime == null) {
+ throw new Error('No originalMtime found')
+ }
+
+ await delay(2000)
+ updatedShardCid = await fs.touch(shardedDirCid)
+ const updatedMtime = (await fs.stat(updatedShardCid)).mtime
+
+ if (updatedMtime == null) {
+ throw new Error('No updatedMtime found')
+ }
+
+ // no bigint support
+ expect(updatedMtime.secs).to.satisfy((s: bigint) => s > originalMtime.secs)
+ })
+
+ it('should update mtime recursively for a hamt-sharded-directory', async function () {
+ this.slow(5 * 1000)
+ const mtime = new Date()
+ const seconds = Math.floor(mtime.getTime() / 1000)
+ const shardedDirCid = await createShardedDirectory(blockstore)
+
+ await delay(2000)
+
+ const updatedCid = await fs.touch(shardedDirCid, {
+ recursive: true
+ })
+
+ await expect(fs.stat(updatedCid)).to.eventually.have.nested.property('mtime.secs')
+ // no bigint support
+ .that.satisfies((s: bigint) => s > seconds)
+
+ for await (const file of fs.ls(updatedCid)) {
+ expect(file).to.have.nested.property('unixfs.mtime.secs')
+ // no bigint support
+ .that.satisfies((s: bigint) => s > seconds)
+ }
+ })
+})
diff --git a/tsconfig.json b/tsconfig.json
new file mode 100644
index 0000000..13a3599
--- /dev/null
+++ b/tsconfig.json
@@ -0,0 +1,10 @@
+{
+ "extends": "aegir/src/config/tsconfig.aegir.json",
+ "compilerOptions": {
+ "outDir": "dist"
+ },
+ "include": [
+ "src",
+ "test"
+ ]
+}