Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: use @ipld/dag-pb instead of ipld-dag-pb #116

Merged
merged 26 commits into from
Jul 9, 2021
Merged
Show file tree
Hide file tree
Changes from 21 commits
Commits
Show all changes
26 commits
Select commit Hold shift + click to select a range
2f24a1c
fix: use @ipld/dag-pb instead of ipld-dag-pb
vmx Feb 16, 2021
8779482
WIP: all in using js-multiformats for everying
vmx Feb 24, 2021
9663d14
feat: replace hashAlg option with actual hasher
vmx Mar 26, 2021
ce3d324
fixup with WIP commit: things almost work, make it fully work once js…
vmx Mar 26, 2021
e746040
chore: format builder.spec.js properly
vmx Mar 27, 2021
052eb3f
fix: fix typescript typechecking errors
vmx Mar 27, 2021
5e250c8
fix: more typescript type fixes
vmx Mar 29, 2021
566e5f9
fix: more typescript type fixes
vmx Mar 29, 2021
1f898d2
chore: update dependencies
vmx Apr 1, 2021
11d8d74
fix: use correct size of the node
vmx Apr 5, 2021
75ac9ff
chore: fix linter errors
vmx Apr 7, 2021
7b58b00
chore: remove/comment some ts-ignores
vmx Apr 7, 2021
2fd91c1
chore: use latest multiformats, dag-pb and dag-cbor w/ types
rvagg Apr 22, 2021
1145e02
chore: fix lint errors
rvagg Apr 22, 2021
6109622
chore: remove more @ts-ignores
rvagg Apr 23, 2021
3ffbb5b
chore: update travis env to focal
rvagg Apr 23, 2021
426cc8b
TEMPORARY dist/ directories for gitpkg
rvagg Apr 24, 2021
856f6de
chore: update deps
achingbrain May 4, 2021
e0c5e27
chore: remove dist dirs
achingbrain May 7, 2021
09bb43e
chore: ignore dist dirs
achingbrain May 7, 2021
63827a4
chore: update deps
achingbrain May 7, 2021
c553824
chore: swap ipfs-block-service for interface-blockstore and remove mu…
achingbrain Jun 23, 2021
9d20ba9
chore: record bundle size
achingbrain Jun 23, 2021
e443a5c
chore: import type not namespace
achingbrain Jun 23, 2021
a794648
chore: add missing deps
achingbrain Jun 24, 2021
350c9c1
chore: update bundle size
achingbrain Jul 9, 2021
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .travis.yml
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
language: node_js
cache: npm
dist: focal

branches:
only:
Expand Down
10 changes: 4 additions & 6 deletions packages/ipfs-unixfs-exporter/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -43,15 +43,10 @@
"events": "^3.3.0",
"ipfs-core-types": "^0.3.1",
"ipfs-unixfs-importer": "^7.0.3",
"ipld": "^0.29.0",
"ipld-block": "^0.11.1",
"ipld-dag-pb": "^0.22.2",
"ipld-in-memory": "^8.0.0",
"it-all": "^1.0.5",
"it-buffer-stream": "^2.0.0",
"it-first": "^1.0.6",
"merge-options": "^3.0.4",
"multicodec": "^3.0.1",
"native-abort-controller": "^1.0.3",
"nyc": "^15.0.0",
"readable-stream": "^3.6.0",
Expand All @@ -61,11 +56,14 @@
"util": "^0.12.3"
},
"dependencies": {
"cids": "^1.1.5",
"@ipld/dag-cbor": "^5.0.0",
"@ipld/dag-pb": "^1.1.0",
"err-code": "^3.0.1",
"hamt-sharding": "^2.0.0",
"ipfs-unixfs": "^4.0.3",
"it-last": "^1.0.5",
"multicodec": "^3.0.1",
"multiformats": "^8.0.3",
"multihashing-async": "^2.1.0"
},
"types": "dist/src/index.d.ts",
Expand Down
32 changes: 16 additions & 16 deletions packages/ipfs-unixfs-exporter/src/index.js
Original file line number Diff line number Diff line change
@@ -1,14 +1,13 @@
'use strict'

const errCode = require('err-code')
const CID = require('cids')
const { CID } = require('multiformats/cid')
const resolve = require('./resolvers')
const last = require('it-last')

/**
* @typedef {import('ipfs-unixfs').UnixFS} UnixFS
* @typedef {import('ipld-dag-pb').DAGNode} DAGNode
* @typedef {import('ipld')} IPLD
* @typedef {import('ipfs-unixfs-importer/src/types').BlockAPI} BlockAPI
* @typedef {import('./types').ExporterOptions} ExporterOptions
* @typedef {import('./types').UnixFSFile} UnixFSFile
* @typedef {import('./types').UnixFSDirectory} UnixFSDirectory
Expand All @@ -32,14 +31,15 @@ const toPathComponents = (path = '') => {
const cidAndRest = (path) => {
if (path instanceof Uint8Array) {
return {
cid: new CID(path),
cid: CID.decode(path),
toResolve: []
}
}

if (CID.isCID(path)) {
const cid = CID.asCID(path)
if (cid) {
return {
cid: path,
cid,
toResolve: []
}
}
Expand All @@ -52,7 +52,7 @@ const cidAndRest = (path) => {
const output = toPathComponents(path)

return {
cid: new CID(output[0]),
cid: CID.parse(output[0]),
toResolve: output.slice(1)
}
}
Expand All @@ -62,10 +62,10 @@ const cidAndRest = (path) => {

/**
* @param {string | CID} path
* @param {IPLD} ipld
* @param {BlockAPI} blockService
* @param {ExporterOptions} [options]
*/
async function * walkPath (path, ipld, options = {}) {
async function * walkPath (path, blockService, options = {}) {
let {
cid,
toResolve
Expand All @@ -75,7 +75,7 @@ async function * walkPath (path, ipld, options = {}) {
const startingDepth = toResolve.length

while (true) {
const result = await resolve(cid, name, entryPath, toResolve, startingDepth, ipld, options)
const result = await resolve(cid, name, entryPath, toResolve, startingDepth, blockService, options)

if (!result.entry && !result.next) {
throw errCode(new Error(`Could not resolve ${path}`), 'ERR_NOT_FOUND')
Expand All @@ -99,11 +99,11 @@ async function * walkPath (path, ipld, options = {}) {

/**
* @param {string | CID} path
* @param {IPLD} ipld
* @param {BlockAPI} blockService
* @param {ExporterOptions} [options]
*/
async function exporter (path, ipld, options = {}) {
const result = await last(walkPath(path, ipld, options))
async function exporter (path, blockService, options = {}) {
const result = await last(walkPath(path, blockService, options))

if (!result) {
throw errCode(new Error(`Could not resolve ${path}`), 'ERR_NOT_FOUND')
Expand All @@ -114,11 +114,11 @@ async function exporter (path, ipld, options = {}) {

/**
* @param {string | CID} path
* @param {IPLD} ipld
* @param {BlockAPI} blockService
* @param {ExporterOptions} [options]
*/
async function * recursive (path, ipld, options = {}) {
const node = await exporter(path, ipld, options)
async function * recursive (path, blockService, options = {}) {
const node = await exporter(path, blockService, options)

if (!node) {
return
Expand Down
18 changes: 10 additions & 8 deletions packages/ipfs-unixfs-exporter/src/resolvers/dag-cbor.js
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
'use strict'

const CID = require('cids')
const { CID } = require('multiformats/cid')
const errCode = require('err-code')
const dagCbor = require('@ipld/dag-cbor')

/**
* @typedef {import('../types').Resolver} Resolver
Expand All @@ -10,9 +11,9 @@ const errCode = require('err-code')
/**
* @type {Resolver}
*/
const resolve = async (cid, name, path, toResolve, resolve, depth, ipld, options) => {
const object = await ipld.get(cid, options)
const block = await ipld.get(new CID(1, 'raw', cid.multihash))
const resolve = async (cid, name, path, toResolve, resolve, depth, blockService, options) => {
const block = await blockService.get(cid)
const object = dagCbor.decode(block.bytes)
let subObject = object
let subPath = path

Expand All @@ -24,22 +25,23 @@ const resolve = async (cid, name, path, toResolve, resolve, depth, ipld, options
toResolve.shift()
subPath = `${subPath}/${prop}`

if (CID.isCID(subObject[prop])) {
const subObjectCid = CID.asCID(subObject[prop])
if (subObjectCid) {
return {
entry: {
type: 'object',
name,
path,
cid,
node: block,
node: block.bytes,
depth,
size: block.length,
content: async function * () {
yield object
}
},
next: {
cid: subObject[prop],
cid: subObjectCid,
name: prop,
path: subPath,
toResolve
Expand All @@ -60,7 +62,7 @@ const resolve = async (cid, name, path, toResolve, resolve, depth, ipld, options
name,
path,
cid,
node: block,
node: block.bytes,
depth,
size: block.length,
content: async function * () {
Expand Down
9 changes: 4 additions & 5 deletions packages/ipfs-unixfs-exporter/src/resolvers/identity.js
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
const errCode = require('err-code')
const extractDataFromBlock = require('../utils/extract-data-from-block')
const validateOffsetAndLength = require('../utils/validate-offset-and-length')
const mh = require('multihashing-async').multihash
const mh = require('multiformats/hashes/digest')

/**
* @typedef {import('../types').ExporterOptions} ExporterOptions
Expand Down Expand Up @@ -32,12 +32,11 @@ const rawContent = (node) => {
/**
* @type {Resolver}
*/
const resolve = async (cid, name, path, toResolve, resolve, depth, ipld, options) => {
const resolve = async (cid, name, path, toResolve, resolve, depth, blockService, options) => {
if (toResolve.length) {
throw errCode(new Error(`No link named ${path} found in raw node ${cid}`), 'ERR_NOT_FOUND')
}

const buf = await mh.decode(cid.multihash)
const buf = await mh.decode(cid.multihash.bytes)

return {
entry: {
Expand All @@ -47,7 +46,7 @@ const resolve = async (cid, name, path, toResolve, resolve, depth, ipld, options
cid,
content: rawContent(buf.digest),
depth,
size: buf.length,
size: buf.digest.length,
node: buf.digest
}
}
Expand Down
21 changes: 11 additions & 10 deletions packages/ipfs-unixfs-exporter/src/resolvers/index.js
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
'use strict'

const errCode = require('err-code')
const multicodec = require('multicodec')

/**
* @typedef {import('cids')} CID
* @typedef {import('ipld')} IPLD
* @typedef {import('../').BlockAPI} BlockAPI
* @typedef {import('../types').ExporterOptions} ExporterOptions
* @typedef {import('../types').UnixFSEntry} UnixFSEntry
* @typedef {import('../types').Resolver} Resolver
Expand All @@ -15,23 +15,24 @@ const errCode = require('err-code')
* @type {{ [ key: string ]: Resolver }}
*/
const resolvers = {
'dag-pb': require('./unixfs-v1'),
raw: require('./raw'),
'dag-cbor': require('./dag-cbor'),
identity: require('./identity')
[multicodec.DAG_PB]: require('./unixfs-v1'),
[multicodec.RAW]: require('./raw'),
[multicodec.DAG_CBOR]: require('./dag-cbor'),
[multicodec.IDENTITY]: require('./identity')
}

/**
* @type {Resolve}
*/
function resolve (cid, name, path, toResolve, depth, ipld, options) {
const resolver = resolvers[cid.codec]
function resolve (cid, name, path, toResolve, depth, blockService, options) {
const resolver = resolvers[cid.code]

if (!resolver) {
throw errCode(new Error(`No resolver for codec ${cid.codec}`), 'ERR_NO_RESOLVER')
// @ts-ignore - A `CodecCode` is expected, but a number is just fine
throw errCode(new Error(`No resolver for codec ${multicodec.getName(cid.code)}`), 'ERR_NO_RESOLVER')
}

return resolver(cid, name, path, toResolve, resolve, depth, ipld, options)
return resolver(cid, name, path, toResolve, resolve, depth, blockService, options)
}

module.exports = resolve
10 changes: 5 additions & 5 deletions packages/ipfs-unixfs-exporter/src/resolvers/raw.js
Original file line number Diff line number Diff line change
Expand Up @@ -30,23 +30,23 @@ const rawContent = (node) => {
/**
* @type {import('../types').Resolver}
*/
const resolve = async (cid, name, path, toResolve, resolve, depth, ipld, options) => {
const resolve = async (cid, name, path, toResolve, resolve, depth, blockService, options) => {
if (toResolve.length) {
throw errCode(new Error(`No link named ${path} found in raw node ${cid}`), 'ERR_NOT_FOUND')
}

const buf = await ipld.get(cid, options)
const block = await blockService.get(cid, options)

return {
entry: {
type: 'raw',
name,
path,
cid,
content: rawContent(buf),
content: rawContent(block.bytes),
depth,
size: buf.length,
node: buf
size: block.bytes.length,
node: block.bytes
}
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
/**
* @type {UnixfsV1Resolver}
*/
const directoryContent = (cid, node, unixfs, path, resolve, depth, ipld) => {
const directoryContent = (cid, node, unixfs, path, resolve, depth, blockService) => {
/**
* @param {ExporterOptions} [options]
* @returns {UnixfsV1DirectoryContent}
Expand All @@ -20,7 +20,7 @@ const directoryContent = (cid, node, unixfs, path, resolve, depth, ipld) => {
const links = node.Links.slice(offset, length)

for (const link of links) {
const result = await resolve(link.Hash, link.Name, `${path}/${link.Name}`, [], depth + 1, ipld, options)
const result = await resolve(link.Hash, link.Name || '', `${path}/${link.Name || ''}`, [], depth + 1, blockService, options)

if (result.entry) {
yield result.entry
Expand Down
Loading