Skip to content
This repository has been archived by the owner on Oct 1, 2021. It is now read-only.

chore: upgrade to new multiformats module #98

Merged
merged 14 commits into from
Jul 7, 2021
Merged

chore: upgrade to new multiformats module #98

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
f98d385
chore: upgrade to new multiformats module
achingbrain May 15, 2021
e22501b
chore: include globals for tests to run old level
achingbrain May 15, 2021
17248ac
chore: remove commented code
achingbrain May 17, 2021
155bc90
chore: update cid import style
achingbrain May 24, 2021
d4d09e7
chore: use exported CIDVersion instead of 0|1
achingbrain May 27, 2021
ef6c211
chore: take instances of datastores instead of controlling their life…
achingbrain Jun 29, 2021
3546fb9
chore: update deps
achingbrain Jun 29, 2021
6c38340
chore: fix up types
achingbrain Jun 30, 2021
9b5a552
chore: upgrade deps
achingbrain Jul 1, 2021
c9d01ea
chore: update deps
achingbrain Jul 6, 2021
a2a7a59
chore: faster without batch
achingbrain Jul 6, 2021
a4ce9c6
chore: ensure base58btc is always used
achingbrain Jul 6, 2021
90a0e78
chore: catch error when closing repo
achingbrain Jul 6, 2021
f402825
chore: revert base68btc
achingbrain Jul 6, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 2 additions & 12 deletions .aegir.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,17 +4,7 @@ const path = require('path')

const esbuild = {
// this will inject all the named exports from 'node-globals.js' as globals
inject: [path.join(__dirname, 'scripts/node-globals.js')],
plugins: [
{
name: 'node built ins', // this will make the bundler resolve node builtins to the respective browser polyfill
setup (build) {
build.onResolve({ filter: /^stream$/ }, () => {
return { path: require.resolve('readable-stream') }
})
}
}
]
inject: [path.join(__dirname, 'scripts/node-globals.js')]
}

/** @type {import('aegir').PartialOptions} */
Expand All @@ -27,6 +17,6 @@ module.exports = {
}
},
build: {
config: esbuild
bundlesizeMax: '37kB'
}
}
22 changes: 11 additions & 11 deletions migrations/migration-8/index.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,13 @@
'use strict'

const CID = require('cids')
const { CID } = require('multiformats')
achingbrain marked this conversation as resolved.
Show resolved Hide resolved
const Key = require('interface-datastore').Key
const mb = require('multibase')
const log = require('debug')('ipfs:repo:migrator:migration-8')
const uint8ArrayToString = require('uint8arrays/to-string')
const { createStore } = require('../../src/utils')
const length = require('it-length')
const { base32 } = require('multiformats/bases/base32')
const raw = require('multiformats/codecs/raw')
const mhd = require('multiformats/hashes/digest')

/**
* @typedef {import('../../src/types').Migration} Migration
Expand All @@ -16,16 +17,14 @@ const length = require('it-length')
* @param {Key} key
*/
function keyToMultihash (key) {
const buf = mb.decode(`b${key.toString().slice(1)}`)
const buf = base32.decode(`b${key.toString().toLowerCase().slice(1)}`)

// Extract multihash from CID
let multihash = new CID(buf).multihash
let multihash = CID.decode(buf).multihash.bytes

// Encode and slice off multibase codec
multihash = mb.encode('base32', multihash).slice(1)

// Should be uppercase for interop with go
const multihashStr = uint8ArrayToString(multihash).toUpperCase()
const multihashStr = base32.encode(multihash).slice(1).toUpperCase()

return new Key(`/${multihashStr}`, false)
}
Expand All @@ -34,12 +33,13 @@ function keyToMultihash (key) {
* @param {Key} key
*/
function keyToCid (key) {
const buf = mb.decode(`b${key.toString().slice(1)}`)
const buf = base32.decode(`b${key.toString().toLowerCase().slice(1)}`)
const digest = mhd.decode(buf)

// CID to Key
const multihash = mb.encode('base32', new CID(1, 'raw', buf).bytes).slice(1)
const multihash = base32.encode(CID.createV1(raw.code, digest).bytes).slice(1)

return new Key(`/${uint8ArrayToString(multihash)}`.toUpperCase(), false)
return new Key(`/${multihash.toUpperCase()}`, false)
}

/**
Expand Down
52 changes: 29 additions & 23 deletions migrations/migration-9/index.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,20 +1,20 @@
'use strict'

const CID = require('cids')
const dagpb = require('ipld-dag-pb')
const { CID } = require('multiformats')
achingbrain marked this conversation as resolved.
Show resolved Hide resolved
const dagPb = require('@ipld/dag-pb')
const cbor = require('cborg')
const multicodec = require('multicodec')
const multibase = require('multibase')
const pinset = require('./pin-set')
const { createStore } = require('../../src/utils')
const { cidToKey, PIN_DS_KEY, PinTypes } = require('./utils')
const length = require('it-length')
const { sha256 } = require('multiformats/hashes/sha2')
const mhd = require('multiformats/hashes/digest')
const { base32 } = require('multiformats/bases/base32')

/**
* @typedef {import('../../src/types').Migration} Migration
* @typedef {import('../../src/types').MigrationProgressCallback} MigrationProgressCallback
* @typedef {import('interface-datastore').Datastore} Datastore
* @typedef {import('multicodec').CodecCode} CodecCode
*/

/**
Expand All @@ -29,9 +29,9 @@ async function pinsToDatastore (blockstore, datastore, pinstore, onProgress) {
}

const mh = await datastore.get(PIN_DS_KEY)
const cid = new CID(mh)
const cid = CID.decode(mh)
const pinRootBuf = await blockstore.get(cidToKey(cid))
const pinRoot = dagpb.util.deserialize(pinRootBuf)
const pinRoot = dagPb.decode(pinRootBuf)
let counter = 0
let pinCount

Expand All @@ -40,7 +40,7 @@ async function pinsToDatastore (blockstore, datastore, pinstore, onProgress) {
for await (const cid of pinset.loadSet(blockstore, pinRoot, PinTypes.recursive)) {
counter++

/** @type {{ depth: number, version?: CID.CIDVersion, codec?: CodecCode }} */
/** @type {{ depth: number, version?: 0 | 1, codec?: number }} */
achingbrain marked this conversation as resolved.
Show resolved Hide resolved
const pin = {
depth: Infinity
}
Expand All @@ -49,8 +49,8 @@ async function pinsToDatastore (blockstore, datastore, pinstore, onProgress) {
pin.version = cid.version
}

if (cid.codec !== 'dag-pb') {
pin.codec = multicodec.getNumber(cid.codec)
if (cid.code !== dagPb.code) {
pin.codec = cid.code
rvagg marked this conversation as resolved.
Show resolved Hide resolved
}

await pinstore.put(cidToKey(cid), cbor.encode(pin))
Expand All @@ -61,7 +61,7 @@ async function pinsToDatastore (blockstore, datastore, pinstore, onProgress) {
for await (const cid of pinset.loadSet(blockstore, pinRoot, PinTypes.direct)) {
counter++

/** @type {{ depth: number, version?: CID.CIDVersion, codec?: CodecCode }} */
/** @type {{ depth: number, version?: 0 | 1, codec?: number }} */
const pin = {
depth: 0
}
Expand All @@ -70,8 +70,8 @@ async function pinsToDatastore (blockstore, datastore, pinstore, onProgress) {
pin.version = cid.version
}

if (cid.codec !== 'dag-pb') {
pin.codec = multicodec.getNumber(cid.codec)
if (cid.code !== dagPb.code) {
pin.codec = cid.code
}

await pinstore.put(cidToKey(cid), cbor.encode(pin))
Expand All @@ -98,7 +98,11 @@ async function pinsToDAG (blockstore, datastore, pinstore, onProgress) {
for await (const { key, value } of pinstore.query({})) {
counter++
const pin = cbor.decode(value)
const cid = new CID(pin.version || 0, pin.codec && multicodec.getName(pin.codec) || 'dag-pb', multibase.decode('b' + key.toString().split('/').pop()))
const cid = CID.create(
pin.version || 0,
pin.codec || dagPb.code,
mhd.decode(base32.decode('b' + key.toString().toLowerCase().split('/').pop()))
)

if (pin.depth === 0) {
onProgress((counter / pinCount) * 100, `Reverted direct pin ${cid}`)
Expand All @@ -112,16 +116,18 @@ async function pinsToDAG (blockstore, datastore, pinstore, onProgress) {
}

onProgress(100, 'Updating pin root')
const pinRoot = new dagpb.DAGNode(new Uint8Array(), [
await pinset.storeSet(blockstore, PinTypes.recursive, recursivePins),
await pinset.storeSet(blockstore, PinTypes.direct, directPins)
])
const buf = pinRoot.serialize()
const cid = await dagpb.util.cid(buf, {
cidVersion: 0
})
const pinRoot = {
Links: [
await pinset.storeSet(blockstore, PinTypes.direct, directPins),
rvagg marked this conversation as resolved.
Show resolved Hide resolved
await pinset.storeSet(blockstore, PinTypes.recursive, recursivePins)
]
}
const buf = dagPb.encode(pinRoot)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

const block = Block.encode({ value: pinRoot, codec: dagPb, hasher: sha256 }) is available to shorten this slightly, then you can pick block.cid.bytes out of it.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It looks like Block.encode will only ever give me a v1 CID back and we need a v0 CID here?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could just pull the multihash bytes out of the v1 CID, I guess..

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

mmm, block.cid.multihash should do it. Alternatives include block.cid.toV0().bytes (which is more explicit but a little wasteful) or adding a cidVersion argument to Block.encode.

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think Block.encode will save that much, and probably it's best to not add support for v0 there if we eventually want to break from v0. I would personally leave it as is.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This repo will always have to handle v0 as it needs to be able to migrate repos from before v1 was a thing.

const digest = await sha256.digest(buf)
const cid = CID.createV0(digest)

await blockstore.put(cidToKey(cid), buf)
await datastore.put(PIN_DS_KEY, cid.multihash)
await datastore.put(PIN_DS_KEY, cid.bytes)
}

/**
Expand Down
Loading