chore: switch to ESM (#161)

Switches the codebase to ESM and exports ESM and CJS code. The main goal is to enable ipfs-unixfs to be built on next generation bundlers like skypack: codepen.io/vascosantos/pen/NWjaYqP?editors=0011 This is a problem that has not been fixed in skypack yet. More details: skypackjs/skypack-cdn#171

Given we will need to export ESM and we will, let's get this to circumvent the issue.

We also need to manually change the generated protobuf code due to protobufjs/protobuf.js#1230

BREAKING CHANGE: ./src/dir-sharded is not in the exports map so cannot be imported

Co-authored-by: Alex Potsides <alex@achingbrain.net>

.travis.yml

@@ -87,7 +87,10 @@ jobs:
       name: electron main
       addons:
         firefox: latest
-      script: npm run test -- -- -- -t electron-main
+      script: 
+        - npm run build
+        - npx lerna link # use publishConfig.directory
+        - npm run test -- -- -- -t electron-main -f dist/cjs/node-test/*js
 
     - stage: release
       # only run on changes to master

packages/ipfs-unixfs-exporter/package.json

@@ -4,13 +4,14 @@
   "description": "JavaScript implementation of the UnixFs exporter used by IPFS",
   "leadMaintainer": "Alex Potsides <alex.potsides@protocol.ai>",
   "main": "src/index.js",
+  "type": "module",
   "browser": {
     "fs": false
   },
   "scripts": {
     "prepare": "aegir build --no-bundle",
     "test": "aegir test",
-    "build": "aegir build",
+    "build": "aegir build --esm-tests",
     "clean": "rimraf ./dist",
     "lint": "aegir ts -p check && aegir lint",
     "coverage": "nyc -s npm run test -t node && nyc report --reporter=html",
@@ -32,14 +33,16 @@
     "npm": ">=7.0.0"
   },
   "homepage": "https://github.com/ipfs/js-ipfs-unixfs#readme",
+  "publishConfig": {
+    "directory": "dist"
+  },
   "devDependencies": {
     "@types/mocha": "^8.2.1",
     "@types/sinon": "^10.0.0",
     "abort-controller": "^3.0.0",
-    "aegir": "^34.0.0",
+    "aegir": "^35.0.1",
     "copy": "^0.3.2",
     "crypto-browserify": "^3.12.0",
-    "detect-node": "^2.0.4",
     "events": "^3.3.0",
     "ipfs-unixfs-importer": "^8.0.2",
     "it-all": "^1.0.5",
@@ -52,7 +55,6 @@
     "readable-stream": "^3.6.0",
     "rimraf": "^3.0.2",
     "sinon": "^11.1.1",
-    "uint8arrays": "^2.1.2",
     "util": "^0.12.3"
   },
   "dependencies": {
@@ -65,14 +67,13 @@
     "it-last": "^1.0.5",
     "multiformats": "^9.4.2",
     "murmurhash3js-revisited": "^3.0.0",
-    "uint8arrays": "^2.1.7"
+    "uint8arrays": "^3.0.0"
   },
   "types": "dist/src/index.d.ts",
-  "files": [
-    "src",
-    "dist"
-  ],
   "eslintConfig": {
-    "extends": "ipfs"
+    "extends": "ipfs",
+    "parserOptions": {
+      "sourceType": "module"
+    }
   }
 }

packages/ipfs-unixfs-exporter/src/index.js

@@ -1,9 +1,7 @@
-'use strict'
-
-const errCode = require('err-code')
-const { CID } = require('multiformats/cid')
-const resolve = require('./resolvers')
-const last = require('it-last')
+import errCode from 'err-code'
+import { CID } from 'multiformats/cid'
+import resolve from './resolvers/index.js'
+import last from 'it-last'
 
 /**
  * @typedef {import('ipfs-unixfs').UnixFS} UnixFS
@@ -65,7 +63,7 @@ const cidAndRest = (path) => {
  * @param {Blockstore} blockstore
  * @param {ExporterOptions} [options]
  */
-async function * walkPath (path, blockstore, options = {}) {
+export async function * walkPath (path, blockstore, options = {}) {
   let {
     cid,
     toResolve
@@ -102,7 +100,7 @@ async function * walkPath (path, blockstore, options = {}) {
  * @param {Blockstore} blockstore
  * @param {ExporterOptions} [options]
  */
-async function exporter (path, blockstore, options = {}) {
+export async function exporter (path, blockstore, options = {}) {
   const result = await last(walkPath(path, blockstore, options))
 
   if (!result) {
@@ -117,7 +115,7 @@ async function exporter (path, blockstore, options = {}) {
  * @param {Blockstore} blockstore
  * @param {ExporterOptions} [options]
  */
-async function * recursive (path, blockstore, options = {}) {
+export async function * recursive (path, blockstore, options = {}) {
   const node = await exporter(path, blockstore, options)
 
   if (!node) {
@@ -151,9 +149,3 @@ async function * recursive (path, blockstore, options = {}) {
     }
   }
 }
-
-module.exports = {
-  exporter,
-  walkPath,
-  recursive
-}

packages/ipfs-unixfs-exporter/src/resolvers/dag-cbor.js

@@ -1,8 +1,6 @@
-'use strict'
-
-const { CID } = require('multiformats/cid')
-const errCode = require('err-code')
-const dagCbor = require('@ipld/dag-cbor')
+import { CID } from 'multiformats/cid'
+import errCode from 'err-code'
+import * as dagCbor from '@ipld/dag-cbor'
 
 /**
  * @typedef {import('../types').Resolver} Resolver
@@ -72,4 +70,4 @@ const resolve = async (cid, name, path, toResolve, resolve, depth, blockstore, o
   }
 }
 
-module.exports = resolve
+export default resolve

packages/ipfs-unixfs-exporter/src/resolvers/identity.js

@@ -1,9 +1,7 @@
-'use strict'
-
-const errCode = require('err-code')
-const extractDataFromBlock = require('../utils/extract-data-from-block')
-const validateOffsetAndLength = require('../utils/validate-offset-and-length')
-const mh = require('multiformats/hashes/digest')
+import errCode from 'err-code'
+import extractDataFromBlock from '../utils/extract-data-from-block.js'
+import validateOffsetAndLength from '../utils/validate-offset-and-length.js'
+import * as mh from 'multiformats/hashes/digest'
 
 /**
  * @typedef {import('../types').ExporterOptions} ExporterOptions
@@ -52,4 +50,4 @@ const resolve = async (cid, name, path, toResolve, resolve, depth, blockstore, o
   }
 }
 
-module.exports = resolve
+export default resolve

packages/ipfs-unixfs-exporter/src/resolvers/index.js

@@ -1,11 +1,14 @@
-'use strict'
+import errCode from 'err-code'
 
-const errCode = require('err-code')
+import * as dagPb from '@ipld/dag-pb'
+import * as dagCbor from '@ipld/dag-cbor'
+import * as raw from 'multiformats/codecs/raw'
+import { identity } from 'multiformats/hashes/identity'
 
-const dagPb = require('@ipld/dag-pb')
-const dagCbor = require('@ipld/dag-cbor')
-const raw = require('multiformats/codecs/raw')
-const { identity } = require('multiformats/hashes/identity')
+import dagPbResolver from './unixfs-v1/index.js'
+import rawResolver from './raw.js'
+import dagCborResolver from './dag-cbor.js'
+import identifyResolver from './identity.js'
 
 /**
  * @typedef {import('../types').Resolver} Resolver
@@ -16,10 +19,10 @@ const { identity } = require('multiformats/hashes/identity')
  * @type {{ [ key: string ]: Resolver }}
  */
 const resolvers = {
-  [dagPb.code]: require('./unixfs-v1'),
-  [raw.code]: require('./raw'),
-  [dagCbor.code]: require('./dag-cbor'),
-  [identity.code]: require('./identity')
+  [dagPb.code]: dagPbResolver,
+  [raw.code]: rawResolver,
+  [dagCbor.code]: dagCborResolver,
+  [identity.code]: identifyResolver
 }
 
 /**
@@ -35,4 +38,4 @@ function resolve (cid, name, path, toResolve, depth, blockstore, options) {
   return resolver(cid, name, path, toResolve, resolve, depth, blockstore, options)
 }
 
-module.exports = resolve
+export default resolve

packages/ipfs-unixfs-exporter/src/resolvers/raw.js

@@ -1,8 +1,6 @@
-'use strict'
-
-const errCode = require('err-code')
-const extractDataFromBlock = require('../utils/extract-data-from-block')
-const validateOffsetAndLength = require('../utils/validate-offset-and-length')
+import errCode from 'err-code'
+import extractDataFromBlock from '../utils/extract-data-from-block.js'
+import validateOffsetAndLength from '../utils/validate-offset-and-length.js'
 
 /**
  * @typedef {import('../types').ExporterOptions} ExporterOptions
@@ -51,4 +49,4 @@ const resolve = async (cid, name, path, toResolve, resolve, depth, blockstore, o
   }
 }
 
-module.exports = resolve
+export default resolve

packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/directory.js

@@ -1,5 +1,3 @@
-'use strict'
-
 /**
  * @typedef {import('../../../types').ExporterOptions} ExporterOptions
  * @typedef {import('../../../types').UnixfsV1DirectoryContent} UnixfsV1DirectoryContent
@@ -31,4 +29,4 @@ const directoryContent = (cid, node, unixfs, path, resolve, depth, blockstore) =
   return yieldDirectoryContent
 }
 
-module.exports = directoryContent
+export default directoryContent

packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/file.js

@@ -1,12 +1,10 @@
-'use strict'
-
-const extractDataFromBlock = require('../../../utils/extract-data-from-block')
-const validateOffsetAndLength = require('../../../utils/validate-offset-and-length')
-const { UnixFS } = require('ipfs-unixfs')
-const errCode = require('err-code')
-const dagPb = require('@ipld/dag-pb')
-const dagCbor = require('@ipld/dag-cbor')
-const raw = require('multiformats/codecs/raw')
+import extractDataFromBlock from '../../../utils/extract-data-from-block.js'
+import validateOffsetAndLength from '../../../utils/validate-offset-and-length.js'
+import { UnixFS } from 'ipfs-unixfs'
+import errCode from 'err-code'
+import * as dagPb from '@ipld/dag-pb'
+import * as dagCbor from '@ipld/dag-cbor'
+import * as raw from 'multiformats/codecs/raw'
 
 /**
  * @typedef {import('../../../types').ExporterOptions} ExporterOptions
@@ -126,4 +124,4 @@ const fileContent = (cid, node, unixfs, path, resolve, depth, blockstore) => {
   return yieldFileContent
 }
 
-module.exports = fileContent
+export default fileContent

packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/hamt-sharded-directory.js

@@ -1,6 +1,4 @@
-'use strict'
-
-const { decode } = require('@ipld/dag-pb')
+import { decode } from '@ipld/dag-pb'
 
 /**
  * @typedef {import('interface-blockstore').Blockstore} Blockstore
@@ -58,4 +56,4 @@ async function * listDirectory (node, path, resolve, depth, blockstore, options)
   }
 }
 
-module.exports = hamtShardedDirectoryContent
+export default hamtShardedDirectoryContent

packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/raw.js

@@ -1,7 +1,5 @@
-'use strict'
-
-const extractDataFromBlock = require('../../../utils/extract-data-from-block')
-const validateOffsetAndLength = require('../../../utils/validate-offset-and-length')
+import extractDataFromBlock from '../../../utils/extract-data-from-block.js'
+import validateOffsetAndLength from '../../../utils/validate-offset-and-length.js'
 
 /**
  * @typedef {import('../../../types').ExporterOptions} ExporterOptions
@@ -33,4 +31,4 @@ const rawContent = (cid, node, unixfs, path, resolve, depth, blockstore) => {
   return yieldRawContent
 }
 
-module.exports = rawContent
+export default rawContent

packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/index.js

@@ -1,9 +1,11 @@
-'use strict'
+import errCode from 'err-code'
+import { UnixFS } from 'ipfs-unixfs'
+import findShardCid from '../../utils/find-cid-in-shard.js'
+import { decode } from '@ipld/dag-pb'
 
-const errCode = require('err-code')
-const { UnixFS } = require('ipfs-unixfs')
-const findShardCid = require('../../utils/find-cid-in-shard')
-const { decode } = require('@ipld/dag-pb')
+import contentFile from './content/file.js'
+import contentDirectory from './content/directory.js'
+import contentHamtShardedDirectory from './content/hamt-sharded-directory.js'
 
 /**
  * @typedef {import('../../types').Resolve} Resolve
@@ -26,10 +28,10 @@ const findLinkCid = (node, name) => {
  * @type {{ [key: string]: UnixfsV1Resolver }}
  */
 const contentExporters = {
-  raw: require('./content/file'),
-  file: require('./content/file'),
-  directory: require('./content/directory'),
-  'hamt-sharded-directory': require('./content/hamt-sharded-directory'),
+  raw: contentFile,
+  file: contentFile,
+  directory: contentDirectory,
+  'hamt-sharded-directory': contentHamtShardedDirectory,
   metadata: (cid, node, unixfs, path, resolve, depth, blockstore) => {
     return () => []
   },
@@ -109,4 +111,4 @@ const unixFsResolver = async (cid, name, path, toResolve, resolve, depth, blocks
   }
 }
 
-module.exports = unixFsResolver
+export default unixFsResolver

packages/ipfs-unixfs-exporter/src/utils/extract-data-from-block.js

@@ -1,12 +1,10 @@
-'use strict'
-
 /**
  * @param {Uint8Array} block
  * @param {number} blockStart
  * @param {number} requestedStart
  * @param {number} requestedEnd
  */
-module.exports = function extractDataFromBlock (block, blockStart, requestedStart, requestedEnd) {
+function extractDataFromBlock (block, blockStart, requestedStart, requestedEnd) {
   const blockLength = block.length
   const blockEnd = blockStart + blockLength
 
@@ -28,3 +26,5 @@ module.exports = function extractDataFromBlock (block, blockStart, requestedStar
 
   return block
 }
+
+export default extractDataFromBlock

packages/ipfs-unixfs-exporter/src/utils/find-cid-in-shard.js

@@ -1,10 +1,9 @@
-'use strict'
 
-const { Bucket, createHAMT } = require('hamt-sharding')
-const { decode } = require('@ipld/dag-pb')
+import { Bucket, createHAMT } from 'hamt-sharding'
+import { decode } from '@ipld/dag-pb'
 // @ts-ignore - no types available
-const mur = require('murmurhash3js-revisited')
-const uint8ArrayFromString = require('uint8arrays/from-string')
+import mur from 'murmurhash3js-revisited'
+import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string'
 
 /**
  * @typedef {import('interface-blockstore').Blockstore} Blockstore
@@ -152,4 +151,4 @@ const findShardCid = async (node, name, blockstore, context, options) => {
   return findShardCid(node, name, blockstore, context, options)
 }
 
-module.exports = findShardCid
+export default findShardCid

packages/ipfs-unixfs-exporter/src/utils/validate-offset-and-length.js

@@ -1,6 +1,4 @@
-'use strict'
-
-const errCode = require('err-code')
+import errCode from 'err-code'
 
 /**
  * @param {number} size
@@ -38,4 +36,4 @@ const validateOffsetAndLength = (size, offset, length) => {
   }
 }
 
-module.exports = validateOffsetAndLength
+export default validateOffsetAndLength