feat!: use Helia's blockBroker interface

package.json

@@ -118,29 +118,28 @@
     "storybook:build": "storybook build"
   },
   "dependencies": {
-    "@chainsafe/libp2p-gossipsub": "^8.0.0",
-    "@chainsafe/libp2p-noise": "^12.0.0",
-    "@chainsafe/libp2p-yamux": "^4.0.2",
-    "@ipld/car": "^5.1.1",
-    "@ipld/dag-cbor": "^9.0.1",
-    "@ipld/dag-json": "^10.0.1",
-    "@ipld/dag-pb": "^4.0.3",
-    "@libp2p/delegated-content-routing": "^4.0.3",
-    "@libp2p/delegated-peer-routing": "^4.0.6",
+    "@chainsafe/libp2p-gossipsub": "^10.1.1",
+    "@chainsafe/libp2p-noise": "^13.0.5",
+    "@chainsafe/libp2p-yamux": "^5.0.3",
+    "@ipld/car": "^5.2.4",
+    "@ipld/dag-cbor": "^9.0.6",
+    "@ipld/dag-json": "^10.1.5",
+    "@ipld/dag-pb": "^4.0.6",
+    "@libp2p/delegated-content-routing": "^4.0.11",
+    "@libp2p/delegated-peer-routing": "^4.0.14",
     "@libp2p/ipni-content-routing": "^1.0.1",
     "@loadable/component": "^5.14.1",
     "@tableflip/react-inspector": "^2.3.0",
     "cytoscape": "^3.18.1",
     "cytoscape-dagre": "^2.3.2",
     "filesize": "^6.1.0",
-    "helia": "^1.1.2",
-    "ipfs-unixfs": "^4.0.1",
+    "helia": "^2.1.0",
+    "ipfs-unixfs": "^11.1.0",
     "ipld-git": "^0.6.6",
     "ipld-raw": "7.0.0",
-    "kubo-rpc-client": "^3.0.1",
-    "libp2p": "^0.45.1",
+    "libp2p": "^0.46.21",
     "milliseconds": "^1.0.3",
-    "multiformats": "^11.0.2",
+    "multiformats": "^12.1.3",
     "react-joyride": "^2.3.0",
     "stream-to-it": "^0.2.4",
     "web-encoding": "^1.1.5"
@@ -157,9 +156,9 @@
     "@esbuild-plugins/esm-externals": "^0.1.2",
     "@esbuild-plugins/node-globals-polyfill": "^0.2.3",
     "@esbuild-plugins/node-modules-polyfill": "^0.2.2",
-    "@helia/interface": "^1.0.0",
-    "@libp2p/bootstrap": "^8.0.0",
-    "@libp2p/websockets": "^6.0.1",
+    "@helia/interface": "^2.1.0",
+    "@libp2p/bootstrap": "^9.0.12",
+    "@libp2p/websockets": "^7.0.13",
     "@rollup/plugin-commonjs": "^25.0.0",
     "@rollup/plugin-json": "^6.0.0",
     "@rollup/plugin-node-resolve": "^15.0.2",
@@ -197,7 +196,7 @@
     "dag-jose": "^4.0.0",
     "datastore-core": "^9.1.1",
     "eslint": "^8.41.0",
-    "eslint-config-ipfs": "^4.0.3",
+    "eslint-config-ipfs": "^6.0.0",
     "eslint-config-react-app": "^7.0.1",
     "eslint-config-standard": "^17.0.0",
     "eslint-config-standard-react": "^13.0.0",
@@ -217,7 +216,7 @@
     "interface-datastore": "^8.2.0",
     "intl-messageformat": "^10.3.5",
     "ipfs-core": "^0.18.0",
-    "ipfs-css": "^1.3.0",
+    "ipfs-css": "^1.4.0",
     "jsdom": "^22.1.0",
     "npm-run-all": "^4.1.5",
     "prop-types": "^15.8.1",

src/bundles/explore.js

@@ -40,7 +40,7 @@ const makeBundle = () => {
           localPath,
           nodes,
           pathBoundaries
-        } = await resolveIpldPath(store.selectHelia(), store.selectKuboClient(), cid, rest)
+        } = await resolveIpldPath(store.selectHelia(), cid, rest)
 
         return {
           path,

src/bundles/helia.ts

@@ -1,25 +1,23 @@
 /* globals globalThis */
 import { type Helia } from '@helia/interface'
-import { create as createKuboClient, type IPFSHTTPClient } from 'kubo-rpc-client'
 
-import initHelia from '../lib/init-helia'
+import initHelia from '../lib/init-helia.js'
+import type { KuboGatewayOpts } from '../types.d.js'
 
 interface HeliaBundleState {
-  apiOpts: Record<string, string>
+  apiOpts: KuboGatewayOpts
   instance: Helia | null
   error: Error | null
-  kuboClient: IPFSHTTPClient | null
 }
 
 const defaultState: HeliaBundleState = {
   apiOpts: {
     host: '127.0.0.1',
-    port: '5001',
+    port: '8080',
     protocol: 'http'
   },
   instance: null,
-  error: null,
-  kuboClient: null
+  error: null
 }
 
 function getUserOpts (key: string): Record<string, unknown> {
@@ -42,7 +40,6 @@
     if (type === 'HELIA_INIT_FINISHED') {
       return Object.assign({}, state, {
         instance: payload.instance ?? state.instance,
-        kuboClient: payload.kuboClient ?? state.kuboClient,
         apiOpts: payload.apiOpts ?? state.apiOpts,
         error: null
       })
@@ -55,7 +52,6 @@
     return state
   },
 
-  selectKuboClient: ({ helia }: { helia: HeliaBundleState }): IPFSHTTPClient | null => helia.kuboClient,
   selectHelia: ({ helia }: { helia: HeliaBundleState }) => helia.instance,
 
   selectHeliaReady: ({ helia }: { helia: HeliaBundleState }) => helia.instance !== null,
@@ -66,28 +62,26 @@
     return identifyService?.host?.agentVersion.split(' ')[0] ?? 'null'
   },
 
   doInitHelia: () => async ({ dispatch, getState }: any) => {
     dispatch({ type: 'HELIA_INIT_STARTED' })
 
     const apiOpts = Object.assign(
       {},
       getState().helia.apiOpts,
-      getUserOpts('ipfsApi')
+      getUserOpts('kuboGateway')
     )
-    // TRY helia!
+
     try {
       console.info(
-        "🎛️ Customise your kubo-rpc-client opts by setting an `ipfsApi` value in localStorage. e.g. localStorage.setItem('ipfsApi', JSON.stringify({port: '1337'}))"
+        "🎛️ Customise your Kubo gateway opts by setting an `kuboGateway` value in localStorage. e.g. localStorage.setItem('kuboGateway', JSON.stringify({port: '1337'}))"
       )
-      const kuboClient = createKuboClient(apiOpts)
       console.time('HELIA_INIT')
-      const helia = await initHelia(kuboClient)
+      const helia = await initHelia(apiOpts)
       console.timeEnd('HELIA_INIT')
       return dispatch({
         type: 'HELIA_INIT_FINISHED',
         payload: {
           apiOpts,
-          kuboClient,
           instance: helia,
           provider: 'helia'
         }

src/global.d.ts

@@ -3,16 +3,16 @@
  */
 declare module 'stream-to-it' {
   interface toIterable {
-    source: <T>(stream: ReadableStream<T>) => AsyncIterable<T>
+    source<T>(stream: ReadableStream<T>): AsyncIterable<T>
   }
   const toIterable: toIterable
   export default toIterable
 }
 
 interface OldIpldFormat {
   util: {
-    serialize: (obj: unknown) => Promise<Uint8Array>
-    deserialize: (bytes: Uint8Array) => Promise<unknown>
+    serialize(obj: unknown): Promise<Uint8Array>
+    deserialize(bytes: Uint8Array): Promise<unknown>
 
     codec: number
     defaultHashAlg: number
@@ -21,8 +21,8 @@ interface OldIpldFormat {
   }
   codec: number
   resolver: {
-    resolve: (bytes: Uint8Array, path: string) => Promise<unknown>
-    tree: (bytes: Uint8Array) => Promise<unknown>
+    resolve(bytes: Uint8Array, path: string): Promise<unknown>
+    tree(bytes: Uint8Array): Promise<unknown>
   }
   defaultHashAlg: number
 }

src/lib/codec-importer.ts

@@ -13,14 +13,12 @@ export default async function codecImporter<T extends CodecDataTypes = CodecData
     case 'dag-cbor':
       return import('@ipld/dag-cbor')
     case 'dag-pb':
-      // @ts-expect-error - return types need normalizing
       return import('@ipld/dag-pb')
     case 'git-raw':
       return {
         decode: (await import('ipld-git')).default.util.deserialize
       }
     case 'raw':
-      // @ts-expect-error - return types need normalizing
       return import('multiformats/codecs/raw')
     case 'json':
       return import('multiformats/codecs/json')

src/lib/get-codec-for-cid.ts

@@ -8,8 +8,8 @@ import { ensureLeadingSlash } from './helpers'
 import type { ResolveType } from '../types'
 
 interface CodecWrapper<DecodedType = any> {
-  decode: (bytes: Uint8Array) => DecodedType
-  resolve: (path: string, bytes: Uint8Array) => Promise<ResolveType<DecodedType>>
+  decode(bytes: Uint8Array): DecodedType
+  resolve(path: string, bytes: Uint8Array): Promise<ResolveType<DecodedType>>
 }
 
 interface DecodeFn<T = any> {

src/lib/get-raw-block.ts

@@ -1,133 +1,19 @@
 import type { Helia } from '@helia/interface'
-import { type IPFSHTTPClient } from 'kubo-rpc-client'
-import { CID } from 'multiformats'
-import type { Version as CIDVersion } from 'multiformats/cid'
+import { type CID } from 'multiformats'
 
-import { BlockFetchTimeoutError } from './errors'
-import getHasherForCode from './hash-importer.js'
-
-async function getCidFromBytes (bytes: Uint8Array, cidVersion: CIDVersion, codecCode: number, multihashCode: number): Promise<CID> {
-  const hasher = await getHasherForCode(multihashCode)
-
-  try {
-    const hash = await hasher.digest(bytes)
-    return CID.create(cidVersion, codecCode, hash)
-  } catch (err) {
-    console.error('could not create cid from bytes', err)
-  }
-
-  return '' as any as CID
-}
-
-async function getRawBlockFromGateway (url: string | URL, cid: CID, signal: AbortSignal): Promise<Uint8Array> {
-  const gwUrl = new URL(url)
-
-  gwUrl.pathname = `/ipfs/${cid.toString()}`
-  gwUrl.search = '?format=raw' // necessary as not every gateway supports dag-cbor, but every should support sending raw block as-is
-  try {
-    const res = await fetch(gwUrl.toString(), {
-      signal,
-      headers: {
-        // also set header, just in case ?format= is filtered out by some reverse proxy
-        Accept: 'application/vnd.ipld.raw'
-      },
-      cache: 'force-cache'
-    })
-    if (!res.ok) {
-      throw new Error(`unable to fetch raw block for CID ${cid} from gateway ${gwUrl.toString()}`)
-    }
-    return new Uint8Array(await res.arrayBuffer())
-  } catch (cause) {
-    console.error('cause', cause)
-    throw new Error(`unable to fetch raw block for CID ${cid}`)
-  }
-}
+import { BlockFetchTimeoutError } from './errors.js'
 
 /**
- * This method validates that the block we got from the gateway has the same CID as the one we requested
+ * Method for getting a raw block either with helia from trustless gateways or a local Kubo gateway.
  */
-export async function verifyBytes (providedCid: CID, bytes: Uint8Array): Promise<void> {
-  try {
-    const cid = await getCidFromBytes(bytes, providedCid.version, providedCid.code, providedCid.multihash.code)
-
-    if (cid.toString() !== providedCid.toString()) {
-      throw new Error(`CID mismatch, expected '${providedCid.toString()}' but got '${cid.toString()}'`)
-    }
-  } catch (err) {
-    console.error('unable to verify bytes', err)
-    throw err
-  }
-}
-
-/**
- * allow users to disable fetching from gateways by setting 'explore.ipld.gatewayEnabled' to false in localStorage
- *
- * @returns {boolean}
- */
-function ensureGatewayFetchEnabled (): boolean {
-  console.info('import.meta.env.NODE_ENV: ', import.meta.env.NODE_ENV)
-  console.info(
-    "🎛️ Customise whether ipld-explorer-components fetches content from gateways by setting an `explore.ipld.gatewayEnabled` value to true/false in localStorage. e.g. localStorage.setItem('explore.ipld.gatewayEnabled', false) -- NOTE: defaults to true"
-  )
-  const gatewayEnabledSetting = localStorage.getItem('explore.ipld.gatewayEnabled')
-
-  return gatewayEnabledSetting != null ? JSON.parse(gatewayEnabledSetting) : true
-}
-
-export async function getBlockFromAnyGateway (cid: CID, signal: AbortSignal, moreGateways: string[] = []): Promise<Uint8Array> {
-  if (!ensureGatewayFetchEnabled()) {
-    throw new Error('Fetching from gateways is disabled')
-  }
-  const gateways = moreGateways.concat(defaultGateways)
-  for (const url of gateways) {
-    if (signal.aborted) {
-      throw new Error('aborted')
-    }
-    try {
-      const rawBlock = await getRawBlockFromGateway(url, cid, signal)
-      try {
-        await verifyBytes(cid, rawBlock)
-        return rawBlock
-      } catch (err) {
-        console.error('unable to verify block from gateway', url)
-        continue
-      }
-    } catch (err) {
-      console.error('unable to get block from gateway', err)
-      // ignore the error
-    }
-  }
-  throw new Error('Could not get block from any gateway')
-}
-
-const defaultGateways = ['https://ipfs.io', 'https://dweb.link']
-
-/**
- * Method for getting a raw block either with helia or https://docs.ipfs.tech/reference/http/gateway/#trusted-vs-trustless
- * inspiration from https://github.com/ipfs-shipyard/ipfs-geoip/blob/466cd9d6454098c0fcf998b2217225099a654695/src/lookup.js#L18
- */
-export async function getRawBlock (helia: Helia, kuboClient: IPFSHTTPClient, cid: CID, timeout = 30000): Promise<Uint8Array> {
+export async function getRawBlock (helia: Helia, cid: CID, timeout = 30000): Promise<Uint8Array> {
   const abortController = new AbortController()
 
   try {
-    if (await helia.blockstore.has(cid)) {
-      // If we've gotten the block before, we can just return it.
-      return await helia.blockstore.get(cid)
-    }
-
     const timeoutId = setTimeout(() => { abortController.abort('Request timed out') }, timeout)
-    const rawBlock = await Promise.any([kuboClient.block.get(cid), getBlockFromAnyGateway(cid, abortController.signal), helia.blockstore.get(cid, { signal: abortController.signal })])
-    abortController.abort('Content obtained') // abort any other requests.
+    const rawBlock = await helia.blockstore.get(cid, { signal: abortController.signal })
     clearTimeout(timeoutId)
 
-    /**
-     * if we got the block from the gateway, verifyBytes is called, and we can safely store the block.
-     * if we got the block from helia, helia's blockstore should already have the block.
-     */
-    if (!await helia.blockstore.has(cid)) {
-      await helia.blockstore.put(cid, rawBlock)
-    }
-
     return rawBlock
   } catch (err) {
     console.error('unable to get raw block', err)

src/lib/helpers.ts

@@ -11,7 +11,7 @@ export interface DigestFn {
   (data: Uint8Array): Promise<MultihashDigest<number>>
 }
 
-export async function addDagNodeToHelia <T> (helia: Helia, codec: { encode: (n: T) => Uint8Array, code: number }, node: T, digestFn?: DigestFn): Promise<CID> {
+export async function addDagNodeToHelia <T> (helia: Helia, codec: { encode(n: T): Uint8Array, code: number }, node: T, digestFn?: DigestFn): Promise<CID> {
   const encodedNode = codec.encode(node)
   const hash = digestFn != null ? await digestFn(encodedNode) : await sha256.digest(encodedNode)
   const cid = CID.createV1(codec.code, hash)