Add options to configure IPFS gateway to use to fetch data

.husky/commit-msg

@@ -1,4 +1 @@
-#!/usr/bin/env sh
-. "$(dirname -- "$0")/_/husky.sh"
-
 npm run commitlint

package.json

@@ -48,41 +48,41 @@
   },
   "dependencies": {
     "async-retry": "^1.3.3",
-    "blockstore-core": "^4.3.3",
-    "ipfs-unixfs-importer": "^15.1.7",
+    "blockstore-core": "^5.0.2",
+    "ipfs-unixfs-importer": "^15.3.1",
     "multiformats": "^9.9.0"
   },
   "peerDependencies": {
-    "@makerx/node-cache": "^1.0.2"
+    "@makerx/node-cache": "^1.1.0"
   },
   "devDependencies": {
-    "@commitlint/cli": "^18.4.3",
-    "@commitlint/config-conventional": "^18.4.3",
-    "@makerx/eslint-config": "^3.1.0",
-    "@makerx/prettier-config": "^2.0.0",
+    "@commitlint/cli": "^19.5.0",
+    "@commitlint/config-conventional": "^19.5.0",
+    "@makerx/eslint-config": "^3.1.1",
+    "@makerx/prettier-config": "^2.0.1",
     "@makerx/ts-config": "^1.0.1",
-    "@types/async-retry": "^1.4.8",
-    "@types/jest": "^29.5.11",
-    "@types/node": "^18.13.0",
-    "@typescript-eslint/eslint-plugin": "^6.15.0",
-    "@typescript-eslint/parser": "^6.15.0",
-    "conventional-changelog-conventionalcommits": "^7.0.2",
-    "eslint": "^8.56.0",
+    "@types/async-retry": "^1.4.9",
+    "@types/jest": "^29.5.14",
+    "@types/node": "^22.9.0",
+    "@typescript-eslint/eslint-plugin": "^8.14.0",
+    "@typescript-eslint/parser": "^8.14.0",
+    "conventional-changelog-conventionalcommits": "^8.0.0",
+    "eslint": "^8.57.1",
     "eslint-config-prettier": "^9.1.0",
     "eslint-plugin-node": "^11.1.0",
-    "eslint-plugin-prettier": "^5.1.1",
-    "husky": "^8.0.3",
+    "eslint-plugin-prettier": "^5.2.1",
+    "husky": "^9.1.6",
     "jest": "^29.7.0",
     "npm-run-all": "^4.1.5",
-    "prettier": "^3.1.1",
-    "rimraf": "^5.0.5",
-    "semantic-release": "^22.0.12",
-    "ts-jest": "^29.1.1",
-    "ts-jest-mocker": "^0.5.0",
+    "prettier": "^3.3.3",
+    "rimraf": "^6.0.1",
+    "semantic-release": "^24.2.0",
+    "ts-jest": "^29.2.5",
+    "ts-jest-mocker": "^1.0.0",
     "ts-node": "^10.9.2",
-    "typedoc": "^0.25.4",
-    "typedoc-plugin-markdown": "^3.17.1",
-    "typescript": "^5.3.3"
+    "typedoc": "^0.26.11",
+    "typedoc-plugin-markdown": "^4.2.10",
+    "typescript": "^5.6.3"
   },
   "release": {
     "branches": [

src/http.ts

@@ -20,7 +20,7 @@ async function retryer(url: string, fetchOptions?: RequestInit, retries: number
       retries: retries,
       onRetry: (e, num) => {
         // eslint-disable-next-line no-console
-        console.debug(`HTTP request failed. Retrying for #${num} time: ${e.message}`)
+        console.debug(`HTTP request failed. Retrying for #${num} time: ${(e as Error).message}`)
       },
     },
   )

src/index.ts

@@ -1 +1 @@
-export * from './ipfs'
+export { CacheOnlyIPFS, IPFS, InMemoryIPFS, PinataStorage, PinataStorageWithCache } from './ipfs'

src/ipfs.spec.ts

@@ -1,4 +1,4 @@
-import { PinataStorageWithCache } from './ipfs'
+import { getCIDUrl, PinataStorageWithCache } from './ipfs'
 import { ObjectCache } from '@makerx/node-cache'
 import { mock, Mock } from 'ts-jest-mocker'
 
@@ -16,7 +16,11 @@ describe('PinataStorageWithCache', () => {
 
   beforeEach(() => {
     cache = mock<ObjectCache>()
-    ipfs = new PinataStorageWithCache(testToken, cache)
+    cache.put.mockReturnValue(Promise.resolve())
+
+    ipfs = new PinataStorageWithCache(testToken, cache, {
+      baseUrl: 'https://dummy-ipfs-gateway.com',
+    })
     fetch.mockReset()
     fetch.mockResolvedValueOnce({
       ok: true,
@@ -98,3 +102,32 @@ describe('PinataStorageWithCache', () => {
     })
   })
 })
+
+describe('getCIDUrl', () => {
+  it('correctly constructs the absolute URL for a CID', () => {
+    const baseUrl = 'https://ipfs.pinata.cloud/ipfs/'
+    const cid = 'my-cid'
+
+    const value = getCIDUrl(baseUrl, cid)
+
+    expect(value).toBe('https://ipfs.pinata.cloud/ipfs/my-cid')
+  })
+
+  it(`correctly constructs the absolute URL for a CID if the base URL doesn't have a trailing slash`, () => {
+    const baseUrl = 'https://ipfs.pinata.cloud/ipfs'
+    const cid = 'my-cid'
+
+    const value = getCIDUrl(baseUrl, cid)
+
+    expect(value).toBe('https://ipfs.pinata.cloud/ipfs/my-cid')
+  })
+
+  it('correctly constructs the absolute URL for a CID if the base URL has no path', () => {
+    const baseUrl = 'https://ipfs.pinata.cloud'
+    const cid = 'my-cid'
+
+    const value = getCIDUrl(baseUrl, cid)
+
+    expect(value).toBe('https://ipfs.pinata.cloud/my-cid')
+  })
+})

src/ipfs.ts

@@ -68,21 +68,21 @@ export class InMemoryIPFS implements IPFS {
 
 export class CacheOnlyIPFS implements IPFS {
   private cache: ObjectCache
-  private getFromIpfs: boolean
+  private ipfsGatewayOptions?: IPFSGatewayOptions
 
-  constructor(cache: ObjectCache, getFromIpfs?: boolean) {
+  constructor(cache: ObjectCache, ipfsGatewayOptions?: IPFSGatewayOptions) {
     this.cache = cache
-    this.getFromIpfs = getFromIpfs ?? false
+    this.ipfsGatewayOptions = ipfsGatewayOptions
   }
 
   async get<T>(cid: string): Promise<T> {
     return await this.cache.getAndCache<T>(
       `ipfs-${cid}`,
       async (_e) => {
-        if (!this.getFromIpfs) {
+        if (!this.ipfsGatewayOptions) {
           throw new Error('404')
         }
-        const response = await fetchWithRetry(`https://${cid}.ipfs.cf-ipfs.com/`)
+        const response = await fetchWithRetry(getCIDUrl(this.ipfsGatewayOptions.baseUrl, cid))
         const json = await response.json()
         return json as T
       },
@@ -112,10 +112,10 @@ export class CacheOnlyIPFS implements IPFS {
     return await this.cache.getAndCache<Uint8Array>(
       `ipfs-${cid}`,
       async (_e) => {
-        if (!this.getFromIpfs) {
+        if (!this.ipfsGatewayOptions) {
           throw new Error('404')
         }
-        const response = await fetchWithRetry(`https://${cid}.ipfs.cf-ipfs.com/`)
+        const response = await fetchWithRetry(getCIDUrl(this.ipfsGatewayOptions.baseUrl, cid))
         return Buffer.from(await response.arrayBuffer())
       },
       {
@@ -160,24 +160,40 @@ type PinataMetadata = {
   [key: string]: string | undefined
 }
 
+type IPFSGatewayOptions = {
+  /**
+   * The base URL to use when fetching assets from the IPFS gateway.
+   *
+   * The expected format is such that the URL should be correct if another URL segment with the CID is appended.
+   * @example `https://ipfs.algonode.dev/ipfs`
+   */
+  baseUrl: string
+}
+
+const defaultIPFSGatewayOptions: IPFSGatewayOptions = {
+  baseUrl: 'https://ipfs.algonode.dev/ipfs',
+}
+
 export class PinataStorageWithCache implements IPFS {
   private cache: ObjectCache
   private token: string
+  private ipfsGatewayOptions: IPFSGatewayOptions
   private pinataBaseUrl = 'https://api.pinata.cloud/pinning'
 
   // We've chosen to use the Pinata API directly rather than their JS SDK,
   // as it currently uses a really old version of axios, that has security vulnerabilities.
 
-  constructor(pinataToken: string, cache: ObjectCache) {
+  constructor(pinataToken: string, cache: ObjectCache, ipfsGatewayOptions?: IPFSGatewayOptions) {
     this.token = pinataToken
     this.cache = cache
+    this.ipfsGatewayOptions = ipfsGatewayOptions ?? defaultIPFSGatewayOptions
   }
 
   async get<T>(cid: string): Promise<T> {
     return await this.cache.getAndCache<T>(
       `ipfs-${cid}`,
       async (_e) => {
-        const response = await fetchWithRetry(`https://${cid}.ipfs.cf-ipfs.com/`)
+        const response = await fetchWithRetry(getCIDUrl(this.ipfsGatewayOptions.baseUrl, cid))
         // eslint-disable-next-line no-console
         console.debug(`Cache miss for ${cid}, fetching from IPFS`)
         const json = await response.json()
@@ -225,7 +241,7 @@ export class PinataStorageWithCache implements IPFS {
     return await this.cache.getAndCache<Uint8Array>(
       `ipfs-${cid}`,
       async (_e) => {
-        const response = await fetchWithRetry(`https://${cid}.ipfs.cf-ipfs.com/`)
+        const response = await fetchWithRetry(getCIDUrl(this.ipfsGatewayOptions.baseUrl, cid))
         // eslint-disable-next-line no-console
         console.debug(`Cache miss for ${cid}, fetching from IPFS`)
         return Buffer.from(await response.arrayBuffer())
@@ -330,10 +346,21 @@ class NoOpCache implements ObjectCache {
   putBinary(_cacheKey: string, _data: Uint8Array, _mimeType?: string | undefined): Promise<void> {
     return Promise.resolve()
   }
+  clearCache(_cacheKey: string): void {}
 }
 
 export class PinataStorage extends PinataStorageWithCache {
-  constructor(pinataToken: string) {
-    super(pinataToken, new NoOpCache())
+  constructor(pinataToken: string, ipfsGatewayOptions?: IPFSGatewayOptions) {
+    super(pinataToken, new NoOpCache(), ipfsGatewayOptions)
   }
 }
+
+export function getCIDUrl(ipfsGatewayBaseUrl: string, cid: string) {
+  // The trailing slash is important as it allows us to append the CID segment
+  // using URL instead of concatenating strings ourselves.
+  // Without the trailing slash, the whole path would be replaced.
+  const baseUrl = ipfsGatewayBaseUrl.endsWith('/') ? new URL(ipfsGatewayBaseUrl) : new URL(`${ipfsGatewayBaseUrl}/`)
+  const cidUrl = new URL(cid, baseUrl)
+
+  return cidUrl.toString()
+}