From 77c127ab0d7e97fc250c2ae9a6ef9466d0ee4972 Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Wed, 21 Feb 2024 10:24:30 -0800 Subject: [PATCH 01/12] feat(verified-fetch): customize ipns dnsResolvers --- packages/verified-fetch/src/verified-fetch.ts | 5 ++-- .../test/verified-fetch.spec.ts | 27 +++++++++++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/packages/verified-fetch/src/verified-fetch.ts b/packages/verified-fetch/src/verified-fetch.ts index 9f319ad5..89b3b379 100644 --- a/packages/verified-fetch/src/verified-fetch.ts +++ b/packages/verified-fetch/src/verified-fetch.ts @@ -1,4 +1,4 @@ -import { ipns as heliaIpns, type IPNS } from '@helia/ipns' +import { ipns as heliaIpns, type DNSResolver, type IPNS } from '@helia/ipns' import { dnsJsonOverHttps } from '@helia/ipns/dns-resolvers' import { unixfs as heliaUnixFs, type UnixFS as HeliaUnixFs, type UnixFSStats } from '@helia/unixfs' import { code as dagCborCode } from '@ipld/dag-cbor' @@ -32,6 +32,7 @@ interface VerifiedFetchComponents { */ interface VerifiedFetchInit { contentTypeParser?: ContentTypeParser + dnsResolvers?: DNSResolver[] } interface FetchHandlerFunctionArg { @@ -86,7 +87,7 @@ export class VerifiedFetch { this.helia = helia this.log = helia.logger.forComponent('helia:verified-fetch') this.ipns = ipns ?? heliaIpns(helia, { - resolvers: [ + resolvers: init?.dnsResolvers ?? [ dnsJsonOverHttps('https://mozilla.cloudflare-dns.com/dns-query'), dnsJsonOverHttps('https://dns.google/resolve') ] diff --git a/packages/verified-fetch/test/verified-fetch.spec.ts b/packages/verified-fetch/test/verified-fetch.spec.ts index d495c031..8089b996 100644 --- a/packages/verified-fetch/test/verified-fetch.spec.ts +++ b/packages/verified-fetch/test/verified-fetch.spec.ts @@ -523,4 +523,31 @@ describe('@helia/verifed-fetch', () => { await expect(resp.text()).to.eventually.equal('hello world') }) }) + + describe('custom dns-resolvers', () => { + it('uses custom dnsResolvers if provided', async () => { + const customResolver1 = Sinon.stub() + const customResolver2 = Sinon.stub() + const onProgress = Sinon.stub() + + customResolver1.rejects(new Error('Could not resolve PeerId "mydomain.com"')) + customResolver2.returns(Promise.resolve('/ipfs/QmVP2ip92jQuMDezVSzQBWDqWFbp9nyCHNQSiciRauPLDg')) + + const verifiedFetch = new VerifiedFetch({ + helia + }, { + dnsResolvers: [customResolver1, customResolver2] + }) + try { + await verifiedFetch.fetch('ipns://mydomain.com', { onProgress }) + } catch { + // ignoring error of walking the CID because we haven't actually added the block to the blockstore + } + expect(customResolver1.callCount).to.equal(1) + expect(customResolver1.getCall(0).args).to.deep.equal(['mydomain.com', { onProgress }]) + await expect(customResolver1.getCall(0).returnValue).to.eventually.be.rejectedWith('Could not resolve PeerId "mydomain.com"') + expect(customResolver2.callCount).to.equal(1) + expect(customResolver2.getCall(0).args).to.deep.equal(['mydomain.com', { onProgress }]) + }) + }) }) From 29bf8fc697147946e8488729ccf849ae48f6f4c0 Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Wed, 21 Feb 2024 10:28:25 -0800 Subject: [PATCH 02/12] chore: pr comment Co-authored-by: Alex Potsides --- packages/verified-fetch/test/verified-fetch.spec.ts | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/packages/verified-fetch/test/verified-fetch.spec.ts b/packages/verified-fetch/test/verified-fetch.spec.ts index 8089b996..21bd09d3 100644 --- a/packages/verified-fetch/test/verified-fetch.spec.ts +++ b/packages/verified-fetch/test/verified-fetch.spec.ts @@ -538,11 +538,9 @@ describe('@helia/verifed-fetch', () => { }, { dnsResolvers: [customResolver1, customResolver2] }) - try { - await verifiedFetch.fetch('ipns://mydomain.com', { onProgress }) - } catch { - // ignoring error of walking the CID because we haven't actually added the block to the blockstore - } + // ignoring error of walking the CID because we haven't actually added the block to the blockstore + await expect(verifiedFetch.fetch('ipns://mydomain.com', { onProgress })).to.eventually.be.rejected + .with.property('code', 'ERR_EXPECTED_ERR_CODE') expect(customResolver1.callCount).to.equal(1) expect(customResolver1.getCall(0).args).to.deep.equal(['mydomain.com', { onProgress }]) await expect(customResolver1.getCall(0).returnValue).to.eventually.be.rejectedWith('Could not resolve PeerId "mydomain.com"') From fcc5c326c61cb7f6c75777bd48d1e01b80476926 Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Wed, 21 Feb 2024 12:18:15 -0800 Subject: [PATCH 03/12] test: fix dns-resolvers test --- packages/verified-fetch/test/verified-fetch.spec.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/verified-fetch/test/verified-fetch.spec.ts b/packages/verified-fetch/test/verified-fetch.spec.ts index 21bd09d3..1b3366c2 100644 --- a/packages/verified-fetch/test/verified-fetch.spec.ts +++ b/packages/verified-fetch/test/verified-fetch.spec.ts @@ -538,9 +538,9 @@ describe('@helia/verifed-fetch', () => { }, { dnsResolvers: [customResolver1, customResolver2] }) - // ignoring error of walking the CID because we haven't actually added the block to the blockstore - await expect(verifiedFetch.fetch('ipns://mydomain.com', { onProgress })).to.eventually.be.rejected - .with.property('code', 'ERR_EXPECTED_ERR_CODE') + // error of walking the CID/dag because we haven't actually added the block to the blockstore + await expect(verifiedFetch.fetch('ipns://mydomain.com', { onProgress })).to.eventually.be.rejectedWith('All promises were rejected') + expect(customResolver1.callCount).to.equal(1) expect(customResolver1.getCall(0).args).to.deep.equal(['mydomain.com', { onProgress }]) await expect(customResolver1.getCall(0).returnValue).to.eventually.be.rejectedWith('Could not resolve PeerId "mydomain.com"') From fc1e5b820428759ac9fcb8528a668f458b86c355 Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Wed, 21 Feb 2024 12:31:42 -0800 Subject: [PATCH 04/12] chore: harden custom dns resolver test --- .../verified-fetch/test/verified-fetch.spec.ts | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/packages/verified-fetch/test/verified-fetch.spec.ts b/packages/verified-fetch/test/verified-fetch.spec.ts index 1b3366c2..2f03d2e7 100644 --- a/packages/verified-fetch/test/verified-fetch.spec.ts +++ b/packages/verified-fetch/test/verified-fetch.spec.ts @@ -526,26 +526,21 @@ describe('@helia/verifed-fetch', () => { describe('custom dns-resolvers', () => { it('uses custom dnsResolvers if provided', async () => { - const customResolver1 = Sinon.stub() - const customResolver2 = Sinon.stub() + const customDnsResolver = Sinon.stub() const onProgress = Sinon.stub() - customResolver1.rejects(new Error('Could not resolve PeerId "mydomain.com"')) - customResolver2.returns(Promise.resolve('/ipfs/QmVP2ip92jQuMDezVSzQBWDqWFbp9nyCHNQSiciRauPLDg')) + customDnsResolver.returns(Promise.resolve('/ipfs/QmVP2ip92jQuMDezVSzQBWDqWFbp9nyCHNQSiciRauPLDg')) const verifiedFetch = new VerifiedFetch({ helia }, { - dnsResolvers: [customResolver1, customResolver2] + dnsResolvers: [customDnsResolver] }) // error of walking the CID/dag because we haven't actually added the block to the blockstore await expect(verifiedFetch.fetch('ipns://mydomain.com', { onProgress })).to.eventually.be.rejectedWith('All promises were rejected') - expect(customResolver1.callCount).to.equal(1) - expect(customResolver1.getCall(0).args).to.deep.equal(['mydomain.com', { onProgress }]) - await expect(customResolver1.getCall(0).returnValue).to.eventually.be.rejectedWith('Could not resolve PeerId "mydomain.com"') - expect(customResolver2.callCount).to.equal(1) - expect(customResolver2.getCall(0).args).to.deep.equal(['mydomain.com', { onProgress }]) + expect(customDnsResolver.callCount).to.equal(1) + expect(customDnsResolver.getCall(0).args).to.deep.equal(['mydomain.com', { onProgress }]) }) }) }) From 348e4831e75f827715b08c356424319bb316cb0a Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Wed, 21 Feb 2024 16:09:21 -0800 Subject: [PATCH 05/12] test: actually fix the custom dns resolver test The test was passing when ran in isolation, but not in the group. dns caching was causing problems so i customized the url --- packages/verified-fetch/test/verified-fetch.spec.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/verified-fetch/test/verified-fetch.spec.ts b/packages/verified-fetch/test/verified-fetch.spec.ts index 2f03d2e7..97985391 100644 --- a/packages/verified-fetch/test/verified-fetch.spec.ts +++ b/packages/verified-fetch/test/verified-fetch.spec.ts @@ -527,7 +527,6 @@ describe('@helia/verifed-fetch', () => { describe('custom dns-resolvers', () => { it('uses custom dnsResolvers if provided', async () => { const customDnsResolver = Sinon.stub() - const onProgress = Sinon.stub() customDnsResolver.returns(Promise.resolve('/ipfs/QmVP2ip92jQuMDezVSzQBWDqWFbp9nyCHNQSiciRauPLDg')) @@ -537,10 +536,10 @@ describe('@helia/verifed-fetch', () => { dnsResolvers: [customDnsResolver] }) // error of walking the CID/dag because we haven't actually added the block to the blockstore - await expect(verifiedFetch.fetch('ipns://mydomain.com', { onProgress })).to.eventually.be.rejectedWith('All promises were rejected') + await expect(verifiedFetch.fetch('ipns://some-non-cached-domain.com')).to.eventually.be.rejectedWith('All promises were rejected') expect(customDnsResolver.callCount).to.equal(1) - expect(customDnsResolver.getCall(0).args).to.deep.equal(['mydomain.com', { onProgress }]) + expect(customDnsResolver.getCall(0).args).to.deep.equal(['some-non-cached-domain.com', { onProgress: undefined }]) }) }) }) From f1800477056ebc3b2633ebd4cc6f05f91c63855a Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Wed, 21 Feb 2024 16:18:12 -0800 Subject: [PATCH 06/12] chore: firefox promise.any aggregate name is different --- packages/verified-fetch/test/verified-fetch.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/verified-fetch/test/verified-fetch.spec.ts b/packages/verified-fetch/test/verified-fetch.spec.ts index 97985391..5499ca1b 100644 --- a/packages/verified-fetch/test/verified-fetch.spec.ts +++ b/packages/verified-fetch/test/verified-fetch.spec.ts @@ -536,7 +536,7 @@ describe('@helia/verifed-fetch', () => { dnsResolvers: [customDnsResolver] }) // error of walking the CID/dag because we haven't actually added the block to the blockstore - await expect(verifiedFetch.fetch('ipns://some-non-cached-domain.com')).to.eventually.be.rejectedWith('All promises were rejected') + await expect(verifiedFetch.fetch('ipns://some-non-cached-domain.com')).to.eventually.be.rejected.with.property('errors').that.has.lengthOf(0) expect(customDnsResolver.callCount).to.equal(1) expect(customDnsResolver.getCall(0).args).to.deep.equal(['some-non-cached-domain.com', { onProgress: undefined }]) From 8f3459e3307c43b3bcd1809e913b3761cd163ce7 Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Wed, 21 Feb 2024 17:16:59 -0800 Subject: [PATCH 07/12] fix: createVerifiedFetch passes through custom dnsResolvers --- packages/verified-fetch/src/index.ts | 43 ++++++++++++++++++++-- packages/verified-fetch/test/index.spec.ts | 20 ++++++++++ 2 files changed, 59 insertions(+), 4 deletions(-) diff --git a/packages/verified-fetch/src/index.ts b/packages/verified-fetch/src/index.ts index 786f2573..d9c9b184 100644 --- a/packages/verified-fetch/src/index.ts +++ b/packages/verified-fetch/src/index.ts @@ -138,6 +138,29 @@ * }) * ``` * + * ### Custom DNS resolvers + * + * If you don't want to leek DNS queries to the default resolvers, you can provide your own list of DNS resolvers to `createVerifiedFetch`. + * + * Note that you do not need to provide both a DNS-over-HTTPS and a DNS-over-JSON resolver, and you should prefer `dnsJsonOverHttps` resolvers for usage in the browser for a smaller bundle size. See https://github.com/ipfs/helia/tree/main/packages/ipns#example---using-dns-json-over-https for more information. + * + * @example Customizing DNS resolvers + * + * ```typescript + * import { createVerifiedFetch } from '@helia/verified-fetch' + * import { dnsJsonOverHttps, dnsOverHttps } from '@helia/ipns/dns-resolvers' + * + * const fetch = await createVerifiedFetch({ + * gateways: ['https://trustless-gateway.link'], + * routers: ['http://delegated-ipfs.dev'] + * }, { + * dnsResolvers: [ + * dnsJsonOverHttps('https://my-dns-resolver.example.com/dns-json'), + * dnsOverHttps('https://my-dns-resolver.example.com/dns-query') + * ] + * }) + * ``` + * * ### IPLD codec handling * * IPFS supports several data formats (typically referred to as codecs) which are included in the CID. `@helia/verified-fetch` attempts to abstract away some of the details for easier consumption. @@ -439,7 +462,7 @@ import { createHeliaHTTP } from '@helia/http' import { delegatedHTTPRouting } from '@helia/routers' import { VerifiedFetch as VerifiedFetchClass } from './verified-fetch.js' import type { Helia } from '@helia/interface' -import type { IPNSRoutingEvents, ResolveDnsLinkProgressEvents, ResolveProgressEvents } from '@helia/ipns' +import type { DNSResolver, IPNSRoutingEvents, ResolveDnsLinkProgressEvents, ResolveProgressEvents } from '@helia/ipns' import type { GetEvents } from '@helia/unixfs' import type { CID } from 'multiformats/cid' import type { ProgressEvent, ProgressOptions } from 'progress-events' @@ -479,8 +502,22 @@ export interface CreateVerifiedFetchOptions { * provide will be passed the first set of bytes we receive from the network, * and should return a string that will be used as the value for the * `Content-Type` header in the response. + * + * @default undefined */ contentTypeParser?: ContentTypeParser + + /** + * In order to parse DNSLink records, we need to resolve DNS queries. You can + * pass a list of DNS resolvers that we will provide to the @helia/ipns + * instance for you. You must construct them using the `dnsJsonOverHttps` or + * `dnsOverHttps` functions exported from `@helia/ipns/dns-resolvers`. + * + * We use cloudflare and google's dnsJsonOverHttps resolvers by default. + * + * @default [dnsJsonOverHttps('https://mozilla.cloudflare-dns.com/dns-query'),dnsJsonOverHttps('https://dns.google/resolve')] + */ + dnsResolvers?: DNSResolver[] } /** @@ -524,8 +561,6 @@ export interface VerifiedFetchInit extends RequestInit, ProgressOptions { - const contentTypeParser: ContentTypeParser | undefined = options?.contentTypeParser - if (!isHelia(init)) { init = await createHeliaHTTP({ blockBrokers: [ @@ -537,7 +572,7 @@ export async function createVerifiedFetch (init?: Helia | CreateVerifiedFetchIni }) } - const verifiedFetchInstance = new VerifiedFetchClass({ helia: init }, { contentTypeParser }) + const verifiedFetchInstance = new VerifiedFetchClass({ helia: init }, options) async function verifiedFetch (resource: Resource, options?: VerifiedFetchInit): Promise { return verifiedFetchInstance.fetch(resource, options) } diff --git a/packages/verified-fetch/test/index.spec.ts b/packages/verified-fetch/test/index.spec.ts index e79d0437..8043fb94 100644 --- a/packages/verified-fetch/test/index.spec.ts +++ b/packages/verified-fetch/test/index.spec.ts @@ -1,5 +1,6 @@ /* eslint-env mocha */ import { createHeliaHTTP } from '@helia/http' +import { dnsJsonOverHttps, dnsOverHttps } from '@helia/ipns/dns-resolvers' import { expect } from 'aegir/chai' import { createHelia } from 'helia' import { createVerifiedFetch, verifiedFetch } from '../src/index.js' @@ -51,4 +52,23 @@ describe('createVerifiedFetch', () => { expect(verifiedFetch.stop).to.be.a('function') expect(verifiedFetch.start).to.be.a('function') }) + + it('can be passed a contentTypeParser', async () => { + const contentTypeParser = (_bytes: Uint8Array): string => 'application/json' + const verifiedFetch = await createVerifiedFetch(undefined, { + contentTypeParser + }) + expect(verifiedFetch).to.be.ok() + await verifiedFetch.stop() + }) + + it('can be passed custom dnsResolvers', async () => { + const dnsResolver = dnsOverHttps('https://example.com/dns-query') + const dnsJsonResolver = dnsJsonOverHttps('https://example.com/dns-json') + const verifiedFetch = await createVerifiedFetch(undefined, { + dnsResolvers: [dnsResolver, dnsJsonResolver] + }) + expect(verifiedFetch).to.be.ok() + await verifiedFetch.stop() + }) }) From cd9a008730411408bcb6365972d61e9e318f45ca Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Thu, 22 Feb 2024 10:35:45 -0800 Subject: [PATCH 08/12] docs: jsdoc fix from pr comment Co-authored-by: Alex Potsides --- packages/verified-fetch/src/index.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/verified-fetch/src/index.ts b/packages/verified-fetch/src/index.ts index d9c9b184..bd78c3a3 100644 --- a/packages/verified-fetch/src/index.ts +++ b/packages/verified-fetch/src/index.ts @@ -151,13 +151,13 @@ * import { dnsJsonOverHttps, dnsOverHttps } from '@helia/ipns/dns-resolvers' * * const fetch = await createVerifiedFetch({ - * gateways: ['https://trustless-gateway.link'], - * routers: ['http://delegated-ipfs.dev'] + * gateways: ['https://trustless-gateway.link'], + * routers: ['http://delegated-ipfs.dev'] * }, { - * dnsResolvers: [ - * dnsJsonOverHttps('https://my-dns-resolver.example.com/dns-json'), - * dnsOverHttps('https://my-dns-resolver.example.com/dns-query') - * ] + * dnsResolvers: [ + * dnsJsonOverHttps('https://my-dns-resolver.example.com/dns-json'), + * dnsOverHttps('https://my-dns-resolver.example.com/dns-query') + * ] * }) * ``` * From a7b5b89020c72cd8aa8852e84530f3dc2640bc92 Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Fri, 23 Feb 2024 12:56:38 -0800 Subject: [PATCH 09/12] test: move custom-dns-resolvers tests --- packages/verified-fetch/src/verified-fetch.ts | 2 +- .../test/custom-dns-resolvers.spec.ts | 51 +++++++++++++++++++ packages/verified-fetch/test/index.spec.ts | 10 ---- .../test/verified-fetch.spec.ts | 19 ------- 4 files changed, 52 insertions(+), 30 deletions(-) create mode 100644 packages/verified-fetch/test/custom-dns-resolvers.spec.ts diff --git a/packages/verified-fetch/src/verified-fetch.ts b/packages/verified-fetch/src/verified-fetch.ts index ec03e626..8935ad16 100644 --- a/packages/verified-fetch/src/verified-fetch.ts +++ b/packages/verified-fetch/src/verified-fetch.ts @@ -1,5 +1,5 @@ import { car } from '@helia/car' -import { ipns as heliaIpns, type IPNS } from '@helia/ipns' +import { ipns as heliaIpns, type DNSResolver, type IPNS } from '@helia/ipns' import { dnsJsonOverHttps } from '@helia/ipns/dns-resolvers' import { unixfs as heliaUnixFs, type UnixFS as HeliaUnixFs, type UnixFSStats } from '@helia/unixfs' import * as ipldDagCbor from '@ipld/dag-cbor' diff --git a/packages/verified-fetch/test/custom-dns-resolvers.spec.ts b/packages/verified-fetch/test/custom-dns-resolvers.spec.ts new file mode 100644 index 00000000..29b6888f --- /dev/null +++ b/packages/verified-fetch/test/custom-dns-resolvers.spec.ts @@ -0,0 +1,51 @@ +import { stop } from '@libp2p/interface' +import { expect } from 'aegir/chai' +import Sinon from 'sinon' +import { createVerifiedFetch } from '../src/index.js' +import { VerifiedFetch } from '../src/verified-fetch.js' +import { createHelia } from './fixtures/create-offline-helia.js' +import type { Helia } from '@helia/interface' + +describe('custom dns-resolvers', () => { + let helia: Helia + + beforeEach(async () => { + helia = await createHelia() + }) + + afterEach(async () => { + await stop(helia) + }) + + it('is used when passed to createVerifiedFetch', async () => { + const customDnsResolver = Sinon.stub() + + customDnsResolver.returns(Promise.resolve('/ipfs/QmVP2ip92jQuMDezVSzQBWDqWFbp9nyCHNQSiciRauPLDg')) + + const fetch = await createVerifiedFetch(helia, { + dnsResolvers: [customDnsResolver] + }) + // error of walking the CID/dag because we haven't actually added the block to the blockstore + await expect(fetch('ipns://some-non-cached-domain.com')).to.eventually.be.rejected.with.property('errors').that.has.lengthOf(0) + + expect(customDnsResolver.callCount).to.equal(1) + expect(customDnsResolver.getCall(0).args).to.deep.equal(['some-non-cached-domain.com', { onProgress: undefined }]) + }) + + it('is used when passed to VerifiedFetch', async () => { + const customDnsResolver = Sinon.stub() + + customDnsResolver.returns(Promise.resolve('/ipfs/QmVP2ip92jQuMDezVSzQBWDqWFbp9nyCHNQSiciRauPLDg')) + + const verifiedFetch = new VerifiedFetch({ + helia + }, { + dnsResolvers: [customDnsResolver] + }) + // error of walking the CID/dag because we haven't actually added the block to the blockstore + await expect(verifiedFetch.fetch('ipns://some-non-cached-domain2.com')).to.eventually.be.rejected.with.property('errors').that.has.lengthOf(0) + + expect(customDnsResolver.callCount).to.equal(1) + expect(customDnsResolver.getCall(0).args).to.deep.equal(['some-non-cached-domain2.com', { onProgress: undefined }]) + }) +}) diff --git a/packages/verified-fetch/test/index.spec.ts b/packages/verified-fetch/test/index.spec.ts index 68f5e99a..b5cf11e8 100644 --- a/packages/verified-fetch/test/index.spec.ts +++ b/packages/verified-fetch/test/index.spec.ts @@ -60,14 +60,4 @@ describe('createVerifiedFetch', () => { expect(verifiedFetch).to.be.ok() await verifiedFetch.stop() }) - - it('can be passed custom dnsResolvers', async () => { - const dnsResolver = dnsOverHttps('https://example.com/dns-query') - const dnsJsonResolver = dnsJsonOverHttps('https://example.com/dns-json') - const verifiedFetch = await createVerifiedFetch(undefined, { - dnsResolvers: [dnsResolver, dnsJsonResolver] - }) - expect(verifiedFetch).to.be.ok() - await verifiedFetch.stop() - }) }) diff --git a/packages/verified-fetch/test/verified-fetch.spec.ts b/packages/verified-fetch/test/verified-fetch.spec.ts index bbe454b8..9dc49c7e 100644 --- a/packages/verified-fetch/test/verified-fetch.spec.ts +++ b/packages/verified-fetch/test/verified-fetch.spec.ts @@ -551,23 +551,4 @@ describe('@helia/verifed-fetch', () => { expect(output).to.deep.equal(obj) }) }) - - describe('custom dns-resolvers', () => { - it('uses custom dnsResolvers if provided', async () => { - const customDnsResolver = Sinon.stub() - - customDnsResolver.returns(Promise.resolve('/ipfs/QmVP2ip92jQuMDezVSzQBWDqWFbp9nyCHNQSiciRauPLDg')) - - const verifiedFetch = new VerifiedFetch({ - helia - }, { - dnsResolvers: [customDnsResolver] - }) - // error of walking the CID/dag because we haven't actually added the block to the blockstore - await expect(verifiedFetch.fetch('ipns://some-non-cached-domain.com')).to.eventually.be.rejected.with.property('errors').that.has.lengthOf(0) - - expect(customDnsResolver.callCount).to.equal(1) - expect(customDnsResolver.getCall(0).args).to.deep.equal(['some-non-cached-domain.com', { onProgress: undefined }]) - }) - }) }) From 07a4ad414f052cfdb1f8e1a2bb883e8666dd7b41 Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Fri, 23 Feb 2024 13:00:14 -0800 Subject: [PATCH 10/12] test: move content-type-parser test --- .../verified-fetch/test/content-type-parser.spec.ts | 13 +++++++++++++ packages/verified-fetch/test/index.spec.ts | 10 ---------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/packages/verified-fetch/test/content-type-parser.spec.ts b/packages/verified-fetch/test/content-type-parser.spec.ts index 8754c9f2..b614529b 100644 --- a/packages/verified-fetch/test/content-type-parser.spec.ts +++ b/packages/verified-fetch/test/content-type-parser.spec.ts @@ -4,7 +4,9 @@ import { stop } from '@libp2p/interface' import { fileTypeFromBuffer } from '@sgtpooki/file-type' import { expect } from 'aegir/chai' import { filetypemime } from 'magic-bytes.js' +import Sinon from 'sinon' import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' +import { createVerifiedFetch } from '../src/index.js' import { VerifiedFetch } from '../src/verified-fetch.js' import type { Helia } from '@helia/interface' import type { CID } from 'multiformats/cid' @@ -26,6 +28,17 @@ describe('content-type-parser', () => { await stop(verifiedFetch) }) + it('is used when passed to createVerifiedFetch', async () => { + const contentTypeParser = Sinon.stub().resolves('text/plain') + const fetch = await createVerifiedFetch(helia, { + contentTypeParser + }) + expect(fetch).to.be.ok() + const resp = await fetch(cid) + expect(resp.headers.get('content-type')).to.equal('text/plain') + await fetch.stop() + }) + it('sets default content type if contentTypeParser is not passed', async () => { verifiedFetch = new VerifiedFetch({ helia diff --git a/packages/verified-fetch/test/index.spec.ts b/packages/verified-fetch/test/index.spec.ts index b5cf11e8..90cd1054 100644 --- a/packages/verified-fetch/test/index.spec.ts +++ b/packages/verified-fetch/test/index.spec.ts @@ -1,5 +1,4 @@ import { createHeliaHTTP } from '@helia/http' -import { dnsJsonOverHttps, dnsOverHttps } from '@helia/ipns/dns-resolvers' import { expect } from 'aegir/chai' import { createHelia } from 'helia' import { createVerifiedFetch, verifiedFetch } from '../src/index.js' @@ -51,13 +50,4 @@ describe('createVerifiedFetch', () => { expect(verifiedFetch.stop).to.be.a('function') expect(verifiedFetch.start).to.be.a('function') }) - - it('can be passed a contentTypeParser', async () => { - const contentTypeParser = (_bytes: Uint8Array): string => 'application/json' - const verifiedFetch = await createVerifiedFetch(undefined, { - contentTypeParser - }) - expect(verifiedFetch).to.be.ok() - await verifiedFetch.stop() - }) }) From 17fdb486bdd829b608c0a17b7049ec6aab60e9eb Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Mon, 26 Feb 2024 09:16:08 -0800 Subject: [PATCH 11/12] docs: typo Co-authored-by: Alex Potsides --- packages/verified-fetch/src/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/verified-fetch/src/index.ts b/packages/verified-fetch/src/index.ts index cb388ec5..6eb558c5 100644 --- a/packages/verified-fetch/src/index.ts +++ b/packages/verified-fetch/src/index.ts @@ -140,7 +140,7 @@ * * ### Custom DNS resolvers * - * If you don't want to leek DNS queries to the default resolvers, you can provide your own list of DNS resolvers to `createVerifiedFetch`. + * If you don't want to leak DNS queries to the default resolvers, you can provide your own list of DNS resolvers to `createVerifiedFetch`. * * Note that you do not need to provide both a DNS-over-HTTPS and a DNS-over-JSON resolver, and you should prefer `dnsJsonOverHttps` resolvers for usage in the browser for a smaller bundle size. See https://github.com/ipfs/helia/tree/main/packages/ipns#example---using-dns-json-over-https for more information. * From cca38414603c7c27e4d2251c80638d2a16a6eef4 Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Mon, 26 Feb 2024 12:56:52 -0700 Subject: [PATCH 12/12] fix: dnsResolvers is passed in first arg to createVerifiedFetch --- packages/verified-fetch/src/index.ts | 31 ++++++++++--------- .../test/custom-dns-resolvers.spec.ts | 5 +-- 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/packages/verified-fetch/src/index.ts b/packages/verified-fetch/src/index.ts index 6eb558c5..ef54ce29 100644 --- a/packages/verified-fetch/src/index.ts +++ b/packages/verified-fetch/src/index.ts @@ -152,8 +152,7 @@ * * const fetch = await createVerifiedFetch({ * gateways: ['https://trustless-gateway.link'], - * routers: ['http://delegated-ipfs.dev'] - * }, { + * routers: ['http://delegated-ipfs.dev'], * dnsResolvers: [ * dnsJsonOverHttps('https://my-dns-resolver.example.com/dns-json'), * dnsOverHttps('https://my-dns-resolver.example.com/dns-query') @@ -531,18 +530,6 @@ export interface VerifiedFetch { export interface CreateVerifiedFetchInit { gateways: string[] routers?: string[] -} - -export interface CreateVerifiedFetchOptions { - /** - * A function to handle parsing content type from bytes. The function you - * provide will be passed the first set of bytes we receive from the network, - * and should return a string that will be used as the value for the - * `Content-Type` header in the response. - * - * @default undefined - */ - contentTypeParser?: ContentTypeParser /** * In order to parse DNSLink records, we need to resolve DNS queries. You can @@ -557,6 +544,18 @@ export interface CreateVerifiedFetchOptions { dnsResolvers?: DNSResolver[] } +export interface CreateVerifiedFetchOptions { + /** + * A function to handle parsing content type from bytes. The function you + * provide will be passed the first set of bytes we receive from the network, + * and should return a string that will be used as the value for the + * `Content-Type` header in the response. + * + * @default undefined + */ + contentTypeParser?: ContentTypeParser +} + /** * A ContentTypeParser attempts to return the mime type of a given file. It * receives the first chunk of the file data and the file name, if it is @@ -598,7 +597,9 @@ export interface VerifiedFetchInit extends RequestInit, ProgressOptions { + let dnsResolvers: DNSResolver[] | undefined if (!isHelia(init)) { + dnsResolvers = init?.dnsResolvers init = await createHeliaHTTP({ blockBrokers: [ trustlessGateway({ @@ -609,7 +610,7 @@ export async function createVerifiedFetch (init?: Helia | CreateVerifiedFetchIni }) } - const verifiedFetchInstance = new VerifiedFetchClass({ helia: init }, options) + const verifiedFetchInstance = new VerifiedFetchClass({ helia: init }, { dnsResolvers, ...options }) async function verifiedFetch (resource: Resource, options?: VerifiedFetchInit): Promise { return verifiedFetchInstance.fetch(resource, options) } diff --git a/packages/verified-fetch/test/custom-dns-resolvers.spec.ts b/packages/verified-fetch/test/custom-dns-resolvers.spec.ts index 29b6888f..a37c5292 100644 --- a/packages/verified-fetch/test/custom-dns-resolvers.spec.ts +++ b/packages/verified-fetch/test/custom-dns-resolvers.spec.ts @@ -22,11 +22,12 @@ describe('custom dns-resolvers', () => { customDnsResolver.returns(Promise.resolve('/ipfs/QmVP2ip92jQuMDezVSzQBWDqWFbp9nyCHNQSiciRauPLDg')) - const fetch = await createVerifiedFetch(helia, { + const fetch = await createVerifiedFetch({ + gateways: ['http://127.0.0.1:8080'], dnsResolvers: [customDnsResolver] }) // error of walking the CID/dag because we haven't actually added the block to the blockstore - await expect(fetch('ipns://some-non-cached-domain.com')).to.eventually.be.rejected.with.property('errors').that.has.lengthOf(0) + await expect(fetch('ipns://some-non-cached-domain.com')).to.eventually.be.rejected.with.property('errors') expect(customDnsResolver.callCount).to.equal(1) expect(customDnsResolver.getCall(0).args).to.deep.equal(['some-non-cached-domain.com', { onProgress: undefined }])