From b87549ecaf229f4304ceb9b468ae336f6a79f47d Mon Sep 17 00:00:00 2001 From: achingbrain Date: Wed, 13 Mar 2024 17:02:04 +0100 Subject: [PATCH 1/2] fix: update @helia/ipns and dns config Updates dns config to allow specifiying per-TLD dns resolvers without needing to add things to the second options object. --- packages/verified-fetch/README.md | 25 +++++++- packages/verified-fetch/package.json | 1 + packages/verified-fetch/src/index.ts | 57 ++++++++++++++++--- .../src/utils/parse-resource.ts | 4 +- .../src/utils/parse-url-string.ts | 6 +- packages/verified-fetch/src/verified-fetch.ts | 11 +--- .../test/custom-dns-resolvers.spec.ts | 32 +++++++++-- .../test/fixtures/create-offline-helia.ts | 6 +- .../test/parse-resource.spec.ts | 2 +- .../test/utils/parse-url-string.spec.ts | 20 +++---- 10 files changed, 123 insertions(+), 41 deletions(-) diff --git a/packages/verified-fetch/README.md b/packages/verified-fetch/README.md index 7783eaa2..bb3ef513 100644 --- a/packages/verified-fetch/README.md +++ b/packages/verified-fetch/README.md @@ -177,7 +177,7 @@ Note that you do not need to provide both a DNS-over-HTTPS and a DNS-over-JSON r ```typescript import { createVerifiedFetch } from '@helia/verified-fetch' -import { dnsJsonOverHttps, dnsOverHttps } from '@helia/ipns/dns-resolvers' +import { dnsJsonOverHttps, dnsOverHttps } from '@multiformats/dns/resolvers' const fetch = await createVerifiedFetch({ gateways: ['https://trustless-gateway.link'], @@ -189,6 +189,29 @@ const fetch = await createVerifiedFetch({ }) ``` +## Example - Customizing DNS per-TLD resolvers + +DNS resolvers can be configured to only service DNS queries for specific +TLDs: + +```typescript +import { createVerifiedFetch } from '@helia/verified-fetch' +import { dnsJsonOverHttps, dnsOverHttps } from '@multiformats/dns/resolvers' + +const fetch = await createVerifiedFetch({ + gateways: ['https://trustless-gateway.link'], + routers: ['http://delegated-ipfs.dev'], + dnsResolvers: { + // this resolver will only be used for `.com` domains (note - this could + // also be an array of resolvers) + 'com.': dnsJsonOverHttps('https://my-dns-resolver.example.com/dns-json'), + // this resolver will be used for everything else (note - this could + // also be an array of resolvers) + '.': dnsOverHttps('https://my-dns-resolver.example.com/dns-query') + } +}) +``` + ### IPLD codec handling IPFS supports several data formats (typically referred to as codecs) which are included in the CID. `@helia/verified-fetch` attempts to abstract away some of the details for easier consumption. diff --git a/packages/verified-fetch/package.json b/packages/verified-fetch/package.json index 38403fe7..3e84571f 100644 --- a/packages/verified-fetch/package.json +++ b/packages/verified-fetch/package.json @@ -70,6 +70,7 @@ "@libp2p/interface": "^1.1.4", "@libp2p/kad-dht": "^12.0.8", "@libp2p/peer-id": "^4.0.7", + "@multiformats/dns": "^1.0.2", "cborg": "^4.0.9", "hashlru": "^2.3.0", "interface-blockstore": "^5.2.10", diff --git a/packages/verified-fetch/src/index.ts b/packages/verified-fetch/src/index.ts index aebbb441..5c95bcb4 100644 --- a/packages/verified-fetch/src/index.ts +++ b/packages/verified-fetch/src/index.ts @@ -148,7 +148,7 @@ * * ```typescript * import { createVerifiedFetch } from '@helia/verified-fetch' - * import { dnsJsonOverHttps, dnsOverHttps } from '@helia/ipns/dns-resolvers' + * import { dnsJsonOverHttps, dnsOverHttps } from '@multiformats/dns/resolvers' * * const fetch = await createVerifiedFetch({ * gateways: ['https://trustless-gateway.link'], @@ -160,6 +160,29 @@ * }) * ``` * + * @example Customizing DNS per-TLD resolvers + * + * DNS resolvers can be configured to only service DNS queries for specific + * TLDs: + * + * ```typescript + * import { createVerifiedFetch } from '@helia/verified-fetch' + * import { dnsJsonOverHttps, dnsOverHttps } from '@multiformats/dns/resolvers' + * + * const fetch = await createVerifiedFetch({ + * gateways: ['https://trustless-gateway.link'], + * routers: ['http://delegated-ipfs.dev'], + * dnsResolvers: { + * // this resolver will only be used for `.com` domains (note - this could + * // also be an array of resolvers) + * 'com.': dnsJsonOverHttps('https://my-dns-resolver.example.com/dns-json'), + * // this resolver will be used for everything else (note - this could + * // also be an array of resolvers) + * '.': dnsOverHttps('https://my-dns-resolver.example.com/dns-query') + * } + * }) + * ``` + * * ### IPLD codec handling * * IPFS supports several data formats (typically referred to as codecs) which are included in the CID. `@helia/verified-fetch` attempts to abstract away some of the details for easier consumption. @@ -569,10 +592,13 @@ import { trustlessGateway } from '@helia/block-brokers' import { createHeliaHTTP } from '@helia/http' import { delegatedHTTPRouting } from '@helia/routers' +import { dns } from '@multiformats/dns' import { VerifiedFetch as VerifiedFetchClass } from './verified-fetch.js' import type { Helia } from '@helia/interface' -import type { DNSResolver, IPNSRoutingEvents, ResolveDnsLinkProgressEvents, ResolveProgressEvents } from '@helia/ipns' +import type { ResolveDNSLinkProgressEvents } from '@helia/ipns' import type { GetEvents } from '@helia/unixfs' +import type { DNSResolvers, DNS } from '@multiformats/dns' +import type { DNSResolver } from '@multiformats/dns/resolvers' import type { CID } from 'multiformats/cid' import type { ProgressEvent, ProgressOptions } from 'progress-events' @@ -618,7 +644,7 @@ export interface CreateVerifiedFetchInit { * * @default [dnsJsonOverHttps('https://mozilla.cloudflare-dns.com/dns-query'),dnsJsonOverHttps('https://dns.google/resolve')] */ - dnsResolvers?: DNSResolver[] + dnsResolvers?: DNSResolver[] | DNSResolvers } export interface CreateVerifiedFetchOptions { @@ -651,7 +677,7 @@ export type BubbledProgressEvents = // unixfs GetEvents | // ipns - ResolveProgressEvents | ResolveDnsLinkProgressEvents | IPNSRoutingEvents + ResolveDNSLinkProgressEvents export type VerifiedFetchProgressEvents = ProgressEvent<'verified-fetch:request:start', CIDDetail> | @@ -674,20 +700,19 @@ export interface VerifiedFetchInit extends RequestInit, ProgressOptions { - let dnsResolvers: DNSResolver[] | undefined if (!isHelia(init)) { - dnsResolvers = init?.dnsResolvers init = await createHeliaHTTP({ blockBrokers: [ trustlessGateway({ gateways: init?.gateways }) ], - routers: (init?.routers ?? ['https://delegated-ipfs.dev']).map((routerUrl) => delegatedHTTPRouting(routerUrl)) + routers: (init?.routers ?? ['https://delegated-ipfs.dev']).map((routerUrl) => delegatedHTTPRouting(routerUrl)), + dns: createDns(init?.dnsResolvers) }) } - const verifiedFetchInstance = new VerifiedFetchClass({ helia: init }, { dnsResolvers, ...options }) + const verifiedFetchInstance = new VerifiedFetchClass({ helia: init }, options) async function verifiedFetch (resource: Resource, options?: VerifiedFetchInit): Promise { return verifiedFetchInstance.fetch(resource, options) } @@ -707,3 +732,19 @@ function isHelia (obj: any): obj is Helia { obj?.stop != null && obj?.start != null } + +function createDns (resolvers?: DNSResolver[] | DNSResolvers): DNS | undefined { + if (resolvers == null) { + return + } + + if (Array.isArray(resolvers)) { + return dns({ + resolvers: { + '.': resolvers + } + }) + } + + return dns({ resolvers }) +} diff --git a/packages/verified-fetch/src/utils/parse-resource.ts b/packages/verified-fetch/src/utils/parse-resource.ts index a20e82d4..49e0b6d3 100644 --- a/packages/verified-fetch/src/utils/parse-resource.ts +++ b/packages/verified-fetch/src/utils/parse-resource.ts @@ -2,7 +2,7 @@ import { CID } from 'multiformats/cid' import { parseUrlString } from './parse-url-string.js' import type { ParsedUrlStringResults } from './parse-url-string.js' import type { Resource } from '../index.js' -import type { IPNS, IPNSRoutingEvents, ResolveDnsLinkProgressEvents, ResolveProgressEvents } from '@helia/ipns' +import type { IPNS, IPNSRoutingEvents, ResolveDNSLinkProgressEvents, ResolveProgressEvents } from '@helia/ipns' import type { ComponentLogger } from '@libp2p/interface' import type { ProgressOptions } from 'progress-events' @@ -11,7 +11,7 @@ export interface ParseResourceComponents { logger: ComponentLogger } -export interface ParseResourceOptions extends ProgressOptions { +export interface ParseResourceOptions extends ProgressOptions { } /** diff --git a/packages/verified-fetch/src/utils/parse-url-string.ts b/packages/verified-fetch/src/utils/parse-url-string.ts index 6b5ff903..6866f7f1 100644 --- a/packages/verified-fetch/src/utils/parse-url-string.ts +++ b/packages/verified-fetch/src/utils/parse-url-string.ts @@ -2,7 +2,7 @@ import { peerIdFromString } from '@libp2p/peer-id' import { CID } from 'multiformats/cid' import { TLRU } from './tlru.js' import type { RequestFormatShorthand } from '../types.js' -import type { IPNS, IPNSRoutingEvents, ResolveDnsLinkProgressEvents, ResolveProgressEvents, ResolveResult } from '@helia/ipns' +import type { IPNS, ResolveDNSLinkProgressEvents, ResolveResult } from '@helia/ipns' import type { ComponentLogger } from '@libp2p/interface' import type { ProgressOptions } from 'progress-events' @@ -13,7 +13,7 @@ export interface ParseUrlStringInput { ipns: IPNS logger: ComponentLogger } -export interface ParseUrlStringOptions extends ProgressOptions { +export interface ParseUrlStringOptions extends ProgressOptions { } @@ -134,7 +134,7 @@ export async function parseUrlString ({ urlString, ipns, logger }: ParseUrlStrin log.trace('Attempting to resolve DNSLink for %s', decodedDnsLinkLabel) try { - resolveResult = await ipns.resolveDns(decodedDnsLinkLabel, { onProgress: options?.onProgress }) + resolveResult = await ipns.resolveDNSLink(decodedDnsLinkLabel, { onProgress: options?.onProgress }) cid = resolveResult?.cid resolvedPath = resolveResult?.path log.trace('resolved %s to %c', decodedDnsLinkLabel, cid) diff --git a/packages/verified-fetch/src/verified-fetch.ts b/packages/verified-fetch/src/verified-fetch.ts index ca8c6dbc..4153d882 100644 --- a/packages/verified-fetch/src/verified-fetch.ts +++ b/packages/verified-fetch/src/verified-fetch.ts @@ -1,6 +1,5 @@ import { car } from '@helia/car' -import { ipns as heliaIpns, type DNSResolver, type IPNS } from '@helia/ipns' -import { dnsJsonOverHttps } from '@helia/ipns/dns-resolvers' +import { ipns as heliaIpns, type IPNS } from '@helia/ipns' import { unixfs as heliaUnixFs, type UnixFS as HeliaUnixFs, type UnixFSStats } from '@helia/unixfs' import * as ipldDagCbor from '@ipld/dag-cbor' import * as ipldDagJson from '@ipld/dag-json' @@ -29,6 +28,7 @@ import type { CIDDetail, ContentTypeParser, Resource, VerifiedFetchInit as Verif import type { RequestFormatShorthand } from './types.js' import type { Helia } from '@helia/interface' import type { AbortOptions, Logger, PeerId } from '@libp2p/interface' +import type { DNSResolver } from '@multiformats/dns/resolvers' import type { UnixFSEntry } from 'ipfs-unixfs-exporter' import type { CID } from 'multiformats/cid' @@ -126,12 +126,7 @@ export class VerifiedFetch { constructor ({ helia, ipns, unixfs }: VerifiedFetchComponents, init?: VerifiedFetchInit) { this.helia = helia this.log = helia.logger.forComponent('helia:verified-fetch') - this.ipns = ipns ?? heliaIpns(helia, { - resolvers: init?.dnsResolvers ?? [ - dnsJsonOverHttps('https://mozilla.cloudflare-dns.com/dns-query'), - dnsJsonOverHttps('https://dns.google/resolve') - ] - }) + this.ipns = ipns ?? heliaIpns(helia) this.unixfs = unixfs ?? heliaUnixFs(helia) this.contentTypeParser = init?.contentTypeParser this.log.trace('created VerifiedFetch instance') diff --git a/packages/verified-fetch/test/custom-dns-resolvers.spec.ts b/packages/verified-fetch/test/custom-dns-resolvers.spec.ts index a37c5292..ae44cde8 100644 --- a/packages/verified-fetch/test/custom-dns-resolvers.spec.ts +++ b/packages/verified-fetch/test/custom-dns-resolvers.spec.ts @@ -1,4 +1,5 @@ import { stop } from '@libp2p/interface' +import { dns, RecordType } from '@multiformats/dns' import { expect } from 'aegir/chai' import Sinon from 'sinon' import { createVerifiedFetch } from '../src/index.js' @@ -30,23 +31,42 @@ describe('custom dns-resolvers', () => { await expect(fetch('ipns://some-non-cached-domain.com')).to.eventually.be.rejected.with.property('errors') expect(customDnsResolver.callCount).to.equal(1) - expect(customDnsResolver.getCall(0).args).to.deep.equal(['some-non-cached-domain.com', { onProgress: undefined }]) + expect(customDnsResolver.getCall(0).args).to.deep.equal(['_dnslink.some-non-cached-domain.com', { + onProgress: undefined, + types: [ + RecordType.TXT + ] + }]) }) it('is used when passed to VerifiedFetch', async () => { - const customDnsResolver = Sinon.stub() + const customDnsResolver = Sinon.stub().withArgs('_dnslink.some-non-cached-domain2.com').resolves({ + Answer: [{ + data: 'dnslink=/ipfs/QmVP2ip92jQuMDezVSzQBWDqWFbp9nyCHNQSiciRauPLDg' + }] + }) - customDnsResolver.returns(Promise.resolve('/ipfs/QmVP2ip92jQuMDezVSzQBWDqWFbp9nyCHNQSiciRauPLDg')) + await stop(helia) + helia = await createHelia({ + dns: dns({ + resolvers: { + '.': customDnsResolver + } + }) + }) const verifiedFetch = new VerifiedFetch({ helia - }, { - dnsResolvers: [customDnsResolver] }) // error of walking the CID/dag because we haven't actually added the block to the blockstore await expect(verifiedFetch.fetch('ipns://some-non-cached-domain2.com')).to.eventually.be.rejected.with.property('errors').that.has.lengthOf(0) expect(customDnsResolver.callCount).to.equal(1) - expect(customDnsResolver.getCall(0).args).to.deep.equal(['some-non-cached-domain2.com', { onProgress: undefined }]) + expect(customDnsResolver.getCall(0).args).to.deep.equal(['_dnslink.some-non-cached-domain2.com', { + onProgress: undefined, + types: [ + RecordType.TXT + ] + }]) }) }) diff --git a/packages/verified-fetch/test/fixtures/create-offline-helia.ts b/packages/verified-fetch/test/fixtures/create-offline-helia.ts index 2c6ab4af..389662ed 100644 --- a/packages/verified-fetch/test/fixtures/create-offline-helia.ts +++ b/packages/verified-fetch/test/fixtures/create-offline-helia.ts @@ -1,9 +1,10 @@ import { Helia as HeliaClass } from '@helia/utils' import { MemoryBlockstore } from 'blockstore-core' import { MemoryDatastore } from 'datastore-core' +import type { HeliaHTTPInit } from '@helia/http' import type { Helia } from '@helia/interface' -export async function createHelia (): Promise { +export async function createHelia (init: Partial = {}): Promise { const datastore = new MemoryDatastore() const blockstore = new MemoryBlockstore() @@ -11,7 +12,8 @@ export async function createHelia (): Promise { datastore, blockstore, blockBrokers: [], - routers: [] + routers: [], + ...init }) await helia.start() diff --git a/packages/verified-fetch/test/parse-resource.spec.ts b/packages/verified-fetch/test/parse-resource.spec.ts index 6e59817f..05a30192 100644 --- a/packages/verified-fetch/test/parse-resource.spec.ts +++ b/packages/verified-fetch/test/parse-resource.spec.ts @@ -13,7 +13,7 @@ describe('parseResource', () => { const shouldNotBeCalled2 = sinon.stub().throws(new Error('should not be called')) const { cid, path, query } = await parseResource(testCID, { ipns: stubInterface({ - resolveDns: shouldNotBeCalled1, + resolveDNSLink: shouldNotBeCalled1, resolve: shouldNotBeCalled2 }), logger: defaultLogger() diff --git a/packages/verified-fetch/test/utils/parse-url-string.spec.ts b/packages/verified-fetch/test/utils/parse-url-string.spec.ts index ad750224..93184f2b 100644 --- a/packages/verified-fetch/test/utils/parse-url-string.spec.ts +++ b/packages/verified-fetch/test/utils/parse-url-string.spec.ts @@ -155,7 +155,7 @@ describe('parseUrlString', () => { describe('ipns:// URLs', () => { it('handles invalid DNSLinkDomains', async () => { ipns.resolve.rejects(new Error('Unexpected failure from ipns resolve method')) - ipns.resolveDns.rejects(new Error('Unexpected failure from ipns dns query')) + ipns.resolveDNSLink.rejects(new Error('Unexpected failure from ipns dns query')) await expect(parseUrlString({ urlString: 'ipns://mydomain.com', ipns, logger })).to.eventually.be.rejected .with.property('errors').that.deep.equals([ @@ -165,7 +165,7 @@ describe('parseUrlString', () => { }) it('can parse a URL with DNSLinkDomain only', async () => { - ipns.resolveDns.withArgs('mydomain.com').resolves({ + ipns.resolveDNSLink.withArgs('mydomain.com').resolves({ cid: CID.parse('QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr'), path: '' }) @@ -181,7 +181,7 @@ describe('parseUrlString', () => { }) it('can parse a URL with DNSLinkDomain+path', async () => { - ipns.resolveDns.withArgs('mydomain.com').resolves({ + ipns.resolveDNSLink.withArgs('mydomain.com').resolves({ cid: CID.parse('QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr'), path: '' }) @@ -197,7 +197,7 @@ describe('parseUrlString', () => { }) it('can parse a URL with DNSLinkDomain+queryString', async () => { - ipns.resolveDns.withArgs('mydomain.com').resolves({ + ipns.resolveDNSLink.withArgs('mydomain.com').resolves({ cid: CID.parse('QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr'), path: '' }) @@ -215,7 +215,7 @@ describe('parseUrlString', () => { }) it('can parse a URL with DNSLinkDomain+path+queryString', async () => { - ipns.resolveDns.withArgs('mydomain.com').resolves({ + ipns.resolveDNSLink.withArgs('mydomain.com').resolves({ cid: CID.parse('QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr'), path: '' }) @@ -233,7 +233,7 @@ describe('parseUrlString', () => { }) it('can parse a URL with DNSLinkDomain+directoryPath+queryString', async () => { - ipns.resolveDns.withArgs('mydomain.com').resolves({ + ipns.resolveDNSLink.withArgs('mydomain.com').resolves({ cid: CID.parse('QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr'), path: '' }) @@ -410,7 +410,7 @@ describe('parseUrlString', () => { it('handles invalid PeerIds', async () => { ipns.resolve.rejects(new Error('Unexpected failure from ipns resolve method')) - ipns.resolveDns.rejects(new Error('Unexpected failure from ipns dns query')) + ipns.resolveDNSLink.rejects(new Error('Unexpected failure from ipns dns query')) await expect(parseUrlString({ urlString: 'ipns://123PeerIdIsFake456', ipns, logger })).to.eventually.be.rejected .with.property('errors').that.deep.equals([ @@ -421,7 +421,7 @@ describe('parseUrlString', () => { it('handles valid PeerId resolve failures', async () => { ipns.resolve.rejects(new Error('Unexpected failure from ipns resolve method')) - ipns.resolveDns.rejects(new Error('Unexpected failure from ipns dns query')) + ipns.resolveDNSLink.rejects(new Error('Unexpected failure from ipns dns query')) await expect(parseUrlString({ urlString: `ipns://${testPeerId}`, ipns, logger })).to.eventually.be.rejected .with.property('errors').that.deep.equals([ @@ -779,12 +779,12 @@ describe('parseUrlString', () => { }) } else if (type === 'dnslink-encoded') { const matchValue = (value as string).replace(/-/g, '.') - ipns.resolveDns.withArgs(match(matchValue)).resolves({ + ipns.resolveDNSLink.withArgs(match(matchValue)).resolves({ cid, path: '' }) } else { - ipns.resolveDns.withArgs(match(value as string)).resolves({ + ipns.resolveDNSLink.withArgs(match(value as string)).resolves({ cid, path: '' }) From 5cba641c1b56d9f64d7a9b53359eae25e12bc2e9 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Thu, 14 Mar 2024 12:00:59 +0100 Subject: [PATCH 2/2] chore: update deps --- packages/verified-fetch/package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/verified-fetch/package.json b/packages/verified-fetch/package.json index 3e84571f..deabd498 100644 --- a/packages/verified-fetch/package.json +++ b/packages/verified-fetch/package.json @@ -61,7 +61,7 @@ "@helia/car": "^3.1.0", "@helia/http": "^1.0.2", "@helia/interface": "^4.0.1", - "@helia/ipns": "^6.0.1", + "@helia/ipns": "^7.0.0", "@helia/routers": "^1.0.1", "@helia/unixfs": "^3.0.1", "@ipld/dag-cbor": "^9.2.0", @@ -89,7 +89,7 @@ "@helia/dag-cbor": "^3.0.1", "@helia/dag-json": "^3.0.1", "@helia/json": "^3.0.1", - "@helia/utils": "^0.0.2", + "@helia/utils": "^0.1.0", "@ipld/car": "^5.3.0", "@libp2p/interface-compliance-tests": "^5.3.2", "@libp2p/logger": "^4.0.7",