From 7291eab02e67a9d126c16925c8097ee0821a4f9b Mon Sep 17 00:00:00 2001 From: Yusef Napora Date: Wed, 10 Mar 2021 12:56:44 -0500 Subject: [PATCH 01/21] stub out remote pin api --- .../ipfs-core/src/components/pin/index.js | 2 + .../src/components/pin/remote/index.js | 131 ++++++++++++++++++ 2 files changed, 133 insertions(+) create mode 100644 packages/ipfs-core/src/components/pin/remote/index.js diff --git a/packages/ipfs-core/src/components/pin/index.js b/packages/ipfs-core/src/components/pin/index.js index 2e3ee7c66b..2e3cb04aec 100644 --- a/packages/ipfs-core/src/components/pin/index.js +++ b/packages/ipfs-core/src/components/pin/index.js @@ -5,6 +5,7 @@ const createAddAll = require('./add-all') const createLs = require('./ls') const createRm = require('./rm') const createRmAll = require('./rm-all') +const RemotePinAPI = require('./remote') class PinAPI { /** @@ -21,6 +22,7 @@ class PinAPI { this.rmAll = rmAll this.rm = createRm({ rmAll }) this.ls = createLs({ dagReader, pinManager }) + this.remote = new RemotePinAPI() } } module.exports = PinAPI diff --git a/packages/ipfs-core/src/components/pin/remote/index.js b/packages/ipfs-core/src/components/pin/remote/index.js new file mode 100644 index 0000000000..ecd1d533e0 --- /dev/null +++ b/packages/ipfs-core/src/components/pin/remote/index.js @@ -0,0 +1,131 @@ +'use strict' + +/** + * @typedef {import('cids')} CID + * + * @typedef {'queued'|'pinning'|'pinned'|'failed'} PinStatus + * + * @typedef {object} RemotePin + * @property {string} [cid] + * @property {PinStatus} [status] + * @property {?string} [name] + * @property {?object} [meta] + */ + +class RemotePinAPI { + + constructor() { + this.service = new RemotePinServiceAPI() + } + + /** + * Asks a remote pinning service to pin an IPFS object from a given path + * + * @typedef {object} RemotePinAddOptions + * @property {string} [service] name of a configured remote pinning service + * @property {?string} [name] optional descriptive name for pin + * @property {?Object} [meta] optional metadata to attach to pin + * @property {?number} [timeout] request timeout (seconds) + * @property {?boolean} [background] If true, add returns remote a pin object as soon as the remote service responds. + * The returned pin object may have a status of 'queued' or 'pinning'. + * If false, the add method will not resolve until the pin status is 'pinned' before returning. + * When background==false and the remote service returns a status of 'failed', an Error will be thrown. + * + * @param {string|CID} cid + * @param {RemotePinAddOptions} options + * @returns {Promise} + */ + async add(cid, options) { + throw new Error('not yet implemented') + } + + /** + * List objects that are pinned by a remote service. + * + * @typedef {object} RemotePinLsOptions + * @property {string} [service] name of a configured remote pinning service + * @property {?Array} [cid] return pins for the specified CID(s) + * @property {?string} [name] return pins that contain the provided value (case-sensitive, exact match) + * @property {?Array} [status] return pins with the specified statuses (queued, pinning, pinned, failed). Default: pinned + * @property {?number} [timeout] request timeout (seconds) + + * @param {RemotePinLsOptions} options + * @returns {AsyncGenerator} + */ + async * ls(options) { + throw new Error('not yet implemented') + } + + /** + * Remove a single pin from a remote pinning service. + * Fails if multiple pins match the specified criteria. Use rmAll to remove all pins that match. + * + * @typedef {object} RemotePinRmOptions + * @property {string} [service] name of a configured remote pinning service + * @property {Array} [cid] CID(s) to remove from remote pinning service + * @property {?Array} [status] only remove pins that have one of the specified statuses (queued, pinning, pinned, failed). Default: pinned + * @property {?number} [timeout] request timeout (seconds) + * + * @param {RemotePinRmOptions} options + * @returns {Promise} + */ + async rm(options) { + throw new Error('not yet implemented') + } + + /** + * Remove all pins that match the given criteria from a remote pinning service. + * + * @param {RemotePinRmOptions} options + * @returns {Promise} + */ + async rmAll(options) { + throw new Error('not yet implemented') + } +} + +/** + * RemotePinServiceAPI provides methods to add, remove, and list the configured + * remote pinning services that are used by the RemotePinAPI. + */ +class RemotePinServiceAPI { + + /** + * Adds a new remote pinning service to the set of configured services. + * + * @typedef {object} RemotePinServiceAddOptions + * @property {string|URL} [endpoint] the remote API endpoint URL + * @property {string} [key] an API key that authorizes use of the remote pinning service + * + * @param {string} name the name of the pinning service. Used to identify the service in future remote pinning API calls. + * @param {RemotePinServiceAddOptions} options + */ + async add(name, options) { + throw new Error('not yet implemented') + } + + /** + * List the configured remote pinning services. + * + * @typedef {object} RemotePinningServiceDescription + * @property {string} name + * @property {URL} endpoint + * + * @return {Promise>} + */ + async ls() { + throw new Error('not yet implemented') + } + + /** + * Remove a remote pinning service from the set of configured services. + * + * @param {string} name the name of the pinning service to remove + * @returns {Promise} + */ + async rm(name) { + throw new Error('not yet implemented') + } +} + +module.exports = RemotePinAPI From 9c9a1e7eed12b2536f3e3998c19900da619d10e4 Mon Sep 17 00:00:00 2001 From: Yusef Napora Date: Wed, 10 Mar 2021 15:51:55 -0500 Subject: [PATCH 02/21] implement remote pinning API --- packages/ipfs-core/package.json | 1 + packages/ipfs-core/src/components/index.js | 10 +- .../ipfs-core/src/components/pin/index.js | 7 +- .../src/components/pin/remote/index.js | 405 +++++++++++++++--- 4 files changed, 363 insertions(+), 60 deletions(-) diff --git a/packages/ipfs-core/package.json b/packages/ipfs-core/package.json index 7adb43adbc..a106f20ea4 100644 --- a/packages/ipfs-core/package.json +++ b/packages/ipfs-core/package.json @@ -89,6 +89,7 @@ "it-first": "^1.0.4", "it-last": "^1.0.4", "it-pipe": "^1.1.0", + "js-ipfs-pinning-service-client": "github:yusefnapora/js-ipfs-pinning-service-client#master", "libp2p": "^0.30.7", "libp2p-bootstrap": "^0.12.1", "libp2p-crypto": "^0.19.0", diff --git a/packages/ipfs-core/src/components/index.js b/packages/ipfs-core/src/components/index.js index 20652c29a5..498094428b 100644 --- a/packages/ipfs-core/src/components/index.js +++ b/packages/ipfs-core/src/components/index.js @@ -29,6 +29,7 @@ const createIDAPI = require('./id') const createConfigAPI = require('./config') const DagAPI = require('./dag') const PinManagerAPI = require('./pin/pin-manager') +const PinRemoteAPI = require('./pin/remote') const createPreloadAPI = require('../preload') const createMfsPreloadAPI = require('../mfs-preload') const createFilesAPI = require('./files') @@ -57,6 +58,8 @@ class IPFS { constructor ({ print, storage, options }) { const { peerId, repo, keychain } = storage const network = Service.create(Network) + const swarm = new SwarmAPI({ network }) + const config = createConfigAPI({ repo }) const preload = createPreloadAPI(options.preload) @@ -86,7 +89,8 @@ class IPFS { }) const resolve = createResolveAPI({ ipld, name }) const pinManager = new PinManagerAPI({ repo, dagReader }) - const pin = new PinAPI({ gcLock, pinManager, dagReader }) + const pinRemote = new PinRemoteAPI({ swarm, config, peerId }) + const pin = new PinAPI({ gcLock, pinManager, dagReader, pinRemote }) const block = new BlockAPI({ blockService, preload, gcLock, pinManager, pin }) const dag = new DagAPI({ ipld, preload, gcLock, pin, dagReader }) const refs = Object.assign(createRefsAPI({ ipld, resolve, preload }), { @@ -155,7 +159,7 @@ class IPFS { this.version = createVersionAPI({ repo }) this.bitswap = new BitswapAPI({ network }) this.bootstrap = new BootstrapAPI({ repo }) - this.config = createConfigAPI({ repo }) + this.config = config this.ping = createPingAPI({ network }) this.add = add @@ -170,7 +174,7 @@ class IPFS { this.object = new ObjectAPI({ ipld, preload, gcLock, dag }) this.repo = new RepoAPI({ gcLock, pin, repo, refs }) this.stats = new StatsAPI({ repo, network }) - this.swarm = new SwarmAPI({ network }) + this.swarm = swarm // For the backwards compatibility Object.defineProperty(this, 'libp2p', { diff --git a/packages/ipfs-core/src/components/pin/index.js b/packages/ipfs-core/src/components/pin/index.js index 2e3cb04aec..beeb582b55 100644 --- a/packages/ipfs-core/src/components/pin/index.js +++ b/packages/ipfs-core/src/components/pin/index.js @@ -5,7 +5,6 @@ const createAddAll = require('./add-all') const createLs = require('./ls') const createRm = require('./rm') const createRmAll = require('./rm-all') -const RemotePinAPI = require('./remote') class PinAPI { /** @@ -13,8 +12,9 @@ class PinAPI { * @param {GCLock} config.gcLock * @param {DagReader} config.dagReader * @param {PinManager} config.pinManager + * @param {PinRemoteAPI} config.pinRemote */ - constructor ({ gcLock, dagReader, pinManager }) { + constructor ({ gcLock, dagReader, pinManager, pinRemote }) { const addAll = createAddAll({ gcLock, dagReader, pinManager }) this.addAll = addAll this.add = createAdd({ addAll }) @@ -22,7 +22,7 @@ class PinAPI { this.rmAll = rmAll this.rm = createRm({ rmAll }) this.ls = createLs({ dagReader, pinManager }) - this.remote = new RemotePinAPI() + this.remote = pinRemote } } module.exports = PinAPI @@ -34,4 +34,5 @@ module.exports = PinAPI * @typedef {import('..').PinManager} PinManager * @typedef {import('..').AbortOptions} AbortOptions * @typedef {import('..').CID} CID + * @typedef {import('./remote')} PinRemoteAPI */ diff --git a/packages/ipfs-core/src/components/pin/remote/index.js b/packages/ipfs-core/src/components/pin/remote/index.js index ecd1d533e0..d1dca3137e 100644 --- a/packages/ipfs-core/src/components/pin/remote/index.js +++ b/packages/ipfs-core/src/components/pin/remote/index.js @@ -1,120 +1,169 @@ 'use strict' +const multiaddr = require('multiaddr') +const PinningClient = require('js-ipfs-pinning-service-client') + /** - * @typedef {import('cids')} CID - * - * @typedef {'queued'|'pinning'|'pinned'|'failed'} PinStatus - * - * @typedef {object} RemotePin - * @property {string} [cid] - * @property {PinStatus} [status] - * @property {?string} [name] - * @property {?object} [meta] + * PinRemoteAPI provides an API for pinning content to remote services. */ +class PinRemoteAPI { -class RemotePinAPI { - - constructor() { - this.service = new RemotePinServiceAPI() + /** + * @param {Object} opts + * @param {SwarmAPI} opts.swarm + * @param {Config} opts.config + * @param {PeerId} opts.peerId + */ + constructor({swarm, config, peerId}) { + this.swarm = swarm + this.service = new PinRemoteServiceAPI({config, swarm, peerId}) } /** * Asks a remote pinning service to pin an IPFS object from a given path * - * @typedef {object} RemotePinAddOptions - * @property {string} [service] name of a configured remote pinning service - * @property {?string} [name] optional descriptive name for pin - * @property {?Object} [meta] optional metadata to attach to pin - * @property {?number} [timeout] request timeout (seconds) - * @property {?boolean} [background] If true, add returns remote a pin object as soon as the remote service responds. + * @param {string|CID} cid + * @param {object} options + * @param {string} options.service name of a configured remote pinning service + * @param {?string} options.name optional descriptive name for pin + * @param {?Object} options.meta optional metadata to attach to pin + * @param {?number} options.timeout request timeout (seconds) + * @param {?boolean} options.background If true, add returns remote a pin object as soon as the remote service responds. * The returned pin object may have a status of 'queued' or 'pinning'. * If false, the add method will not resolve until the pin status is 'pinned' before returning. * When background==false and the remote service returns a status of 'failed', an Error will be thrown. - * - * @param {string|CID} cid - * @param {RemotePinAddOptions} options * @returns {Promise} */ async add(cid, options) { - throw new Error('not yet implemented') + const {service, ...addOpts} = options + if (!service) { + throw new Error('service name must be passed') + } + const svc = this.service.serviceNamed(service) + return svc.add(cid, addOpts) } /** * List objects that are pinned by a remote service. * - * @typedef {object} RemotePinLsOptions - * @property {string} [service] name of a configured remote pinning service - * @property {?Array} [cid] return pins for the specified CID(s) - * @property {?string} [name] return pins that contain the provided value (case-sensitive, exact match) - * @property {?Array} [status] return pins with the specified statuses (queued, pinning, pinned, failed). Default: pinned - * @property {?number} [timeout] request timeout (seconds) - - * @param {RemotePinLsOptions} options + * @param {object} options + * @param {string} options.service name of a configured remote pinning service + * @param {?Array} options.cid return pins for the specified CID(s) + * @param {?string} options.name return pins that contain the provided value (case-sensitive, exact match) + * @param {?Array} options.status return pins with the specified statuses (queued, pinning, pinned, failed). Default: pinned + * @param {?number} options.timeout request timeout (seconds) * @returns {AsyncGenerator} */ async * ls(options) { - throw new Error('not yet implemented') + const {service, ...lsOpts} = options + if (!service) { + throw new Error('service name must be passed') + } + const svc = this.service.serviceNamed(service) + return svc.ls(lsOpts) } /** * Remove a single pin from a remote pinning service. * Fails if multiple pins match the specified criteria. Use rmAll to remove all pins that match. * - * @typedef {object} RemotePinRmOptions - * @property {string} [service] name of a configured remote pinning service - * @property {Array} [cid] CID(s) to remove from remote pinning service - * @property {?Array} [status] only remove pins that have one of the specified statuses (queued, pinning, pinned, failed). Default: pinned - * @property {?number} [timeout] request timeout (seconds) - * - * @param {RemotePinRmOptions} options + * @param {object} options + * @param {string} options.service name of a configured remote pinning service + * @param {Array} options.cid CID(s) to remove from remote pinning service + * @param {?Array} options.status only remove pins that have one of the specified statuses (queued, pinning, pinned, failed). Default: pinned + * @param {?number} options.timeout request timeout (seconds) * @returns {Promise} */ async rm(options) { - throw new Error('not yet implemented') + const {service, ...rmOpts} = options + if (!service) { + throw new Error('service name must be passed') + } + const svc = this.service.serviceNamed(service) + return svc.rm(rmOpts) } /** * Remove all pins that match the given criteria from a remote pinning service. * - * @param {RemotePinRmOptions} options + * @param {object} options + * @param {string} options.service name of a configured remote pinning service + * @param {Array} options.cid CID(s) to remove from remote pinning service + * @param {?Array} options.status only remove pins that have one of the specified statuses (queued, pinning, pinned, failed). Default: pinned + * @param {?number} options.timeout request timeout (seconds) * @returns {Promise} */ async rmAll(options) { - throw new Error('not yet implemented') + const {service, ...rmOpts} = options + if (!service) { + throw new Error('service name must be passed') } + const svc = this.service.serviceNamed(service) + return svc.rmAll(rmOpts) + } } /** * RemotePinServiceAPI provides methods to add, remove, and list the configured - * remote pinning services that are used by the RemotePinAPI. + * remote pinning services that are used by the remote pinning api. */ -class RemotePinServiceAPI { +class PinRemoteServiceAPI { + + constructor({config, swarm, peerId}) { + this.config = config + this.swarm = swarm + this.peerId = peerId + + // TODO: read service config from IPFS config to construct remote service at init + this._services = {} + } /** * Adds a new remote pinning service to the set of configured services. * - * @typedef {object} RemotePinServiceAddOptions - * @property {string|URL} [endpoint] the remote API endpoint URL - * @property {string} [key] an API key that authorizes use of the remote pinning service - * * @param {string} name the name of the pinning service. Used to identify the service in future remote pinning API calls. - * @param {RemotePinServiceAddOptions} options + * @param {Object} options + * @param {string|URL} options.endpoint the remote API endpoint URL + * @param {string} options.key an API key that authorizes use of the remote pinning service */ async add(name, options) { - throw new Error('not yet implemented') + if (this._services[name]) { + throw new Error('service already present: ' + name) + } + + const svcOpts = Object.assign({swarm: this.swarm, peerId: this.peerId}, options) + this._services[name] = new RemotePinningService(name, svcOpts) } /** * List the configured remote pinning services. * + * @typedef {object} PinCounts + * @property {number} queued + * @property {number} pinning + * @property {number} pinned + * @property {number} failed + * * @typedef {object} RemotePinningServiceDescription * @property {string} name * @property {URL} endpoint + * @property {?object} stat + * @property {PinServiceStatus} stat.status + * @property {PinCounts} stat.pinCount * + * @param {object} opts + * @param {?boolean} opts.stat if true, include status info for each pinning service * @return {Promise>} */ - async ls() { - throw new Error('not yet implemented') + async ls(opts) { + const {stat} = (opts || {}) + + const promises = [] + for (const name of this._services.keys()) { + const svc = this._services[name] + promises.push(svc.info(stat)) + } + return Promise.all(promises) } /** @@ -124,8 +173,256 @@ class RemotePinServiceAPI { * @returns {Promise} */ async rm(name) { - throw new Error('not yet implemented') + delete this._services[name] + } + + /** + * Returns a RemotePinningService object for the given service name. Throws if no service has been configured with the given name. + * @param {string} name + * @returns {RemotePinningService} + */ + serviceNamed(name) { + if (!this._services[name]) { + throw new Error('no remote pinning service configured with name: ' + name) + } + return this._services[name] + } +} + + +/** + * RemotePinningService provides add, ls, and rm operations for a single remote pinning service. + */ + class RemotePinningService { + + /** + * + * @param {string} name pinning service name + * @param {object} config + * @param {string|URL} config.endpoint the remote API endpoint URL + * @param {string} config.key an API key that authorizes use of the remote pinning service + * @param {SwarmAPI} config.swarm + * @param {PeerId} config.peerId + */ + constructor(name, {endpoint, key, swarm, peerId}) { + this.name = name + this.endpoint = endpoint + this.swarm = swarm + this.peerId = peerId + this.client = new PinningClient({name, endpoint, accessToken: key}) + } + + async info(includeStats=false) { + let stat = undefined + if (includeStats) { + stat = await this.stat() + } + const {name, endpoint} = this + return {name, endpoint, stat} + } + + async stat() { + try { + const promises = [] + for (const pinStatus of ['queued', 'pinning', 'pinned', 'failed']) { + promises.push(this._countForStatus(pinStatus)) + } + const [queued, pinning, pinned, failed] = await Promise.all(promises) + return { + status: 'valid', + pinCount: { queued, pinning, pinned, failed } + } + } catch (e) { + // TODO: log error + return { + status: 'invalid' + } + } + } + + /** + * + * @param {object} options + * @param {?string} options.name optional descriptive name for pin + * @param {?Object} options.meta optional metadata to attach to pin + * @param {?number} options.timeout request timeout (seconds) + * @param {?boolean} options.background If true, add returns remote a pin object as soon as the remote service responds. + * The returned pin object may have a status of 'queued' or 'pinning'. + * If false, the add method will not resolve until the pin status is 'pinned' before returning. + * When background==false and the remote service returns a status of 'failed', an Error will be thrown. + * + * @returns {Promise} + */ + async add(cid, options) { + const {name, meta, background} = options + const origins = await this._originAddresses() + const response = await this.client.add({cid, name, meta, origins}) + + // TODO: implement timeout + + const {status, pin, delegates} = response + this._connectToDelegates(delegates) + + if (!background) { + return this._awaitPinCompletion(response) + } + + return { + status, + name: pin.name, + meta: pin.meta, + } + } + + async _awaitPinCompletion(pinResponse) { + const pollIntervalMs = 500 + + let {status, requestid} = pinResponse + while (status !== 'pinned') { + switch (status) { + case 'failed': + throw new Error('pin failed: ' + JSON.stringify(pinResponse.info)) + + case 'queued': + // fallthrough + case 'pinning': + await delay(pollIntervalMs) + pinResponse = await this.client.get(requestid) + status = pinResponse.status + } + } + + const {pin} = pinResponse + return { + status, + name: pin.name, + meta: pin.meta, + } + } + + /** + * + * @param {object} options + * @param {?Array} options.cid return pins for the specified CID(s) + * @param {?string} options.name return pins that contain the provided value (case-sensitive, exact match) + * @param {?Array} options.status return pins with the specified statuses (queued, pinning, pinned, failed). Default: pinned + * @param {?number} options.timeout request timeout (seconds) + * + * @returns {AsyncGenerator} + */ + async * ls(options) { + // TODO: implement timeout + + const {cid, name, status} = options + for (const pinInfo of this.client.list({cid, name, status})) { + const {status, pin} = pinInfo + const {cid, name, meta} = pin + const result = { + status, + cid, + name, + meta + } + yield result + } + } + + /** + * Remove a single pin from a remote pinning service. + * Fails if multiple pins match the specified criteria. Use rmAll to remove all pins that match. + * + * @param {object} options + * @param {Array} options.cid CID(s) to remove from remote pinning service + * @param {?Array} options.status only remove pins that have one of the specified statuses (queued, pinning, pinned, failed). Default: pinned + * @param {?number} options.timeout request timeout (seconds) + */ + async rm(options) { + // TODO: implement timeout + + // the pinning service API only supports deletion by requestid, so we need to lookup the pins first + const {cid, status} = options + const resp = await this.client.ls({cid, status}) + if (resp.count > 1) { + throw new Error('multiple remote pins are matching this query') + } + + const requestid = resp.results[0].requestid + await this.client.delete(requestid) + } + + /** + * Remove all pins that match the given criteria from a remote pinning service. + * + * @param {object} options + * @param {Array} options.cid CID(s) to remove from remote pinning service + * @param {?Array} options.status only remove pins that have one of the specified statuses (queued, pinning, pinned, failed). Default: pinned + * @param {?number} options.timeout request timeout (seconds) + * @returns {Promise} + */ + async rmAll(options) { + // TODO: implement timeout + const {cid, status} = options + const requestIds = new Set() + for (const result of this.client.list({cid, status})) { + requestIds.add(result.requestid) + } + + const promises = [] + for (const requestid of requestIds.entries()) { + promises.push(this.client.delete(requestid)) + } + await Promise.all(promises) + } + + async _originAddresses() { + const addrs = await this.swarm.localAddrs() + const id = this.peerId + return addrs.map(ma => { + const str = ma.toString() + + // some relay-style transports add our peer id to the ma for us + // so don't double-add + if (str.endsWith(`/p2p/${id}`)) { + return str + } + + return `${str}/p2p/${id}` + }) + } + + async _countForStatus(status) { + const response = await this.client.ls({status: [status], limit: 0}) + return response.count + } + + async _connectToDelegates(delegates) { + const addrs = delegates.map(multiaddr) + for (const addr of addrs) { + try { + this.swarm.connect(addr) + } catch (e) { + // TODO: log connection error + } + } } } -module.exports = RemotePinAPI +const delay = ms => new Promise(resolve => setTimeout(resolve, ms)) + +/** + * @typedef {import('cids')} CID + * @typedef {import('../..').PeerId} PeerId + * @typedef {import('../../swarm')} SwarmAPI + * @typedef {import('../../config').Config} Config + * + * @typedef {'queued'|'pinning'|'pinned'|'failed'} PinStatus + * @typedef {'valid'|'invalid'} PinServiceStatus + * + * @typedef {object} RemotePin + * @property {string} [cid] + * @property {PinStatus} [status] + * @property {?string} [name] + * @property {?object} [meta] + */ + +module.exports = PinRemoteAPI From e3d96836d8d76d35c8879638cb6270b215ebdb64 Mon Sep 17 00:00:00 2001 From: Yusef Napora Date: Wed, 10 Mar 2021 16:02:30 -0500 Subject: [PATCH 03/21] implement timeouts for remote pin methods --- .../src/components/pin/remote/index.js | 21 +++++++++---------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/packages/ipfs-core/src/components/pin/remote/index.js b/packages/ipfs-core/src/components/pin/remote/index.js index d1dca3137e..005a7a34c1 100644 --- a/packages/ipfs-core/src/components/pin/remote/index.js +++ b/packages/ipfs-core/src/components/pin/remote/index.js @@ -2,6 +2,7 @@ const multiaddr = require('multiaddr') const PinningClient = require('js-ipfs-pinning-service-client') +const withTimeoutOption = require('ipfs-core-utils/src/with-timeout-option') /** * PinRemoteAPI provides an API for pinning content to remote services. @@ -210,6 +211,11 @@ class PinRemoteServiceAPI { this.swarm = swarm this.peerId = peerId this.client = new PinningClient({name, endpoint, accessToken: key}) + + this.add = withTimeoutOption(this._add) + this.ls = withTimeoutOption(this._ls) + this.rm = withTimeoutOption(this._rm) + this.rmAll = withTimeoutOption(this._rmAll) } async info(includeStats=false) { @@ -253,12 +259,10 @@ class PinRemoteServiceAPI { * * @returns {Promise} */ - async add(cid, options) { + async _add(cid, options) { const {name, meta, background} = options const origins = await this._originAddresses() const response = await this.client.add({cid, name, meta, origins}) - - // TODO: implement timeout const {status, pin, delegates} = response this._connectToDelegates(delegates) @@ -310,9 +314,7 @@ class PinRemoteServiceAPI { * * @returns {AsyncGenerator} */ - async * ls(options) { - // TODO: implement timeout - + async * _ls(options) { const {cid, name, status} = options for (const pinInfo of this.client.list({cid, name, status})) { const {status, pin} = pinInfo @@ -336,9 +338,7 @@ class PinRemoteServiceAPI { * @param {?Array} options.status only remove pins that have one of the specified statuses (queued, pinning, pinned, failed). Default: pinned * @param {?number} options.timeout request timeout (seconds) */ - async rm(options) { - // TODO: implement timeout - + async _rm(options) { // the pinning service API only supports deletion by requestid, so we need to lookup the pins first const {cid, status} = options const resp = await this.client.ls({cid, status}) @@ -359,8 +359,7 @@ class PinRemoteServiceAPI { * @param {?number} options.timeout request timeout (seconds) * @returns {Promise} */ - async rmAll(options) { - // TODO: implement timeout + async _rmAll(options) { const {cid, status} = options const requestIds = new Set() for (const result of this.client.list({cid, status})) { From 93cb6eefdafefa8f0d087232af71e13ba24b9fde Mon Sep 17 00:00:00 2001 From: Yusef Napora Date: Wed, 10 Mar 2021 18:00:53 -0500 Subject: [PATCH 04/21] fix pin.remote.ls --- .../src/components/pin/remote/index.js | 91 ++++++++++--------- 1 file changed, 47 insertions(+), 44 deletions(-) diff --git a/packages/ipfs-core/src/components/pin/remote/index.js b/packages/ipfs-core/src/components/pin/remote/index.js index 005a7a34c1..dacc5a1453 100644 --- a/packages/ipfs-core/src/components/pin/remote/index.js +++ b/packages/ipfs-core/src/components/pin/remote/index.js @@ -3,6 +3,7 @@ const multiaddr = require('multiaddr') const PinningClient = require('js-ipfs-pinning-service-client') const withTimeoutOption = require('ipfs-core-utils/src/with-timeout-option') +const log = require('debug')('ipfs:components:pin:remote') /** * PinRemoteAPI provides an API for pinning content to remote services. @@ -10,7 +11,7 @@ const withTimeoutOption = require('ipfs-core-utils/src/with-timeout-option') class PinRemoteAPI { /** - * @param {Object} opts + * @param {Object} opts * @param {SwarmAPI} opts.swarm * @param {Config} opts.config * @param {PeerId} opts.peerId @@ -22,7 +23,7 @@ class PinRemoteAPI { /** * Asks a remote pinning service to pin an IPFS object from a given path - * + * * @param {string|CID} cid * @param {object} options * @param {string} options.service name of a configured remote pinning service @@ -30,7 +31,7 @@ class PinRemoteAPI { * @param {?Object} options.meta optional metadata to attach to pin * @param {?number} options.timeout request timeout (seconds) * @param {?boolean} options.background If true, add returns remote a pin object as soon as the remote service responds. - * The returned pin object may have a status of 'queued' or 'pinning'. + * The returned pin object may have a status of 'queued' or 'pinning'. * If false, the add method will not resolve until the pin status is 'pinned' before returning. * When background==false and the remote service returns a status of 'failed', an Error will be thrown. * @returns {Promise} @@ -43,17 +44,17 @@ class PinRemoteAPI { const svc = this.service.serviceNamed(service) return svc.add(cid, addOpts) } - + /** * List objects that are pinned by a remote service. - * + * * @param {object} options * @param {string} options.service name of a configured remote pinning service * @param {?Array} options.cid return pins for the specified CID(s) * @param {?string} options.name return pins that contain the provided value (case-sensitive, exact match) * @param {?Array} options.status return pins with the specified statuses (queued, pinning, pinned, failed). Default: pinned * @param {?number} options.timeout request timeout (seconds) - * @returns {AsyncGenerator} + * @returns {AsyncGenerator} */ async * ls(options) { const {service, ...lsOpts} = options @@ -61,13 +62,15 @@ class PinRemoteAPI { throw new Error('service name must be passed') } const svc = this.service.serviceNamed(service) - return svc.ls(lsOpts) + for await (const res of svc.ls(lsOpts)) { + yield res + } } /** - * Remove a single pin from a remote pinning service. + * Remove a single pin from a remote pinning service. * Fails if multiple pins match the specified criteria. Use rmAll to remove all pins that match. - * + * * @param {object} options * @param {string} options.service name of a configured remote pinning service * @param {Array} options.cid CID(s) to remove from remote pinning service @@ -85,8 +88,8 @@ class PinRemoteAPI { } /** - * Remove all pins that match the given criteria from a remote pinning service. - * + * Remove all pins that match the given criteria from a remote pinning service. + * * @param {object} options * @param {string} options.service name of a configured remote pinning service * @param {Array} options.cid CID(s) to remove from remote pinning service @@ -121,7 +124,7 @@ class PinRemoteServiceAPI { /** * Adds a new remote pinning service to the set of configured services. - * + * * @param {string} name the name of the pinning service. Used to identify the service in future remote pinning API calls. * @param {Object} options * @param {string|URL} options.endpoint the remote API endpoint URL @@ -138,20 +141,20 @@ class PinRemoteServiceAPI { /** * List the configured remote pinning services. - * + * * @typedef {object} PinCounts * @property {number} queued * @property {number} pinning * @property {number} pinned * @property {number} failed - * + * * @typedef {object} RemotePinningServiceDescription * @property {string} name * @property {URL} endpoint * @property {?object} stat * @property {PinServiceStatus} stat.status * @property {PinCounts} stat.pinCount - * + * * @param {object} opts * @param {?boolean} opts.stat if true, include status info for each pinning service * @return {Promise>} @@ -160,7 +163,7 @@ class PinRemoteServiceAPI { const {stat} = (opts || {}) const promises = [] - for (const name of this._services.keys()) { + for (const name of Object.keys(this._services)) { const svc = this._services[name] promises.push(svc.info(stat)) } @@ -169,7 +172,7 @@ class PinRemoteServiceAPI { /** * Remove a remote pinning service from the set of configured services. - * + * * @param {string} name the name of the pinning service to remove * @returns {Promise} */ @@ -179,7 +182,7 @@ class PinRemoteServiceAPI { /** * Returns a RemotePinningService object for the given service name. Throws if no service has been configured with the given name. - * @param {string} name + * @param {string} name * @returns {RemotePinningService} */ serviceNamed(name) { @@ -197,7 +200,7 @@ class PinRemoteServiceAPI { class RemotePinningService { /** - * + * * @param {string} name pinning service name * @param {object} config * @param {string|URL} config.endpoint the remote API endpoint URL @@ -212,10 +215,10 @@ class PinRemoteServiceAPI { this.peerId = peerId this.client = new PinningClient({name, endpoint, accessToken: key}) - this.add = withTimeoutOption(this._add) - this.ls = withTimeoutOption(this._ls) - this.rm = withTimeoutOption(this._rm) - this.rmAll = withTimeoutOption(this._rmAll) + this.add = withTimeoutOption(this._add.bind(this)) + this.ls = withTimeoutOption(this._ls.bind(this)) + this.rm = withTimeoutOption(this._rm.bind(this)) + this.rmAll = withTimeoutOption(this._rmAll.bind(this)) } async info(includeStats=false) { @@ -239,7 +242,7 @@ class PinRemoteServiceAPI { pinCount: { queued, pinning, pinned, failed } } } catch (e) { - // TODO: log error + log('error getting stats: ', e) return { status: 'invalid' } @@ -247,23 +250,23 @@ class PinRemoteServiceAPI { } /** - * + * * @param {object} options * @param {?string} options.name optional descriptive name for pin * @param {?Object} options.meta optional metadata to attach to pin * @param {?number} options.timeout request timeout (seconds) * @param {?boolean} options.background If true, add returns remote a pin object as soon as the remote service responds. - * The returned pin object may have a status of 'queued' or 'pinning'. + * The returned pin object may have a status of 'queued' or 'pinning'. * If false, the add method will not resolve until the pin status is 'pinned' before returning. * When background==false and the remote service returns a status of 'failed', an Error will be thrown. - * + * * @returns {Promise} */ async _add(cid, options) { const {name, meta, background} = options const origins = await this._originAddresses() - const response = await this.client.add({cid, name, meta, origins}) - + const response = await this.client.add({cid: cid.toString(), name, meta, origins}) + const {status, pin, delegates} = response this._connectToDelegates(delegates) @@ -286,7 +289,7 @@ class PinRemoteServiceAPI { switch (status) { case 'failed': throw new Error('pin failed: ' + JSON.stringify(pinResponse.info)) - + case 'queued': // fallthrough case 'pinning': @@ -305,18 +308,18 @@ class PinRemoteServiceAPI { } /** - * + * * @param {object} options * @param {?Array} options.cid return pins for the specified CID(s) * @param {?string} options.name return pins that contain the provided value (case-sensitive, exact match) * @param {?Array} options.status return pins with the specified statuses (queued, pinning, pinned, failed). Default: pinned * @param {?number} options.timeout request timeout (seconds) - * + * * @returns {AsyncGenerator} */ async * _ls(options) { const {cid, name, status} = options - for (const pinInfo of this.client.list({cid, name, status})) { + for await (const pinInfo of this.client.list({cid, name, status})) { const {status, pin} = pinInfo const {cid, name, meta} = pin const result = { @@ -330,9 +333,9 @@ class PinRemoteServiceAPI { } /** - * Remove a single pin from a remote pinning service. + * Remove a single pin from a remote pinning service. * Fails if multiple pins match the specified criteria. Use rmAll to remove all pins that match. - * + * * @param {object} options * @param {Array} options.cid CID(s) to remove from remote pinning service * @param {?Array} options.status only remove pins that have one of the specified statuses (queued, pinning, pinned, failed). Default: pinned @@ -351,8 +354,8 @@ class PinRemoteServiceAPI { } /** - * Remove all pins that match the given criteria from a remote pinning service. - * + * Remove all pins that match the given criteria from a remote pinning service. + * * @param {object} options * @param {Array} options.cid CID(s) to remove from remote pinning service * @param {?Array} options.status only remove pins that have one of the specified statuses (queued, pinning, pinned, failed). Default: pinned @@ -390,19 +393,19 @@ class PinRemoteServiceAPI { } async _countForStatus(status) { - const response = await this.client.ls({status: [status], limit: 0}) + const response = await this.client.ls({status: [status], limit: 1}) return response.count } async _connectToDelegates(delegates) { const addrs = delegates.map(multiaddr) + const promises = [] for (const addr of addrs) { - try { - this.swarm.connect(addr) - } catch (e) { - // TODO: log connection error - } + promises.push(this.swarm.connect(addr).catch(e => { + log('error connecting to pinning service delegate: ', e) + })) } + await Promise.all(promises) } } @@ -416,7 +419,7 @@ const delay = ms => new Promise(resolve => setTimeout(resolve, ms)) * * @typedef {'queued'|'pinning'|'pinned'|'failed'} PinStatus * @typedef {'valid'|'invalid'} PinServiceStatus - * + * * @typedef {object} RemotePin * @property {string} [cid] * @property {PinStatus} [status] From bdb3d1ef236935edc21df8e3c7953fecef5ff998 Mon Sep 17 00:00:00 2001 From: Yusef Napora Date: Thu, 11 Mar 2021 11:24:30 -0500 Subject: [PATCH 05/21] add pin.remote tests to tests-interface-core.js --- packages/ipfs/test/interface-core.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/ipfs/test/interface-core.js b/packages/ipfs/test/interface-core.js index 6e220dcd80..d37c424677 100644 --- a/packages/ipfs/test/interface-core.js +++ b/packages/ipfs/test/interface-core.js @@ -79,6 +79,8 @@ describe('interface-ipfs-core tests', function () { tests.pin(commonFactory) + tests.pin.remote(commonFactory) + tests.ping(commonFactory) tests.pubsub(factory({}, { From 2e232b4f30b08ced3c7ead0c9013950817910a4d Mon Sep 17 00:00:00 2001 From: Yusef Napora Date: Thu, 11 Mar 2021 12:00:04 -0500 Subject: [PATCH 06/21] fix lint issues --- .../src/components/pin/remote/index.js | 391 +++++++++--------- 1 file changed, 194 insertions(+), 197 deletions(-) diff --git a/packages/ipfs-core/src/components/pin/remote/index.js b/packages/ipfs-core/src/components/pin/remote/index.js index dacc5a1453..d0e9df79d8 100644 --- a/packages/ipfs-core/src/components/pin/remote/index.js +++ b/packages/ipfs-core/src/components/pin/remote/index.js @@ -9,16 +9,15 @@ const log = require('debug')('ipfs:components:pin:remote') * PinRemoteAPI provides an API for pinning content to remote services. */ class PinRemoteAPI { - /** * @param {Object} opts * @param {SwarmAPI} opts.swarm * @param {Config} opts.config * @param {PeerId} opts.peerId */ - constructor({swarm, config, peerId}) { + constructor ({ swarm, config, peerId }) { this.swarm = swarm - this.service = new PinRemoteServiceAPI({config, swarm, peerId}) + this.service = new PinRemoteServiceAPI({ config, swarm, peerId }) } /** @@ -26,18 +25,18 @@ class PinRemoteAPI { * * @param {string|CID} cid * @param {object} options - * @param {string} options.service name of a configured remote pinning service - * @param {?string} options.name optional descriptive name for pin - * @param {?Object} options.meta optional metadata to attach to pin - * @param {?number} options.timeout request timeout (seconds) - * @param {?boolean} options.background If true, add returns remote a pin object as soon as the remote service responds. - * The returned pin object may have a status of 'queued' or 'pinning'. - * If false, the add method will not resolve until the pin status is 'pinned' before returning. - * When background==false and the remote service returns a status of 'failed', an Error will be thrown. + * @param {string} options.service - name of a configured remote pinning service + * @param {?string} options.name - optional descriptive name for pin + * @param {?Object} options.meta - optional metadata to attach to pin + * @param {?number} options.timeout - request timeout (seconds) + * @param {?boolean} options.background - If true, add returns remote a pin object as soon as the remote service responds. + * The returned pin object may have a status of 'queued' or 'pinning'. + * If false, the add method will not resolve until the pin status is 'pinned' before returning. + * When background==false and the remote service returns a status of 'failed', an Error will be thrown. * @returns {Promise} */ - async add(cid, options) { - const {service, ...addOpts} = options + async add (cid, options) { + const { service, ...addOpts } = options if (!service) { throw new Error('service name must be passed') } @@ -49,15 +48,15 @@ class PinRemoteAPI { * List objects that are pinned by a remote service. * * @param {object} options - * @param {string} options.service name of a configured remote pinning service - * @param {?Array} options.cid return pins for the specified CID(s) - * @param {?string} options.name return pins that contain the provided value (case-sensitive, exact match) - * @param {?Array} options.status return pins with the specified statuses (queued, pinning, pinned, failed). Default: pinned - * @param {?number} options.timeout request timeout (seconds) + * @param {string} options.service - name of a configured remote pinning service + * @param {?Array} options.cid - return pins for the specified CID(s) + * @param {?string} options.name - return pins that contain the provided value (case-sensitive, exact match) + * @param {?Array} options.status - return pins with the specified statuses (queued, pinning, pinned, failed). Default: pinned + * @param {?number} options.timeout - request timeout (seconds) * @returns {AsyncGenerator} */ - async * ls(options) { - const {service, ...lsOpts} = options + async * ls (options) { + const { service, ...lsOpts } = options if (!service) { throw new Error('service name must be passed') } @@ -72,14 +71,14 @@ class PinRemoteAPI { * Fails if multiple pins match the specified criteria. Use rmAll to remove all pins that match. * * @param {object} options - * @param {string} options.service name of a configured remote pinning service - * @param {Array} options.cid CID(s) to remove from remote pinning service - * @param {?Array} options.status only remove pins that have one of the specified statuses (queued, pinning, pinned, failed). Default: pinned - * @param {?number} options.timeout request timeout (seconds) + * @param {string} options.service - name of a configured remote pinning service + * @param {Array} options.cid - CID(s) to remove from remote pinning service + * @param {?Array} options.status - only remove pins that have one of the specified statuses (queued, pinning, pinned, failed). Default: pinned + * @param {?number} options.timeout - request timeout (seconds) * @returns {Promise} */ - async rm(options) { - const {service, ...rmOpts} = options + async rm (options) { + const { service, ...rmOpts } = options if (!service) { throw new Error('service name must be passed') } @@ -91,14 +90,14 @@ class PinRemoteAPI { * Remove all pins that match the given criteria from a remote pinning service. * * @param {object} options - * @param {string} options.service name of a configured remote pinning service - * @param {Array} options.cid CID(s) to remove from remote pinning service - * @param {?Array} options.status only remove pins that have one of the specified statuses (queued, pinning, pinned, failed). Default: pinned - * @param {?number} options.timeout request timeout (seconds) + * @param {string} options.service - name of a configured remote pinning service + * @param {Array} options.cid - CID(s) to remove from remote pinning service + * @param {?Array} options.status - only remove pins that have one of the specified statuses (queued, pinning, pinned, failed). Default: pinned + * @param {?number} options.timeout - request timeout (seconds) * @returns {Promise} */ - async rmAll(options) { - const {service, ...rmOpts} = options + async rmAll (options) { + const { service, ...rmOpts } = options if (!service) { throw new Error('service name must be passed') } @@ -112,8 +111,7 @@ class PinRemoteAPI { * remote pinning services that are used by the remote pinning api. */ class PinRemoteServiceAPI { - - constructor({config, swarm, peerId}) { + constructor ({ config, swarm, peerId }) { this.config = config this.swarm = swarm this.peerId = peerId @@ -125,17 +123,17 @@ class PinRemoteServiceAPI { /** * Adds a new remote pinning service to the set of configured services. * - * @param {string} name the name of the pinning service. Used to identify the service in future remote pinning API calls. + * @param {string} name - the name of the pinning service. Used to identify the service in future remote pinning API calls. * @param {Object} options - * @param {string|URL} options.endpoint the remote API endpoint URL - * @param {string} options.key an API key that authorizes use of the remote pinning service + * @param {string|URL} options.endpoint - the remote API endpoint URL + * @param {string} options.key - an API key that authorizes use of the remote pinning service */ - async add(name, options) { + async add (name, options) { if (this._services[name]) { throw new Error('service already present: ' + name) } - const svcOpts = Object.assign({swarm: this.swarm, peerId: this.peerId}, options) + const svcOpts = Object.assign({ swarm: this.swarm, peerId: this.peerId }, options) this._services[name] = new RemotePinningService(name, svcOpts) } @@ -156,11 +154,11 @@ class PinRemoteServiceAPI { * @property {PinCounts} stat.pinCount * * @param {object} opts - * @param {?boolean} opts.stat if true, include status info for each pinning service - * @return {Promise>} + * @param {?boolean} opts.stat - if true, include status info for each pinning service + * @returns {Promise>} */ - async ls(opts) { - const {stat} = (opts || {}) + async ls (opts) { + const { stat } = (opts || {}) const promises = [] for (const name of Object.keys(this._services)) { @@ -173,19 +171,20 @@ class PinRemoteServiceAPI { /** * Remove a remote pinning service from the set of configured services. * - * @param {string} name the name of the pinning service to remove + * @param {string} name - the name of the pinning service to remove * @returns {Promise} */ - async rm(name) { + async rm (name) { delete this._services[name] } /** * Returns a RemotePinningService object for the given service name. Throws if no service has been configured with the given name. + * * @param {string} name * @returns {RemotePinningService} */ - serviceNamed(name) { + serviceNamed (name) { if (!this._services[name]) { throw new Error('no remote pinning service configured with name: ' + name) } @@ -193,143 +192,141 @@ class PinRemoteServiceAPI { } } - /** * RemotePinningService provides add, ls, and rm operations for a single remote pinning service. */ - class RemotePinningService { - +class RemotePinningService { /** * - * @param {string} name pinning service name + * @param {string} name - pinning service name * @param {object} config - * @param {string|URL} config.endpoint the remote API endpoint URL - * @param {string} config.key an API key that authorizes use of the remote pinning service - * @param {SwarmAPI} config.swarm - * @param {PeerId} config.peerId + * @param {string|URL} config.endpoint - the remote API endpoint URL + * @param {string} config.key - an API key that authorizes use of the remote pinning service + * @param {SwarmAPI} config.swarm - SwarmAPI instance for the local IPFS node + * @param {PeerId} config.peerId - PeerId of the local IPFS node */ - constructor(name, {endpoint, key, swarm, peerId}) { - this.name = name - this.endpoint = endpoint - this.swarm = swarm - this.peerId = peerId - this.client = new PinningClient({name, endpoint, accessToken: key}) - - this.add = withTimeoutOption(this._add.bind(this)) - this.ls = withTimeoutOption(this._ls.bind(this)) - this.rm = withTimeoutOption(this._rm.bind(this)) - this.rmAll = withTimeoutOption(this._rmAll.bind(this)) + constructor (name, { endpoint, key, swarm, peerId }) { + this.name = name + this.endpoint = endpoint + this.swarm = swarm + this.peerId = peerId + this.client = new PinningClient({ name, endpoint, accessToken: key }) + + this.add = withTimeoutOption(this._add.bind(this)) + this.ls = withTimeoutOption(this._ls.bind(this)) + this.rm = withTimeoutOption(this._rm.bind(this)) + this.rmAll = withTimeoutOption(this._rmAll.bind(this)) } - async info(includeStats=false) { - let stat = undefined - if (includeStats) { - stat = await this.stat() - } - const {name, endpoint} = this - return {name, endpoint, stat} + async info (includeStats = false) { + let stat + if (includeStats) { + stat = await this.stat() + } + const { name, endpoint } = this + return { name, endpoint, stat } } - async stat() { - try { - const promises = [] - for (const pinStatus of ['queued', 'pinning', 'pinned', 'failed']) { - promises.push(this._countForStatus(pinStatus)) - } - const [queued, pinning, pinned, failed] = await Promise.all(promises) - return { - status: 'valid', - pinCount: { queued, pinning, pinned, failed } - } - } catch (e) { - log('error getting stats: ', e) - return { - status: 'invalid' - } + async stat () { + try { + const promises = [] + for (const pinStatus of ['queued', 'pinning', 'pinned', 'failed']) { + promises.push(this._countForStatus(pinStatus)) } + const [queued, pinning, pinned, failed] = await Promise.all(promises) + return { + status: 'valid', + pinCount: { queued, pinning, pinned, failed } + } + } catch (e) { + log('error getting stats: ', e) + return { + status: 'invalid' + } + } } /** + * Request that the remote service add a pin for the given CID. * + * @param {CID|string} cid - CID to pin to remote service * @param {object} options - * @param {?string} options.name optional descriptive name for pin - * @param {?Object} options.meta optional metadata to attach to pin - * @param {?number} options.timeout request timeout (seconds) - * @param {?boolean} options.background If true, add returns remote a pin object as soon as the remote service responds. - * The returned pin object may have a status of 'queued' or 'pinning'. - * If false, the add method will not resolve until the pin status is 'pinned' before returning. - * When background==false and the remote service returns a status of 'failed', an Error will be thrown. + * @param {?string} options.name - optional descriptive name for pin + * @param {?Object} options.meta - optional metadata to attach to pin + * @param {?number} options.timeout - request timeout (seconds) + * @param {?boolean} options.background - If background==true, add returns remote a pin object as soon as the remote service responds. + * The returned pin object may have a status of 'queued' or 'pinning'. + * If background==false, the add method will not resolve until the pin status is 'pinned' before returning. + * When background==false and the remote service returns a status of 'failed', an Error will be thrown. * * @returns {Promise} */ - async _add(cid, options) { - const {name, meta, background} = options - const origins = await this._originAddresses() - const response = await this.client.add({cid: cid.toString(), name, meta, origins}) + async _add (cid, options) { + const { name, meta, background } = options + const origins = await this._originAddresses() + const response = await this.client.add({ cid: cid.toString(), name, meta, origins }) - const {status, pin, delegates} = response - this._connectToDelegates(delegates) + const { status, pin, delegates } = response + this._connectToDelegates(delegates) - if (!background) { - return this._awaitPinCompletion(response) - } + if (!background) { + return this._awaitPinCompletion(response) + } - return { - status, - name: pin.name, - meta: pin.meta, - } + return { + status, + name: pin.name, + meta: pin.meta + } } - async _awaitPinCompletion(pinResponse) { - const pollIntervalMs = 500 - - let {status, requestid} = pinResponse - while (status !== 'pinned') { - switch (status) { - case 'failed': - throw new Error('pin failed: ' + JSON.stringify(pinResponse.info)) - - case 'queued': - // fallthrough - case 'pinning': - await delay(pollIntervalMs) - pinResponse = await this.client.get(requestid) - status = pinResponse.status - } - } + async _awaitPinCompletion (pinResponse) { + const pollIntervalMs = 500 - const {pin} = pinResponse - return { - status, - name: pin.name, - meta: pin.meta, + let { status, requestid } = pinResponse + while (status !== 'pinned') { + if (status === 'failed') { + throw new Error('pin failed: ' + JSON.stringify(pinResponse.info)) } + + log(`pin status for CID ${pinResponse.pin.cid} (request id ${requestid}): ${status}. Waiting ${pollIntervalMs}ms to refresh status.`) + await delay(pollIntervalMs) + pinResponse = await this.client.get(requestid) + status = pinResponse.status + } + + const { pin } = pinResponse + return { + status, + name: pin.name, + meta: pin.meta + } } /** + * List pins from the remote service that match the given criteria. If no criteria are provided, returns all pins with the status 'pinned'. * * @param {object} options - * @param {?Array} options.cid return pins for the specified CID(s) - * @param {?string} options.name return pins that contain the provided value (case-sensitive, exact match) - * @param {?Array} options.status return pins with the specified statuses (queued, pinning, pinned, failed). Default: pinned - * @param {?number} options.timeout request timeout (seconds) + * @param {?Array} options.cid - return pins for the specified CID(s) + * @param {?string} options.name - return pins that contain the provided value (case-sensitive, exact match) + * @param {?Array} options.status - return pins with the specified statuses (queued, pinning, pinned, failed). Default: pinned + * @param {?number} options.timeout - request timeout (seconds) * * @returns {AsyncGenerator} */ - async * _ls(options) { - const {cid, name, status} = options - for await (const pinInfo of this.client.list({cid, name, status})) { - const {status, pin} = pinInfo - const {cid, name, meta} = pin - const result = { - status, - cid, - name, - meta - } - yield result + async * _ls (options) { + const { cid, name, status } = options + for await (const pinInfo of this.client.list({ cid, name, status })) { + const { status, pin } = pinInfo + const { cid, name, meta } = pin + const result = { + status, + cid, + name, + meta } + yield result + } } /** @@ -337,75 +334,75 @@ class PinRemoteServiceAPI { * Fails if multiple pins match the specified criteria. Use rmAll to remove all pins that match. * * @param {object} options - * @param {Array} options.cid CID(s) to remove from remote pinning service - * @param {?Array} options.status only remove pins that have one of the specified statuses (queued, pinning, pinned, failed). Default: pinned - * @param {?number} options.timeout request timeout (seconds) + * @param {Array} options.cid - CID(s) to remove from remote pinning service + * @param {?Array} options.status - only remove pins that have one of the specified statuses (queued, pinning, pinned, failed). Default: pinned + * @param {?number} options.timeout - request timeout (seconds) */ - async _rm(options) { - // the pinning service API only supports deletion by requestid, so we need to lookup the pins first - const {cid, status} = options - const resp = await this.client.ls({cid, status}) - if (resp.count > 1) { - throw new Error('multiple remote pins are matching this query') - } + async _rm (options) { + // the pinning service API only supports deletion by requestid, so we need to lookup the pins first + const { cid, status } = options + const resp = await this.client.ls({ cid, status }) + if (resp.count > 1) { + throw new Error('multiple remote pins are matching this query') + } - const requestid = resp.results[0].requestid - await this.client.delete(requestid) + const requestid = resp.results[0].requestid + await this.client.delete(requestid) } /** * Remove all pins that match the given criteria from a remote pinning service. * * @param {object} options - * @param {Array} options.cid CID(s) to remove from remote pinning service - * @param {?Array} options.status only remove pins that have one of the specified statuses (queued, pinning, pinned, failed). Default: pinned - * @param {?number} options.timeout request timeout (seconds) + * @param {Array} options.cid - CID(s) to remove from remote pinning service + * @param {?Array} options.status - only remove pins that have one of the specified statuses (queued, pinning, pinned, failed). Default: pinned + * @param {?number} options.timeout - request timeout (seconds) * @returns {Promise} */ - async _rmAll(options) { - const {cid, status} = options - const requestIds = new Set() - for (const result of this.client.list({cid, status})) { - requestIds.add(result.requestid) - } + async _rmAll (options) { + const { cid, status } = options + const requestIds = new Set() + for (const result of this.client.list({ cid, status })) { + requestIds.add(result.requestid) + } - const promises = [] - for (const requestid of requestIds.entries()) { - promises.push(this.client.delete(requestid)) - } - await Promise.all(promises) + const promises = [] + for (const requestid of requestIds.entries()) { + promises.push(this.client.delete(requestid)) + } + await Promise.all(promises) } - async _originAddresses() { - const addrs = await this.swarm.localAddrs() - const id = this.peerId - return addrs.map(ma => { - const str = ma.toString() + async _originAddresses () { + const addrs = await this.swarm.localAddrs() + const id = this.peerId + return addrs.map(ma => { + const str = ma.toString() - // some relay-style transports add our peer id to the ma for us - // so don't double-add - if (str.endsWith(`/p2p/${id}`)) { - return str - } + // some relay-style transports add our peer id to the ma for us + // so don't double-add + if (str.endsWith(`/p2p/${id}`)) { + return str + } - return `${str}/p2p/${id}` - }) + return `${str}/p2p/${id}` + }) } - async _countForStatus(status) { - const response = await this.client.ls({status: [status], limit: 1}) - return response.count + async _countForStatus (status) { + const response = await this.client.ls({ status: [status], limit: 1 }) + return response.count } - async _connectToDelegates(delegates) { - const addrs = delegates.map(multiaddr) - const promises = [] - for (const addr of addrs) { - promises.push(this.swarm.connect(addr).catch(e => { - log('error connecting to pinning service delegate: ', e) - })) - } - await Promise.all(promises) + async _connectToDelegates (delegates) { + const addrs = delegates.map(multiaddr) + const promises = [] + for (const addr of addrs) { + promises.push(this.swarm.connect(addr).catch(e => { + log('error connecting to pinning service delegate: ', e) + })) + } + await Promise.all(promises) } } From a1f7d8f14f861a00ed71e20e7c2a15a23327bce1 Mon Sep 17 00:00:00 2001 From: Yusef Napora Date: Thu, 11 Mar 2021 12:06:18 -0500 Subject: [PATCH 07/21] mv remote/index.js remote.js --- .../src/components/pin/{remote/index.js => remote.js} | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) rename packages/ipfs-core/src/components/pin/{remote/index.js => remote.js} (99%) diff --git a/packages/ipfs-core/src/components/pin/remote/index.js b/packages/ipfs-core/src/components/pin/remote.js similarity index 99% rename from packages/ipfs-core/src/components/pin/remote/index.js rename to packages/ipfs-core/src/components/pin/remote.js index d0e9df79d8..2f7c0aac7e 100644 --- a/packages/ipfs-core/src/components/pin/remote/index.js +++ b/packages/ipfs-core/src/components/pin/remote.js @@ -410,9 +410,9 @@ const delay = ms => new Promise(resolve => setTimeout(resolve, ms)) /** * @typedef {import('cids')} CID - * @typedef {import('../..').PeerId} PeerId - * @typedef {import('../../swarm')} SwarmAPI - * @typedef {import('../../config').Config} Config + * @typedef {import('..').PeerId} PeerId + * @typedef {import('../swarm')} SwarmAPI + * @typedef {import('../config').Config} Config * * @typedef {'queued'|'pinning'|'pinned'|'failed'} PinStatus * @typedef {'valid'|'invalid'} PinServiceStatus From a7b6e84ee011693d00a7684ec32e59c64044f81d Mon Sep 17 00:00:00 2001 From: Yusef Napora Date: Thu, 11 Mar 2021 17:49:14 -0500 Subject: [PATCH 08/21] use commit hash for pinning client dependency --- packages/ipfs-core/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ipfs-core/package.json b/packages/ipfs-core/package.json index a106f20ea4..ef76684cba 100644 --- a/packages/ipfs-core/package.json +++ b/packages/ipfs-core/package.json @@ -89,7 +89,7 @@ "it-first": "^1.0.4", "it-last": "^1.0.4", "it-pipe": "^1.1.0", - "js-ipfs-pinning-service-client": "github:yusefnapora/js-ipfs-pinning-service-client#master", + "js-ipfs-pinning-service-client": "github:yusefnapora/js-ipfs-pinning-service-client#afe06981a07ece5841c33e0fd4ae7bcc37473a80", "libp2p": "^0.30.7", "libp2p-bootstrap": "^0.12.1", "libp2p-crypto": "^0.19.0", From 0047a58ddd84ab06212cdcdbcbcfbe6e32c6365c Mon Sep 17 00:00:00 2001 From: Yusef Napora Date: Thu, 11 Mar 2021 17:50:23 -0500 Subject: [PATCH 09/21] make interface-core tests pass - on my machine, at least :) --- .../ipfs-core/src/components/pin/remote.js | 77 +++++++++++-------- 1 file changed, 44 insertions(+), 33 deletions(-) diff --git a/packages/ipfs-core/src/components/pin/remote.js b/packages/ipfs-core/src/components/pin/remote.js index 2f7c0aac7e..37553e6de6 100644 --- a/packages/ipfs-core/src/components/pin/remote.js +++ b/packages/ipfs-core/src/components/pin/remote.js @@ -1,6 +1,7 @@ 'use strict' const multiaddr = require('multiaddr') +const CID = require('cids') const PinningClient = require('js-ipfs-pinning-service-client') const withTimeoutOption = require('ipfs-core-utils/src/with-timeout-option') const log = require('debug')('ipfs:components:pin:remote') @@ -133,6 +134,14 @@ class PinRemoteServiceAPI { throw new Error('service already present: ' + name) } + if (!options.endpoint) { + throw new Error('option "endpoint" is required') + } + + if (!options.key) { + throw new Error('option "key" is required') + } + const svcOpts = Object.assign({ swarm: this.swarm, peerId: this.peerId }, options) this._services[name] = new RemotePinningService(name, svcOpts) } @@ -175,6 +184,9 @@ class PinRemoteServiceAPI { * @returns {Promise} */ async rm (name) { + if (!name) { + throw new Error('parameter "name" is required') + } delete this._services[name] } @@ -206,8 +218,8 @@ class RemotePinningService { * @param {PeerId} config.peerId - PeerId of the local IPFS node */ constructor (name, { endpoint, key, swarm, peerId }) { + this.endpoint = new URL(endpoint.toString()) this.name = name - this.endpoint = endpoint this.swarm = swarm this.peerId = peerId this.client = new PinningClient({ name, endpoint, accessToken: key }) @@ -219,12 +231,12 @@ class RemotePinningService { } async info (includeStats = false) { - let stat + const { name, endpoint } = this + const info = { service: name, endpoint } if (includeStats) { - stat = await this.stat() + info.stat = await this.stat() } - const { name, endpoint } = this - return { name, endpoint, stat } + return info } async stat () { @@ -239,7 +251,7 @@ class RemotePinningService { pinCount: { queued, pinning, pinned, failed } } } catch (e) { - log('error getting stats: ', e) + log(`error getting stats for service ${this.name}: `, e) return { status: 'invalid' } @@ -262,7 +274,8 @@ class RemotePinningService { * @returns {Promise} */ async _add (cid, options) { - const { name, meta, background } = options + const { meta, background } = options + const name = options.name || '' const origins = await this._originAddresses() const response = await this.client.add({ cid: cid.toString(), name, meta, origins }) @@ -273,15 +286,11 @@ class RemotePinningService { return this._awaitPinCompletion(response) } - return { - status, - name: pin.name, - meta: pin.meta - } + return this._formatPinResult(status, pin) } async _awaitPinCompletion (pinResponse) { - const pollIntervalMs = 500 + const pollIntervalMs = 100 let { status, requestid } = pinResponse while (status !== 'pinned') { @@ -289,18 +298,22 @@ class RemotePinningService { throw new Error('pin failed: ' + JSON.stringify(pinResponse.info)) } - log(`pin status for CID ${pinResponse.pin.cid} (request id ${requestid}): ${status}. Waiting ${pollIntervalMs}ms to refresh status.`) await delay(pollIntervalMs) pinResponse = await this.client.get(requestid) status = pinResponse.status } - const { pin } = pinResponse - return { - status, - name: pin.name, - meta: pin.meta + return this._formatPinResult(pinResponse.status, pinResponse.pin) + } + + _formatPinResult(status, pin) { + const name = pin.name || '' + const cid = new CID(pin.cid) + const result = { status, name, cid } + if (pin.meta) { + result.meta = pin.meta } + return result } /** @@ -315,16 +328,12 @@ class RemotePinningService { * @returns {AsyncGenerator} */ async * _ls (options) { - const { cid, name, status } = options + const cid = options.cid || [] + const status = options.status || [] + const name = options.name for await (const pinInfo of this.client.list({ cid, name, status })) { const { status, pin } = pinInfo - const { cid, name, meta } = pin - const result = { - status, - cid, - name, - meta - } + const result = this._formatPinResult(status, pin) yield result } } @@ -342,6 +351,9 @@ class RemotePinningService { // the pinning service API only supports deletion by requestid, so we need to lookup the pins first const { cid, status } = options const resp = await this.client.ls({ cid, status }) + if (resp.count == 0) { + return + } if (resp.count > 1) { throw new Error('multiple remote pins are matching this query') } @@ -361,13 +373,13 @@ class RemotePinningService { */ async _rmAll (options) { const { cid, status } = options - const requestIds = new Set() - for (const result of this.client.list({ cid, status })) { - requestIds.add(result.requestid) + const requestIds = [] + for await (const result of this.client.list({ cid, status })) { + requestIds.push(result.requestid) } const promises = [] - for (const requestid of requestIds.entries()) { + for (const requestid of requestIds) { promises.push(this.client.delete(requestid)) } await Promise.all(promises) @@ -409,7 +421,6 @@ class RemotePinningService { const delay = ms => new Promise(resolve => setTimeout(resolve, ms)) /** - * @typedef {import('cids')} CID * @typedef {import('..').PeerId} PeerId * @typedef {import('../swarm')} SwarmAPI * @typedef {import('../config').Config} Config @@ -418,7 +429,7 @@ const delay = ms => new Promise(resolve => setTimeout(resolve, ms)) * @typedef {'valid'|'invalid'} PinServiceStatus * * @typedef {object} RemotePin - * @property {string} [cid] + * @property {string|CID} [cid] * @property {PinStatus} [status] * @property {?string} [name] * @property {?object} [meta] From a2e9846b3b23ffc3782cdf46ae951746ee0d5670 Mon Sep 17 00:00:00 2001 From: Yusef Napora Date: Thu, 11 Mar 2021 18:02:49 -0500 Subject: [PATCH 10/21] bump client dependency --- packages/ipfs-core/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ipfs-core/package.json b/packages/ipfs-core/package.json index ef76684cba..58154fe032 100644 --- a/packages/ipfs-core/package.json +++ b/packages/ipfs-core/package.json @@ -89,7 +89,7 @@ "it-first": "^1.0.4", "it-last": "^1.0.4", "it-pipe": "^1.1.0", - "js-ipfs-pinning-service-client": "github:yusefnapora/js-ipfs-pinning-service-client#afe06981a07ece5841c33e0fd4ae7bcc37473a80", + "js-ipfs-pinning-service-client": "github:yusefnapora/js-ipfs-pinning-service-client#874c6ec7875ec2b3039c2361638c54158e7e4874", "libp2p": "^0.30.7", "libp2p-bootstrap": "^0.12.1", "libp2p-crypto": "^0.19.0", From cd578537da163f23f5d94b2337bb97a4fab0864c Mon Sep 17 00:00:00 2001 From: Yusef Napora Date: Fri, 12 Mar 2021 10:10:24 -0500 Subject: [PATCH 11/21] use default status "pinned" if none provided in ls --- packages/ipfs-core/src/components/pin/remote.js | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/ipfs-core/src/components/pin/remote.js b/packages/ipfs-core/src/components/pin/remote.js index 37553e6de6..9e693b95a0 100644 --- a/packages/ipfs-core/src/components/pin/remote.js +++ b/packages/ipfs-core/src/components/pin/remote.js @@ -306,7 +306,7 @@ class RemotePinningService { return this._formatPinResult(pinResponse.status, pinResponse.pin) } - _formatPinResult(status, pin) { + _formatPinResult (status, pin) { const name = pin.name || '' const cid = new CID(pin.cid) const result = { status, name, cid } @@ -329,8 +329,11 @@ class RemotePinningService { */ async * _ls (options) { const cid = options.cid || [] - const status = options.status || [] const name = options.name + let status = options.status || [] + if (status.length === 0) { + status = ['pinned'] + } for await (const pinInfo of this.client.list({ cid, name, status })) { const { status, pin } = pinInfo const result = this._formatPinResult(status, pin) @@ -351,7 +354,7 @@ class RemotePinningService { // the pinning service API only supports deletion by requestid, so we need to lookup the pins first const { cid, status } = options const resp = await this.client.ls({ cid, status }) - if (resp.count == 0) { + if (resp.count === 0) { return } if (resp.count > 1) { From 2bb4779a47e88f8a84e9ca0e49721344ae456391 Mon Sep 17 00:00:00 2001 From: Yusef Napora Date: Fri, 26 Mar 2021 15:08:35 -0400 Subject: [PATCH 12/21] wip - refactor code style --- .../src/components/pin/remote/add.js | 30 +++ .../src/components/pin/remote/client.js | 0 .../src/components/pin/remote/index.js | 40 ++++ .../ipfs-core/src/components/pin/remote/ls.js | 33 +++ .../ipfs-core/src/components/pin/remote/rm.js | 30 +++ .../src/components/pin/remote/rmAll.js | 29 +++ .../pin/{remote.js => remote/service.js} | 195 +++--------------- 7 files changed, 193 insertions(+), 164 deletions(-) create mode 100644 packages/ipfs-core/src/components/pin/remote/add.js create mode 100644 packages/ipfs-core/src/components/pin/remote/client.js create mode 100644 packages/ipfs-core/src/components/pin/remote/index.js create mode 100644 packages/ipfs-core/src/components/pin/remote/ls.js create mode 100644 packages/ipfs-core/src/components/pin/remote/rm.js create mode 100644 packages/ipfs-core/src/components/pin/remote/rmAll.js rename packages/ipfs-core/src/components/pin/{remote.js => remote/service.js} (53%) diff --git a/packages/ipfs-core/src/components/pin/remote/add.js b/packages/ipfs-core/src/components/pin/remote/add.js new file mode 100644 index 0000000000..ded8f8a27e --- /dev/null +++ b/packages/ipfs-core/src/components/pin/remote/add.js @@ -0,0 +1,30 @@ +'use strict' + +const withTimeoutOption = require('ipfs-core-utils/src/with-timeout-option') + +module.exports = ({remotePinServices}) => { + /** + * Asks a remote pinning service to pin an IPFS object from a given path + * + * @param {string|CID} cid + * @param {AddOptions & AbortOptions} options + * @returns {Promise} + */ + async function add (cid, options) { + const { service, ...addOpts } = options + if (!service) { + throw new Error('service name must be passed') + } + const svc = remotePinServices.serviceNamed(service) + return svc.add(cid, addOpts) + } + + return withTimeoutOption(add) +} + +/** + * @typedef {import('cids')} CID + * @typedef {import('ipfs-core-types/src/basic').AbortOptions} AbortOptions + * @typedef {import('ipfs-core-types/src/pin/remote').Pin} Pin + * @typedef {import('ipfs-core-types/src/pin/remote').AddOptions} AddOptions + */ diff --git a/packages/ipfs-core/src/components/pin/remote/client.js b/packages/ipfs-core/src/components/pin/remote/client.js new file mode 100644 index 0000000000..e69de29bb2 diff --git a/packages/ipfs-core/src/components/pin/remote/index.js b/packages/ipfs-core/src/components/pin/remote/index.js new file mode 100644 index 0000000000..5cd595b127 --- /dev/null +++ b/packages/ipfs-core/src/components/pin/remote/index.js @@ -0,0 +1,40 @@ +'use strict' + +const PinRemoteServiceAPI = require('./service') +const createAdd = require('./add') +const createLs = require('./ls') +const createRm = require('./rm') +const createRmAll = require('./rmAll') + +/** + * PinRemoteAPI provides an API for pinning content to remote services. + */ +class PinRemoteAPI { + /** + * @param {Object} opts + * @param {SwarmAPI} opts.swarm + * @param {Config} opts.config + * @param {PeerId} opts.peerId + */ + constructor ({ swarm, config, peerId }) { + this.swarm = swarm + this.service = new PinRemoteServiceAPI({ config, swarm, peerId }) + + // TODO: remove this.service & this.swarm once everything is refactored + const remotePinServices = this.service + this.add = createAdd({ remotePinServices }) + this.ls = createLs({ remotePinServices }) + this.rm = createRm({ remotePinServices }) + this.rmAll = createRmAll({ remotePinServices }) + } + +} + + +/** + * @typedef {import('../..').PeerId} PeerId + * @typedef {import('../../swarm')} SwarmAPI + * @typedef {import('../../config').Config} Config + */ + +module.exports = PinRemoteAPI diff --git a/packages/ipfs-core/src/components/pin/remote/ls.js b/packages/ipfs-core/src/components/pin/remote/ls.js new file mode 100644 index 0000000000..c870045f2e --- /dev/null +++ b/packages/ipfs-core/src/components/pin/remote/ls.js @@ -0,0 +1,33 @@ +'use strict' + +const withTimeoutOption = require('ipfs-core-utils/src/with-timeout-option') + +module.exports = ({ remotePinServices }) => { + + /** + * List objects that are pinned by a remote service. + * + * @param {Query & AbortOptions} options + * @returns {AsyncGenerator} + */ + async function* ls (options) { + const { service, ...lsOpts } = options + if (!service) { + throw new Error('service name must be passed') + } + const svc = remotePinServices.serviceNamed(service) + for await (const res of svc.ls(lsOpts)) { + yield res + } + } + + return withTimeoutOption(ls) +} + +/** + * @typedef {import('cids')} CID + * @typedef {import('ipfs-core-types/src/basic').AbortOptions} AbortOptions + * @typedef {import('ipfs-core-types/src/pin/remote').Pin} Pin + * @typedef {import('ipfs-core-types/src/pin/remote').Status} Status + * @typedef {import('ipfs-core-types/src/pin/remote').Query} Query + */ \ No newline at end of file diff --git a/packages/ipfs-core/src/components/pin/remote/rm.js b/packages/ipfs-core/src/components/pin/remote/rm.js new file mode 100644 index 0000000000..7906bd6417 --- /dev/null +++ b/packages/ipfs-core/src/components/pin/remote/rm.js @@ -0,0 +1,30 @@ +'use strict' + +const withTimeoutOption = require('ipfs-core-utils/src/with-timeout-option') + +module.exports = ({ remotePinServices }) => { + /** + * Remove a single pin from a remote pinning service. + * Fails if multiple pins match the specified query. Use rmAll to remove all pins that match. + * + * @param {Query & AbortOptions} options + * @returns {Promise} + */ + async function rm (options) { + const { service, ...rmOpts } = options + if (!service) { + throw new Error('service name must be passed') + } + const svc = remotePinServices.serviceNamed(service) + return svc.rm(rmOpts) + } + + return withTimeoutOption(rm) +} + +/** + * @typedef {import('cids')} CID + * @typedef {import('ipfs-core-types/src/basic').AbortOptions} AbortOptions + * @typedef {import('ipfs-core-types/src/pin/remote').Pin} Pin + * @typedef {import('ipfs-core-types/src/pin/remote').Query} Query + */ diff --git a/packages/ipfs-core/src/components/pin/remote/rmAll.js b/packages/ipfs-core/src/components/pin/remote/rmAll.js new file mode 100644 index 0000000000..e9ca123baf --- /dev/null +++ b/packages/ipfs-core/src/components/pin/remote/rmAll.js @@ -0,0 +1,29 @@ +'use strict' + +const withTimeoutOption = require('ipfs-core-utils/src/with-timeout-option') + +module.exports = ({ remotePinServices }) => { + /** + * Remove all pins that match the given criteria from a remote pinning service. + * + * @param {Query & AbortOptions} options + * @returns {Promise} + */ + async function rmAll (options) { + const { service, ...rmOpts } = options + if (!service) { + throw new Error('service name must be passed') + } + const svc = remotePinServices.serviceNamed(service) + return svc.rmAll(rmOpts) + } + + return withTimeoutOption(rmAll) +} + +/** + * @typedef {import('cids')} CID + * @typedef {import('ipfs-core-types/src/basic').AbortOptions} AbortOptions + * @typedef {import('ipfs-core-types/src/pin/remote').Pin} Pin + * @typedef {import('ipfs-core-types/src/pin/remote').Query} Query + */ diff --git a/packages/ipfs-core/src/components/pin/remote.js b/packages/ipfs-core/src/components/pin/remote/service.js similarity index 53% rename from packages/ipfs-core/src/components/pin/remote.js rename to packages/ipfs-core/src/components/pin/remote/service.js index 9e693b95a0..c23a6b6c93 100644 --- a/packages/ipfs-core/src/components/pin/remote.js +++ b/packages/ipfs-core/src/components/pin/remote/service.js @@ -1,117 +1,17 @@ -'use strict' - const multiaddr = require('multiaddr') const CID = require('cids') const PinningClient = require('js-ipfs-pinning-service-client') const withTimeoutOption = require('ipfs-core-utils/src/with-timeout-option') const log = require('debug')('ipfs:components:pin:remote') -/** - * PinRemoteAPI provides an API for pinning content to remote services. - */ -class PinRemoteAPI { - /** - * @param {Object} opts - * @param {SwarmAPI} opts.swarm - * @param {Config} opts.config - * @param {PeerId} opts.peerId - */ - constructor ({ swarm, config, peerId }) { - this.swarm = swarm - this.service = new PinRemoteServiceAPI({ config, swarm, peerId }) - } - - /** - * Asks a remote pinning service to pin an IPFS object from a given path - * - * @param {string|CID} cid - * @param {object} options - * @param {string} options.service - name of a configured remote pinning service - * @param {?string} options.name - optional descriptive name for pin - * @param {?Object} options.meta - optional metadata to attach to pin - * @param {?number} options.timeout - request timeout (seconds) - * @param {?boolean} options.background - If true, add returns remote a pin object as soon as the remote service responds. - * The returned pin object may have a status of 'queued' or 'pinning'. - * If false, the add method will not resolve until the pin status is 'pinned' before returning. - * When background==false and the remote service returns a status of 'failed', an Error will be thrown. - * @returns {Promise} - */ - async add (cid, options) { - const { service, ...addOpts } = options - if (!service) { - throw new Error('service name must be passed') - } - const svc = this.service.serviceNamed(service) - return svc.add(cid, addOpts) - } - - /** - * List objects that are pinned by a remote service. - * - * @param {object} options - * @param {string} options.service - name of a configured remote pinning service - * @param {?Array} options.cid - return pins for the specified CID(s) - * @param {?string} options.name - return pins that contain the provided value (case-sensitive, exact match) - * @param {?Array} options.status - return pins with the specified statuses (queued, pinning, pinned, failed). Default: pinned - * @param {?number} options.timeout - request timeout (seconds) - * @returns {AsyncGenerator} - */ - async * ls (options) { - const { service, ...lsOpts } = options - if (!service) { - throw new Error('service name must be passed') - } - const svc = this.service.serviceNamed(service) - for await (const res of svc.ls(lsOpts)) { - yield res - } - } - - /** - * Remove a single pin from a remote pinning service. - * Fails if multiple pins match the specified criteria. Use rmAll to remove all pins that match. - * - * @param {object} options - * @param {string} options.service - name of a configured remote pinning service - * @param {Array} options.cid - CID(s) to remove from remote pinning service - * @param {?Array} options.status - only remove pins that have one of the specified statuses (queued, pinning, pinned, failed). Default: pinned - * @param {?number} options.timeout - request timeout (seconds) - * @returns {Promise} - */ - async rm (options) { - const { service, ...rmOpts } = options - if (!service) { - throw new Error('service name must be passed') - } - const svc = this.service.serviceNamed(service) - return svc.rm(rmOpts) - } - - /** - * Remove all pins that match the given criteria from a remote pinning service. - * - * @param {object} options - * @param {string} options.service - name of a configured remote pinning service - * @param {Array} options.cid - CID(s) to remove from remote pinning service - * @param {?Array} options.status - only remove pins that have one of the specified statuses (queued, pinning, pinned, failed). Default: pinned - * @param {?number} options.timeout - request timeout (seconds) - * @returns {Promise} - */ - async rmAll (options) { - const { service, ...rmOpts } = options - if (!service) { - throw new Error('service name must be passed') - } - const svc = this.service.serviceNamed(service) - return svc.rmAll(rmOpts) - } -} /** * RemotePinServiceAPI provides methods to add, remove, and list the configured * remote pinning services that are used by the remote pinning api. + * + * @implements API */ -class PinRemoteServiceAPI { + class PinRemoteServiceAPI { constructor ({ config, swarm, peerId }) { this.config = config this.swarm = swarm @@ -149,22 +49,8 @@ class PinRemoteServiceAPI { /** * List the configured remote pinning services. * - * @typedef {object} PinCounts - * @property {number} queued - * @property {number} pinning - * @property {number} pinned - * @property {number} failed - * - * @typedef {object} RemotePinningServiceDescription - * @property {string} name - * @property {URL} endpoint - * @property {?object} stat - * @property {PinServiceStatus} stat.status - * @property {PinCounts} stat.pinCount - * - * @param {object} opts - * @param {?boolean} opts.stat - if true, include status info for each pinning service - * @returns {Promise>} + * @param {{stat: ?boolean} & AbortOptions} opts + * @returns {Promise | Array>} - a Promise resolving to an array of objects describing the configured remote pinning services. If stat==true, each object will include more detailed status info, including the number of pins for each pin status. */ async ls (opts) { const { stat } = (opts || {}) @@ -262,22 +148,15 @@ class RemotePinningService { * Request that the remote service add a pin for the given CID. * * @param {CID|string} cid - CID to pin to remote service - * @param {object} options - * @param {?string} options.name - optional descriptive name for pin - * @param {?Object} options.meta - optional metadata to attach to pin - * @param {?number} options.timeout - request timeout (seconds) - * @param {?boolean} options.background - If background==true, add returns remote a pin object as soon as the remote service responds. - * The returned pin object may have a status of 'queued' or 'pinning'. - * If background==false, the add method will not resolve until the pin status is 'pinned' before returning. - * When background==false and the remote service returns a status of 'failed', an Error will be thrown. + * @param {AddOptions} options * - * @returns {Promise} + * @returns {Promise} */ async _add (cid, options) { - const { meta, background } = options + const { background } = options const name = options.name || '' const origins = await this._originAddresses() - const response = await this.client.add({ cid: cid.toString(), name, meta, origins }) + const response = await this.client.add({ cid: cid.toString(), name, origins }) const { status, pin, delegates } = response this._connectToDelegates(delegates) @@ -309,23 +188,14 @@ class RemotePinningService { _formatPinResult (status, pin) { const name = pin.name || '' const cid = new CID(pin.cid) - const result = { status, name, cid } - if (pin.meta) { - result.meta = pin.meta - } - return result + return { status, name, cid } } /** * List pins from the remote service that match the given criteria. If no criteria are provided, returns all pins with the status 'pinned'. * - * @param {object} options - * @param {?Array} options.cid - return pins for the specified CID(s) - * @param {?string} options.name - return pins that contain the provided value (case-sensitive, exact match) - * @param {?Array} options.status - return pins with the specified statuses (queued, pinning, pinned, failed). Default: pinned - * @param {?number} options.timeout - request timeout (seconds) - * - * @returns {AsyncGenerator} + * @param {Query} options + * @returns {AsyncGenerator} */ async * _ls (options) { const cid = options.cid || [] @@ -345,10 +215,8 @@ class RemotePinningService { * Remove a single pin from a remote pinning service. * Fails if multiple pins match the specified criteria. Use rmAll to remove all pins that match. * - * @param {object} options - * @param {Array} options.cid - CID(s) to remove from remote pinning service - * @param {?Array} options.status - only remove pins that have one of the specified statuses (queued, pinning, pinned, failed). Default: pinned - * @param {?number} options.timeout - request timeout (seconds) + * @param {Query} options + * @returns {Promise} */ async _rm (options) { // the pinning service API only supports deletion by requestid, so we need to lookup the pins first @@ -368,10 +236,7 @@ class RemotePinningService { /** * Remove all pins that match the given criteria from a remote pinning service. * - * @param {object} options - * @param {Array} options.cid - CID(s) to remove from remote pinning service - * @param {?Array} options.status - only remove pins that have one of the specified statuses (queued, pinning, pinned, failed). Default: pinned - * @param {?number} options.timeout - request timeout (seconds) + * @param {Query} options * @returns {Promise} */ async _rmAll (options) { @@ -423,19 +288,21 @@ class RemotePinningService { const delay = ms => new Promise(resolve => setTimeout(resolve, ms)) +// TODO: refactor all the things +module.exports = PinRemoteServiceAPI + /** - * @typedef {import('..').PeerId} PeerId - * @typedef {import('../swarm')} SwarmAPI - * @typedef {import('../config').Config} Config - * - * @typedef {'queued'|'pinning'|'pinned'|'failed'} PinStatus - * @typedef {'valid'|'invalid'} PinServiceStatus - * - * @typedef {object} RemotePin - * @property {string|CID} [cid] - * @property {PinStatus} [status] - * @property {?string} [name] - * @property {?object} [meta] + * @typedef {import('../..').PeerId} PeerId + * @typedef {import('../../swarm')} SwarmAPI + * @typedef {import('../../config').Config} Config + * + * @typedef {import('ipfs-core-types/src/basic').AbortOptions} AbortOptions + * @typedef {import('ipfs-core-types/src/pin/remote').Status} Status + * @typedef {import('ipfs-core-types/src/pin/remote').Query} Query + * @typedef {import('ipfs-core-types/src/pin/remote').Pin} Pin + * @typedef {import('ipfs-core-types/src/pin/remote').AddOptions} AddOptions + * @typedef {import('ipfs-core-types/src/pin/remote/service').API} API + * @typedef {import('ipfs-core-types/src/pin/remote/service').Credentials} Credentials + * @typedef {import('ipfs-core-types/src/pin/remote/service').RemotePinService} RemotePinService + * @typedef {import('ipfs-core-types/src/pin/remote/service').RemotePinServiceWithStat} RemotePinServiceWithStat */ - -module.exports = PinRemoteAPI From 4ce386fcf548fb5618bcdd555ae74fa3141db8a6 Mon Sep 17 00:00:00 2001 From: Yusef Napora Date: Fri, 26 Mar 2021 16:15:25 -0400 Subject: [PATCH 13/21] wip - more refactoring --- .../src/components/pin/remote/client.js | 222 ++++++++++++++++ .../src/components/pin/remote/service.js | 239 ++---------------- 2 files changed, 241 insertions(+), 220 deletions(-) diff --git a/packages/ipfs-core/src/components/pin/remote/client.js b/packages/ipfs-core/src/components/pin/remote/client.js index e69de29bb2..504d9f8efe 100644 --- a/packages/ipfs-core/src/components/pin/remote/client.js +++ b/packages/ipfs-core/src/components/pin/remote/client.js @@ -0,0 +1,222 @@ +'use strict' + +const CID = require('cids') +const multiaddr = require('multiaddr') +const log = require('debug')('ipfs:components:pin:remote:client') + +// TODO: replace this package with one built using ipfs-utils/src/http to reduce bundle size +const PinningClient = require('js-ipfs-pinning-service-client') +const withTimeoutOption = require('ipfs-core-utils/src/with-timeout-option') + + +module.exports = ({ service, endpoint, key, swarm, peerId }) => { + + // TODO: use HTTP requests directly + const client = new PinningClient({endpoint, accessToken: key}) + + async function serviceInfo(includeStats = false) { + if (includeStats) { + return { + service, + endpoint: new URL(endpoint), + stat: await stat() + } + } + return { service, endpoint: new URL(endpoint) } + } + + async function stat () { + try { + const promises = [] + for (const pinStatus of ['queued', 'pinning', 'pinned', 'failed']) { + promises.push(countForStatus(pinStatus)) + } + const [queued, pinning, pinned, failed] = await Promise.all(promises) + return { + status: 'valid', + pinCount: { queued, pinning, pinned, failed } + } + } catch (e) { + log(`error getting stats for service ${service}: `, e) + return { + status: 'invalid' + } + } + } + + async function countForStatus (status) { + const response = await client.ls({ status: [status], limit: 1 }) + return response.count + } + + async function originAddresses () { + const addrs = await swarm.localAddrs() + return addrs.map(ma => { + const str = ma.toString() + + // some relay-style transports add our peer id to the ma for us + // so don't double-add + if (str.endsWith(`/p2p/${peerId}`)) { + return str + } + + return `${str}/p2p/${peerId}` + }) + } + + async function countForStatus (status) { + const response = await client.ls({ status: [status], limit: 1 }) + return response.count + } + + async function connectToDelegates (delegates) { + const addrs = delegates.map(multiaddr) + const promises = [] + for (const addr of addrs) { + promises.push(swarm.connect(addr).catch(e => { + log('error connecting to pinning service delegate: ', e) + })) + } + await Promise.all(promises) + } + + async function awaitPinCompletion (pinResponse) { + const pollIntervalMs = 100 + + let { status, requestid } = pinResponse + while (status !== 'pinned') { + if (status === 'failed') { + throw new Error('pin failed: ' + JSON.stringify(pinResponse.info)) + } + + await delay(pollIntervalMs) + pinResponse = await client.get(requestid) + status = pinResponse.status + } + + return formatPinResult(pinResponse.status, pinResponse.pin) + } + + function formatPinResult(status, pin) { + const name = pin.name || '' + const cid = new CID(pin.cid) + return { status, name, cid } + } + + /** + * Request that the remote service add a pin for the given CID. + * + * @param {CID|string} cid - CID to pin to remote service + * @param {AddOptions} options + * + * @returns {Promise} + */ + async function add (cid, options) { + const { background } = options + const name = options.name || '' + const origins = await originAddresses() + const response = await client.add({ cid: cid.toString(), name, origins }) + + const { status, pin, delegates } = response + connectToDelegates(delegates) + + if (!background) { + return awaitPinCompletion(response) + } + + return formatPinResult(status, pin) + } + + /** + * List pins from the remote service that match the given criteria. If no criteria are provided, returns all pins with the status 'pinned'. + * + * @param {Query} options + * @returns {AsyncGenerator} + */ + async function* ls (options) { + const cid = options.cid || [] + const name = options.name + let status = options.status || [] + if (status.length === 0) { + status = ['pinned'] + } + for await (const pinInfo of client.list({ cid, name, status })) { + const { status, pin } = pinInfo + yield formatPinResult(status, pin) + } + } + + /** + * Remove a single pin from a remote pinning service. + * Fails if multiple pins match the specified criteria. Use rmAll to remove all pins that match. + * + * @param {Query} options + * @returns {Promise} + */ + async function rm (options) { + // the pinning service API only supports deletion by requestid, so we need to lookup the pins first + const { cid, status } = options + const resp = await client.ls({ cid, status }) + if (resp.count === 0) { + return + } + if (resp.count > 1) { + throw new Error('multiple remote pins are matching this query') + } + + const requestid = resp.results[0].requestid + await client.delete(requestid) + } + + + /** + * Remove all pins that match the given criteria from a remote pinning service. + * + * @param {Query} options + * @returns {Promise} + */ + async function rmAll (options) { + const { cid, status } = options + const requestIds = [] + for await (const result of client.list({ cid, status })) { + requestIds.push(result.requestid) + } + + const promises = [] + for (const requestid of requestIds) { + promises.push(client.delete(requestid)) + } + await Promise.all(promises) + } + + return { + info: serviceInfo, + ls: withTimeoutOption(ls), + add: withTimeoutOption(add), + rm: withTimeoutOption(rm), + rmAll: withTimeoutOption(rmAll), + } +} + +const delay = ms => new Promise(resolve => setTimeout(resolve, ms)) + +/** + * @typedef {import('../..').PeerId} PeerId + * @typedef {import('../../swarm')} SwarmAPI + * @typedef {import('../../config').Config} Config + * + * @typedef {import('ipfs-core-types/src/basic').AbortOptions} AbortOptions + * @typedef {import('ipfs-core-types/src/pin/remote').Status} Status + * @typedef {import('ipfs-core-types/src/pin/remote').Query} Query + * @typedef {import('ipfs-core-types/src/pin/remote').Pin} Pin + * @typedef {import('ipfs-core-types/src/pin/remote').AddOptions} AddOptions + * @typedef {import('ipfs-core-types/src/pin/remote/service').API} API + * @typedef {import('ipfs-core-types/src/pin/remote/service').Credentials} Credentials + * @typedef {import('ipfs-core-types/src/pin/remote/service').RemotePinService} RemotePinService + * @typedef {import('ipfs-core-types/src/pin/remote/service').RemotePinServiceWithStat} RemotePinServiceWithStat + */ + +/** + * @typedef {Object} RemotePinClient + * + */ diff --git a/packages/ipfs-core/src/components/pin/remote/service.js b/packages/ipfs-core/src/components/pin/remote/service.js index c23a6b6c93..fa74573ab2 100644 --- a/packages/ipfs-core/src/components/pin/remote/service.js +++ b/packages/ipfs-core/src/components/pin/remote/service.js @@ -1,9 +1,4 @@ -const multiaddr = require('multiaddr') -const CID = require('cids') -const PinningClient = require('js-ipfs-pinning-service-client') -const withTimeoutOption = require('ipfs-core-utils/src/with-timeout-option') -const log = require('debug')('ipfs:components:pin:remote') - +const createClient = require('./client') /** * RemotePinServiceAPI provides methods to add, remove, and list the configured @@ -18,32 +13,34 @@ const log = require('debug')('ipfs:components:pin:remote') this.peerId = peerId // TODO: read service config from IPFS config to construct remote service at init - this._services = {} + this._clients = {} } /** * Adds a new remote pinning service to the set of configured services. * * @param {string} name - the name of the pinning service. Used to identify the service in future remote pinning API calls. - * @param {Object} options - * @param {string|URL} options.endpoint - the remote API endpoint URL - * @param {string} options.key - an API key that authorizes use of the remote pinning service + * @param {Credentials & AbortOptions} credentials */ - async add (name, options) { - if (this._services[name]) { + async add (name, credentials) { + if (this._clients[name]) { throw new Error('service already present: ' + name) } - if (!options.endpoint) { + if (!credentials.endpoint) { throw new Error('option "endpoint" is required') } - if (!options.key) { + if (!credentials.key) { throw new Error('option "key" is required') } - const svcOpts = Object.assign({ swarm: this.swarm, peerId: this.peerId }, options) - this._services[name] = new RemotePinningService(name, svcOpts) + this._clients[name] = createClient({ + swarm: this.swarm, + peerId: this.peerId, + service: name, + ...credentials + }) } /** @@ -52,12 +49,12 @@ const log = require('debug')('ipfs:components:pin:remote') * @param {{stat: ?boolean} & AbortOptions} opts * @returns {Promise | Array>} - a Promise resolving to an array of objects describing the configured remote pinning services. If stat==true, each object will include more detailed status info, including the number of pins for each pin status. */ + // @ts-ignore: The API type definition is polymorphic on the value of the stat field. I'm not sure how to represent that in jsdoc. async ls (opts) { const { stat } = (opts || {}) const promises = [] - for (const name of Object.keys(this._services)) { - const svc = this._services[name] + for (const svc of Object.values(this._clients)) { promises.push(svc.info(stat)) } return Promise.all(promises) @@ -73,221 +70,23 @@ const log = require('debug')('ipfs:components:pin:remote') if (!name) { throw new Error('parameter "name" is required') } - delete this._services[name] + delete this._clients[name] } /** * Returns a RemotePinningService object for the given service name. Throws if no service has been configured with the given name. * * @param {string} name - * @returns {RemotePinningService} + * @returns {any} */ serviceNamed (name) { - if (!this._services[name]) { + if (!this._clients[name]) { throw new Error('no remote pinning service configured with name: ' + name) } - return this._services[name] + return this._clients[name] } } -/** - * RemotePinningService provides add, ls, and rm operations for a single remote pinning service. - */ -class RemotePinningService { - /** - * - * @param {string} name - pinning service name - * @param {object} config - * @param {string|URL} config.endpoint - the remote API endpoint URL - * @param {string} config.key - an API key that authorizes use of the remote pinning service - * @param {SwarmAPI} config.swarm - SwarmAPI instance for the local IPFS node - * @param {PeerId} config.peerId - PeerId of the local IPFS node - */ - constructor (name, { endpoint, key, swarm, peerId }) { - this.endpoint = new URL(endpoint.toString()) - this.name = name - this.swarm = swarm - this.peerId = peerId - this.client = new PinningClient({ name, endpoint, accessToken: key }) - - this.add = withTimeoutOption(this._add.bind(this)) - this.ls = withTimeoutOption(this._ls.bind(this)) - this.rm = withTimeoutOption(this._rm.bind(this)) - this.rmAll = withTimeoutOption(this._rmAll.bind(this)) - } - - async info (includeStats = false) { - const { name, endpoint } = this - const info = { service: name, endpoint } - if (includeStats) { - info.stat = await this.stat() - } - return info - } - - async stat () { - try { - const promises = [] - for (const pinStatus of ['queued', 'pinning', 'pinned', 'failed']) { - promises.push(this._countForStatus(pinStatus)) - } - const [queued, pinning, pinned, failed] = await Promise.all(promises) - return { - status: 'valid', - pinCount: { queued, pinning, pinned, failed } - } - } catch (e) { - log(`error getting stats for service ${this.name}: `, e) - return { - status: 'invalid' - } - } - } - - /** - * Request that the remote service add a pin for the given CID. - * - * @param {CID|string} cid - CID to pin to remote service - * @param {AddOptions} options - * - * @returns {Promise} - */ - async _add (cid, options) { - const { background } = options - const name = options.name || '' - const origins = await this._originAddresses() - const response = await this.client.add({ cid: cid.toString(), name, origins }) - - const { status, pin, delegates } = response - this._connectToDelegates(delegates) - - if (!background) { - return this._awaitPinCompletion(response) - } - - return this._formatPinResult(status, pin) - } - - async _awaitPinCompletion (pinResponse) { - const pollIntervalMs = 100 - - let { status, requestid } = pinResponse - while (status !== 'pinned') { - if (status === 'failed') { - throw new Error('pin failed: ' + JSON.stringify(pinResponse.info)) - } - - await delay(pollIntervalMs) - pinResponse = await this.client.get(requestid) - status = pinResponse.status - } - - return this._formatPinResult(pinResponse.status, pinResponse.pin) - } - - _formatPinResult (status, pin) { - const name = pin.name || '' - const cid = new CID(pin.cid) - return { status, name, cid } - } - - /** - * List pins from the remote service that match the given criteria. If no criteria are provided, returns all pins with the status 'pinned'. - * - * @param {Query} options - * @returns {AsyncGenerator} - */ - async * _ls (options) { - const cid = options.cid || [] - const name = options.name - let status = options.status || [] - if (status.length === 0) { - status = ['pinned'] - } - for await (const pinInfo of this.client.list({ cid, name, status })) { - const { status, pin } = pinInfo - const result = this._formatPinResult(status, pin) - yield result - } - } - - /** - * Remove a single pin from a remote pinning service. - * Fails if multiple pins match the specified criteria. Use rmAll to remove all pins that match. - * - * @param {Query} options - * @returns {Promise} - */ - async _rm (options) { - // the pinning service API only supports deletion by requestid, so we need to lookup the pins first - const { cid, status } = options - const resp = await this.client.ls({ cid, status }) - if (resp.count === 0) { - return - } - if (resp.count > 1) { - throw new Error('multiple remote pins are matching this query') - } - - const requestid = resp.results[0].requestid - await this.client.delete(requestid) - } - - /** - * Remove all pins that match the given criteria from a remote pinning service. - * - * @param {Query} options - * @returns {Promise} - */ - async _rmAll (options) { - const { cid, status } = options - const requestIds = [] - for await (const result of this.client.list({ cid, status })) { - requestIds.push(result.requestid) - } - - const promises = [] - for (const requestid of requestIds) { - promises.push(this.client.delete(requestid)) - } - await Promise.all(promises) - } - - async _originAddresses () { - const addrs = await this.swarm.localAddrs() - const id = this.peerId - return addrs.map(ma => { - const str = ma.toString() - - // some relay-style transports add our peer id to the ma for us - // so don't double-add - if (str.endsWith(`/p2p/${id}`)) { - return str - } - - return `${str}/p2p/${id}` - }) - } - - async _countForStatus (status) { - const response = await this.client.ls({ status: [status], limit: 1 }) - return response.count - } - - async _connectToDelegates (delegates) { - const addrs = delegates.map(multiaddr) - const promises = [] - for (const addr of addrs) { - promises.push(this.swarm.connect(addr).catch(e => { - log('error connecting to pinning service delegate: ', e) - })) - } - await Promise.all(promises) - } -} - -const delay = ms => new Promise(resolve => setTimeout(resolve, ms)) - // TODO: refactor all the things module.exports = PinRemoteServiceAPI From 2808e3e71e93b4f8a04f8c5c51e75cf161ea1184 Mon Sep 17 00:00:00 2001 From: Yusef Napora Date: Fri, 26 Mar 2021 17:37:16 -0400 Subject: [PATCH 14/21] remove pinning client dependency --- packages/ipfs-core/package.json | 1 - .../src/components/pin/remote/client.js | 134 +++++++++++++----- 2 files changed, 101 insertions(+), 34 deletions(-) diff --git a/packages/ipfs-core/package.json b/packages/ipfs-core/package.json index 58154fe032..7adb43adbc 100644 --- a/packages/ipfs-core/package.json +++ b/packages/ipfs-core/package.json @@ -89,7 +89,6 @@ "it-first": "^1.0.4", "it-last": "^1.0.4", "it-pipe": "^1.1.0", - "js-ipfs-pinning-service-client": "github:yusefnapora/js-ipfs-pinning-service-client#874c6ec7875ec2b3039c2361638c54158e7e4874", "libp2p": "^0.30.7", "libp2p-bootstrap": "^0.12.1", "libp2p-crypto": "^0.19.0", diff --git a/packages/ipfs-core/src/components/pin/remote/client.js b/packages/ipfs-core/src/components/pin/remote/client.js index 504d9f8efe..d1ed2269a3 100644 --- a/packages/ipfs-core/src/components/pin/remote/client.js +++ b/packages/ipfs-core/src/components/pin/remote/client.js @@ -2,17 +2,21 @@ const CID = require('cids') const multiaddr = require('multiaddr') +const HTTP = require('ipfs-utils/src/http') const log = require('debug')('ipfs:components:pin:remote:client') -// TODO: replace this package with one built using ipfs-utils/src/http to reduce bundle size -const PinningClient = require('js-ipfs-pinning-service-client') const withTimeoutOption = require('ipfs-core-utils/src/with-timeout-option') module.exports = ({ service, endpoint, key, swarm, peerId }) => { - // TODO: use HTTP requests directly - const client = new PinningClient({endpoint, accessToken: key}) + const api = new HTTP({ + base: endpoint, + headers: { + 'Authorization': `Bearer ${key}`, + 'Content-Type': 'application/json', + } + }) async function serviceInfo(includeStats = false) { if (includeStats) { @@ -44,9 +48,15 @@ module.exports = ({ service, endpoint, key, swarm, peerId }) => { } } + /** + * @param {string} status + * @returns {Promise} - the number of remote pins with the given status + */ async function countForStatus (status) { - const response = await client.ls({ status: [status], limit: 1 }) - return response.count + const searchParams = new URLSearchParams({ status, limit: '1' }) + const response = await api.get('/pins', {searchParams}) + const body = await response.json() + return body.count } async function originAddresses () { @@ -64,11 +74,6 @@ module.exports = ({ service, endpoint, key, swarm, peerId }) => { }) } - async function countForStatus (status) { - const response = await client.ls({ status: [status], limit: 1 }) - return response.count - } - async function connectToDelegates (delegates) { const addrs = delegates.map(multiaddr) const promises = [] @@ -90,7 +95,8 @@ module.exports = ({ service, endpoint, key, swarm, peerId }) => { } await delay(pollIntervalMs) - pinResponse = await client.get(requestid) + const resp = await api.get(`/pins/${requestid}`) + pinResponse = await resp.json() status = pinResponse.status } @@ -115,13 +121,15 @@ module.exports = ({ service, endpoint, key, swarm, peerId }) => { const { background } = options const name = options.name || '' const origins = await originAddresses() - const response = await client.add({ cid: cid.toString(), name, origins }) + const addOpts = { cid: cid.toString(), name, origins } + const response = await api.post('/pins', { json: addOpts }) + const responseBody = await response.json() - const { status, pin, delegates } = response + const { status, pin, delegates } = responseBody connectToDelegates(delegates) if (!background) { - return awaitPinCompletion(response) + return awaitPinCompletion(responseBody) } return formatPinResult(status, pin) @@ -131,18 +139,59 @@ module.exports = ({ service, endpoint, key, swarm, peerId }) => { * List pins from the remote service that match the given criteria. If no criteria are provided, returns all pins with the status 'pinned'. * * @param {Query} options - * @returns {AsyncGenerator} + * @returns {AsyncGenerator} */ - async function* ls (options) { - const cid = options.cid || [] - const name = options.name + async function* _lsRaw (options) { let status = options.status || [] if (status.length === 0) { status = ['pinned'] } - for await (const pinInfo of client.list({ cid, name, status })) { - const { status, pin } = pinInfo - yield formatPinResult(status, pin) + + const searchParams = new URLSearchParams() + if (options.name) { + searchParams.append('name', options.name) + } + for (const cid of (options.cid || [])) { + searchParams.append('cid', cid.toString()) + } + for (const s of status) { + searchParams.append('status', s) + } + + let resp = await api.get('/pins', { searchParams }) + let body = await resp.json() + const total = body.count + let yielded = 0 + while (true) { + if (body.results.length < 1) { + return + } + for (const result of body.results) { + yield result + yielded += 1 + } + + if (yielded == total) { + return + } + + // if we've run out of results and haven't yielded everything, fetch a page of older results + const oldestResult = body.results[body.results.length - 1] + searchParams.set('before', oldestResult.created) + resp = await api.get('/pins', { searchParams }) + body = await resp.json() + } + } + + /** + * List pins from the remote service that match the given criteria. If no criteria are provided, returns all pins with the status 'pinned'. + * + * @param {Query} options + * @returns {AsyncGenerator} + */ + async function* ls(options) { + for await (const result of _lsRaw(options)) { + yield formatPinResult(result.status, result.pin) } } @@ -155,17 +204,33 @@ module.exports = ({ service, endpoint, key, swarm, peerId }) => { */ async function rm (options) { // the pinning service API only supports deletion by requestid, so we need to lookup the pins first - const { cid, status } = options - const resp = await client.ls({ cid, status }) - if (resp.count === 0) { + const searchParams = new URLSearchParams() + if (options.name) { + searchParams.set('name', options.name) + } + for (const cid of (options.cid || [])) { + searchParams.append('cid', cid.toString()) + } + for (const status of (options.status || [])) { + searchParams.append('status', status) + } + const resp = await api.get('/pins', { searchParams }) + const body = await resp.json() + if (body.count === 0) { return } - if (resp.count > 1) { + if (body.count > 1) { throw new Error('multiple remote pins are matching this query') } - const requestid = resp.results[0].requestid - await client.delete(requestid) + const requestid = body.results[0].requestid + try { + await api.delete(`/pins/${requestid}`) + } catch (e) { + if (e.status !== 404) { + throw e + } + } } @@ -176,15 +241,14 @@ module.exports = ({ service, endpoint, key, swarm, peerId }) => { * @returns {Promise} */ async function rmAll (options) { - const { cid, status } = options const requestIds = [] - for await (const result of client.list({ cid, status })) { + for await (const result of _lsRaw(options)) { requestIds.push(result.requestid) } const promises = [] for (const requestid of requestIds) { - promises.push(client.delete(requestid)) + promises.push(api.delete(`/pins/${requestid}`)) } await Promise.all(promises) } @@ -210,13 +274,17 @@ const delay = ms => new Promise(resolve => setTimeout(resolve, ms)) * @typedef {import('ipfs-core-types/src/pin/remote').Query} Query * @typedef {import('ipfs-core-types/src/pin/remote').Pin} Pin * @typedef {import('ipfs-core-types/src/pin/remote').AddOptions} AddOptions - * @typedef {import('ipfs-core-types/src/pin/remote/service').API} API * @typedef {import('ipfs-core-types/src/pin/remote/service').Credentials} Credentials * @typedef {import('ipfs-core-types/src/pin/remote/service').RemotePinService} RemotePinService * @typedef {import('ipfs-core-types/src/pin/remote/service').RemotePinServiceWithStat} RemotePinServiceWithStat */ /** - * @typedef {Object} RemotePinClient + * @typedef {Object} PinDetails + * @property {string} requestid + * @property {string} created + * @property {Status} status + * @property {Pin} pin + * @property {Array} delegates * */ From a64118eac7ad0aac835be211b50a46590aad0242 Mon Sep 17 00:00:00 2001 From: Yusef Napora Date: Fri, 26 Mar 2021 17:39:16 -0400 Subject: [PATCH 15/21] fix lint issues --- .../src/components/pin/remote/add.js | 14 +- .../src/components/pin/remote/client.js | 145 +++++++++--------- .../src/components/pin/remote/index.js | 2 - .../ipfs-core/src/components/pin/remote/ls.js | 5 +- .../ipfs-core/src/components/pin/remote/rm.js | 2 +- .../src/components/pin/remote/rmAll.js | 2 +- .../src/components/pin/remote/service.js | 16 +- 7 files changed, 91 insertions(+), 95 deletions(-) diff --git a/packages/ipfs-core/src/components/pin/remote/add.js b/packages/ipfs-core/src/components/pin/remote/add.js index ded8f8a27e..b2aac19393 100644 --- a/packages/ipfs-core/src/components/pin/remote/add.js +++ b/packages/ipfs-core/src/components/pin/remote/add.js @@ -2,14 +2,14 @@ const withTimeoutOption = require('ipfs-core-utils/src/with-timeout-option') -module.exports = ({remotePinServices}) => { +module.exports = ({ remotePinServices }) => { /** - * Asks a remote pinning service to pin an IPFS object from a given path - * - * @param {string|CID} cid - * @param {AddOptions & AbortOptions} options - * @returns {Promise} - */ + * Asks a remote pinning service to pin an IPFS object from a given path + * + * @param {string|CID} cid + * @param {AddOptions & AbortOptions} options + * @returns {Promise} + */ async function add (cid, options) { const { service, ...addOpts } = options if (!service) { diff --git a/packages/ipfs-core/src/components/pin/remote/client.js b/packages/ipfs-core/src/components/pin/remote/client.js index d1ed2269a3..36c991031d 100644 --- a/packages/ipfs-core/src/components/pin/remote/client.js +++ b/packages/ipfs-core/src/components/pin/remote/client.js @@ -7,18 +7,16 @@ const log = require('debug')('ipfs:components:pin:remote:client') const withTimeoutOption = require('ipfs-core-utils/src/with-timeout-option') - module.exports = ({ service, endpoint, key, swarm, peerId }) => { - const api = new HTTP({ base: endpoint, headers: { - 'Authorization': `Bearer ${key}`, - 'Content-Type': 'application/json', + Authorization: `Bearer ${key}`, + 'Content-Type': 'application/json' } }) - async function serviceInfo(includeStats = false) { + async function serviceInfo (includeStats = false) { if (includeStats) { return { service, @@ -49,12 +47,12 @@ module.exports = ({ service, endpoint, key, swarm, peerId }) => { } /** - * @param {string} status + * @param {string} status * @returns {Promise} - the number of remote pins with the given status */ async function countForStatus (status) { const searchParams = new URLSearchParams({ status, limit: '1' }) - const response = await api.get('/pins', {searchParams}) + const response = await api.get('/pins', { searchParams }) const body = await response.json() return body.count } @@ -103,7 +101,7 @@ module.exports = ({ service, endpoint, key, swarm, peerId }) => { return formatPinResult(pinResponse.status, pinResponse.pin) } - function formatPinResult(status, pin) { + function formatPinResult (status, pin) { const name = pin.name || '' const cid = new CID(pin.cid) return { status, name, cid } @@ -117,31 +115,31 @@ module.exports = ({ service, endpoint, key, swarm, peerId }) => { * * @returns {Promise} */ - async function add (cid, options) { - const { background } = options - const name = options.name || '' - const origins = await originAddresses() - const addOpts = { cid: cid.toString(), name, origins } - const response = await api.post('/pins', { json: addOpts }) - const responseBody = await response.json() - - const { status, pin, delegates } = responseBody - connectToDelegates(delegates) - - if (!background) { - return awaitPinCompletion(responseBody) - } - - return formatPinResult(status, pin) + async function add (cid, options) { + const { background } = options + const name = options.name || '' + const origins = await originAddresses() + const addOpts = { cid: cid.toString(), name, origins } + const response = await api.post('/pins', { json: addOpts }) + const responseBody = await response.json() + + const { status, pin, delegates } = responseBody + connectToDelegates(delegates) + + if (!background) { + return awaitPinCompletion(responseBody) } + return formatPinResult(status, pin) + } + /** * List pins from the remote service that match the given criteria. If no criteria are provided, returns all pins with the status 'pinned'. * * @param {Query} options * @returns {AsyncGenerator} */ - async function* _lsRaw (options) { + async function * _lsRaw (options) { let status = options.status || [] if (status.length === 0) { status = ['pinned'] @@ -151,7 +149,7 @@ module.exports = ({ service, endpoint, key, swarm, peerId }) => { if (options.name) { searchParams.append('name', options.name) } - for (const cid of (options.cid || [])) { + for (const cid of (options.cid || [])) { searchParams.append('cid', cid.toString()) } for (const s of status) { @@ -171,7 +169,7 @@ module.exports = ({ service, endpoint, key, swarm, peerId }) => { yielded += 1 } - if (yielded == total) { + if (yielded === total) { return } @@ -189,7 +187,7 @@ module.exports = ({ service, endpoint, key, swarm, peerId }) => { * @param {Query} options * @returns {AsyncGenerator} */ - async function* ls(options) { + async function * ls (options) { for await (const result of _lsRaw(options)) { yield formatPinResult(result.status, result.pin) } @@ -202,37 +200,36 @@ module.exports = ({ service, endpoint, key, swarm, peerId }) => { * @param {Query} options * @returns {Promise} */ - async function rm (options) { - // the pinning service API only supports deletion by requestid, so we need to lookup the pins first - const searchParams = new URLSearchParams() - if (options.name) { - searchParams.set('name', options.name) - } - for (const cid of (options.cid || [])) { - searchParams.append('cid', cid.toString()) - } - for (const status of (options.status || [])) { - searchParams.append('status', status) - } - const resp = await api.get('/pins', { searchParams }) - const body = await resp.json() - if (body.count === 0) { - return - } - if (body.count > 1) { - throw new Error('multiple remote pins are matching this query') - } - - const requestid = body.results[0].requestid - try { - await api.delete(`/pins/${requestid}`) - } catch (e) { - if (e.status !== 404) { - throw e - } - } + async function rm (options) { + // the pinning service API only supports deletion by requestid, so we need to lookup the pins first + const searchParams = new URLSearchParams() + if (options.name) { + searchParams.set('name', options.name) + } + for (const cid of (options.cid || [])) { + searchParams.append('cid', cid.toString()) + } + for (const status of (options.status || [])) { + searchParams.append('status', status) + } + const resp = await api.get('/pins', { searchParams }) + const body = await resp.json() + if (body.count === 0) { + return + } + if (body.count > 1) { + throw new Error('multiple remote pins are matching this query') } + const requestid = body.results[0].requestid + try { + await api.delete(`/pins/${requestid}`) + } catch (e) { + if (e.status !== 404) { + throw e + } + } + } /** * Remove all pins that match the given criteria from a remote pinning service. @@ -240,25 +237,25 @@ module.exports = ({ service, endpoint, key, swarm, peerId }) => { * @param {Query} options * @returns {Promise} */ - async function rmAll (options) { - const requestIds = [] - for await (const result of _lsRaw(options)) { - requestIds.push(result.requestid) - } - - const promises = [] - for (const requestid of requestIds) { - promises.push(api.delete(`/pins/${requestid}`)) - } - await Promise.all(promises) + async function rmAll (options) { + const requestIds = [] + for await (const result of _lsRaw(options)) { + requestIds.push(result.requestid) } - + + const promises = [] + for (const requestid of requestIds) { + promises.push(api.delete(`/pins/${requestid}`)) + } + await Promise.all(promises) + } + return { info: serviceInfo, ls: withTimeoutOption(ls), - add: withTimeoutOption(add), - rm: withTimeoutOption(rm), - rmAll: withTimeoutOption(rmAll), + add: withTimeoutOption(add), + rm: withTimeoutOption(rm), + rmAll: withTimeoutOption(rmAll) } } @@ -268,7 +265,7 @@ const delay = ms => new Promise(resolve => setTimeout(resolve, ms)) * @typedef {import('../..').PeerId} PeerId * @typedef {import('../../swarm')} SwarmAPI * @typedef {import('../../config').Config} Config - * + * * @typedef {import('ipfs-core-types/src/basic').AbortOptions} AbortOptions * @typedef {import('ipfs-core-types/src/pin/remote').Status} Status * @typedef {import('ipfs-core-types/src/pin/remote').Query} Query @@ -286,5 +283,5 @@ const delay = ms => new Promise(resolve => setTimeout(resolve, ms)) * @property {Status} status * @property {Pin} pin * @property {Array} delegates - * + * */ diff --git a/packages/ipfs-core/src/components/pin/remote/index.js b/packages/ipfs-core/src/components/pin/remote/index.js index 5cd595b127..d1683764ce 100644 --- a/packages/ipfs-core/src/components/pin/remote/index.js +++ b/packages/ipfs-core/src/components/pin/remote/index.js @@ -27,10 +27,8 @@ class PinRemoteAPI { this.rm = createRm({ remotePinServices }) this.rmAll = createRmAll({ remotePinServices }) } - } - /** * @typedef {import('../..').PeerId} PeerId * @typedef {import('../../swarm')} SwarmAPI diff --git a/packages/ipfs-core/src/components/pin/remote/ls.js b/packages/ipfs-core/src/components/pin/remote/ls.js index c870045f2e..267db781cf 100644 --- a/packages/ipfs-core/src/components/pin/remote/ls.js +++ b/packages/ipfs-core/src/components/pin/remote/ls.js @@ -3,14 +3,13 @@ const withTimeoutOption = require('ipfs-core-utils/src/with-timeout-option') module.exports = ({ remotePinServices }) => { - /** * List objects that are pinned by a remote service. * * @param {Query & AbortOptions} options * @returns {AsyncGenerator} */ - async function* ls (options) { + async function * ls (options) { const { service, ...lsOpts } = options if (!service) { throw new Error('service name must be passed') @@ -30,4 +29,4 @@ module.exports = ({ remotePinServices }) => { * @typedef {import('ipfs-core-types/src/pin/remote').Pin} Pin * @typedef {import('ipfs-core-types/src/pin/remote').Status} Status * @typedef {import('ipfs-core-types/src/pin/remote').Query} Query - */ \ No newline at end of file + */ diff --git a/packages/ipfs-core/src/components/pin/remote/rm.js b/packages/ipfs-core/src/components/pin/remote/rm.js index 7906bd6417..4c7cd73853 100644 --- a/packages/ipfs-core/src/components/pin/remote/rm.js +++ b/packages/ipfs-core/src/components/pin/remote/rm.js @@ -10,7 +10,7 @@ module.exports = ({ remotePinServices }) => { * @param {Query & AbortOptions} options * @returns {Promise} */ - async function rm (options) { + async function rm (options) { const { service, ...rmOpts } = options if (!service) { throw new Error('service name must be passed') diff --git a/packages/ipfs-core/src/components/pin/remote/rmAll.js b/packages/ipfs-core/src/components/pin/remote/rmAll.js index e9ca123baf..0b9ea96528 100644 --- a/packages/ipfs-core/src/components/pin/remote/rmAll.js +++ b/packages/ipfs-core/src/components/pin/remote/rmAll.js @@ -9,7 +9,7 @@ module.exports = ({ remotePinServices }) => { * @param {Query & AbortOptions} options * @returns {Promise} */ - async function rmAll (options) { + async function rmAll (options) { const { service, ...rmOpts } = options if (!service) { throw new Error('service name must be passed') diff --git a/packages/ipfs-core/src/components/pin/remote/service.js b/packages/ipfs-core/src/components/pin/remote/service.js index fa74573ab2..84ca8431c6 100644 --- a/packages/ipfs-core/src/components/pin/remote/service.js +++ b/packages/ipfs-core/src/components/pin/remote/service.js @@ -1,12 +1,14 @@ +'use strict' + const createClient = require('./client') /** * RemotePinServiceAPI provides methods to add, remove, and list the configured * remote pinning services that are used by the remote pinning api. - * + * * @implements API */ - class PinRemoteServiceAPI { +class PinRemoteServiceAPI { constructor ({ config, swarm, peerId }) { this.config = config this.swarm = swarm @@ -35,10 +37,10 @@ const createClient = require('./client') throw new Error('option "key" is required') } - this._clients[name] = createClient({ - swarm: this.swarm, - peerId: this.peerId, - service: name, + this._clients[name] = createClient({ + swarm: this.swarm, + peerId: this.peerId, + service: name, ...credentials }) } @@ -94,7 +96,7 @@ module.exports = PinRemoteServiceAPI * @typedef {import('../..').PeerId} PeerId * @typedef {import('../../swarm')} SwarmAPI * @typedef {import('../../config').Config} Config - * + * * @typedef {import('ipfs-core-types/src/basic').AbortOptions} AbortOptions * @typedef {import('ipfs-core-types/src/pin/remote').Status} Status * @typedef {import('ipfs-core-types/src/pin/remote').Query} Query From 0fa854959b7057fd9aaedc32a64fa5f59eca0504 Mon Sep 17 00:00:00 2001 From: Yusef Napora Date: Fri, 26 Mar 2021 17:48:21 -0400 Subject: [PATCH 16/21] fix timeout for remote service stat --- packages/ipfs-core/src/components/pin/remote/client.js | 4 ++-- packages/ipfs-core/src/components/pin/remote/service.js | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/ipfs-core/src/components/pin/remote/client.js b/packages/ipfs-core/src/components/pin/remote/client.js index 36c991031d..d159d33818 100644 --- a/packages/ipfs-core/src/components/pin/remote/client.js +++ b/packages/ipfs-core/src/components/pin/remote/client.js @@ -16,7 +16,7 @@ module.exports = ({ service, endpoint, key, swarm, peerId }) => { } }) - async function serviceInfo (includeStats = false) { + async function info ({stat: includeStats}) { if (includeStats) { return { service, @@ -251,7 +251,7 @@ module.exports = ({ service, endpoint, key, swarm, peerId }) => { } return { - info: serviceInfo, + info: withTimeoutOption(info), ls: withTimeoutOption(ls), add: withTimeoutOption(add), rm: withTimeoutOption(rm), diff --git a/packages/ipfs-core/src/components/pin/remote/service.js b/packages/ipfs-core/src/components/pin/remote/service.js index 84ca8431c6..3140164400 100644 --- a/packages/ipfs-core/src/components/pin/remote/service.js +++ b/packages/ipfs-core/src/components/pin/remote/service.js @@ -53,11 +53,11 @@ class PinRemoteServiceAPI { */ // @ts-ignore: The API type definition is polymorphic on the value of the stat field. I'm not sure how to represent that in jsdoc. async ls (opts) { - const { stat } = (opts || {}) + opts = opts || { stat: false } const promises = [] for (const svc of Object.values(this._clients)) { - promises.push(svc.info(stat)) + promises.push(svc.info(opts)) } return Promise.all(promises) } From 1648b227ecd3c86a77e200c627dae45be9a56efb Mon Sep 17 00:00:00 2001 From: Yusef Napora Date: Fri, 26 Mar 2021 21:36:54 -0400 Subject: [PATCH 17/21] add Pinning.RemoteServices to IpfsConfig --- packages/ipfs-core/src/components/config.js | 9 +++ .../src/components/pin/remote/service.js | 64 ++++++++++++++++++- 2 files changed, 72 insertions(+), 1 deletion(-) diff --git a/packages/ipfs-core/src/components/config.js b/packages/ipfs-core/src/components/config.js index bc598485ae..905ef1213f 100644 --- a/packages/ipfs-core/src/components/config.js +++ b/packages/ipfs-core/src/components/config.js @@ -328,6 +328,7 @@ module.exports.profiles = profiles * @property {PubsubConfig} [Pubsub] * @property {SwarmConfig} [Swarm] * @property {RoutingConfig} [Routing] + * @property {PinningConfig} [Pinning] * * @typedef {Object} AddressConfig * Contains information about various listener addresses to be used by this node. @@ -524,4 +525,12 @@ module.exports.profiles = profiles * * @typedef {import('ipfs-core-types/src/basic').ToJSON} ToJSON * @typedef {import('.').AbortOptions} AbortOptions + * + * @typedef {Object} PinningConfig + * @property {Object} [RemoteServices] + * + * @typedef {Object} RemotePinningServiceConfig + * @property {Object} API + * @property {string} API.Endpoint + * @property {string} API.Key */ diff --git a/packages/ipfs-core/src/components/pin/remote/service.js b/packages/ipfs-core/src/components/pin/remote/service.js index 3140164400..84ae48f902 100644 --- a/packages/ipfs-core/src/components/pin/remote/service.js +++ b/packages/ipfs-core/src/components/pin/remote/service.js @@ -1,5 +1,8 @@ 'use strict' +const { Errors } = require('interface-datastore') +const ERR_NOT_FOUND = Errors.notFoundError().code + const createClient = require('./client') /** @@ -9,13 +12,56 @@ const createClient = require('./client') * @implements API */ class PinRemoteServiceAPI { + /** + * + * @param {Object} options + * @param {Config} options.config + * @param {SwarmAPI} options.swarm + * @param {PeerId} options.peerId + */ constructor ({ config, swarm, peerId }) { this.config = config this.swarm = swarm this.peerId = peerId - // TODO: read service config from IPFS config to construct remote service at init this._clients = {} + this._configured = false + } + + async _loadConfig() { + if (this._configured) { + return + } + + try { + const pinConfig = /** @type {PinningConfig|null} */ (await this.config.get('Pinning')) + if (pinConfig == null || pinConfig.RemoteServices == null) { + this._configured = true + return + } + + for (const [name, svcConfig] of Object.entries(pinConfig.RemoteServices)) { + if (svcConfig == null) { + continue + } + const { Endpoint: endpoint, Key: key } = svcConfig.API + if (!endpoint || !key) { + continue + } + this._clients[name] = createClient({ + swarm: this.swarm, + peerId: this.peerId, + service: name, + endpoint, + key, + }) + } + this._configured = true + } catch (e) { + if (e.code !== ERR_NOT_FOUND) { + throw e + } + } } /** @@ -25,6 +71,8 @@ class PinRemoteServiceAPI { * @param {Credentials & AbortOptions} credentials */ async add (name, credentials) { + await this._loadConfig() + if (this._clients[name]) { throw new Error('service already present: ' + name) } @@ -37,6 +85,13 @@ class PinRemoteServiceAPI { throw new Error('option "key" is required') } + await this.config.set(`Pinning.RemoteServices.${name}`, { + API: { + Endpoint: credentials.endpoint.toString(), + Key: credentials.key, + } + }) + this._clients[name] = createClient({ swarm: this.swarm, peerId: this.peerId, @@ -53,6 +108,8 @@ class PinRemoteServiceAPI { */ // @ts-ignore: The API type definition is polymorphic on the value of the stat field. I'm not sure how to represent that in jsdoc. async ls (opts) { + await this._loadConfig() + opts = opts || { stat: false } const promises = [] @@ -72,7 +129,11 @@ class PinRemoteServiceAPI { if (!name) { throw new Error('parameter "name" is required') } + await this._loadConfig() delete this._clients[name] + const services = (await this.config.get('Pinning.RemoteServices')) || {} + delete services[name] + await this.config.set(`Pinning.RemoteServices`, services) } /** @@ -96,6 +157,7 @@ module.exports = PinRemoteServiceAPI * @typedef {import('../..').PeerId} PeerId * @typedef {import('../../swarm')} SwarmAPI * @typedef {import('../../config').Config} Config + * @typedef {import('../../config').PinningConfig} PinningConfig * * @typedef {import('ipfs-core-types/src/basic').AbortOptions} AbortOptions * @typedef {import('ipfs-core-types/src/pin/remote').Status} Status From c01fe6cfae72fb49b14dac8a32909ad237d21bb8 Mon Sep 17 00:00:00 2001 From: Yusef Napora Date: Fri, 26 Mar 2021 21:54:30 -0400 Subject: [PATCH 18/21] refactor class into lambda factory fn --- packages/ipfs-core/src/components/config.js | 4 +- .../src/components/pin/remote/add.js | 4 +- .../src/components/pin/remote/client.js | 2 +- .../src/components/pin/remote/index.js | 16 ++- .../ipfs-core/src/components/pin/remote/ls.js | 4 +- .../ipfs-core/src/components/pin/remote/rm.js | 4 +- .../src/components/pin/remote/rmAll.js | 4 +- .../src/components/pin/remote/service.js | 102 +++++++++--------- 8 files changed, 66 insertions(+), 74 deletions(-) diff --git a/packages/ipfs-core/src/components/config.js b/packages/ipfs-core/src/components/config.js index 905ef1213f..c9258eb398 100644 --- a/packages/ipfs-core/src/components/config.js +++ b/packages/ipfs-core/src/components/config.js @@ -525,10 +525,10 @@ module.exports.profiles = profiles * * @typedef {import('ipfs-core-types/src/basic').ToJSON} ToJSON * @typedef {import('.').AbortOptions} AbortOptions - * + * * @typedef {Object} PinningConfig * @property {Object} [RemoteServices] - * + * * @typedef {Object} RemotePinningServiceConfig * @property {Object} API * @property {string} API.Endpoint diff --git a/packages/ipfs-core/src/components/pin/remote/add.js b/packages/ipfs-core/src/components/pin/remote/add.js index b2aac19393..716283f427 100644 --- a/packages/ipfs-core/src/components/pin/remote/add.js +++ b/packages/ipfs-core/src/components/pin/remote/add.js @@ -2,7 +2,7 @@ const withTimeoutOption = require('ipfs-core-utils/src/with-timeout-option') -module.exports = ({ remotePinServices }) => { +module.exports = ({ serviceRegistry }) => { /** * Asks a remote pinning service to pin an IPFS object from a given path * @@ -15,7 +15,7 @@ module.exports = ({ remotePinServices }) => { if (!service) { throw new Error('service name must be passed') } - const svc = remotePinServices.serviceNamed(service) + const svc = serviceRegistry.serviceNamed(service) return svc.add(cid, addOpts) } diff --git a/packages/ipfs-core/src/components/pin/remote/client.js b/packages/ipfs-core/src/components/pin/remote/client.js index d159d33818..add691ba9c 100644 --- a/packages/ipfs-core/src/components/pin/remote/client.js +++ b/packages/ipfs-core/src/components/pin/remote/client.js @@ -16,7 +16,7 @@ module.exports = ({ service, endpoint, key, swarm, peerId }) => { } }) - async function info ({stat: includeStats}) { + async function info ({ stat: includeStats }) { if (includeStats) { return { service, diff --git a/packages/ipfs-core/src/components/pin/remote/index.js b/packages/ipfs-core/src/components/pin/remote/index.js index d1683764ce..efc12b9bf6 100644 --- a/packages/ipfs-core/src/components/pin/remote/index.js +++ b/packages/ipfs-core/src/components/pin/remote/index.js @@ -1,6 +1,6 @@ 'use strict' -const PinRemoteServiceAPI = require('./service') +const createServiceApi = require('./service') const createAdd = require('./add') const createLs = require('./ls') const createRm = require('./rm') @@ -17,15 +17,13 @@ class PinRemoteAPI { * @param {PeerId} opts.peerId */ constructor ({ swarm, config, peerId }) { - this.swarm = swarm - this.service = new PinRemoteServiceAPI({ config, swarm, peerId }) + const serviceRegistry = createServiceApi({ config, swarm, peerId }) - // TODO: remove this.service & this.swarm once everything is refactored - const remotePinServices = this.service - this.add = createAdd({ remotePinServices }) - this.ls = createLs({ remotePinServices }) - this.rm = createRm({ remotePinServices }) - this.rmAll = createRmAll({ remotePinServices }) + this.service = serviceRegistry + this.add = createAdd({ serviceRegistry }) + this.ls = createLs({ serviceRegistry }) + this.rm = createRm({ serviceRegistry }) + this.rmAll = createRmAll({ serviceRegistry }) } } diff --git a/packages/ipfs-core/src/components/pin/remote/ls.js b/packages/ipfs-core/src/components/pin/remote/ls.js index 267db781cf..4f40a8bc74 100644 --- a/packages/ipfs-core/src/components/pin/remote/ls.js +++ b/packages/ipfs-core/src/components/pin/remote/ls.js @@ -2,7 +2,7 @@ const withTimeoutOption = require('ipfs-core-utils/src/with-timeout-option') -module.exports = ({ remotePinServices }) => { +module.exports = ({ serviceRegistry }) => { /** * List objects that are pinned by a remote service. * @@ -14,7 +14,7 @@ module.exports = ({ remotePinServices }) => { if (!service) { throw new Error('service name must be passed') } - const svc = remotePinServices.serviceNamed(service) + const svc = serviceRegistry.serviceNamed(service) for await (const res of svc.ls(lsOpts)) { yield res } diff --git a/packages/ipfs-core/src/components/pin/remote/rm.js b/packages/ipfs-core/src/components/pin/remote/rm.js index 4c7cd73853..9b55297f67 100644 --- a/packages/ipfs-core/src/components/pin/remote/rm.js +++ b/packages/ipfs-core/src/components/pin/remote/rm.js @@ -2,7 +2,7 @@ const withTimeoutOption = require('ipfs-core-utils/src/with-timeout-option') -module.exports = ({ remotePinServices }) => { +module.exports = ({ serviceRegistry }) => { /** * Remove a single pin from a remote pinning service. * Fails if multiple pins match the specified query. Use rmAll to remove all pins that match. @@ -15,7 +15,7 @@ module.exports = ({ remotePinServices }) => { if (!service) { throw new Error('service name must be passed') } - const svc = remotePinServices.serviceNamed(service) + const svc = serviceRegistry.serviceNamed(service) return svc.rm(rmOpts) } diff --git a/packages/ipfs-core/src/components/pin/remote/rmAll.js b/packages/ipfs-core/src/components/pin/remote/rmAll.js index 0b9ea96528..661631eaf9 100644 --- a/packages/ipfs-core/src/components/pin/remote/rmAll.js +++ b/packages/ipfs-core/src/components/pin/remote/rmAll.js @@ -2,7 +2,7 @@ const withTimeoutOption = require('ipfs-core-utils/src/with-timeout-option') -module.exports = ({ remotePinServices }) => { +module.exports = ({ serviceRegistry }) => { /** * Remove all pins that match the given criteria from a remote pinning service. * @@ -14,7 +14,7 @@ module.exports = ({ remotePinServices }) => { if (!service) { throw new Error('service name must be passed') } - const svc = remotePinServices.serviceNamed(service) + const svc = serviceRegistry.serviceNamed(service) return svc.rmAll(rmOpts) } diff --git a/packages/ipfs-core/src/components/pin/remote/service.js b/packages/ipfs-core/src/components/pin/remote/service.js index 84ae48f902..5b47e3def6 100644 --- a/packages/ipfs-core/src/components/pin/remote/service.js +++ b/packages/ipfs-core/src/components/pin/remote/service.js @@ -6,41 +6,30 @@ const ERR_NOT_FOUND = Errors.notFoundError().code const createClient = require('./client') /** - * RemotePinServiceAPI provides methods to add, remove, and list the configured - * remote pinning services that are used by the remote pinning api. * - * @implements API + * @param {Object} options + * @param {Config} options.config + * @param {SwarmAPI} options.swarm + * @param {PeerId} options.peerId + * @returns {API} */ -class PinRemoteServiceAPI { - /** - * - * @param {Object} options - * @param {Config} options.config - * @param {SwarmAPI} options.swarm - * @param {PeerId} options.peerId - */ - constructor ({ config, swarm, peerId }) { - this.config = config - this.swarm = swarm - this.peerId = peerId - - this._clients = {} - this._configured = false - } +module.exports = ({ config, swarm, peerId }) => { + let configured = false + const clients = {} - async _loadConfig() { - if (this._configured) { + async function loadConfig () { + if (configured) { return } try { - const pinConfig = /** @type {PinningConfig|null} */ (await this.config.get('Pinning')) + const pinConfig = /** @type {PinningConfig|null} */ (await config.get('Pinning')) if (pinConfig == null || pinConfig.RemoteServices == null) { - this._configured = true + configured = true return } - for (const [name, svcConfig] of Object.entries(pinConfig.RemoteServices)) { + for (const [service, svcConfig] of Object.entries(pinConfig.RemoteServices)) { if (svcConfig == null) { continue } @@ -48,20 +37,21 @@ class PinRemoteServiceAPI { if (!endpoint || !key) { continue } - this._clients[name] = createClient({ - swarm: this.swarm, - peerId: this.peerId, - service: name, + clients[service] = createClient({ + swarm, + peerId, + service, endpoint, - key, + key }) } - this._configured = true } catch (e) { if (e.code !== ERR_NOT_FOUND) { throw e } } + + configured = true } /** @@ -70,10 +60,10 @@ class PinRemoteServiceAPI { * @param {string} name - the name of the pinning service. Used to identify the service in future remote pinning API calls. * @param {Credentials & AbortOptions} credentials */ - async add (name, credentials) { - await this._loadConfig() + async function add (name, credentials) { + await loadConfig() - if (this._clients[name]) { + if (clients[name]) { throw new Error('service already present: ' + name) } @@ -85,16 +75,16 @@ class PinRemoteServiceAPI { throw new Error('option "key" is required') } - await this.config.set(`Pinning.RemoteServices.${name}`, { + await config.set(`Pinning.RemoteServices.${name}`, { API: { Endpoint: credentials.endpoint.toString(), - Key: credentials.key, + Key: credentials.key } }) - this._clients[name] = createClient({ - swarm: this.swarm, - peerId: this.peerId, + clients[name] = createClient({ + swarm, + peerId, service: name, ...credentials }) @@ -106,14 +96,13 @@ class PinRemoteServiceAPI { * @param {{stat: ?boolean} & AbortOptions} opts * @returns {Promise | Array>} - a Promise resolving to an array of objects describing the configured remote pinning services. If stat==true, each object will include more detailed status info, including the number of pins for each pin status. */ - // @ts-ignore: The API type definition is polymorphic on the value of the stat field. I'm not sure how to represent that in jsdoc. - async ls (opts) { - await this._loadConfig() + async function ls (opts) { + await loadConfig() opts = opts || { stat: false } const promises = [] - for (const svc of Object.values(this._clients)) { + for (const svc of Object.values(clients)) { promises.push(svc.info(opts)) } return Promise.all(promises) @@ -125,33 +114,38 @@ class PinRemoteServiceAPI { * @param {string} name - the name of the pinning service to remove * @returns {Promise} */ - async rm (name) { + async function rm (name) { if (!name) { throw new Error('parameter "name" is required') } - await this._loadConfig() - delete this._clients[name] - const services = (await this.config.get('Pinning.RemoteServices')) || {} + await loadConfig() + delete clients[name] + const services = (await config.get('Pinning.RemoteServices')) || {} delete services[name] - await this.config.set(`Pinning.RemoteServices`, services) + await config.set('Pinning.RemoteServices', services) } /** - * Returns a RemotePinningService object for the given service name. Throws if no service has been configured with the given name. + * Returns a client for the service with the given name. Throws if no service has been configured with the given name. * * @param {string} name * @returns {any} */ - serviceNamed (name) { - if (!this._clients[name]) { + function serviceNamed (name) { + if (!clients[name]) { throw new Error('no remote pinning service configured with name: ' + name) } - return this._clients[name] + return clients[name] } -} -// TODO: refactor all the things -module.exports = PinRemoteServiceAPI + return { + add, + // @ts-ignore: The API type definition for the ls method is polymorphic on the value of the stat field. I'm not sure how to represent that in jsdoc. + ls, + rm, + serviceNamed + } +} /** * @typedef {import('../..').PeerId} PeerId From 47713cc4a79584b83983eceff4adbe1182b5dc6d Mon Sep 17 00:00:00 2001 From: Yusef Napora Date: Fri, 26 Mar 2021 21:58:10 -0400 Subject: [PATCH 19/21] rm unused type imports --- packages/ipfs-core/src/components/pin/remote/client.js | 6 +----- packages/ipfs-core/src/components/pin/remote/service.js | 4 ---- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/packages/ipfs-core/src/components/pin/remote/client.js b/packages/ipfs-core/src/components/pin/remote/client.js index add691ba9c..c17fd56b34 100644 --- a/packages/ipfs-core/src/components/pin/remote/client.js +++ b/packages/ipfs-core/src/components/pin/remote/client.js @@ -272,11 +272,7 @@ const delay = ms => new Promise(resolve => setTimeout(resolve, ms)) * @typedef {import('ipfs-core-types/src/pin/remote').Pin} Pin * @typedef {import('ipfs-core-types/src/pin/remote').AddOptions} AddOptions * @typedef {import('ipfs-core-types/src/pin/remote/service').Credentials} Credentials - * @typedef {import('ipfs-core-types/src/pin/remote/service').RemotePinService} RemotePinService - * @typedef {import('ipfs-core-types/src/pin/remote/service').RemotePinServiceWithStat} RemotePinServiceWithStat - */ - -/** + * * @typedef {Object} PinDetails * @property {string} requestid * @property {string} created diff --git a/packages/ipfs-core/src/components/pin/remote/service.js b/packages/ipfs-core/src/components/pin/remote/service.js index 5b47e3def6..d81cd6614b 100644 --- a/packages/ipfs-core/src/components/pin/remote/service.js +++ b/packages/ipfs-core/src/components/pin/remote/service.js @@ -154,10 +154,6 @@ module.exports = ({ config, swarm, peerId }) => { * @typedef {import('../../config').PinningConfig} PinningConfig * * @typedef {import('ipfs-core-types/src/basic').AbortOptions} AbortOptions - * @typedef {import('ipfs-core-types/src/pin/remote').Status} Status - * @typedef {import('ipfs-core-types/src/pin/remote').Query} Query - * @typedef {import('ipfs-core-types/src/pin/remote').Pin} Pin - * @typedef {import('ipfs-core-types/src/pin/remote').AddOptions} AddOptions * @typedef {import('ipfs-core-types/src/pin/remote/service').API} API * @typedef {import('ipfs-core-types/src/pin/remote/service').Credentials} Credentials * @typedef {import('ipfs-core-types/src/pin/remote/service').RemotePinService} RemotePinService From da1ef9d5bdfbf87d4ee1dec2679e6ead728115d0 Mon Sep 17 00:00:00 2001 From: Yusef Napora Date: Fri, 26 Mar 2021 22:08:02 -0400 Subject: [PATCH 20/21] move serviceNamed fn outside of pin.remote.service api object --- .../src/components/pin/remote/index.js | 4 ++-- .../src/components/pin/remote/service.js | 19 ++++++++++++------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/packages/ipfs-core/src/components/pin/remote/index.js b/packages/ipfs-core/src/components/pin/remote/index.js index efc12b9bf6..a271b114cd 100644 --- a/packages/ipfs-core/src/components/pin/remote/index.js +++ b/packages/ipfs-core/src/components/pin/remote/index.js @@ -17,9 +17,9 @@ class PinRemoteAPI { * @param {PeerId} opts.peerId */ constructor ({ swarm, config, peerId }) { - const serviceRegistry = createServiceApi({ config, swarm, peerId }) + const { service, serviceRegistry } = createServiceApi({ config, swarm, peerId }) - this.service = serviceRegistry + this.service = service this.add = createAdd({ serviceRegistry }) this.ls = createLs({ serviceRegistry }) this.rm = createRm({ serviceRegistry }) diff --git a/packages/ipfs-core/src/components/pin/remote/service.js b/packages/ipfs-core/src/components/pin/remote/service.js index d81cd6614b..7f66d64aad 100644 --- a/packages/ipfs-core/src/components/pin/remote/service.js +++ b/packages/ipfs-core/src/components/pin/remote/service.js @@ -11,7 +11,7 @@ const createClient = require('./client') * @param {Config} options.config * @param {SwarmAPI} options.swarm * @param {PeerId} options.peerId - * @returns {API} + * @returns {{service: ServiceAPI, serviceRegistry: { serviceNamed: function }}} */ module.exports = ({ config, swarm, peerId }) => { let configured = false @@ -139,11 +139,16 @@ module.exports = ({ config, swarm, peerId }) => { } return { - add, - // @ts-ignore: The API type definition for the ls method is polymorphic on the value of the stat field. I'm not sure how to represent that in jsdoc. - ls, - rm, - serviceNamed + service: { + add, + rm, + // @ts-ignore: The API type definition for the ls method is polymorphic on the value of the stat field. I'm not sure how to represent that in jsdoc. + ls, + }, + + serviceRegistry: { + serviceNamed + } } } @@ -154,7 +159,7 @@ module.exports = ({ config, swarm, peerId }) => { * @typedef {import('../../config').PinningConfig} PinningConfig * * @typedef {import('ipfs-core-types/src/basic').AbortOptions} AbortOptions - * @typedef {import('ipfs-core-types/src/pin/remote/service').API} API + * @typedef {import('ipfs-core-types/src/pin/remote/service').API} ServiceAPI * @typedef {import('ipfs-core-types/src/pin/remote/service').Credentials} Credentials * @typedef {import('ipfs-core-types/src/pin/remote/service').RemotePinService} RemotePinService * @typedef {import('ipfs-core-types/src/pin/remote/service').RemotePinServiceWithStat} RemotePinServiceWithStat From b8634c3ead4a81441ebbe64e3d0065b98e82e910 Mon Sep 17 00:00:00 2001 From: Yusef Napora Date: Fri, 26 Mar 2021 22:11:16 -0400 Subject: [PATCH 21/21] consolidate input validation --- packages/ipfs-core/src/components/pin/remote/add.js | 7 ++----- packages/ipfs-core/src/components/pin/remote/ls.js | 7 ++----- packages/ipfs-core/src/components/pin/remote/rm.js | 7 ++----- packages/ipfs-core/src/components/pin/remote/rmAll.js | 7 ++----- packages/ipfs-core/src/components/pin/remote/service.js | 3 +++ 5 files changed, 11 insertions(+), 20 deletions(-) diff --git a/packages/ipfs-core/src/components/pin/remote/add.js b/packages/ipfs-core/src/components/pin/remote/add.js index 716283f427..178ba73d03 100644 --- a/packages/ipfs-core/src/components/pin/remote/add.js +++ b/packages/ipfs-core/src/components/pin/remote/add.js @@ -11,12 +11,9 @@ module.exports = ({ serviceRegistry }) => { * @returns {Promise} */ async function add (cid, options) { - const { service, ...addOpts } = options - if (!service) { - throw new Error('service name must be passed') - } + const { service } = options const svc = serviceRegistry.serviceNamed(service) - return svc.add(cid, addOpts) + return svc.add(cid, options) } return withTimeoutOption(add) diff --git a/packages/ipfs-core/src/components/pin/remote/ls.js b/packages/ipfs-core/src/components/pin/remote/ls.js index 4f40a8bc74..a3a7d007ec 100644 --- a/packages/ipfs-core/src/components/pin/remote/ls.js +++ b/packages/ipfs-core/src/components/pin/remote/ls.js @@ -10,12 +10,9 @@ module.exports = ({ serviceRegistry }) => { * @returns {AsyncGenerator} */ async function * ls (options) { - const { service, ...lsOpts } = options - if (!service) { - throw new Error('service name must be passed') - } + const { service } = options const svc = serviceRegistry.serviceNamed(service) - for await (const res of svc.ls(lsOpts)) { + for await (const res of svc.ls(options)) { yield res } } diff --git a/packages/ipfs-core/src/components/pin/remote/rm.js b/packages/ipfs-core/src/components/pin/remote/rm.js index 9b55297f67..fc9a66b399 100644 --- a/packages/ipfs-core/src/components/pin/remote/rm.js +++ b/packages/ipfs-core/src/components/pin/remote/rm.js @@ -11,12 +11,9 @@ module.exports = ({ serviceRegistry }) => { * @returns {Promise} */ async function rm (options) { - const { service, ...rmOpts } = options - if (!service) { - throw new Error('service name must be passed') - } + const { service } = options const svc = serviceRegistry.serviceNamed(service) - return svc.rm(rmOpts) + return svc.rm(options) } return withTimeoutOption(rm) diff --git a/packages/ipfs-core/src/components/pin/remote/rmAll.js b/packages/ipfs-core/src/components/pin/remote/rmAll.js index 661631eaf9..e5968aebeb 100644 --- a/packages/ipfs-core/src/components/pin/remote/rmAll.js +++ b/packages/ipfs-core/src/components/pin/remote/rmAll.js @@ -10,12 +10,9 @@ module.exports = ({ serviceRegistry }) => { * @returns {Promise} */ async function rmAll (options) { - const { service, ...rmOpts } = options - if (!service) { - throw new Error('service name must be passed') - } + const { service } = options const svc = serviceRegistry.serviceNamed(service) - return svc.rmAll(rmOpts) + return svc.rmAll(options) } return withTimeoutOption(rmAll) diff --git a/packages/ipfs-core/src/components/pin/remote/service.js b/packages/ipfs-core/src/components/pin/remote/service.js index 7f66d64aad..697f7bf6a1 100644 --- a/packages/ipfs-core/src/components/pin/remote/service.js +++ b/packages/ipfs-core/src/components/pin/remote/service.js @@ -132,6 +132,9 @@ module.exports = ({ config, swarm, peerId }) => { * @returns {any} */ function serviceNamed (name) { + if (!name) { + throw new Error('service name must be passed') + } if (!clients[name]) { throw new Error('no remote pinning service configured with name: ' + name) }