diff --git a/packages/ipfs-core/src/components/config.js b/packages/ipfs-core/src/components/config.js index bc598485ae..c9258eb398 100644 --- a/packages/ipfs-core/src/components/config.js +++ b/packages/ipfs-core/src/components/config.js @@ -328,6 +328,7 @@ module.exports.profiles = profiles * @property {PubsubConfig} [Pubsub] * @property {SwarmConfig} [Swarm] * @property {RoutingConfig} [Routing] + * @property {PinningConfig} [Pinning] * * @typedef {Object} AddressConfig * Contains information about various listener addresses to be used by this node. @@ -524,4 +525,12 @@ module.exports.profiles = profiles * * @typedef {import('ipfs-core-types/src/basic').ToJSON} ToJSON * @typedef {import('.').AbortOptions} AbortOptions + * + * @typedef {Object} PinningConfig + * @property {Object} [RemoteServices] + * + * @typedef {Object} RemotePinningServiceConfig + * @property {Object} API + * @property {string} API.Endpoint + * @property {string} API.Key */ diff --git a/packages/ipfs-core/src/components/index.js b/packages/ipfs-core/src/components/index.js index 20652c29a5..498094428b 100644 --- a/packages/ipfs-core/src/components/index.js +++ b/packages/ipfs-core/src/components/index.js @@ -29,6 +29,7 @@ const createIDAPI = require('./id') const createConfigAPI = require('./config') const DagAPI = require('./dag') const PinManagerAPI = require('./pin/pin-manager') +const PinRemoteAPI = require('./pin/remote') const createPreloadAPI = require('../preload') const createMfsPreloadAPI = require('../mfs-preload') const createFilesAPI = require('./files') @@ -57,6 +58,8 @@ class IPFS { constructor ({ print, storage, options }) { const { peerId, repo, keychain } = storage const network = Service.create(Network) + const swarm = new SwarmAPI({ network }) + const config = createConfigAPI({ repo }) const preload = createPreloadAPI(options.preload) @@ -86,7 +89,8 @@ class IPFS { }) const resolve = createResolveAPI({ ipld, name }) const pinManager = new PinManagerAPI({ repo, dagReader }) - const pin = new PinAPI({ gcLock, pinManager, dagReader }) + const pinRemote = new PinRemoteAPI({ swarm, config, peerId }) + const pin = new PinAPI({ gcLock, pinManager, dagReader, pinRemote }) const block = new BlockAPI({ blockService, preload, gcLock, pinManager, pin }) const dag = new DagAPI({ ipld, preload, gcLock, pin, dagReader }) const refs = Object.assign(createRefsAPI({ ipld, resolve, preload }), { @@ -155,7 +159,7 @@ class IPFS { this.version = createVersionAPI({ repo }) this.bitswap = new BitswapAPI({ network }) this.bootstrap = new BootstrapAPI({ repo }) - this.config = createConfigAPI({ repo }) + this.config = config this.ping = createPingAPI({ network }) this.add = add @@ -170,7 +174,7 @@ class IPFS { this.object = new ObjectAPI({ ipld, preload, gcLock, dag }) this.repo = new RepoAPI({ gcLock, pin, repo, refs }) this.stats = new StatsAPI({ repo, network }) - this.swarm = new SwarmAPI({ network }) + this.swarm = swarm // For the backwards compatibility Object.defineProperty(this, 'libp2p', { diff --git a/packages/ipfs-core/src/components/pin/index.js b/packages/ipfs-core/src/components/pin/index.js index 2e3ee7c66b..beeb582b55 100644 --- a/packages/ipfs-core/src/components/pin/index.js +++ b/packages/ipfs-core/src/components/pin/index.js @@ -12,8 +12,9 @@ class PinAPI { * @param {GCLock} config.gcLock * @param {DagReader} config.dagReader * @param {PinManager} config.pinManager + * @param {PinRemoteAPI} config.pinRemote */ - constructor ({ gcLock, dagReader, pinManager }) { + constructor ({ gcLock, dagReader, pinManager, pinRemote }) { const addAll = createAddAll({ gcLock, dagReader, pinManager }) this.addAll = addAll this.add = createAdd({ addAll }) @@ -21,6 +22,7 @@ class PinAPI { this.rmAll = rmAll this.rm = createRm({ rmAll }) this.ls = createLs({ dagReader, pinManager }) + this.remote = pinRemote } } module.exports = PinAPI @@ -32,4 +34,5 @@ module.exports = PinAPI * @typedef {import('..').PinManager} PinManager * @typedef {import('..').AbortOptions} AbortOptions * @typedef {import('..').CID} CID + * @typedef {import('./remote')} PinRemoteAPI */ diff --git a/packages/ipfs-core/src/components/pin/remote/add.js b/packages/ipfs-core/src/components/pin/remote/add.js new file mode 100644 index 0000000000..178ba73d03 --- /dev/null +++ b/packages/ipfs-core/src/components/pin/remote/add.js @@ -0,0 +1,27 @@ +'use strict' + +const withTimeoutOption = require('ipfs-core-utils/src/with-timeout-option') + +module.exports = ({ serviceRegistry }) => { + /** + * Asks a remote pinning service to pin an IPFS object from a given path + * + * @param {string|CID} cid + * @param {AddOptions & AbortOptions} options + * @returns {Promise} + */ + async function add (cid, options) { + const { service } = options + const svc = serviceRegistry.serviceNamed(service) + return svc.add(cid, options) + } + + return withTimeoutOption(add) +} + +/** + * @typedef {import('cids')} CID + * @typedef {import('ipfs-core-types/src/basic').AbortOptions} AbortOptions + * @typedef {import('ipfs-core-types/src/pin/remote').Pin} Pin + * @typedef {import('ipfs-core-types/src/pin/remote').AddOptions} AddOptions + */ diff --git a/packages/ipfs-core/src/components/pin/remote/client.js b/packages/ipfs-core/src/components/pin/remote/client.js new file mode 100644 index 0000000000..c17fd56b34 --- /dev/null +++ b/packages/ipfs-core/src/components/pin/remote/client.js @@ -0,0 +1,283 @@ +'use strict' + +const CID = require('cids') +const multiaddr = require('multiaddr') +const HTTP = require('ipfs-utils/src/http') +const log = require('debug')('ipfs:components:pin:remote:client') + +const withTimeoutOption = require('ipfs-core-utils/src/with-timeout-option') + +module.exports = ({ service, endpoint, key, swarm, peerId }) => { + const api = new HTTP({ + base: endpoint, + headers: { + Authorization: `Bearer ${key}`, + 'Content-Type': 'application/json' + } + }) + + async function info ({ stat: includeStats }) { + if (includeStats) { + return { + service, + endpoint: new URL(endpoint), + stat: await stat() + } + } + return { service, endpoint: new URL(endpoint) } + } + + async function stat () { + try { + const promises = [] + for (const pinStatus of ['queued', 'pinning', 'pinned', 'failed']) { + promises.push(countForStatus(pinStatus)) + } + const [queued, pinning, pinned, failed] = await Promise.all(promises) + return { + status: 'valid', + pinCount: { queued, pinning, pinned, failed } + } + } catch (e) { + log(`error getting stats for service ${service}: `, e) + return { + status: 'invalid' + } + } + } + + /** + * @param {string} status + * @returns {Promise} - the number of remote pins with the given status + */ + async function countForStatus (status) { + const searchParams = new URLSearchParams({ status, limit: '1' }) + const response = await api.get('/pins', { searchParams }) + const body = await response.json() + return body.count + } + + async function originAddresses () { + const addrs = await swarm.localAddrs() + return addrs.map(ma => { + const str = ma.toString() + + // some relay-style transports add our peer id to the ma for us + // so don't double-add + if (str.endsWith(`/p2p/${peerId}`)) { + return str + } + + return `${str}/p2p/${peerId}` + }) + } + + async function connectToDelegates (delegates) { + const addrs = delegates.map(multiaddr) + const promises = [] + for (const addr of addrs) { + promises.push(swarm.connect(addr).catch(e => { + log('error connecting to pinning service delegate: ', e) + })) + } + await Promise.all(promises) + } + + async function awaitPinCompletion (pinResponse) { + const pollIntervalMs = 100 + + let { status, requestid } = pinResponse + while (status !== 'pinned') { + if (status === 'failed') { + throw new Error('pin failed: ' + JSON.stringify(pinResponse.info)) + } + + await delay(pollIntervalMs) + const resp = await api.get(`/pins/${requestid}`) + pinResponse = await resp.json() + status = pinResponse.status + } + + return formatPinResult(pinResponse.status, pinResponse.pin) + } + + function formatPinResult (status, pin) { + const name = pin.name || '' + const cid = new CID(pin.cid) + return { status, name, cid } + } + + /** + * Request that the remote service add a pin for the given CID. + * + * @param {CID|string} cid - CID to pin to remote service + * @param {AddOptions} options + * + * @returns {Promise} + */ + async function add (cid, options) { + const { background } = options + const name = options.name || '' + const origins = await originAddresses() + const addOpts = { cid: cid.toString(), name, origins } + const response = await api.post('/pins', { json: addOpts }) + const responseBody = await response.json() + + const { status, pin, delegates } = responseBody + connectToDelegates(delegates) + + if (!background) { + return awaitPinCompletion(responseBody) + } + + return formatPinResult(status, pin) + } + + /** + * List pins from the remote service that match the given criteria. If no criteria are provided, returns all pins with the status 'pinned'. + * + * @param {Query} options + * @returns {AsyncGenerator} + */ + async function * _lsRaw (options) { + let status = options.status || [] + if (status.length === 0) { + status = ['pinned'] + } + + const searchParams = new URLSearchParams() + if (options.name) { + searchParams.append('name', options.name) + } + for (const cid of (options.cid || [])) { + searchParams.append('cid', cid.toString()) + } + for (const s of status) { + searchParams.append('status', s) + } + + let resp = await api.get('/pins', { searchParams }) + let body = await resp.json() + const total = body.count + let yielded = 0 + while (true) { + if (body.results.length < 1) { + return + } + for (const result of body.results) { + yield result + yielded += 1 + } + + if (yielded === total) { + return + } + + // if we've run out of results and haven't yielded everything, fetch a page of older results + const oldestResult = body.results[body.results.length - 1] + searchParams.set('before', oldestResult.created) + resp = await api.get('/pins', { searchParams }) + body = await resp.json() + } + } + + /** + * List pins from the remote service that match the given criteria. If no criteria are provided, returns all pins with the status 'pinned'. + * + * @param {Query} options + * @returns {AsyncGenerator} + */ + async function * ls (options) { + for await (const result of _lsRaw(options)) { + yield formatPinResult(result.status, result.pin) + } + } + + /** + * Remove a single pin from a remote pinning service. + * Fails if multiple pins match the specified criteria. Use rmAll to remove all pins that match. + * + * @param {Query} options + * @returns {Promise} + */ + async function rm (options) { + // the pinning service API only supports deletion by requestid, so we need to lookup the pins first + const searchParams = new URLSearchParams() + if (options.name) { + searchParams.set('name', options.name) + } + for (const cid of (options.cid || [])) { + searchParams.append('cid', cid.toString()) + } + for (const status of (options.status || [])) { + searchParams.append('status', status) + } + const resp = await api.get('/pins', { searchParams }) + const body = await resp.json() + if (body.count === 0) { + return + } + if (body.count > 1) { + throw new Error('multiple remote pins are matching this query') + } + + const requestid = body.results[0].requestid + try { + await api.delete(`/pins/${requestid}`) + } catch (e) { + if (e.status !== 404) { + throw e + } + } + } + + /** + * Remove all pins that match the given criteria from a remote pinning service. + * + * @param {Query} options + * @returns {Promise} + */ + async function rmAll (options) { + const requestIds = [] + for await (const result of _lsRaw(options)) { + requestIds.push(result.requestid) + } + + const promises = [] + for (const requestid of requestIds) { + promises.push(api.delete(`/pins/${requestid}`)) + } + await Promise.all(promises) + } + + return { + info: withTimeoutOption(info), + ls: withTimeoutOption(ls), + add: withTimeoutOption(add), + rm: withTimeoutOption(rm), + rmAll: withTimeoutOption(rmAll) + } +} + +const delay = ms => new Promise(resolve => setTimeout(resolve, ms)) + +/** + * @typedef {import('../..').PeerId} PeerId + * @typedef {import('../../swarm')} SwarmAPI + * @typedef {import('../../config').Config} Config + * + * @typedef {import('ipfs-core-types/src/basic').AbortOptions} AbortOptions + * @typedef {import('ipfs-core-types/src/pin/remote').Status} Status + * @typedef {import('ipfs-core-types/src/pin/remote').Query} Query + * @typedef {import('ipfs-core-types/src/pin/remote').Pin} Pin + * @typedef {import('ipfs-core-types/src/pin/remote').AddOptions} AddOptions + * @typedef {import('ipfs-core-types/src/pin/remote/service').Credentials} Credentials + * + * @typedef {Object} PinDetails + * @property {string} requestid + * @property {string} created + * @property {Status} status + * @property {Pin} pin + * @property {Array} delegates + * + */ diff --git a/packages/ipfs-core/src/components/pin/remote/index.js b/packages/ipfs-core/src/components/pin/remote/index.js new file mode 100644 index 0000000000..a271b114cd --- /dev/null +++ b/packages/ipfs-core/src/components/pin/remote/index.js @@ -0,0 +1,36 @@ +'use strict' + +const createServiceApi = require('./service') +const createAdd = require('./add') +const createLs = require('./ls') +const createRm = require('./rm') +const createRmAll = require('./rmAll') + +/** + * PinRemoteAPI provides an API for pinning content to remote services. + */ +class PinRemoteAPI { + /** + * @param {Object} opts + * @param {SwarmAPI} opts.swarm + * @param {Config} opts.config + * @param {PeerId} opts.peerId + */ + constructor ({ swarm, config, peerId }) { + const { service, serviceRegistry } = createServiceApi({ config, swarm, peerId }) + + this.service = service + this.add = createAdd({ serviceRegistry }) + this.ls = createLs({ serviceRegistry }) + this.rm = createRm({ serviceRegistry }) + this.rmAll = createRmAll({ serviceRegistry }) + } +} + +/** + * @typedef {import('../..').PeerId} PeerId + * @typedef {import('../../swarm')} SwarmAPI + * @typedef {import('../../config').Config} Config + */ + +module.exports = PinRemoteAPI diff --git a/packages/ipfs-core/src/components/pin/remote/ls.js b/packages/ipfs-core/src/components/pin/remote/ls.js new file mode 100644 index 0000000000..a3a7d007ec --- /dev/null +++ b/packages/ipfs-core/src/components/pin/remote/ls.js @@ -0,0 +1,29 @@ +'use strict' + +const withTimeoutOption = require('ipfs-core-utils/src/with-timeout-option') + +module.exports = ({ serviceRegistry }) => { + /** + * List objects that are pinned by a remote service. + * + * @param {Query & AbortOptions} options + * @returns {AsyncGenerator} + */ + async function * ls (options) { + const { service } = options + const svc = serviceRegistry.serviceNamed(service) + for await (const res of svc.ls(options)) { + yield res + } + } + + return withTimeoutOption(ls) +} + +/** + * @typedef {import('cids')} CID + * @typedef {import('ipfs-core-types/src/basic').AbortOptions} AbortOptions + * @typedef {import('ipfs-core-types/src/pin/remote').Pin} Pin + * @typedef {import('ipfs-core-types/src/pin/remote').Status} Status + * @typedef {import('ipfs-core-types/src/pin/remote').Query} Query + */ diff --git a/packages/ipfs-core/src/components/pin/remote/rm.js b/packages/ipfs-core/src/components/pin/remote/rm.js new file mode 100644 index 0000000000..fc9a66b399 --- /dev/null +++ b/packages/ipfs-core/src/components/pin/remote/rm.js @@ -0,0 +1,27 @@ +'use strict' + +const withTimeoutOption = require('ipfs-core-utils/src/with-timeout-option') + +module.exports = ({ serviceRegistry }) => { + /** + * Remove a single pin from a remote pinning service. + * Fails if multiple pins match the specified query. Use rmAll to remove all pins that match. + * + * @param {Query & AbortOptions} options + * @returns {Promise} + */ + async function rm (options) { + const { service } = options + const svc = serviceRegistry.serviceNamed(service) + return svc.rm(options) + } + + return withTimeoutOption(rm) +} + +/** + * @typedef {import('cids')} CID + * @typedef {import('ipfs-core-types/src/basic').AbortOptions} AbortOptions + * @typedef {import('ipfs-core-types/src/pin/remote').Pin} Pin + * @typedef {import('ipfs-core-types/src/pin/remote').Query} Query + */ diff --git a/packages/ipfs-core/src/components/pin/remote/rmAll.js b/packages/ipfs-core/src/components/pin/remote/rmAll.js new file mode 100644 index 0000000000..e5968aebeb --- /dev/null +++ b/packages/ipfs-core/src/components/pin/remote/rmAll.js @@ -0,0 +1,26 @@ +'use strict' + +const withTimeoutOption = require('ipfs-core-utils/src/with-timeout-option') + +module.exports = ({ serviceRegistry }) => { + /** + * Remove all pins that match the given criteria from a remote pinning service. + * + * @param {Query & AbortOptions} options + * @returns {Promise} + */ + async function rmAll (options) { + const { service } = options + const svc = serviceRegistry.serviceNamed(service) + return svc.rmAll(options) + } + + return withTimeoutOption(rmAll) +} + +/** + * @typedef {import('cids')} CID + * @typedef {import('ipfs-core-types/src/basic').AbortOptions} AbortOptions + * @typedef {import('ipfs-core-types/src/pin/remote').Pin} Pin + * @typedef {import('ipfs-core-types/src/pin/remote').Query} Query + */ diff --git a/packages/ipfs-core/src/components/pin/remote/service.js b/packages/ipfs-core/src/components/pin/remote/service.js new file mode 100644 index 0000000000..697f7bf6a1 --- /dev/null +++ b/packages/ipfs-core/src/components/pin/remote/service.js @@ -0,0 +1,169 @@ +'use strict' + +const { Errors } = require('interface-datastore') +const ERR_NOT_FOUND = Errors.notFoundError().code + +const createClient = require('./client') + +/** + * + * @param {Object} options + * @param {Config} options.config + * @param {SwarmAPI} options.swarm + * @param {PeerId} options.peerId + * @returns {{service: ServiceAPI, serviceRegistry: { serviceNamed: function }}} + */ +module.exports = ({ config, swarm, peerId }) => { + let configured = false + const clients = {} + + async function loadConfig () { + if (configured) { + return + } + + try { + const pinConfig = /** @type {PinningConfig|null} */ (await config.get('Pinning')) + if (pinConfig == null || pinConfig.RemoteServices == null) { + configured = true + return + } + + for (const [service, svcConfig] of Object.entries(pinConfig.RemoteServices)) { + if (svcConfig == null) { + continue + } + const { Endpoint: endpoint, Key: key } = svcConfig.API + if (!endpoint || !key) { + continue + } + clients[service] = createClient({ + swarm, + peerId, + service, + endpoint, + key + }) + } + } catch (e) { + if (e.code !== ERR_NOT_FOUND) { + throw e + } + } + + configured = true + } + + /** + * Adds a new remote pinning service to the set of configured services. + * + * @param {string} name - the name of the pinning service. Used to identify the service in future remote pinning API calls. + * @param {Credentials & AbortOptions} credentials + */ + async function add (name, credentials) { + await loadConfig() + + if (clients[name]) { + throw new Error('service already present: ' + name) + } + + if (!credentials.endpoint) { + throw new Error('option "endpoint" is required') + } + + if (!credentials.key) { + throw new Error('option "key" is required') + } + + await config.set(`Pinning.RemoteServices.${name}`, { + API: { + Endpoint: credentials.endpoint.toString(), + Key: credentials.key + } + }) + + clients[name] = createClient({ + swarm, + peerId, + service: name, + ...credentials + }) + } + + /** + * List the configured remote pinning services. + * + * @param {{stat: ?boolean} & AbortOptions} opts + * @returns {Promise | Array>} - a Promise resolving to an array of objects describing the configured remote pinning services. If stat==true, each object will include more detailed status info, including the number of pins for each pin status. + */ + async function ls (opts) { + await loadConfig() + + opts = opts || { stat: false } + + const promises = [] + for (const svc of Object.values(clients)) { + promises.push(svc.info(opts)) + } + return Promise.all(promises) + } + + /** + * Remove a remote pinning service from the set of configured services. + * + * @param {string} name - the name of the pinning service to remove + * @returns {Promise} + */ + async function rm (name) { + if (!name) { + throw new Error('parameter "name" is required') + } + await loadConfig() + delete clients[name] + const services = (await config.get('Pinning.RemoteServices')) || {} + delete services[name] + await config.set('Pinning.RemoteServices', services) + } + + /** + * Returns a client for the service with the given name. Throws if no service has been configured with the given name. + * + * @param {string} name + * @returns {any} + */ + function serviceNamed (name) { + if (!name) { + throw new Error('service name must be passed') + } + if (!clients[name]) { + throw new Error('no remote pinning service configured with name: ' + name) + } + return clients[name] + } + + return { + service: { + add, + rm, + // @ts-ignore: The API type definition for the ls method is polymorphic on the value of the stat field. I'm not sure how to represent that in jsdoc. + ls, + }, + + serviceRegistry: { + serviceNamed + } + } +} + +/** + * @typedef {import('../..').PeerId} PeerId + * @typedef {import('../../swarm')} SwarmAPI + * @typedef {import('../../config').Config} Config + * @typedef {import('../../config').PinningConfig} PinningConfig + * + * @typedef {import('ipfs-core-types/src/basic').AbortOptions} AbortOptions + * @typedef {import('ipfs-core-types/src/pin/remote/service').API} ServiceAPI + * @typedef {import('ipfs-core-types/src/pin/remote/service').Credentials} Credentials + * @typedef {import('ipfs-core-types/src/pin/remote/service').RemotePinService} RemotePinService + * @typedef {import('ipfs-core-types/src/pin/remote/service').RemotePinServiceWithStat} RemotePinServiceWithStat + */ diff --git a/packages/ipfs/test/interface-core.js b/packages/ipfs/test/interface-core.js index 6e220dcd80..d37c424677 100644 --- a/packages/ipfs/test/interface-core.js +++ b/packages/ipfs/test/interface-core.js @@ -79,6 +79,8 @@ describe('interface-ipfs-core tests', function () { tests.pin(commonFactory) + tests.pin.remote(commonFactory) + tests.ping(commonFactory) tests.pubsub(factory({}, {