feat!: refactor upload-api (#154)

This PR depends on storacha/w3up#504, which will require release and update here. With local symlink it works as expected.
storacha · Mar 8, 2023 · ac2a4a7 · ac2a4a7
1 parent cd1db3d
commit ac2a4a7
Show file tree

Hide file tree

Showing 32 changed files with 16,481 additions and 20,505 deletions.
diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
@@ -23,4 +23,4 @@ jobs:
         env:
           AWS_REGION: 'us-west-2'
           AWS_ACCESS_KEY_ID: 'NOSUCH'
-          AWS_SECRET_ACCESS_KEY: 'NOSUCH'
+          AWS_SECRET_ACCESS_KEY: 'NOSUCH'
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -6,6 +6,7 @@
   "scripts": {
     "start": "sst start",
     "build": "sst build",
+    "check": "tsc --build",
     "deploy": "sst deploy --outputs-file .test-env.json",
     "remove": "sst remove",
     "console": "sst console",
@@ -21,22 +22,23 @@
     "@serverless-stack/resources": "^1.18.0",
     "@tsconfig/node16": "^1.0.3",
     "@types/git-rev-sync": "^2.0.0",
-    "@ucanto/client": "^4.2.3",
-    "@ucanto/core": "^4.2.3",
-    "@ucanto/principal": "^4.2.3",
-    "@ucanto/transport": "^4.2.3",
+    "@ucanto/client": "^5.1.0",
+    "@ucanto/core": "^5.1.0",
+    "@ucanto/interface": "^6.0.0",
+    "@ucanto/principal": "^5.0.0",
+    "@ucanto/transport": "^5.1.0",
     "@web-std/blob": "^3.0.4",
     "@web-std/fetch": "^4.1.0",
-    "@web3-storage/access": "^9.2.0",
-    "@web3-storage/w3up-client": "^4.1.0",
+    "@web3-storage/access": "^10.0.0",
+    "@web3-storage/w3up-client": "^4.2.0",
     "ava": "^4.3.3",
     "dotenv": "^16.0.3",
     "git-rev-sync": "^3.0.2",
     "hd-scripts": "^3.0.2",
     "lint-staged": "^13.0.3",
+    "multiformats": "^11.0.1",
     "p-wait-for": "^5.0.0",
-    "typescript": "^4.9.3",
-    "multiformats": "^11.0.1"
+    "typescript": "^4.9.3"
   },
   "dependencies": {
     "@serverless-stack/node": "^1.18.0",
@@ -82,5 +84,10 @@
     "satnav",
     "ucan-invocation",
     "upload-api"
-  ]
+  ],
+  "prettier": {
+    "singleQuote": true,
+    "trailingComma": "es5",
+    "semi": false
+  }
 }
diff --git a/upload-api/access.js b/upload-api/access.js
@@ -1,41 +1,47 @@
-import { info } from '@web3-storage/capabilities/space'
+import * as Space from '@web3-storage/capabilities/space'
 import { connect } from '@ucanto/client'
+import { Failure } from '@ucanto/server'
 import { CAR, CBOR, HTTP } from '@ucanto/transport'
 import fetch from '@web-std/fetch'
 
 /**
  * @param {import('@ucanto/interface').Signer} issuer Issuer of UCAN invocations to the Access service.
  * @param {import('@ucanto/interface').Principal} serviceDID DID of the Access service.
  * @param {URL} serviceURL URL of the Access service.
- * @returns {import('./service/types').AccessClient}
+ * @returns {import('@web3-storage/upload-api').AccessVerifier}
  */
-export function createAccessClient (issuer, serviceDID, serviceURL) {
+export function createAccessClient(issuer, serviceDID, serviceURL) {
   /** @type {import('@ucanto/server').ConnectionView<import('@web3-storage/access/types').Service>} */
   const conn = connect({
     id: serviceDID,
     encoder: CAR,
     decoder: CBOR,
-    channel: HTTP.open({ url: serviceURL, method: 'POST', fetch })
+    channel: HTTP.open({ url: serviceURL, method: 'POST', fetch }),
   })
 
   return {
-    async verifyInvocation (invocation) {
+    async allocateSpace(invocation) {
       if (!invocation.capabilities.length) return true
       // if info capability is derivable from the passed capability, then we'll
       // receive a response and know that the invocation issuer has verified
       // themselves with w3access.
-      const res = await info
-        .invoke({
-          issuer,
-          audience: serviceDID,
-          // @ts-expect-error
-          with: invocation.capabilities[0].with,
-          proofs: [invocation]
-        })
-        .execute(conn)
+      const info = Space.info.invoke({
+        issuer,
+        audience: serviceDID,
+        // @ts-expect-error
+        with: invocation.capabilities[0].with,
+        proofs: [invocation],
+      })
 
-      if (res.error) console.warn('invocation verification failed', res)
-      return !res.error
-    }
+      const result = await info.execute(conn)
+
+      if (result.error) {
+        return result.error && result.name === 'SpaceUnknown'
+          ? new Failure(`Space has no storage provider`, { cause: result })
+          : result
+      } else {
+        return {}
+      }
+    },
   }
 }
diff --git a/upload-api/buckets/car-store.js b/upload-api/buckets/car-store.js
@@ -1,33 +1,46 @@
-import { S3Client, HeadObjectCommand, PutObjectCommand } from '@aws-sdk/client-s3'
+import {
+  S3Client,
+  HeadObjectCommand,
+  PutObjectCommand,
+} from '@aws-sdk/client-s3'
 import { base64pad } from 'multiformats/bases/base64'
-import { getSignedUrl } from "@aws-sdk/s3-request-presigner"
+import { getSignedUrl } from '@aws-sdk/s3-request-presigner'
 
 /**
  * Abstraction layer with Factory to perform operations on bucket storing CAR files.
  *
  * @param {string} region
  * @param {string} bucketName
  * @param {import('@aws-sdk/client-s3').ServiceInputTypes} [options]
- * @returns {import('../service/types').CarStoreBucket}
  */
-export function createCarStore (region, bucketName, options) {
-  const s3 = new S3Client({ 
+export function createCarStore(region, bucketName, options) {
+  const s3 = new S3Client({
     region,
-    ...options
+    ...options,
   })
+  return useCarStore(s3, bucketName)
+}
 
+/**
+ *
+ * @param {S3Client} s3
+ * @param {string} bucketName
+ * @returns {import('@web3-storage/upload-api').CarStoreBucket}
+ */
+export function useCarStore(s3, bucketName) {
   return {
     /**
-     * @param {import('../service/types').AnyLink} link
+     * @param {import('@web3-storage/upload-api').UnknownLink} link
      */
     has: async (link) => {
       const cmd = new HeadObjectCommand({
         Key: `${link}/${link}.car`,
         Bucket: bucketName,
-      }) 
+      })
       try {
         await s3.send(cmd)
-      } catch (cause) { // @ts-expect-error
+      } catch (cause) {
+        // @ts-expect-error
         if (cause?.$metadata?.httpStatusCode === 404) {
           return false
         }
@@ -39,8 +52,8 @@ export function createCarStore (region, bucketName, options) {
     /**
      * Create a presigned s3 url allowing the recipient to upload
      * only the CAR that matches the provided Link
-     * 
-     * @param {import('../service/types').AnyLink} link
+     *
+     * @param {import('@web3-storage/upload-api').UnknownLink} link
      * @param {number} size
      */
     createUploadUrl: async (link, size) => {
@@ -49,20 +62,22 @@ export function createCarStore (region, bucketName, options) {
         Key: `${link}/${link}.car`,
         Bucket: bucketName,
         ChecksumSHA256: checksum,
-        ContentLength: size
+        ContentLength: size,
       })
       const expiresIn = 60 * 60 * 24 // 1 day
-      const url = new URL(await getSignedUrl(s3, cmd, {
-        expiresIn,
-        unhoistableHeaders: new Set(['x-amz-checksum-sha256']),
-      }))
+      const url = new URL(
+        await getSignedUrl(s3, cmd, {
+          expiresIn,
+          unhoistableHeaders: new Set(['x-amz-checksum-sha256']),
+        })
+      )
       return {
         url,
         headers: {
           'x-amz-checksum-sha256': checksum,
-          'content-length': size
-        }
+          'content-length': String(size),
+        },
       }
-    }
+    },
   }
 }
diff --git a/upload-api/buckets/dudewhere-store.js b/upload-api/buckets/dudewhere-store.js
@@ -7,14 +7,21 @@ import { S3Client, PutObjectCommand } from '@aws-sdk/client-s3'
  * @param {string} region
  * @param {string} bucketName
  * @param {import('@aws-sdk/client-s3').ServiceInputTypes} [options]
- * @returns {import('../service/types').DudewhereBucket}
  */
-export function createDudewhereStore (region, bucketName, options = {}) {
+export function createDudewhereStore(region, bucketName, options = {}) {
   const s3client = new S3Client({
     region,
-    ...options
+    ...options,
   })
+  return useDudewhereStore(s3client, bucketName)
+}
 
+/**
+ * @param {S3Client} s3client
+ * @param {string} bucketName
+ * @returns {import('@web3-storage/upload-api').DudewhereBucket}
+ */
+export function useDudewhereStore(s3client, bucketName) {
   return {
     /**
      * Put dataCID -> carCID mapping into the bucket
@@ -29,6 +36,6 @@ export function createDudewhereStore (region, bucketName, options = {}) {
         ContentLength: 0,
       })
       await s3client.send(putCmd)
-    }
+    },
   }
 }
diff --git a/upload-api/buckets/ucan-store.js b/upload-api/buckets/ucan-store.js
@@ -7,14 +7,21 @@ import { S3Client, PutObjectCommand } from '@aws-sdk/client-s3'
  * @param {string} region
  * @param {string} bucketName
  * @param {import('@aws-sdk/client-s3').ServiceInputTypes} [options]
- * @returns {import('../service/types').UcanBucket}
  */
-export function createUcanStore (region, bucketName, options = {}) {
+export function createUcanStore(region, bucketName, options = {}) {
   const s3client = new S3Client({
     region,
-    ...options
+    ...options,
   })
+  return useUcanStore(s3client, bucketName)
+}
 
+/**
+ * @param {S3Client} s3client
+ * @param {string} bucketName
+ * @returns {import('@web3-storage/upload-api').UcanBucket}
+ */
+export const useUcanStore = (s3client, bucketName) => {
   return {
     /**
      * Put UCAN invocation CAR file into bucket
@@ -26,9 +33,9 @@ export function createUcanStore (region, bucketName, options = {}) {
       const putCmd = new PutObjectCommand({
         Bucket: bucketName,
         Key: `${carCid}/${carCid}.car`,
-        Body: bytes
+        Body: bytes,
       })
       await s3client.send(putCmd)
-    }
+    },
   }
 }
diff --git a/upload-api/functions/get.js b/upload-api/functions/get.js
@@ -14,20 +14,26 @@ const repo = 'https://github.com/web3-storage/w3infra'
 /**
  * AWS HTTP Gateway handler for GET /version
  *
- * @param {import('aws-lambda').APIGatewayProxyEventV2} request 
+ * @param {import('aws-lambda').APIGatewayProxyEventV2} request
  */
- export async function versionGet (request) {
-  const { NAME: name , VERSION: version, COMMIT: commit, STAGE: env, UPLOAD_API_DID } = process.env
+export async function versionGet(request) {
+  const {
+    NAME: name,
+    VERSION: version,
+    COMMIT: commit,
+    STAGE: env,
+    UPLOAD_API_DID,
+  } = process.env
   const { PRIVATE_KEY } = Config
   const serviceSigner = getServiceSigner({ UPLOAD_API_DID, PRIVATE_KEY })
   const did = serviceSigner.did()
   const publicKey = serviceSigner.toDIDKey()
   return {
     statusCode: 200,
     headers: {
-      'Content-Type': `application/json`
+      'Content-Type': `application/json`,
     },
-    body: JSON.stringify({ name, version, did, publicKey, repo, commit, env })
+    body: JSON.stringify({ name, version, did, publicKey, repo, commit, env }),
   }
 }
 
@@ -36,9 +42,9 @@ export const version = Sentry.AWSLambda.wrapHandler(versionGet)
 /**
  * AWS HTTP Gateway handler for GET /
  *
- * @param {import('aws-lambda').APIGatewayProxyEventV2} request 
+ * @param {import('aws-lambda').APIGatewayProxyEventV2} request
  */
-export async function homeGet (request) {
+export async function homeGet(request) {
   const { VERSION: version, STAGE: stage, UPLOAD_API_DID } = process.env
   const { PRIVATE_KEY } = Config
   const serviceSigner = getServiceSigner({ UPLOAD_API_DID, PRIVATE_KEY })
@@ -48,9 +54,9 @@ export async function homeGet (request) {
   return {
     statusCode: 200,
     headers: {
-      'Content-Type': 'text/plain; charset=utf-8'
+      'Content-Type': 'text/plain; charset=utf-8',
     },
-    body: `⁂ upload-api v${version} ${env}\n- ${repo}\n- ${did}\n- ${publicKey}`
+    body: `⁂ upload-api v${version} ${env}\n- ${repo}\n- ${did}\n- ${publicKey}`,
   }
 }
 
@@ -59,7 +65,7 @@ export const home = Sentry.AWSLambda.wrapHandler(homeGet)
 /**
  * AWS HTTP Gateway handler for GET /error
  */
- export async function errorGet () {
+export async function errorGet() {
   throw new Error('API Error')
 }