diff --git a/.github/workflows/main-core.yaml b/.github/workflows/main-core.yaml index 9a052aef71..7cde08773c 100644 --- a/.github/workflows/main-core.yaml +++ b/.github/workflows/main-core.yaml @@ -61,7 +61,7 @@ jobs: SECRET_JWKS INTERNAL_DKIM_SELECTOR - + INTERNAL_EMAIL_DISTRIBUTION_KEY SECRET_RELAY_DKIM_PRIVATE_KEY INTERNAL_GOOGLE_OAUTH_CLIENT_ID @@ -94,6 +94,7 @@ jobs: INTERNAL_DKIM_SELECTOR: ${{ secrets.INTERNAL_DKIM_SELECTOR }} + INTERNAL_EMAIL_DISTRIBUTION_KEY: ${{secrets.INTERNAL_EMAIL_DISTRIBUTION_KEY_DEV}} SECRET_RELAY_DKIM_PRIVATE_KEY: ${{ secrets.SECRET_RELAY_DKIM_PRIVATE_KEY }} INTERNAL_GOOGLE_OAUTH_CLIENT_ID: ${{ vars.INTERNAL_GOOGLE_OAUTH_CLIENT_ID_DEV }} diff --git a/.github/workflows/main-emaildistributor.yaml b/.github/workflows/main-emaildistributor.yaml new file mode 100644 index 0000000000..1f72bd3ce8 --- /dev/null +++ b/.github/workflows/main-emaildistributor.yaml @@ -0,0 +1,51 @@ +name: Singleton Email distributor + +on: + push: + branches: + - main + +defaults: + run: + working-directory: platform/emaildistributor + +jobs: + deploy: + runs-on: ubuntu-latest + environment: dev + steps: + - uses: actions/checkout@v3 + + - uses: cachix/install-nix-action@v18 + with: + nix_path: nixpkgs=channel:nixos-unstable + + - run: nix-build ../platform.nix + + - name: Cache Dependencies + id: cache-modules + uses: actions/cache@v3 + with: + path: | + node_modules + .yarn + key: ${{ runner.os }}-node_modules-${{ hashFiles('yarn.lock') }} + + - name: Install Dependencies + run: yarn install + + - name: Test + run: yarn run test + + - name: Deploy + uses: cloudflare/wrangler-action@2.0.0 + with: + wranglerVersion: '3.19.0' + apiToken: ${{ secrets.TOKEN_CLOUDFLARE_API }} + accountId: ${{ secrets.INTERNAL_CLOUDFLARE_ACCOUNT_ID }} + workingDirectory: platform/emaildistributor + command: publish + secrets: | + SECRET_EMAIL_DISTRIBUTION_MAP + env: + SECRET_EMAIL_DISTRIBUTION_MAP: ${{ secrets.SECRET_EMAIL_DISTRIBUTION_MAP }} diff --git a/.github/workflows/main-emailinbounder.yaml b/.github/workflows/main-emailinbounder.yaml new file mode 100644 index 0000000000..4c95022dd7 --- /dev/null +++ b/.github/workflows/main-emailinbounder.yaml @@ -0,0 +1,53 @@ +name: Mail Email inbounder + +on: + push: + branches: + - main + +defaults: + run: + working-directory: platform/emailinbounder + +jobs: + deploy: + runs-on: ubuntu-latest + environment: dev + steps: + - uses: actions/checkout@v3 + + - uses: cachix/install-nix-action@v18 + with: + nix_path: nixpkgs=channel:nixos-unstable + + - run: nix-build ../platform.nix + + - name: Cache Dependencies + id: cache-modules + uses: actions/cache@v3 + with: + path: | + node_modules + .yarn + key: ${{ runner.os }}-node_modules-${{ hashFiles('yarn.lock') }} + + - name: Install Dependencies + run: yarn install + + - name: Test + run: yarn run test + + - name: Deploy + uses: cloudflare/wrangler-action@2.0.0 + with: + wranglerVersion: '3.19.0' + apiToken: ${{ secrets.TOKEN_CLOUDFLARE_API }} + accountId: ${{ secrets.INTERNAL_CLOUDFLARE_ACCOUNT_ID }} + workingDirectory: platform/emailinbounder + command: publish + secrets: | + SECRET_EMAIL_DISTRIBUTION_MAP + INTERNAL_EMAIL_DISTRIBUTION_KEY + env: + SECRET_EMAIL_DISTRIBUTION_MAP: ${{ secrets.SECRET_EMAIL_DISTRIBUTION_MAP }} + INTERNAL_EMAIL_DISTRIBUTION_KEY: ${{secrets.INTERNAL_EMAIL_DISTRIBUTION_KEY_DEV }} diff --git a/.github/workflows/next-core.yaml b/.github/workflows/next-core.yaml index 62422de333..f1b934cb9f 100644 --- a/.github/workflows/next-core.yaml +++ b/.github/workflows/next-core.yaml @@ -61,7 +61,7 @@ jobs: SECRET_JWKS INTERNAL_DKIM_SELECTOR - + INTERNAL_EMAIL_DISTRIBUTION_KEY SECRET_RELAY_DKIM_PRIVATE_KEY INTERNAL_GOOGLE_OAUTH_CLIENT_ID @@ -93,6 +93,7 @@ jobs: SECRET_JWKS: ${{ secrets.SECRET_JWKS_TEST }} INTERNAL_DKIM_SELECTOR: ${{ secrets.INTERNAL_DKIM_SELECTOR }} + INTERNAL_EMAIL_DISTRIBUTION_KEY: ${{secrets.INTERNAL_EMAIL_DISTRIBUTION_KEY_TEST }} SECRET_RELAY_DKIM_PRIVATE_KEY: ${{ secrets.SECRET_RELAY_DKIM_PRIVATE_KEY }} diff --git a/.github/workflows/next-emailinbounder.yaml b/.github/workflows/next-emailinbounder.yaml new file mode 100644 index 0000000000..b295472465 --- /dev/null +++ b/.github/workflows/next-emailinbounder.yaml @@ -0,0 +1,55 @@ +name: Next Email inbounder + +on: + push: + tags: + - '*' + +defaults: + run: + working-directory: platform/emailinbounder + +jobs: + deploy: + runs-on: ubuntu-latest + environment: next + steps: + - uses: actions/checkout@v3 + + - uses: cachix/install-nix-action@v18 + with: + nix_path: nixpkgs=channel:nixos-unstable + + - run: nix-build ../platform.nix + + - name: Cache Dependencies + id: cache-modules + uses: actions/cache@v3 + with: + path: | + node_modules + .yarn + key: ${{ runner.os }}-node_modules-${{ hashFiles('yarn.lock') }} + + - name: Install Dependencies + run: yarn install + + - name: Test + run: yarn run test + + - name: Deploy + uses: cloudflare/wrangler-action@2.0.0 + with: + wranglerVersion: '3.19.0' + apiToken: ${{ secrets.TOKEN_CLOUDFLARE_API }} + accountId: ${{ secrets.INTERNAL_CLOUDFLARE_ACCOUNT_ID }} + workingDirectory: platform/emailinbounder + command: publish --config wrangler.next.toml --env next + environment: 'next' + secrets: | + SECRET_EMAIL_DISTRIBUTION_MAP + INTERNAL_EMAIL_DISTRIBUTION_KEY + + env: + SECRET_EMAIL_DISTRIBUTION_MAP: ${{ secrets.SECRET_EMAIL_DISTRIBUTION_MAP }} + INTERNAL_EMAIL_DISTRIBUTION_KEY: ${{secrets.INTERNAL_EMAIL_DISTRIBUTION_KEY_TEST }} diff --git a/.github/workflows/release-core.yaml b/.github/workflows/release-core.yaml index 73afc19a96..fbb89fdac3 100644 --- a/.github/workflows/release-core.yaml +++ b/.github/workflows/release-core.yaml @@ -60,7 +60,7 @@ jobs: SECRET_JWKS INTERNAL_DKIM_SELECTOR - + INTERNAL_EMAIL_DISTRIBUTION_KEY SECRET_RELAY_DKIM_PRIVATE_KEY INTERNAL_GOOGLE_OAUTH_CLIENT_ID @@ -92,6 +92,7 @@ jobs: SECRET_JWKS: ${{ secrets.SECRET_JWKS_PROD }} INTERNAL_DKIM_SELECTOR: ${{ secrets.INTERNAL_DKIM_SELECTOR }} + INTERNAL_EMAIL_DISTRIBUTION_KEY: ${{secrets.INTERNAL_EMAIL_DISTRIBUTION_KEY_PROD }} SECRET_RELAY_DKIM_PRIVATE_KEY: ${{ secrets.SECRET_RELAY_DKIM_PRIVATE_KEY }} diff --git a/.github/workflows/release-emailinbounder.yaml b/.github/workflows/release-emailinbounder.yaml new file mode 100644 index 0000000000..5e4f9f98eb --- /dev/null +++ b/.github/workflows/release-emailinbounder.yaml @@ -0,0 +1,54 @@ +name: Release Email inbounder + +on: + release: + types: [published] + +defaults: + run: + working-directory: platform/emailinbounder + +jobs: + deploy: + runs-on: ubuntu-latest + environment: prod + steps: + - uses: actions/checkout@v3 + + - uses: cachix/install-nix-action@v18 + with: + nix_path: nixpkgs=channel:nixos-unstable + + - run: nix-build ../platform.nix + + - name: Cache Dependencies + id: cache-modules + uses: actions/cache@v3 + with: + path: | + node_modules + .yarn + key: ${{ runner.os }}-node_modules-${{ hashFiles('yarn.lock') }} + + - name: Install Dependencies + run: yarn install + + - name: Test + run: yarn run test + + - name: Deploy + uses: cloudflare/wrangler-action@2.0.0 + with: + wranglerVersion: '3.19.0' + apiToken: ${{ secrets.TOKEN_CLOUDFLARE_API }} + accountId: ${{ secrets.INTERNAL_CLOUDFLARE_ACCOUNT_ID }} + workingDirectory: platform/emailinbounder + command: publish --config wrangler.current.toml --env current + environment: 'current' + secrets: | + SECRET_EMAIL_DISTRIBUTION_MAP + INTERNAL_EMAIL_DISTRIBUTION_KEY + + env: + SECRET_EMAIL_DISTRIBUTION_MAP: ${{ secrets.SECRET_EMAIL_DISTRIBUTION_MAP }} + INTERNAL_EMAIL_DISTRIBUTION_KEY: ${{secrets.INTERNAL_EMAIL_DISTRIBUTION_KEY_PROD }} diff --git a/.yarn/cache/@cloudflare-workers-types-npm-4.20221111.1-69d0ea0925-6ee1ba28ee.zip b/.yarn/cache/@cloudflare-workers-types-npm-4.20221111.1-69d0ea0925-6ee1ba28ee.zip new file mode 100644 index 0000000000..5fbdf4ae35 Binary files /dev/null and b/.yarn/cache/@cloudflare-workers-types-npm-4.20221111.1-69d0ea0925-6ee1ba28ee.zip differ diff --git a/packages/types/email.ts b/packages/types/email.ts index f7a3009195..fe01152c3f 100644 --- a/packages/types/email.ts +++ b/packages/types/email.ts @@ -2,3 +2,15 @@ export enum ReconciliationNotificationType { Billing = 'BILLING', Dev = 'DEV', } + +/** CF EmailMessage type; not provided in CF types lib */ +export interface CloudflareEmailMessage { + readonly from: string + readonly to: string + readonly headers: Headers + readonly raw: ReadableStream + readonly rawSize: number + + setReject(reason: string): void + forward(rcptTo: string, headers?: Headers): Promise +} diff --git a/platform/account/src/jsonrpc/methods/getMaskedAddress.ts b/platform/account/src/jsonrpc/methods/getMaskedAddress.ts index 964b0e33ab..f41f86c547 100644 --- a/platform/account/src/jsonrpc/methods/getMaskedAddress.ts +++ b/platform/account/src/jsonrpc/methods/getMaskedAddress.ts @@ -41,5 +41,9 @@ export const getMaskedAddressMethod: GetMaskedAddressMethod = async ({ } const node = new EmailAccount(ctx.account, ctx.env) - return node.getMaskedAddress(input.clientId) + return node.getMaskedAddress( + input.clientId, + ctx.env.INTERNAL_EMAIL_DISTRIBUTION_KEY, + ctx.env.INTERNAL_RELAY_DKIM_DOMAIN + ) } diff --git a/platform/account/src/jsonrpc/methods/getSourceFromMaskedAddress.ts b/platform/account/src/jsonrpc/methods/getSourceFromMaskedAddress.ts new file mode 100644 index 0000000000..3aaaacc2eb --- /dev/null +++ b/platform/account/src/jsonrpc/methods/getSourceFromMaskedAddress.ts @@ -0,0 +1,63 @@ +import { z } from 'zod' +import { router } from '@proofzero/platform.core' +import { EDGE_ACCOUNT } from '@proofzero/platform.account/src/constants' +import { Context } from '../../context' +import { EmailAccountType } from '@proofzero/types/account' +import { AccountURN, AccountURNSpace } from '@proofzero/urns/account' +import { + BadRequestError, + InternalServerError, + NotFoundError, +} from '@proofzero/errors' +import { generateHashedIDRef } from '@proofzero/packages/urns/idref' +import { EmailAccount, initAccountNodeByName } from '../../nodes' + +export const GetSourceByMaskedAddressInput = z.object({ + maskedEmail: z.string(), +}) + +export const GetSourceByMaskedAddressOutput = z.object({ + nickname: z.string(), + sourceEmail: z.string(), +}) + +type GetSourceByMaskedAddressParams = z.infer< + typeof GetSourceByMaskedAddressInput +> +type GetSourceByMaskedAddressResult = z.infer< + typeof GetSourceByMaskedAddressOutput +> + +export const getSourceFromMaskedAddressMethod = async ({ + input, + ctx, +}: { + input: GetSourceByMaskedAddressParams + ctx: Context +}): Promise => { + const nss = generateHashedIDRef(EmailAccountType.Mask, input.maskedEmail) + const urn = AccountURNSpace.componentizedUrn(nss) + const node = initAccountNodeByName(urn, ctx.env.Account) + + const emailNode = new EmailAccount(node, ctx.env) + const sourceAccountURN = await emailNode.getSourceAccount() + if (!sourceAccountURN) + throw new NotFoundError({ + message: `Could not find hidden address ${input.maskedEmail}`, + }) + + const sourceAccountNode = initAccountNodeByName( + sourceAccountURN, + ctx.env.Account + ) + + const name = (await sourceAccountNode.class.getNickname()) || '' + const address = await sourceAccountNode.class.getAddress() + + if (!address) + throw new InternalServerError({ + message: `Could not find source address for masked email ${input.maskedEmail}`, + }) + + return { sourceEmail: address, nickname: name } +} diff --git a/platform/account/src/jsonrpc/router.ts b/platform/account/src/jsonrpc/router.ts index a4420ae852..5a60eade30 100644 --- a/platform/account/src/jsonrpc/router.ts +++ b/platform/account/src/jsonrpc/router.ts @@ -147,6 +147,11 @@ import { SetSourceAccountInput, SetSourceAccountOutput, } from './methods/setSourceAccount' +import { + GetSourceByMaskedAddressInput, + GetSourceByMaskedAddressOutput, + getSourceFromMaskedAddressMethod, +} from './methods/getSourceFromMaskedAddress' const t = initTRPC.context().create({ errorFormatter }) @@ -403,4 +408,10 @@ export const appRouter = t.router({ .input(SetSourceAccountInput) .output(SetSourceAccountOutput) .mutation(setSourceAccountMethod), + getSourceFromMaskedAddress: t.procedure + .use(LogUsage) + .use(Analytics) + .input(GetSourceByMaskedAddressInput) + .output(GetSourceByMaskedAddressOutput) + .query(getSourceFromMaskedAddressMethod), }) diff --git a/platform/account/src/nodes/email.ts b/platform/account/src/nodes/email.ts index c6291f97a3..bbd32627d5 100644 --- a/platform/account/src/nodes/email.ts +++ b/platform/account/src/nodes/email.ts @@ -247,13 +247,19 @@ export default class EmailAccount { return this.node.storage.put('source-account', accountURN) } - async getMaskedAddress(clientId: string): Promise { + async getMaskedAddress( + clientId: string, + distributionKey: string, + relayDomain: string + ): Promise { + const relayPattern = `.${distributionKey}@${relayDomain}` const key = `masked-address/${clientId}` const stored = await this.node.storage.get(key) - if (stored) return stored + //Generates new masked email if there's an existing one using old pattern + if (stored && stored.endsWith(relayPattern)) return stored const bits = generateRandomString(6) const words = randomWords.generate(3).join('-') - const address = `${words}-${bits}@rollup.email` + const address = `${words}-${bits}${relayPattern}` await this.node.storage.put(key, address) return address } diff --git a/platform/core/.dev.vars.example b/platform/core/.dev.vars.example index 9185e1e08d..56f21e52f0 100644 --- a/platform/core/.dev.vars.example +++ b/platform/core/.dev.vars.example @@ -3,6 +3,7 @@ SECRET_JWKS = '[]' INTERNAL_DKIM_SELECTOR = "" +INTERNAL_RELAY_DISTRIBUTION_KEY = "" INTERNAL_RELAY_DKIM_DOMAIN = "" INTERNAL_RELAY_DKIM_SELECTOR = "" SECRET_RELAY_DKIM_PRIVATE_KEY = "" diff --git a/platform/core/src/index.ts b/platform/core/src/index.ts index e736ce9a59..b3a00045bc 100644 --- a/platform/core/src/index.ts +++ b/platform/core/src/index.ts @@ -7,7 +7,6 @@ import { serverOnError as onError } from '@proofzero/utils/trpc' import { createContext, type Context, createContextInner } from './context' import router from './router' -import relay, { type CloudflareEmailMessage } from './relay' import { CoreQueueMessage, CoreQueueMessageType, @@ -47,19 +46,6 @@ export default { }, }) }, - async email(message: CloudflareEmailMessage, env: Environment) { - const decoder = new TextDecoder() - const reader = message.raw.getReader() - - let content = '' - let { done, value } = await reader.read() - while (!done) { - content += decoder.decode(value) - ;({ done, value } = await reader.read()) - } - - return relay(content, env) - }, async queue( batch: MessageBatch, env: Environment, diff --git a/platform/core/src/types.ts b/platform/core/src/types.ts index e196cbfb17..d82b398ce3 100644 --- a/platform/core/src/types.ts +++ b/platform/core/src/types.ts @@ -71,7 +71,7 @@ export interface Environment { MAX_ATTEMPTS_TIME_PERIOD_IN_MS: number INTERNAL_DKIM_SELECTOR: string - + INTERNAL_EMAIL_DISTRIBUTION_KEY: string INTERNAL_RELAY_DKIM_DOMAIN: string INTERNAL_RELAY_DKIM_SELECTOR: string SECRET_RELAY_DKIM_PRIVATE_KEY: string diff --git a/platform/email/src/emailFunctions.ts b/platform/email/src/emailFunctions.ts index 8fa5cb93ed..153c9ef892 100644 --- a/platform/email/src/emailFunctions.ts +++ b/platform/email/src/emailFunctions.ts @@ -1,4 +1,3 @@ -import { InternalServerError } from '@proofzero/errors' import { EmailTemplateBillingReconciledEntitlements, EmailTemplateDevReconciledEntitlements, @@ -9,7 +8,7 @@ import { EmailTemplateSuccessfulPayment, } from '../emailTemplate' import { EmailMessage, EmailNotification } from './types' -import { CloudflareEmailMessage, EmailContent, Environment } from './types' +import { EmailContent, Environment } from './types' import { Context } from './context' import { z } from 'zod' import { type EmailPlans } from './jsonrpc/methods/sendSuccesfullPaymentNotification' @@ -157,11 +156,6 @@ export async function sendNotification( await send(message, env) } -async function forward(message: CloudflareEmailMessage, env: Environment) { - //TODO: Implement for masked email - throw new InternalServerError({ message: 'Not implemented yet' }) -} - const adjustEmailParams = (params?: Partial) => { if (!params) { params = { diff --git a/platform/email/src/index.ts b/platform/email/src/index.ts index 1955df95d0..d6a01058eb 100644 --- a/platform/email/src/index.ts +++ b/platform/email/src/index.ts @@ -8,7 +8,7 @@ import { serverOnError as onError } from '@proofzero/utils/trpc' import { createContext } from './context' import { appRouter } from './jsonrpc/router' -import type { Environment, CloudflareEmailMessage } from './types' +import type { Environment } from './types' export default { async fetch(request: Request, env: Environment): Promise { @@ -21,10 +21,4 @@ export default { createContext(opts as FetchCreateContextFnOptions, env), }) }, - - async email(message: CloudflareEmailMessage, env: Environment) { - //TODO: Implement email masking - //This is where you'd receive an email, check destination - //address, lookup unmasked address and forward - }, } diff --git a/platform/email/src/types.ts b/platform/email/src/types.ts index 694b97855f..e6a9e793c2 100644 --- a/platform/email/src/types.ts +++ b/platform/email/src/types.ts @@ -14,18 +14,6 @@ export interface Environment { SECRET_TEST_API_TEST_TOKEN: string | undefined } -/** CF EmailMessage type; not provided in CF types lib */ -export interface CloudflareEmailMessage { - readonly from: string - readonly to: string - readonly headers: Headers - readonly raw: ReadableStream - readonly rawSize: number - - setReject(reason: String): void - forward(rcptTo: string, headers?: Headers): Promise -} - export type EmailContentType = 'text/plain' | 'text/html' export type EmailAddressComponents = { name: string diff --git a/platform/emaildistributor/.dev.vars.example b/platform/emaildistributor/.dev.vars.example new file mode 100644 index 0000000000..0d80cb6bef --- /dev/null +++ b/platform/emaildistributor/.dev.vars.example @@ -0,0 +1,4 @@ +#This is only set in the CF environments, as a secret. Locally it can be set to anything +#JSON object containing mapping between env specific identifier in the email address +#and the target internal (hence, secret) email to be forwarded to for env-specific processor +SECRET_RELAY_DISTRIBUTION_MAP="{'local':'supersecretemail@example.com'}" \ No newline at end of file diff --git a/platform/emaildistributor/.eslintrc.json b/platform/emaildistributor/.eslintrc.json new file mode 100644 index 0000000000..d9d41509c1 --- /dev/null +++ b/platform/emaildistributor/.eslintrc.json @@ -0,0 +1,18 @@ +{ + "env": { + "browser": true, + "es2022": true, + "node": true + }, + "extends": [ + "eslint:recommended", + "plugin:@typescript-eslint/recommended", + "prettier" + ], + "parser": "@typescript-eslint/parser", + "parserOptions": { + "ecmaVersion": "latest", + "sourceType": "module" + }, + "plugins": ["@typescript-eslint"] +} diff --git a/platform/emaildistributor/.gitignore b/platform/emaildistributor/.gitignore new file mode 100644 index 0000000000..1f73d4b091 --- /dev/null +++ b/platform/emaildistributor/.gitignore @@ -0,0 +1,132 @@ +# platform/app/starbse/.gitignore + +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* +lerna-debug.log* +.pnpm-debug.log* + +# Diagnostic reports (https://nodejs.org/api/report.html) +report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json + +# Runtime data +pids +*.pid +*.seed +*.pid.lock + +# Directory for instrumented libs generated by jscoverage/JSCover +lib-cov + +# Coverage directory used by tools like istanbul +coverage +*.lcov + +# nyc test coverage +.nyc_output + +# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) +.grunt + +# Bower dependency directory (https://bower.io/) +bower_components + +# node-waf configuration +.lock-wscript + +# Compiled binary addons (https://nodejs.org/api/addons.html) +build/Release + +# Dependency directories +node_modules/ +jspm_packages/ + +# Snowpack dependency directory (https://snowpack.dev/) +web_modules/ + +# TypeScript cache +*.tsbuildinfo + +# Optional npm cache directory +.npm + +# Optional eslint cache +.eslintcache + +# Optional stylelint cache +.stylelintcache + +# Microbundle cache +.rpt2_cache/ +.rts2_cache_cjs/ +.rts2_cache_es/ +.rts2_cache_umd/ + +# Optional REPL history +.node_repl_history + +# Output of 'npm pack' +*.tgz + +# Yarn Integrity file +.yarn-integrity + +# dotenv environment variable files +.env +.env.development.local +.env.test.local +.env.production.local +.env.local + +# parcel-bundler cache (https://parceljs.org/) +.cache +.parcel-cache + +# Next.js build output +.next +out + +# Nuxt.js build / generate output +.nuxt +dist + +# Gatsby files +.cache/ +# Comment in the public line in if your project uses Gatsby and not Next.js +# https://nextjs.org/blog/next-9-1#public-directory-support +# public + +# vuepress build output +.vuepress/dist + +# vuepress v2.x temp and cache directory +.temp +.cache + +# Docusaurus cache and generated files +.docusaurus + +# Serverless directories +.serverless/ + +# FuseBox cache +.fusebox/ + +# DynamoDB Local files +.dynamodb/ + +# TernJS port file +.tern-port + +# Stores VSCode versions used for testing VSCode extensions +.vscode-test + +# yarn v2 +.yarn/cache +.yarn/unplugged +.yarn/build-state.yml +.yarn/install-state.gz +.pnp.* diff --git a/platform/emaildistributor/README.md b/platform/emaildistributor/README.md new file mode 100644 index 0000000000..678d69ee61 --- /dev/null +++ b/platform/emaildistributor/README.md @@ -0,0 +1,11 @@ +# Email Distributor Worker + +## Overview + +This worker distributes inbound emails received globally in our relay infrastructure to env-specific workers + +## Setup + +### Local Env + +1. Copy `.dev.vars.example` to `.dev.vars` and fill in the values. diff --git a/platform/emaildistributor/package.json b/platform/emaildistributor/package.json new file mode 100644 index 0000000000..5223a890dd --- /dev/null +++ b/platform/emaildistributor/package.json @@ -0,0 +1,33 @@ +{ + "name": "@proofzero/services.emaildistributor", + "version": "0.0.0", + "devDependencies": { + "@cloudflare/workers-types": "4.20221111.1", + "@types/node": "18.15.3", + "eslint": "8.28.0", + "eslint-config-prettier": "8.8.0", + "npm-run-all": "4.1.5", + "prettier": "2.8.8", + "typescript": "5.0.4", + "wrangler": "3.18" + }, + "private": true, + "scripts": { + "build": "wrangler publish --dry-run --outdir=dist", + "check": "run-s format:check lint:check types:check", + "format": "run-s format:src", + "format:src": "prettier --write src", + "format:check": "run-s format:check:src", + "format:check:src": "prettier --check src", + "lint": "eslint --fix src test", + "lint:check": "run-s lint:check:src", + "lint:check:src": "eslint src", + "types:check": "tsc --project tsconfig.json", + "dev": "env-cmd --file .dev.env wrangler dev", + "deploy": "wrangler publish", + "test": "run-s check" + }, + "dependencies": { + "@proofzero/utils": "workspace:*" + } +} diff --git a/platform/emaildistributor/src/index.ts b/platform/emaildistributor/src/index.ts new file mode 100644 index 0000000000..a03a035e62 --- /dev/null +++ b/platform/emaildistributor/src/index.ts @@ -0,0 +1,37 @@ +import type { Environment } from './types' +import type { CloudflareEmailMessage } from '@proofzero/packages/types/email' + +export default { + async email(message: CloudflareEmailMessage, env: Environment) { + //Format is loosely aaaa-bbb-ccc-ddd.eee@relaydomain + const emailParts = message.to.split('@') + if (emailParts.length !== 2) { + console.error('Unparsable email received', message.to) + return + } + const userParts = emailParts[0].split('.') + if (userParts.length !== 2) { + console.error('Unparsable masked email relay address', message.to) + return + } + const envPrefix = userParts[1] + + const distEmailEntries = Object.entries( + JSON.parse(env.SECRET_EMAIL_DISTRIBUTION_MAP) + ) as [string, string][] + + const envKeyPair = distEmailEntries.filter( + ([distributorEnvPrefix]) => distributorEnvPrefix === envPrefix + ) + if (envKeyPair.length !== 1) { + console.error('Incorrect relay distribution map configuration') + return + } else { + const [addressEnvSuffix, targetEnvEmail] = envKeyPair[0] + console.info( + `Forwarding to env suffix ${addressEnvSuffix} for ${message.to}` + ) + await message.forward(targetEnvEmail) + } + }, +} diff --git a/platform/emaildistributor/src/types.ts b/platform/emaildistributor/src/types.ts new file mode 100644 index 0000000000..b155ed5cc2 --- /dev/null +++ b/platform/emaildistributor/src/types.ts @@ -0,0 +1,3 @@ +export interface Environment { + SECRET_EMAIL_DISTRIBUTION_MAP: string //containing JSON +} diff --git a/platform/emaildistributor/tsconfig.json b/platform/emaildistributor/tsconfig.json new file mode 100644 index 0000000000..f8f68fe69d --- /dev/null +++ b/platform/emaildistributor/tsconfig.json @@ -0,0 +1,19 @@ +{ + "compilerOptions": { + "target": "esnext", + "lib": ["esnext"], + "jsx": "react", + "module": "es2022", + "moduleResolution": "node", + "types": ["@cloudflare/workers-types", "node"], + "resolveJsonModule": true, + "allowJs": true, + "checkJs": false, + "noEmit": true, + "isolatedModules": true, + "allowSyntheticDefaultImports": true, + "forceConsistentCasingInFileNames": true, + "strict": true, + "skipLibCheck": true + } +} diff --git a/platform/emaildistributor/wrangler.toml b/platform/emaildistributor/wrangler.toml new file mode 100644 index 0000000000..6d6f67fd55 --- /dev/null +++ b/platform/emaildistributor/wrangler.toml @@ -0,0 +1,11 @@ +# This is a singleton service and doesn't require per-env configs +name = "emaildistributor" +main = "src/index.ts" +compatibility_date = "2022-10-05" +logpush = true +workers_dev = false + +[dev] +port = 10145 +inspector_port = 11145 +local_protocol = "http" diff --git a/platform/emailinbounder/.dev.vars.example b/platform/emailinbounder/.dev.vars.example new file mode 100644 index 0000000000..0401cde80f --- /dev/null +++ b/platform/emailinbounder/.dev.vars.example @@ -0,0 +1,5 @@ +#This is only set in the CF environments, as a secret. Locally it can be set to anything +#JSON object containing mapping between env specific identifier in the email address +#and the target internal (hence, secret) email to be forwarded to for env-specific processor +SECRET_RELAY_DISTRIBUTION_MAP="{'local':'supersecretemail@example.com'}" +INTERNAL_RELAY_DISTRIBUTION_KEY = "local" \ No newline at end of file diff --git a/platform/emailinbounder/.eslintrc.json b/platform/emailinbounder/.eslintrc.json new file mode 100644 index 0000000000..d9d41509c1 --- /dev/null +++ b/platform/emailinbounder/.eslintrc.json @@ -0,0 +1,18 @@ +{ + "env": { + "browser": true, + "es2022": true, + "node": true + }, + "extends": [ + "eslint:recommended", + "plugin:@typescript-eslint/recommended", + "prettier" + ], + "parser": "@typescript-eslint/parser", + "parserOptions": { + "ecmaVersion": "latest", + "sourceType": "module" + }, + "plugins": ["@typescript-eslint"] +} diff --git a/platform/emailinbounder/.gitignore b/platform/emailinbounder/.gitignore new file mode 100644 index 0000000000..1f73d4b091 --- /dev/null +++ b/platform/emailinbounder/.gitignore @@ -0,0 +1,132 @@ +# platform/app/starbse/.gitignore + +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* +lerna-debug.log* +.pnpm-debug.log* + +# Diagnostic reports (https://nodejs.org/api/report.html) +report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json + +# Runtime data +pids +*.pid +*.seed +*.pid.lock + +# Directory for instrumented libs generated by jscoverage/JSCover +lib-cov + +# Coverage directory used by tools like istanbul +coverage +*.lcov + +# nyc test coverage +.nyc_output + +# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) +.grunt + +# Bower dependency directory (https://bower.io/) +bower_components + +# node-waf configuration +.lock-wscript + +# Compiled binary addons (https://nodejs.org/api/addons.html) +build/Release + +# Dependency directories +node_modules/ +jspm_packages/ + +# Snowpack dependency directory (https://snowpack.dev/) +web_modules/ + +# TypeScript cache +*.tsbuildinfo + +# Optional npm cache directory +.npm + +# Optional eslint cache +.eslintcache + +# Optional stylelint cache +.stylelintcache + +# Microbundle cache +.rpt2_cache/ +.rts2_cache_cjs/ +.rts2_cache_es/ +.rts2_cache_umd/ + +# Optional REPL history +.node_repl_history + +# Output of 'npm pack' +*.tgz + +# Yarn Integrity file +.yarn-integrity + +# dotenv environment variable files +.env +.env.development.local +.env.test.local +.env.production.local +.env.local + +# parcel-bundler cache (https://parceljs.org/) +.cache +.parcel-cache + +# Next.js build output +.next +out + +# Nuxt.js build / generate output +.nuxt +dist + +# Gatsby files +.cache/ +# Comment in the public line in if your project uses Gatsby and not Next.js +# https://nextjs.org/blog/next-9-1#public-directory-support +# public + +# vuepress build output +.vuepress/dist + +# vuepress v2.x temp and cache directory +.temp +.cache + +# Docusaurus cache and generated files +.docusaurus + +# Serverless directories +.serverless/ + +# FuseBox cache +.fusebox/ + +# DynamoDB Local files +.dynamodb/ + +# TernJS port file +.tern-port + +# Stores VSCode versions used for testing VSCode extensions +.vscode-test + +# yarn v2 +.yarn/cache +.yarn/unplugged +.yarn/build-state.yml +.yarn/install-state.gz +.pnp.* diff --git a/platform/emailinbounder/README.md b/platform/emailinbounder/README.md new file mode 100644 index 0000000000..9e328272e9 --- /dev/null +++ b/platform/emailinbounder/README.md @@ -0,0 +1,11 @@ +# Email Inbounder Worker + +## Overview + +This worker processes inbound emails received from the global distributor worker, in an env-specific way + +## Setup + +### Local Env + +1. Copy `.dev.vars.example` to `.dev.vars` and fill in the values. diff --git a/platform/emailinbounder/package.json b/platform/emailinbounder/package.json new file mode 100644 index 0000000000..ddda875e22 --- /dev/null +++ b/platform/emailinbounder/package.json @@ -0,0 +1,33 @@ +{ + "name": "@proofzero/services.emailinbounder", + "version": "0.0.0", + "devDependencies": { + "@cloudflare/workers-types": "4.20221111.1", + "@types/node": "18.15.3", + "eslint": "8.28.0", + "eslint-config-prettier": "8.8.0", + "npm-run-all": "4.1.5", + "prettier": "2.8.8", + "typescript": "5.0.4", + "wrangler": "3.18" + }, + "private": true, + "scripts": { + "build": "wrangler publish --dry-run --outdir=dist", + "check": "run-s format:check lint:check types:check", + "format": "run-s format:src", + "format:src": "prettier --write src", + "format:check": "run-s format:check:src", + "format:check:src": "prettier --check src", + "lint": "eslint --fix src test", + "lint:check": "run-s lint:check:src", + "lint:check:src": "eslint src", + "types:check": "tsc --project tsconfig.json", + "dev": "env-cmd --file .dev.env wrangler dev", + "deploy": "wrangler publish", + "test": "run-s check" + }, + "dependencies": { + "@proofzero/utils": "workspace:*" + } +} diff --git a/platform/core/src/relay.ts b/platform/emailinbounder/src/index.ts similarity index 59% rename from platform/core/src/relay.ts rename to platform/emailinbounder/src/index.ts index ff4f0f24ec..e9bc5828d4 100644 --- a/platform/core/src/relay.ts +++ b/platform/emailinbounder/src/index.ts @@ -1,21 +1,34 @@ +import { CloudflareEmailMessage } from '@proofzero/packages/types/email' +import type { Environment } from './types' +import createCoreClient from '@proofzero/platform-clients/core' + +import { + generateTraceContextHeaders, + generateTraceSpan, +} from '@proofzero/platform-middleware/trace' import PostalMime, { type Address, Email } from 'postal-mime' -import { AccountURN, AccountURNSpace } from '@proofzero/urns/account' -import { generateHashedIDRef } from '@proofzero/urns/idref' -import { EmailAccountType } from '@proofzero/types/account' -import { initAccountNodeByName } from '@proofzero/platform.account/src/nodes' +export default { + async email(message: CloudflareEmailMessage, env: Environment) { + const decoder = new TextDecoder() + const reader = message.raw.getReader() -import type { Environment } from './types' + let content = '' + let { done, value } = await reader.read() + while (!done) { + content += decoder.decode(value) + ;({ done, value } = await reader.read()) + } -export interface CloudflareEmailMessage { - readonly from: string - readonly to: string - readonly headers: Headers - readonly raw: ReadableStream - readonly rawSize: number + return relay(content, env) + }, +} + +const getCoreClient = (env: Environment) => { + //New trace as entrypoint is the email trigger and not an HTTP request + const headers = generateTraceContextHeaders(generateTraceSpan()) - setReject(reason: string): void - forward(rcptTo: string, headers?: Headers): Promise + return createCoreClient(env.Core, headers) } interface MailChannelAddress { @@ -29,7 +42,7 @@ interface DKIM { dkim_private_key: string } -export default async (message: string, env: Environment) => { +const relay = async (message: string, env: Environment) => { const postalMime = new PostalMime() const email = await postalMime.parse(message) @@ -43,27 +56,18 @@ export default async (message: string, env: Environment) => { .concat(email.to || []) .concat(email.cc || []) .filter((recipient) => - recipient.address.endsWith(`@${env.INTERNAL_RELAY_DKIM_DOMAIN}`) + recipient.address.endsWith( + `.${env.INTERNAL_EMAIL_DISTRIBUTION_KEY}@${env.INTERNAL_RELAY_DKIM_DOMAIN}` + ) ) for (const recipient of recipients) { - const nss = generateHashedIDRef(EmailAccountType.Mask, recipient.address) - const urn = AccountURNSpace.componentizedUrn(nss) - const node = initAccountNodeByName(urn, env.Account) - - const sourceAccountURN = await node.storage.get( - 'source-account' - ) - if (!sourceAccountURN) continue - - const sourceAccountNode = initAccountNodeByName( - sourceAccountURN, - env.Account - ) - - const name = (await sourceAccountNode.class.getNickname()) || '' - const address = await sourceAccountNode.class.getAddress() - if (!address) continue + const coreClient = getCoreClient(env) + const { sourceEmail, nickname } = + await coreClient.account.getSourceFromMaskedAddress.query({ + maskedEmail: recipient.address, + }) + if (!sourceEmail) continue const from: MailChannelAddress = { name: email.from.name, @@ -72,11 +76,10 @@ export default async (message: string, env: Environment) => { const to: MailChannelAddress[] = [ { - name, - email: address, + name: nickname, + email: sourceEmail, }, ] - await send(email, from, to, dkim) } } diff --git a/platform/emailinbounder/src/types.ts b/platform/emailinbounder/src/types.ts new file mode 100644 index 0000000000..574972e50c --- /dev/null +++ b/platform/emailinbounder/src/types.ts @@ -0,0 +1,8 @@ +export interface Environment { + SECRET_EMAIL_DISTRIBUTION_MAP: { [addressEnvSuffix: string]: string } + INTERNAL_EMAIL_DISTRIBUTION_KEY: string + Core: Fetcher + INTERNAL_RELAY_DKIM_DOMAIN: string + INTERNAL_RELAY_DKIM_SELECTOR: string + SECRET_RELAY_DKIM_PRIVATE_KEY: string +} diff --git a/platform/emailinbounder/tsconfig.json b/platform/emailinbounder/tsconfig.json new file mode 100644 index 0000000000..f8f68fe69d --- /dev/null +++ b/platform/emailinbounder/tsconfig.json @@ -0,0 +1,19 @@ +{ + "compilerOptions": { + "target": "esnext", + "lib": ["esnext"], + "jsx": "react", + "module": "es2022", + "moduleResolution": "node", + "types": ["@cloudflare/workers-types", "node"], + "resolveJsonModule": true, + "allowJs": true, + "checkJs": false, + "noEmit": true, + "isolatedModules": true, + "allowSyntheticDefaultImports": true, + "forceConsistentCasingInFileNames": true, + "strict": true, + "skipLibCheck": true + } +} diff --git a/platform/emailinbounder/wrangler.current.toml b/platform/emailinbounder/wrangler.current.toml new file mode 100644 index 0000000000..7c7d534294 --- /dev/null +++ b/platform/emailinbounder/wrangler.current.toml @@ -0,0 +1,17 @@ +# This is a singleton service and doesn't require per-env configs +name = "emailinbounder" +main = "src/index.ts" +compatibility_date = "2022-10-05" +logpush = true +workers_dev = false + +[env.current] +port = 10146 +inspector_port = 11146 +local_protocol = "http" + +services = [{ binding = "Core", service = "core-current" }] + +[env.current.vars] +INTERNAL_RELAY_DKIM_DOMAIN = "rollup.email" +INTERNAL_RELAY_DKIM_SELECTOR = "mailchannels" diff --git a/platform/emailinbounder/wrangler.dev.toml b/platform/emailinbounder/wrangler.dev.toml new file mode 100644 index 0000000000..08477a8ed0 --- /dev/null +++ b/platform/emailinbounder/wrangler.dev.toml @@ -0,0 +1,17 @@ +# This is a singleton service and doesn't require per-env configs +name = "emailinbounder" +main = "src/index.ts" +compatibility_date = "2022-10-05" +logpush = true +workers_dev = false + +[env.dev] +port = 10146 +inspector_port = 11146 +local_protocol = "http" + +services = [{ binding = "Core", service = "core-dev" }] + +[env.dev.vars] +INTERNAL_RELAY_DKIM_DOMAIN = "rollup.email" +INTERNAL_RELAY_DKIM_SELECTOR = "mailchannels" diff --git a/platform/emailinbounder/wrangler.next.toml b/platform/emailinbounder/wrangler.next.toml new file mode 100644 index 0000000000..772c6ca61b --- /dev/null +++ b/platform/emailinbounder/wrangler.next.toml @@ -0,0 +1,17 @@ +# This is a singleton service and doesn't require per-env configs +name = "emailinbounder" +main = "src/index.ts" +compatibility_date = "2022-10-05" +logpush = true +workers_dev = false + +[env.next] +port = 10146 +inspector_port = 11146 +local_protocol = "http" + +services = [{ binding = "Core", service = "core-next" }] + +[env.next.vars] +INTERNAL_RELAY_DKIM_DOMAIN = "rollup.email" +INTERNAL_RELAY_DKIM_SELECTOR = "mailchannels" diff --git a/platform/emailinbounder/wrangler.toml b/platform/emailinbounder/wrangler.toml new file mode 100644 index 0000000000..2a6e507dff --- /dev/null +++ b/platform/emailinbounder/wrangler.toml @@ -0,0 +1,17 @@ +# This is a singleton service and doesn't require per-env configs +name = "emailinbounder" +main = "src/index.ts" +compatibility_date = "2022-10-05" +logpush = true +workers_dev = false + +services = [{ binding = "Core", service = "core" }] + +[dev] +port = 10146 +inspector_port = 11146 +local_protocol = "http" + +[vars] +INTERNAL_RELAY_DKIM_DOMAIN = "rollup.email" +INTERNAL_RELAY_DKIM_SELECTOR = "mailchannels" diff --git a/platform/test/src/index.ts b/platform/test/src/index.ts index 2c683da355..8999ef3b0c 100644 --- a/platform/test/src/index.ts +++ b/platform/test/src/index.ts @@ -1,6 +1,7 @@ import { IRequest, RequestLike, Router } from 'itty-router' +import type { CloudflareEmailMessage } from '@proofzero/packages/types/email' -import type { Environment, CloudflareEmailMessage } from './types' +import type { Environment } from './types' const router = Router() // no "new", as this is not a real class diff --git a/platform/test/src/types.ts b/platform/test/src/types.ts index 213bba2092..1c4cc40747 100644 --- a/platform/test/src/types.ts +++ b/platform/test/src/types.ts @@ -1,15 +1,3 @@ -/** CF EmailMessage type; not provided in CF types lib */ -export interface CloudflareEmailMessage { - readonly from: string - readonly to: string - readonly headers: Headers - readonly raw: ReadableStream - readonly rawSize: number - - setReject(reason: string): void - forward(rcptTo: string, headers?: Headers): Promise -} - export type Environment = { SECRET_TEST_API_TOKEN: string otp_test: KVNamespace diff --git a/yarn.lock b/yarn.lock index 229b7317d8..e14aa8b421 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2262,6 +2262,13 @@ __metadata: languageName: node linkType: hard +"@cloudflare/workers-types@npm:4.20221111.1": + version: 4.20221111.1 + resolution: "@cloudflare/workers-types@npm:4.20221111.1" + checksum: 6ee1ba28ee3c59926f0f49da7023a887846bb637d43da0d8787ed87b2ed624bd83471feba9519a404a58cecad73f3a74a67bc3f52ccfda916947d297cfaa988c + languageName: node + linkType: hard + "@cloudflare/workers-types@npm:4.20231121.0": version: 4.20231121.0 resolution: "@cloudflare/workers-types@npm:4.20231121.0" @@ -7043,6 +7050,38 @@ __metadata: languageName: unknown linkType: soft +"@proofzero/services.emaildistributor@workspace:platform/emaildistributor": + version: 0.0.0-use.local + resolution: "@proofzero/services.emaildistributor@workspace:platform/emaildistributor" + dependencies: + "@cloudflare/workers-types": 4.20221111.1 + "@proofzero/utils": "workspace:*" + "@types/node": 18.15.3 + eslint: 8.28.0 + eslint-config-prettier: 8.8.0 + npm-run-all: 4.1.5 + prettier: 2.8.8 + typescript: 5.0.4 + wrangler: 3.18 + languageName: unknown + linkType: soft + +"@proofzero/services.emailinbounder@workspace:platform/emailinbounder": + version: 0.0.0-use.local + resolution: "@proofzero/services.emailinbounder@workspace:platform/emailinbounder" + dependencies: + "@cloudflare/workers-types": 4.20221111.1 + "@proofzero/utils": "workspace:*" + "@types/node": 18.15.3 + eslint: 8.28.0 + eslint-config-prettier: 8.8.0 + npm-run-all: 4.1.5 + prettier: 2.8.8 + typescript: 5.0.4 + wrangler: 3.18 + languageName: unknown + linkType: soft + "@proofzero/services.images@workspace:platform/images": version: 0.0.0-use.local resolution: "@proofzero/services.images@workspace:platform/images"