From ade600c4161dfa0bed9fd79d6ecfb4a926553c13 Mon Sep 17 00:00:00 2001 From: James Culveyhouse Date: Thu, 27 Jul 2023 16:36:57 -0500 Subject: [PATCH] C3: Add e2e coverage for deployment (#3658) * C3: Add e2e coverage for deployment * PR feedback * Adding cleanup script for test projects * C3: Run framework e2e tests concurrently * Fixing project cleanup and tweaking test concurrency --- .github/workflows/test-c3.yml | 3 + .../create-cloudflare/e2e-tests/helpers.ts | 33 +++- .../create-cloudflare/e2e-tests/pages.test.ts | 187 ++++++++++++------ .../scripts/cleanupPagesProject.mjs | 47 +++++ packages/create-cloudflare/src/common.ts | 2 +- .../src/frameworks/next/index.ts | 2 +- .../src/helpers/interactive.ts | 6 +- .../create-cloudflare/vitest-e2e.config.ts | 3 +- 8 files changed, 211 insertions(+), 72 deletions(-) create mode 100644 packages/create-cloudflare/scripts/cleanupPagesProject.mjs diff --git a/.github/workflows/test-c3.yml b/.github/workflows/test-c3.yml index 5431eed34165..53dcb3c739b9 100644 --- a/.github/workflows/test-c3.yml +++ b/.github/workflows/test-c3.yml @@ -64,3 +64,6 @@ jobs: - name: E2E Tests run: npm run test:e2e -w create-cloudflare + env: + CLOUDFLARE_API_TOKEN: ${{ secrets.TEST_CLOUDFLARE_API_TOKEN }} + CLOUDFLARE_ACCOUNT_ID: ${{ secrets.TEST_CLOUDFLARE_ACCOUNT_ID }} diff --git a/packages/create-cloudflare/e2e-tests/helpers.ts b/packages/create-cloudflare/e2e-tests/helpers.ts index 31ed9d7433c4..200c9b10663a 100644 --- a/packages/create-cloudflare/e2e-tests/helpers.ts +++ b/packages/create-cloudflare/e2e-tests/helpers.ts @@ -1,4 +1,5 @@ import { spawn } from "cross-spawn"; +import { spinnerFrames } from "helpers/interactive"; export const keys = { enter: "\x0d", @@ -37,6 +38,10 @@ export const runC3 = async ({ const currentDialog = promptHandlers[0]; lines.forEach((line) => { + // Uncomment to debug test output + // if (filterLine(line)) { + // console.log(line); + // } stdout.push(line); if (currentDialog && currentDialog.matcher.test(line)) { @@ -64,17 +69,39 @@ export const runC3 = async ({ if (code === 0) { resolve(null); } else { + console.log(stderr.join("\n").trim()); rejects(code); } }); - proc.on("error", (err) => { - rejects(err); + proc.on("error", (exitCode) => { + rejects({ + exitCode, + output: condenseOutput(stdout).join("\n").trim(), + errors: stderr.join("\n").trim(), + }); }); }); return { - output: stdout.join("\n").trim(), + output: condenseOutput(stdout).join("\n").trim(), errors: stderr.join("\n").trim(), }; }; + +// Removes lines from the output of c3 that aren't particularly useful for debugging tests +export const condenseOutput = (lines: string[]) => { + return lines.filter(filterLine); +}; + +const filterLine = (line: string) => { + // Remove all lines with spinners + for (const frame of spinnerFrames) { + if (line.includes(frame)) return false; + } + + // Remove empty lines + if (line.replace(/\s/g, "").length == 0) return false; + + return true; +}; diff --git a/packages/create-cloudflare/e2e-tests/pages.test.ts b/packages/create-cloudflare/e2e-tests/pages.test.ts index aa0bfabb5eab..a2f937592cc9 100644 --- a/packages/create-cloudflare/e2e-tests/pages.test.ts +++ b/packages/create-cloudflare/e2e-tests/pages.test.ts @@ -1,63 +1,95 @@ import { existsSync, mkdtempSync, realpathSync, rmSync } from "fs"; +import crypto from "node:crypto"; import { tmpdir } from "os"; import { join } from "path"; +import spawn from "cross-spawn"; import { FrameworkMap } from "frameworks/index"; import { readJSON } from "helpers/files"; +import { fetch } from "undici"; import { describe, expect, test, afterEach, beforeEach } from "vitest"; import { keys, runC3 } from "./helpers"; import type { RunnerConfig } from "./helpers"; +export const TEST_PREFIX = "c3-e2e-"; + /* Areas for future improvement: -- Make these actually e2e by verifying that deployment works - Add support for frameworks with global installs (like docusaurus, gatsby, etc) */ -describe("E2E: Web frameworks", () => { +type FrameworkTestConfig = RunnerConfig & { + expectResponseToContain: string; +}; + +describe(`E2E: Web frameworks`, () => { const tmpDirPath = realpathSync(mkdtempSync(join(tmpdir(), "c3-tests"))); - const projectPath = join(tmpDirPath, "pages-tests"); + const baseProjectName = `c3-e2e-${crypto.randomBytes(3).toString("hex")}`; - beforeEach(() => { + const getProjectName = (framework: string) => + `${baseProjectName}-${framework}`; + const getProjectPath = (framework: string) => + join(tmpDirPath, getProjectName(framework)); + + beforeEach((ctx) => { + const framework = ctx.meta.name; + const projectPath = getProjectPath(framework); rmSync(projectPath, { recursive: true, force: true }); }); - afterEach(() => { + afterEach((ctx) => { + const framework = ctx.meta.name; + const projectPath = getProjectPath(framework); + const projectName = getProjectName(framework); + if (existsSync(projectPath)) { rmSync(projectPath, { recursive: true }); } + + try { + const { output } = spawn.sync("npx", [ + "wrangler", + "pages", + "project", + "delete", + "-y", + projectName, + ]); + + if (!output.toString().includes(`Successfully deleted ${projectName}`)) { + console.error(output.toString()); + } + } catch (error) { + console.error(`Failed to cleanup project: ${projectName}`); + console.error(error); + } }); const runCli = async ( framework: string, - { argv = [], promptHandlers = [], overrides = {} }: RunnerConfig + { argv = [], promptHandlers = [], overrides }: RunnerConfig ) => { + const projectPath = getProjectPath(framework); + const args = [ projectPath, "--type", "webFramework", "--framework", framework, - "--no-deploy", + "--deploy", + "--no-open", ]; - if (argv.length > 0) { - args.push(...argv); - } else { - args.push("--no-git"); - } - - // For debugging purposes, uncomment the following to see the exact - // command the test uses. You can then run this via the command line. - // console.log("COMMAND: ", `node ${["./dist/cli.js", ...args].join(" ")}`); + args.push(...argv); - await runC3({ argv: args, promptHandlers }); + const { output } = await runC3({ argv: args, promptHandlers }); // Relevant project files should have been created expect(projectPath).toExist(); - const pkgJsonPath = join(projectPath, "package.json"); expect(pkgJsonPath).toExist(); + // Wrangler should be installed const wranglerPath = join(projectPath, "node_modules/wrangler"); expect(wranglerPath).toExist(); @@ -68,7 +100,7 @@ describe("E2E: Web frameworks", () => { ...frameworkConfig.packageScripts, } as Record; - if (overrides.packageScripts) { + if (overrides && overrides.packageScripts) { // override packageScripts with testing provided scripts Object.entries(overrides.packageScripts).forEach(([target, cmd]) => { frameworkTargetPackageScripts[target] = cmd; @@ -79,46 +111,77 @@ describe("E2E: Web frameworks", () => { Object.entries(frameworkTargetPackageScripts).forEach(([target, cmd]) => { expect(pkgJson.scripts[target]).toEqual(cmd); }); + + return { output }; }; - test.each(["astro", "hono", "react", "remix", "vue"])("%s", async (name) => { - await runCli(name, {}); - }); + const runCliWithDeploy = async (framework: string) => { + const projectName = `${baseProjectName}-${framework}`; - test("Nuxt", async () => { - await runCli("nuxt", { - overrides: { - packageScripts: { - build: "NITRO_PRESET=cloudflare-pages nuxt build", - }, - }, + const { argv, overrides, promptHandlers, expectResponseToContain } = + frameworkTests[framework]; + + await runCli(framework, { + overrides, + promptHandlers, + argv: [...(argv ?? []), "--deploy", "--no-git"], }); - }); - test("next", async () => { - await runCli("next", { + // Verify deployment + const projectUrl = `https://${projectName}.pages.dev/`; + + const res = await fetch(projectUrl); + expect(res.status).toBe(200); + + const body = await res.text(); + expect( + body, + `(${framework}) Deployed page (${projectUrl}) didn't contain expected string: "${expectResponseToContain}"` + ).toContain(expectResponseToContain); + }; + + // These are ordered based on speed and reliability for ease of debugging + const frameworkTests: Record = { + astro: { + expectResponseToContain: "Hello, Astronaut!", + }, + hono: { + expectResponseToContain: "/api/hello", + }, + qwik: { + expectResponseToContain: "Welcome to Qwik", promptHandlers: [ { - matcher: /Do you want to use the next-on-pages eslint-plugin\?/, - input: ["y"], + matcher: /Yes looks good, finish update/, + input: [keys.enter], }, ], - }); - }); - - test("qwik", async () => { - await runCli("qwik", { + }, + remix: { + expectResponseToContain: "Welcome to Remix", + }, + next: { + expectResponseToContain: "Create Next App", promptHandlers: [ { - matcher: /Yes looks good, finish update/, - input: [keys.enter], + matcher: /Do you want to use the next-on-pages eslint-plugin\?/, + input: ["y"], }, ], - }); - }); - - test("solid", async () => { - await runCli("solid", { + }, + nuxt: { + expectResponseToContain: "Welcome to Nuxt!", + overrides: { + packageScripts: { + build: "NITRO_PRESET=cloudflare-pages nuxt build", + }, + }, + }, + react: { + expectResponseToContain: "React App", + }, + solid: { + expectResponseToContain: "Hello world", promptHandlers: [ { matcher: /Which template do you want to use/, @@ -133,11 +196,9 @@ describe("E2E: Web frameworks", () => { input: [keys.enter], }, ], - }); - }); - - test("svelte", async () => { - await runCli("svelte", { + }, + svelte: { + expectResponseToContain: "SvelteKit app", promptHandlers: [ { matcher: /Which Svelte app template/, @@ -152,19 +213,21 @@ describe("E2E: Web frameworks", () => { input: [keys.enter], }, ], - }); - }); + }, + vue: { + expectResponseToContain: "Vite App", + }, + }; + + test.concurrent.each(Object.keys(frameworkTests))( + "%s", + async (name) => { + await runCliWithDeploy(name); + }, + { retry: 3 } + ); - // This test blows up in CI due to Github providing an unusual git user email address. - // E.g. - // ``` - // fatal: empty ident name (for ) not allowed - // ``` test.skip("Hono (wrangler defaults)", async () => { await runCli("hono", { argv: ["--wrangler-defaults"] }); - - // verify that wrangler-defaults defaults to `true` for using git - expect(join(projectPath, ".git")).toExist(); }); }); diff --git a/packages/create-cloudflare/scripts/cleanupPagesProject.mjs b/packages/create-cloudflare/scripts/cleanupPagesProject.mjs new file mode 100644 index 000000000000..f6e7f07a3404 --- /dev/null +++ b/packages/create-cloudflare/scripts/cleanupPagesProject.mjs @@ -0,0 +1,47 @@ +import { execa } from "execa"; + +if (!process.env.CLOUDFLARE_API_TOKEN) { + console.error("CLOUDFLARE_API_TOKEN must be set"); + process.exit(1); +} + +if (!process.env.CLOUDFLARE_ACCOUNT_ID) { + console.error("CLOUDFLARE_ACCOUNT_ID must be set"); + process.exit(1); +} + +const npx = async (args) => { + const argv = args.split(" "); + return execa("npx", argv); +}; + +const listProjectsToDelete = async () => { + const toDelete = []; + + const { stdout } = await npx("wrangler pages project list"); + + for (const line of stdout.split("\n")) { + const c3ProjectRe = /(c3-e2e-[\w-]*)\s?│/; + const match = line.match(c3ProjectRe); + + if (match) { + toDelete.push(match[1]); + } + } + + return toDelete; +}; + +const deleteProjects = async (projects) => { + for (const project of projects) { + try { + console.log(`Deleting project: ${project}`); + await npx(`wrangler pages project delete -y ${project}`); + } catch (error) { + console.error(error); + } + } +}; + +const projects = await listProjectsToDelete(); +deleteProjects(projects); diff --git a/packages/create-cloudflare/src/common.ts b/packages/create-cloudflare/src/common.ts index e306a37159a7..6577419dc900 100644 --- a/packages/create-cloudflare/src/common.ts +++ b/packages/create-cloudflare/src/common.ts @@ -93,7 +93,7 @@ export const runDeploy = async (ctx: PagesGeneratorContext) => { const result = await runCommand(deployCmd, { silent: true, cwd: ctx.project.path, - env: { CLOUDFLARE_ACCOUNT_ID: ctx.account.id }, + env: { CLOUDFLARE_ACCOUNT_ID: ctx.account.id, NODE_ENV: "production" }, startText: `Deploying your application`, doneText: `${brandColor("deployed")} ${dim(`via \`${deployCmd}\``)}`, }); diff --git a/packages/create-cloudflare/src/frameworks/next/index.ts b/packages/create-cloudflare/src/frameworks/next/index.ts index 6e93819f32c7..503a6cd9e2a3 100644 --- a/packages/create-cloudflare/src/frameworks/next/index.ts +++ b/packages/create-cloudflare/src/frameworks/next/index.ts @@ -157,7 +157,7 @@ const config: FrameworkConfig = { "--src-dir", "--app", "--import-alias", - '"@/*"', + "@/*", ], compatibilityFlags: ["nodejs_compat"], }; diff --git a/packages/create-cloudflare/src/helpers/interactive.ts b/packages/create-cloudflare/src/helpers/interactive.ts index e106dbe0874b..eda0fdc55c1c 100644 --- a/packages/create-cloudflare/src/helpers/interactive.ts +++ b/packages/create-cloudflare/src/helpers/interactive.ts @@ -259,10 +259,10 @@ const getConfirmRenderers = (config: ConfirmPromptConfig) => { }; }; -export const spinner = () => { - const spinnerFrames = ["┤", "┘", "┴", "└", "├", "┌", "┬", "┐"]; - const ellipsisFrames = ["", ".", "..", "...", " ..", " .", ""]; +export const spinnerFrames = ["┤", "┘", "┴", "└", "├", "┌", "┬", "┐"]; +const ellipsisFrames = ["", ".", "..", "...", " ..", " .", ""]; +export const spinner = () => { // Alternative animations we considered. Keeping around in case we // introduce different animations for different use cases. // const frames = ["▁", "▃", "▄", "▅", "▆", "▇", "▆", "▅", "▄", "▃"]; diff --git a/packages/create-cloudflare/vitest-e2e.config.ts b/packages/create-cloudflare/vitest-e2e.config.ts index 419a30ef2503..deff0c668f7f 100644 --- a/packages/create-cloudflare/vitest-e2e.config.ts +++ b/packages/create-cloudflare/vitest-e2e.config.ts @@ -7,9 +7,8 @@ export default defineConfig({ include: ["e2e-tests/**/*.test.ts"], cache: false, root: ".", - singleThread: true, - threads: false, testTimeout: 1000 * 60 * 3, // 3 min for lengthy installs + maxConcurrency: 4, setupFiles: ["e2e-tests/setup.ts"], }, });