chore(tests): e2e improvements (#17310)

* rename ecosystem to e2e

* rename utils folder

* improve cli for starting tests

* add hook for buildkite

* remove comment

* simplify skip build feature

* fix renamed path

* use github action instead

* add setup phase

* make e2e tests verbose

* attempt rename _example to example

* display how many tests passed

* remove --build in favor of --skipBuild

* add a way to cleanup files that cannot be removed on linux

.buildkite/test/run.sh

@@ -43,7 +43,7 @@ pnpm run --filter "@prisma/client" test:functional
 # Run test for all packages
 pnpm run test
 
-# Client memory test suite
-# Note: we run it last as DB is not isolated and will be dropped after memory tests, which in turn will fail subsequent tests.
-# We should fix it in a similar way we did for functional tests, eventually.
+# Client memory test suite Note: we run it last as DB is not isolated and will
+# be dropped after memory tests, which in turn will fail subsequent tests.  We
+# should fix it in a similar way we did for functional tests, eventually.
 pnpm run --filter "@prisma/client" test:memory

.github/workflows/test.yml

@@ -197,6 +197,51 @@ jobs:
         run: pnpm run test:functional:code --relation-mode-tests-only relationMode/tests_m-to-n-MongoDB.ts
         working-directory: packages/client
 
+  #
+  # CLIENT test:e2e
+  #
+  client-e2e:
+    timeout-minutes: 15
+    runs-on: ${{ matrix.os }}
+
+    needs: detect_jobs_to_run
+    if: ${{ contains(needs.detect_jobs_to_run.outputs.jobs, '-all-') || contains(needs.detect_jobs_to_run.outputs.jobs, '-client-') }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest]
+        node: [16]
+
+    steps:
+      - uses: actions/checkout@v3
+
+      # Not currently needed, but we might need it in the future
+      # - run: docker-compose -f docker/docker-compose.yml up --detach postgres mysql mssql mongo cockroachdb vitess-8
+
+      - uses: pnpm/action-setup@v2.2.4
+        with:
+          version: 7
+
+      - uses: actions/setup-node@v3
+        with:
+          cache: 'pnpm'
+          node-version: ${{ matrix.node }}
+
+      - run: bash .github/workflows/scripts/setup.sh
+        env:
+          CI: true
+          SKIP_GIT: true
+          GITHUB_CONTEXT: ${{ toJson(github) }}
+
+      - run: pnpm run test:e2e --verbose
+        working-directory: packages/client
+        env:
+          CI: true
+          SKIP_GIT: true
+          GITHUB_CONTEXT: ${{ toJson(github) }}
+          NODE_OPTIONS: '--max-old-space-size=8096'
+
   #
   # CLIENT (without types test)
   #

helpers/compile/build.ts

@@ -145,10 +145,10 @@ async function dependencyCheck(options: BuildOptions) {
  * @param options
  */
 export async function build(options: BuildOptions[]) {
-  // When we trigger pnpm pack we actually don't want to build again to go faster.
-  // We re-use what has been already build; and also implies you ran pnpm run watch/dev/build before hand.
-  // In the future, I think I'll enable a full build + packaging mode to be much closer to reality, this could be done on CI.
-  if (process.env.ECOSYSTEM === 'true') return
+  // When we trigger pnpm pack for e2e tests we actually don't want to always
+  // build again to go faster. We re-use what has been already build; and also
+  // implies you ran pnpm run watch/dev/build before hand.
+  if (process.env.SKIP_BUILD === 'true') return
 
   void transduce.async(options, dependencyCheck)
 

packages/client/helpers/functional-test/run-tests.ts

@@ -3,7 +3,7 @@ import * as miniProxy from '@prisma/mini-proxy'
 import execa, { ExecaChildProcess } from 'execa'
 import fs from 'fs'
 
-import { setupQueryEngine } from '../../tests/commonUtils/setupQueryEngine'
+import { setupQueryEngine } from '../../tests/_utils/setupQueryEngine'
 import { Providers } from '../../tests/functional/_utils/providers'
 import { JestCli } from './JestCli'
 

packages/client/package.json

@@ -38,7 +38,7 @@
     "dev": "DEV=true node -r esbuild-register helpers/build.ts",
     "build": "node -r esbuild-register helpers/build.ts",
     "test": "jest --verbose",
-    "test:ecosystem": "node -r esbuild-register tests/ecosystem/_utils/run.ts",
+    "test:e2e": "node -r esbuild-register tests/e2e/_utils/run.ts",
     "test:functional": "node -r esbuild-register helpers/functional-test/run-tests.ts",
     "test:memory": "node -r esbuild-register helpers/memory-tests.ts",
     "test:functional:code": "node -r esbuild-register helpers/functional-test/run-tests.ts --no-types",

packages/client/tests/e2e/_utils/docker-compose-clean.yml

@@ -0,0 +1,15 @@
+version: '3.7'
+
+services:
+  # not an actual service, just a helper to clean up the workspace
+  clean:
+    image: alpine
+    command:
+      - /bin/sh
+      - -c
+      - |
+        rm -fr /e2e/.cache
+        rm -fr /e2e/*.tgz
+        find /e2e -name ".logs.*.txt" -type f -exec rm -fr {} +
+    volumes:
+      - ../:/e2e

packages/client/tests/ecosystem/_utils/docker-compose-common.yml → packages/client/tests/e2e/_utils/docker-compose-common.yml

@@ -2,10 +2,8 @@ services:
   test:
     build:
       dockerfile: ./standard.dockerfile
-      args:
-        - NAME=_example
     volumes:
-      - ../:/ecosystem
+      - ../:/e2e
       - ../../../:/client
       - ../.cache:/root/.cache
       - ../.cache/pnpmcache:/root/.local/share/pnpm/store/v3

packages/client/tests/ecosystem/_utils/docker-compose.yml → packages/client/tests/e2e/_utils/docker-compose.yml

@@ -1,9 +1,9 @@
 version: '3.7'
 
 services:
-  _example:
+  example:
     extends:
       file: docker-compose-common.yml
       service: test
     environment:
-      - NAME=_example
+      - NAME=example

packages/client/tests/ecosystem/_utils/run.ts → packages/client/tests/e2e/_utils/run.ts

@@ -1,11 +1,48 @@
+import { arg } from '@prisma/internals'
 import fs from 'fs/promises'
 import os from 'os'
 import path from 'path'
 import { $ } from 'zx'
 
+const args = arg(
+  process.argv.slice(2),
+  {
+    '--verbose': Boolean,
+    // do not fully build cli and client packages before packing
+    '--skipBuild': Boolean,
+    // a way to cleanup created files that also works on linux
+    '--clean': Boolean,
+  },
+  true,
+  true,
+)
+
 async function main() {
-  console.log('🎠 Preparing ecosystem tests')
-  // we first get all the paths we are going to need to run ecosystem
+  if (args instanceof Error) {
+    console.log(args.message)
+    process.exit(1)
+  }
+
+  args['--verbose'] = args['--verbose'] ?? false
+  args['--skipBuild'] = args['--skipBuild'] ?? false
+  args['--clean'] = args['--clean'] ?? false
+  $.verbose = args['--verbose']
+
+  if (args['--verbose'] === true) {
+    await $`docker -v`
+  }
+
+  if (args['--clean'] === true) {
+    console.log('🧹 Cleaning up created files')
+    await $`docker compose -f ${__dirname}/docker-compose-clean.yml down --remove-orphans`
+    await $`docker compose -f ${__dirname}/docker-compose-clean.yml build`
+    await $`docker compose -f ${__dirname}/docker-compose-clean.yml up`
+
+    return
+  }
+
+  console.log('🎠 Preparing e2e tests')
+  // we first get all the paths we are going to need to run e2e tests
   const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'prisma-build'))
   const cliPkgPath = path.join(__dirname, '..', '..', '..', '..', 'cli')
   const clientPkgPath = path.join(__dirname, '..', '..', '..', '..', 'client')
@@ -16,15 +53,15 @@ async function main() {
   const clientPkgJson = require(clientPkgJsonPath)
 
   // this process will need to modify some package.json, we save copies
-  await $`cd ${tmpDir} && cp ${cliPkgJsonPath} cli.package.json`.quiet()
-  await $`cd ${tmpDir} && cp ${clientPkgJsonPath} client.package.json`.quiet()
-  await $`cd ${tmpDir} && cp ${clientRuntimeDtsPath} client.runtime.d.ts`.quiet()
+  await $`cd ${tmpDir} && cp ${cliPkgJsonPath} cli.package.json`
+  await $`cd ${tmpDir} && cp ${clientPkgJsonPath} client.package.json`
+  await $`cd ${tmpDir} && cp ${clientRuntimeDtsPath} client.runtime.d.ts`
 
   // we provide a function that can revert modified package.json back
   const restoreOriginal = async () => {
-    await $`cd ${tmpDir} && cp cli.package.json ${cliPkgJsonPath}`.quiet()
-    await $`cd ${tmpDir} && cp client.package.json ${clientPkgJsonPath}`.quiet()
-    await $`cd ${tmpDir} && cp client.runtime.d.ts ${clientRuntimeDtsPath}`.quiet()
+    await $`cd ${tmpDir} && cp cli.package.json ${cliPkgJsonPath}`
+    await $`cd ${tmpDir} && cp client.package.json ${clientPkgJsonPath}`
+    await $`cd ${tmpDir} && cp client.runtime.d.ts ${clientRuntimeDtsPath}`
   }
 
   // if process is killed by hand, ensure that package.json is restored
@@ -37,15 +74,16 @@ async function main() {
   // write the modified package.json to overwrite the original package.json
   await fs.writeFile(cliPkgJsonPath, JSON.stringify(cliPkgJson, null, 2))
   await fs.writeFile(clientPkgJsonPath, JSON.stringify(clientPkgJson, null, 2))
-  // this is to avoid bundling types and locally link directly to the sources
-  await fs.writeFile(clientRuntimeDtsPath, `export * from '/client/src/runtime/index'`)
-  // TODO: have a fast mode (this one) and a full mode with type bundling
+
+  if (args['--skipBuild'] === true) {
+    // this is to avoid bundling types and locally link directly to the sources
+    await fs.writeFile(clientRuntimeDtsPath, `export * from '/client/src/runtime/index'`)
+  }
 
   try {
     console.log('📦 Packing package tarballs')
-    // pack package's tarballs while actually skip the compilation via ECOSYSTEM
-    await $`cd ${cliPkgPath} && ECOSYSTEM=true pnpm pack --pack-destination ${__dirname}/../`.quiet()
-    await $`cd ${clientPkgPath} && ECOSYSTEM=true pnpm pack --pack-destination ${__dirname}/../`.quiet()
+    await $`cd ${clientPkgPath} && SKIP_BUILD=${args['--skipBuild']} pnpm pack --pack-destination ${__dirname}/../`
+    await $`cd ${cliPkgPath} && SKIP_BUILD=${args['--skipBuild']} pnpm pack --pack-destination ${__dirname}/../`
   } catch (e) {
     console.log(e.message)
     console.log('🛑 Failed to pack one or more of the packages')
@@ -55,19 +93,25 @@ async function main() {
   }
 
   console.log('🐳 Starting tests in docker')
-  // tarball was created, ready to send it to docker and begin ecosystem tests
-  await $`docker compose -f ${__dirname}/docker-compose.yml build`.quiet()
-  await $`docker compose -f ${__dirname}/docker-compose.yml down`.quiet()
-  await $`docker compose -f ${__dirname}/docker-compose.yml up`.quiet()
+  // tarball was created, ready to send it to docker and begin e2e tests
+  const testNames = args._.join(' ')
+  await $`docker compose -f ${__dirname}/docker-compose.yml down --remove-orphans`
+  await $`docker compose -f ${__dirname}/docker-compose.yml build ${testNames}`
+  await $`docker compose -f ${__dirname}/docker-compose.yml up ${testNames}`
 
   // let the tests run and gather a list of logs for containers that have failed
-  const findErrors = await $`find "$(pwd)" -not -name .logs.0.txt -name .logs.*.txt`.quiet()
+  const findErrors = await $`find "$(pwd)" -not -name ".logs.0.txt" -name ".logs.*.txt"`
+  const findSuccess = await $`find "$(pwd)" -name ".logs.0.txt"`
   const errors = findErrors.stdout.split('\n').filter((v) => v.length > 0)
+  const success = findSuccess.stdout.split('\n').filter((v) => v.length > 0)
   if (errors.length > 0) {
     console.log(`🛑 ${errors.length} tests failed with`, errors)
   } else {
-    console.log(`✅ All tests passed`)
+    console.log(`✅ All ${success.length} tests passed`)
   }
 }
 
-void main()
+void main().catch((e) => {
+  console.log(e)
+  process.exit(1)
+})

packages/client/tests/e2e/_utils/standard.cmd.sh

@@ -0,0 +1,16 @@
+#!/bin/sh
+
+(
+  rm -fr /e2e/$NAME/.logs.* && \
+  cp -r /e2e/_utils /_utils && \
+  cp -r /e2e/$NAME /test && \
+  cp /e2e/prisma-0.0.0.tgz prisma-0.0.0.tgz && \
+  cp /e2e/prisma-client-0.0.0.tgz prisma-client-0.0.0.tgz && \
+  cp /e2e/jest.config.js /test/jest.config.js && \
+  cd /test && rm -fr node_modules && \
+  export NODE_PATH="$(npm root --quiet -g)" && \
+  node -r 'esbuild-register' _steps.ts \
+) > /$NAME.logs.txt 2>&1 ; \
+EXIT_CODE=$? && \
+mv /$NAME.logs.txt /e2e/$NAME/.logs.$EXIT_CODE.txt && \
+exit $EXIT_CODE

packages/client/tests/ecosystem/_utils/standard.dockerfile → packages/client/tests/e2e/_utils/standard.dockerfile

@@ -1,4 +1,4 @@
 FROM node:16
 
 RUN npm i -g pnpm zx typescript jest ts-node esbuild esbuild-register @swc/jest
-CMD chmod +x ./ecosystem/_utils/standard.cmd.sh && ./ecosystem/_utils/standard.cmd.sh
+CMD chmod +x ./e2e/_utils/standard.cmd.sh && ./e2e/_utils/standard.cmd.sh

packages/client/tests/ecosystem/_example/_steps.ts → packages/client/tests/e2e/example/_steps.ts

@@ -16,5 +16,5 @@ void executeSteps({
   finish: async () => {
     await $`echo "done"`
   },
-  // keep: true,
+  // keep: true, // keep docker open to debug it
 })

packages/client/tests/functional/_utils/globalSetup.js

@@ -1,7 +1,7 @@
 'use strict'
 const glob = require('globby')
 const fs = require('fs-extra')
-const { setupQueryEngine } = require('../../commonUtils/setupQueryEngine')
+const { setupQueryEngine } = require('../../_utils/setupQueryEngine')
 
 module.exports = async () => {
   await setupQueryEngine()

packages/client/tests/memory/_utils/runAllMemoryTests.ts

@@ -3,7 +3,7 @@ import { existsSync } from 'fs'
 import fs from 'fs/promises'
 import path from 'path'
 
-import { setupQueryEngine } from '../../commonUtils/setupQueryEngine'
+import { setupQueryEngine } from '../../_utils/setupQueryEngine'
 import { generateMemoryUsageReport } from './generateMemoryUsageReport'
 import { MemoryTestDir } from './MemoryTestDir'
 import { runMemoryTest, TestResult } from './runMemoryTest'