chore(gatsby): Move memory benchmark to generic benchmark, add suppor…

…t for develop and custom site (#35703)

* Move memory to generic benchmark, add support for develop

* Fix develop build

* Fix stats command

* Rename benchmark and minor updates

* Fix memory script

* Stop container at end again

benchmarks/memory/Dockerfile → benchmarks/docker-runner/Dockerfile

@@ -1,6 +1,5 @@
 FROM node:14-buster
 ARG jemalloc
-ENV NODE_ENV=production
 ENV CI=1
 ENV GATSBY_CPU_COUNT=4
 RUN apt-get update -y && apt-get upgrade -y && apt-get install git curl npm -y
@@ -9,7 +8,7 @@ RUN npm i -g gatsby-cli gatsby-dev-cli
 # set heap to 16gb just to catch all test cases
 ENV NODE_OPTIONS="--max-old-space-size=16368"
 
-RUN echo "\n\necho \"Welcome to the Gatsby Memory benchmark container!\\n  - /usr/src/gatsby : Your local gatsby repo\\n  - /usr/src/app : The memory benchmark gatsby site\\n\"" > /root/.bashrc
+RUN echo "\n\necho \"Welcome to the Gatsby benchmark container!\\n  - /usr/src/gatsby : Your local gatsby repo\\n  - /usr/src/app : The benchmark gatsby site\\n\"" > /root/.bashrc
 
 RUN if [ "$jemalloc" = "1" ]; then \
   echo "Using jemalloc for memory allocation" && \

benchmarks/memory/README.md → benchmarks/docker-runner/README.md

@@ -10,25 +10,28 @@ It has ports 9000 (for hosting gatsby) and 9229 (for debugging) exposed.
 Within the container, two points to your local filesystem are mounted:
 
 - /usr/src/gatsby : Your local gatsby repo
-- /usr/src/site : The memory benchmark gatsby site
+- /usr/src/app : The benchmark gatsby site
 
 If you'd like to configure `jemalloc` to run within the container, set the `JEMALLOC=1` env var when building the docker container.
 
 ## Commands
 
 ### Tests
 
-#### yarn test --memory X --num-nodes Y --node-size Z
+#### yarn test --memory X --num-nodes Y --node-size Z --command [build, develop]
 
 Runs a test build within a docker container with the given memory allotment.
 Within our gatsby-node, we'll create X nodes with a string property of size Y.
 
 Example: running a build with 1000 nodes of 1mb each, in a docker container with 8gb of memory.
 
 ```
-$ yarn test --memory 8g --num-nodes 500 --node-size 1m
+$ yarn test --memory 8g --num-nodes 500 --node-size 1m --command build
 ```
 
+You may optionally pass a `--site` flag as well to test a specific site at the given path. This will, however,
+not respond to the `num-nodes` or `node-size` arguments, as those are configured in the benchmark's gatsby-node.
+
 #### yarn test-suite --name some-name --suite [incremental|exhaustive]
 
 Runs through test suites defined in `scripts/test-suite.js` and outputs results to `output/some-name`.

benchmarks/memory/package.json → benchmarks/docker-runner/package.json

@@ -1,5 +1,5 @@
 {
-  "name": "memory-usage-benchmark",
+  "name": "gatsby-generic-benchmark",
   "private": true,
   "version": "1.0.0",
   "description": "Test site stress testing memory usage",
@@ -10,8 +10,8 @@
     "gatsby:develop": "NODE_ENV=development yarn gatsby develop -H 0.0.0.0 -p 9000",
     "gatsby:build:debug": "node --nolazy --inspect-brk=0.0.0.0:9229 node_modules/.bin/gatsby build",
     "gatsby:develop:debug": "NODE_ENV=development node --nolazy --inspect-brk=0.0.0.0:9229 node_modules/.bin/gatsby develop -H 0.0.0.0 -p 9000",
-    "docker:build": "docker build -t gatsby-memory . --build-arg jemalloc=$JEMALLOC",
-    "docker:remove": "docker image rm -f gatsby-memory",
+    "docker:build": "docker build -t gatsby-generic-benchmark . --build-arg jemalloc=$JEMALLOC",
+    "docker:remove": "docker image rm -f gatsby-generic-benchmark",
     "docker:rebuild": "yarn docker:stop; yarn docker:remove && yarn docker:build",
     "docker:start": "./scripts/docker-start",
     "docker:connect": "./scripts/docker-connect",
@@ -23,7 +23,7 @@
   },
   "repository": {
     "type": "git",
-    "url": "https://github.com/gatsbyjs/gatsby/tree/master/benchmarks/memory"
+    "url": "https://github.com/gatsbyjs/gatsby/tree/master/benchmarks/generic"
   },
   "bugs": {
     "url": "https://github.com/gatsbyjs/gatsby/issues"

benchmarks/memory/scripts/docker-connect → benchmarks/docker-runner/scripts/docker-connect

@@ -1,7 +1,7 @@
 DOCKER_ID=$(./scripts/docker-get-id)
 
 if [ -z "$DOCKER_ID" ]; then
-  echo "\nNo gatsby-memory is running. Start one with \`yarn docker:start\`.\n"
+  echo "\nNo gatsby-generic-benchmark is running. Start one with \`yarn docker:start\`.\n"
   return 1
 fi
 

benchmarks/memory/scripts/docker-get-id → benchmarks/docker-runner/scripts/docker-get-id

@@ -1,8 +1,8 @@
 DOCKER_ID=$(\
   docker ps --format '{{.Image}}:{{.ID}}' | \
-  grep "gatsby-memory" | \
+  grep "gatsby-generic-benchmark" | \
   head -n 1 | \
-  sed 's/gatsby\-memory://'\
+  sed 's/gatsby\-generic\-benchmark://'\
 )
 
 echo $DOCKER_ID

benchmarks/memory/scripts/docker-start → benchmarks/docker-runner/scripts/docker-start

@@ -1,25 +1,26 @@
 DOCKER_ID=$(./scripts/docker-get-id)
 if [ -n "$DOCKER_ID" ]; then
-  echo "\nA gatsby-memory container is already running with id $DOCKER_ID."
+  echo "\nA gatsby-generic-benchmark container is already running with id $DOCKER_ID."
   echo "Please use that container, or run \`yarn docker:stop\` to stop it.\n"
   return 1
 fi
 
 MEMORY_LIMIT="${DOCKER_MEMORY_LIMIT:-2g}"
+BENCHMARK_SITE="${DOCKER_BENCHMARK_SITE:-$(pwd)}"
 
 DOCKER_ID=$(\
   docker run -td \
   --mount type=bind,source="$(pwd)/../..",target=/usr/src/gatsby \
-  --mount type=bind,source="$(pwd)",target=/usr/src/app \
+  --mount type=bind,source="${BENCHMARK_SITE}",target=/usr/src/app \
   --publish 9229:9229 \
   --publish 9000:9000 \
   --memory="${MEMORY_LIMIT}" \
   --memory-swap="${MEMORY_LIMIT}" \
-  gatsby-memory \
+  gatsby-generic-benchmark \
   | head -c 12 \
 )
 
 sleep 1
-docker exec $DOCKER_ID bash -c "/usr/src/app/scripts/docker-write-memory &"
+docker exec $DOCKER_ID bash -c "/usr/src/gatsby/benchmarks/docker-runner/scripts/docker-write-memory &"
 
 echo "\nStarted container id ${DOCKER_ID} with ${MEMORY_LIMIT} of memory! Run \`yarn docker:connect\` to connect to the container.\n"

benchmarks/memory/scripts/docker-stats → benchmarks/docker-runner/scripts/docker-stats

@@ -2,7 +2,7 @@
 
 DOCKER_ID=$(./scripts/docker-get-id)
 if [ -z "$DOCKER_ID" ]; then
-  echo -e "\nNo gatsby-memory container was found. Run \`yarn docker:start\` to start one.\n"
+  echo -e "\nNo gatsby-generic-benchmark container was found. Run \`yarn docker:start\` to start one.\n"
   exit 1
 fi
 

benchmarks/memory/scripts/docker-stop → benchmarks/docker-runner/scripts/docker-stop

@@ -1,7 +1,7 @@
 DOCKER_ID=$(./scripts/docker-get-id)
 
 if [ -z "$DOCKER_ID" ]; then
-  echo "\nNo gatsby-memory is running.\n"
+  echo "\nNo gatsby-generic-benchmark is running.\n"
   return 1
 fi
 

benchmarks/docker-runner/scripts/test.js

@@ -0,0 +1,185 @@
+const yargs = require(`yargs`)
+const util = require('util')
+const { exit } = require('process')
+const exec = require('child_process').exec
+const execPromise = util.promisify(exec)
+
+const args = yargs
+  .option(`memory`, {
+    default: '2g',
+    describe: `The memory limit for the docker container`,
+  })
+  .option(`num-nodes`, {
+    default: '100',
+    describe: `The number of nodes to generate in the build`,
+  })
+  .option(`node-size`, {
+    default: '1k',
+    describe: `The rough size of each node generated in the build`,
+  })
+  .option(`command`, {
+    default: 'build',
+    choices: ['build', 'develop'],
+    describe: `The gatsby command to run for measurement`,
+  })
+  .option(`site`, {
+    default: process.cwd(),
+    describe: `The path to the site to use for testing`,
+  })
+  .option(`status`, {
+    default: 'cold',
+    choices: ['cold'],
+    describe: `The status of the build`,
+  }).parse()
+
+const hostExec = (cmd, promise = true) => {
+  console.log(`\x1b[2m$ ${cmd}\x1b[0m`)
+  return promise ? execPromise(cmd) : exec(cmd)
+}
+
+const getDockerExec = (dockerId) => {
+  return (cmd, env = {}, promise = true) => {
+    const envVars = Object.keys(env).map(key => `-e ${key}=${env[key]}`)
+    return hostExec(`docker exec ${envVars.join(' ')} ${dockerId} ${cmd}`, promise)
+  }
+}
+
+async function build(dockerExec) {
+  let code = 139
+  let start = 0
+
+  // there's something buggy with the node/exec/docker-exec integration
+  // we're getting seg faults, so this loop is just a patch for that
+  // so we don't have to fix it right now
+
+  while(code === 139) {
+    try {    
+      console.log(` - clearing cache`)
+      await dockerExec(`rm -rf .cache`)
+
+      console.log(` - running build with ${args.numNodes} nodes of size ${args.nodeSize}`)
+
+      start = Date.now()
+      const { err  } = await dockerExec(
+        `yarn gatsby build`, 
+        {
+          BUILD_NUM_NODES: args.numNodes, 
+          BUILD_STRING_NODE_SIZE: args.nodeSize, 
+          NUM_KEYS_IN_LARGE_SIZE_OBJ: 1,
+        }
+      )
+      code = err.code
+
+    } catch (e) {
+      code = e.code
+    }
+
+    if (code === 139) {
+      console.log(`     \x1b[31mSeg fault, trying again\x1b[0m`)
+    }
+  }
+
+  return {
+    code,
+    time: Math.round((Date.now() - start) / 10) / 100
+  }
+}
+
+async function develop(dockerExec) {
+  let code = undefined
+  let message = undefined
+  let start = 0
+
+  while(code === undefined) {
+    console.log(` - clearing cache`)
+    await dockerExec(`rm -rf .cache`)
+
+    console.log(` - running develop with ${args.numNodes} nodes of size ${args.nodeSize}`)
+
+    start = Date.now()
+    const process = dockerExec(
+      `yarn gatsby develop -H 0.0.0.0 -p 9000`, 
+      {
+        BUILD_NUM_NODES: args.numNodes, 
+        BUILD_STRING_NODE_SIZE: args.nodeSize, 
+        NUM_KEYS_IN_LARGE_SIZE_OBJ: 1
+      },
+      false
+    )
+
+    // wait until we see development bundle was successfully built, then exit
+    process.stdout.on('data', (data) => {
+      const line = data.toString()
+      if (line.indexOf("Building development bundle -") >= 0) {
+        message = line.trim()
+        code = line.indexOf("success") >= 0 ? 0 : 1
+        process.kill()
+      }
+    })
+
+    process.on('exit', (c) => {
+      if (!message) {
+        code = c
+      }
+    })
+
+    while(code === undefined) {
+      await new Promise(r => setTimeout(r, 100));
+    }
+
+    if (code === 139) {
+      console.log(`     \x1b[31mSeg fault, trying again\x1b[0m`)
+      code = undefined
+    }
+  }
+
+  console.log(`\nFinal development message: ${message}`)
+
+  return {
+    code,
+    time: Math.round((Date.now() - start) / 10) / 100
+  }
+}
+
+async function runTest() {
+
+  const { stdout: testDockerId } = await hostExec(`./scripts/docker-get-id`)
+  if (testDockerId.trim()) {
+    console.log(`Docker container ${testDockerId.trim()} is currently running, shutting it down.`)
+    await hostExec(`./scripts/docker-stop`);
+  }
+
+  console.log(`\nStarting container with ${args.memory} memory.`)
+  const { stderr: dockerStartError } = await hostExec(`DOCKER_MEMORY_LIMIT=${args.memory} DOCKER_BENCHMARK_SITE=${args.site} ./scripts/docker-start`)
+  if (dockerStartError) {
+    console.log("Encountered an error:")
+    console.log(dockerStartError)
+    exit(1)
+  }
+
+  let { stdout: dockerId } = await hostExec(`./scripts/docker-get-id`)
+  dockerId = dockerId.trim()
+
+  const dockerExec = getDockerExec(dockerId)
+
+  console.log(`\nStarting test using container ${dockerId}.`)
+
+  const command = args.command === 'build' ? build : develop
+  let {code, time} = await command(dockerExec)
+
+  console.log(`Finished test in ${time}s`)
+
+  if (code) {
+    console.log(`\nFailed with code ${code}\n`)
+  } else {
+    console.log(`\nSuccess!\n`)
+  }
+
+  console.log(`Stopping container ${dockerId}`)
+  await hostExec(`./scripts/docker-stop`)
+
+  exit(code)
+}
+
+
+runTest();