Bot test driver script revamp, squash

.github/workflows/bottest.yml

@@ -0,0 +1,135 @@
+on:
+  push:
+    branches:
+      - master
+      - jgilles/bottest2
+
+  workflow_dispatch:
+    inputs:
+      pr_number:
+        description: 'Pull Request Number'
+        required: false
+        default: ''
+
+  issue_comment:
+    types: [created]
+
+name: Benchmarks
+
+env:
+  GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  GH_REPO: ${{ github.repository }}
+
+jobs:
+  benchmark:
+    name: run 30-bot bot test
+    runs-on: bots-runner
+    # filter for a comment containing 'benchmarks please'
+    if: ${{ github.event_name != 'issue_comment' || (github.event.issue.pull_request && contains(github.event.comment.body, 'bots please')) }}
+    env:
+      PR_NUMBER: ${{ github.event.inputs.pr_number || github.event.issue.number || null }}
+    steps:
+      - name: Enable CPU boost
+        run: echo "1" | sudo tee /sys/devices/system/cpu/cpufreq/boost
+
+      - name: Check membership
+        if: ${{ github.event_name == 'issue_comment' }}
+        env:
+          CONTRIB_ORG: clockworklabs
+          COMMENT_AUTHOR: ${{ github.event.comment.user.login }}
+          ORG_READ_TOKEN: ${{ secrets.ORG_READ_TOKEN }}
+        run: |
+          curl -OL https://github.com/cli/cli/releases/download/v2.37.0/gh_2.37.0_linux_amd64.deb && sudo dpkg -i gh_2.37.0_linux_amd64.deb
+          if [[ $(GH_TOKEN=$ORG_READ_TOKEN gh api --paginate /orgs/{owner}/members --jq 'any(.login == env.COMMENT_AUTHOR)') != true ]]; then
+            gh pr comment $PR_NUMBER -b "Sorry, you don't have permission to run benchmarks."
+            exit 1
+          fi
+
+      - name: Post initial comment
+        run: |
+          if [[ $PR_NUMBER ]]; then
+            comment_parent=issues/$PR_NUMBER
+            comment_update=issues/comments
+          else
+            comment_parent=commits/$GITHUB_SHA
+            comment_update=comments
+          fi
+          comment_body="Bot test in progress..."
+          comment_id=$(gh api "/repos/{owner}/{repo}/$comment_parent/comments" -f body="$comment_body" --jq .id)
+          echo "COMMENT_UPDATE_URL=/repos/{owner}/{repo}/$comment_update/$comment_id" >>$GITHUB_ENV
+
+      - name: find PR branch
+        if: ${{ env.PR_NUMBER }}
+        run: echo "PR_REF=$(gh pr view $PR_NUMBER --json headRefName --jq .headRefName)" >>"$GITHUB_ENV"
+
+      - name: Checkout sources
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ env.PR_REF || github.ref }}
+          # if we're on master we want to know what the sha of HEAD~1 is so
+          # that we can compare results from it to HEAD (in the "Fetch markdown
+          # summary PR" step). otherwise, we can use a fully shallow checkout
+          fetch-depth: ${{ env.PR_NUMBER && 1 || 2 }}
+
+      - name: Install stable toolchain
+        uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          components: clippy
+          toolchain: stable
+          target: wasm32-unknown-unknown
+          override: true
+
+      - name: Build
+        run: |
+          cargo build --release --bin spacetime
+
+      - name: Install latest wasm-opt for module optimisations
+        run: |
+          curl https://github.com/WebAssembly/binaryen/releases/download/version_116/binaryen-version_116-x86_64-linux.tar.gz -L | sudo tar xz -C /usr/local --strip-components=1
+
+      - name: Disable CPU boost
+        run: echo "0" | sudo tee /sys/devices/system/cpu/cpufreq/boost
+
+      - name: Run bot test script
+        run: |
+          echo "We're on a bots runner, so expecting to find bots and tracy in their expected locations"
+          export BOTS_DIR="$HOME/bots"
+          export TRACY_CAPTURE_BIN="$HOME/tracy/capture/build/unix/capture-release"
+          crates/bench/bottest.sh
+          mkdir bottest-results
+          cp crates/bench/bottest.sh bottest-results/$RESULTS_NAME.json
+
+      # this will work for both PR and master
+      - name: Upload bot test results to DO spaces
+        uses: shallwefootball/s3-upload-action@master
+        with:
+          aws_key_id: ${{ secrets.AWS_KEY_ID }}
+          aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY}}
+          aws_bucket: "spacetimedb-ci-benchmarks"
+          source_dir: criterion-results
+          endpoint: https://nyc3.digitaloceanspaces.com
+          destination_dir: bottests
+
+      - name: Post comment
+        run: |
+          BODY="<details><summary>Criterion benchmark results</summary>
+
+          $(cat report.md)
+
+          </details>"
+
+          gh api "$COMMENT_UPDATE_URL" -X PATCH -f body="$BODY"
+
+      - name: Post failure comment
+        if: ${{ failure() && env.COMMENT_UPDATE_URL }}
+        run: |
+          BODY="Benchmarking failed. Please check [the workflow run](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details."
+          gh api "$COMMENT_UPDATE_URL" -X PATCH -f body="$BODY"
+
+      - name: Clean up
+        if: always()
+        run: |
+          rm -fr /stdb/*
+          rm -fr bottest-results/
+          rm -fr crates/bench/bottest/

crates/bench/.gitignore

@@ -1,2 +1,3 @@
 .spacetime/
 target/
+bottest/

crates/bench/bottest.sh

@@ -0,0 +1,112 @@
+#!/bin/bash
+
+# move into project root, keep paths stable
+cd "$(dirname "$0")/../.."
+
+# color constants
+RED='\033[0;31m'
+NC='\033[0m'
+
+function assert_dir {
+    if [ ! -d "$1" ]; then
+        echo -e "${RED}$1 does not exist${NC}"
+        exit 1
+    fi
+}
+function assert_file {
+    if [ ! -f "$1" ]; then
+        echo -e "${RED}$1 does not exist${NC}"
+        exit 1
+    fi
+}
+function cleanup {
+    kill $SPACETIME_PROCESS
+    kill $TRACY_PROCESS
+    git checkout -- .
+}
+
+BOTS_DIR="${BOTS_DIR:-$HOME/bots}"
+TRACY_CAPTURE_BIN="${TRACY_CAPTURE_BIN:-$HOME/tracy/capture/build/unix/capture-release}"
+OUT_DIR="$PWD/crates/bench/bottest"
+
+echo "Expecting to find an unzipped bots directory at $BOTS_DIR (set BOTS_DIR env var to change)"
+echo "(Ask John for a copy of the bots directory if you need one)"
+assert_dir "$BOTS_DIR"
+assert_file "$BOTS_DIR/bitcraft_spacetimedb_with_wasm_opt.wasm"
+assert_file "$BOTS_DIR/bitcraft-bots.tar"
+assert_file "$BOTS_DIR/deploy_world.py"
+assert_file "$BOTS_DIR/docker-compose.yml"
+echo "Expecting to find tracy capture executable $TRACY_CAPTURE_BIN (set TRACY_CAPTURE_BIN env var to change) (tracy should be on git tag v0.10)"
+assert_file "$TRACY_CAPTURE_BIN"
+if [ -z "$(git status --porcelain)" ]; then 
+    # Working directory clean
+    echo "Git is clean, can apply patch"
+else 
+    echo -e "${RED}Git has changes, cannot apply bot test patch. Please commit or stash your uncommitted changes${NC}"
+    exit 1
+fi
+
+set -exo pipefail
+
+echo ------- PRELIMINARIES -------
+
+docker --version
+docker-compose --version
+rustc --version
+
+# run cleanup when script terminates
+trap cleanup EXIT
+
+rm -rf "$OUT_DIR"
+mkdir -p "$OUT_DIR/.spacetime"
+
+# load bot docker image	
+docker load --input "$BOTS_DIR/bitcraft-bots.tar"
+
+git apply << EOF
+diff --git a/Cargo.toml b/Cargo.toml
+index 7caba831..8863989b 100644
+--- a/Cargo.toml
++++ b/Cargo.toml
+@@ -210,7 +210,7 @@ tokio-tungstenite = { version = "0.21", features = ["native-tls"] }
+ tokio-util = { version = "0.7.4", features = ["time"] }
+ toml = "0.8"
+ tower-http = { version = "0.5", features = ["cors"] }
+-tracing = { version = "0.1.37", features = ["release_max_level_off"] }
++tracing = { version = "0.1.37" } #, features = ["release_max_level_off"] }
+ tracing-appender = "0.2.2"
+ tracing-core = "0.1.31"
+ tracing-flame = "0.2.0"
+EOF
+
+cargo build --bin spacetime --release
+
+echo ------- PREPARING WORLD -------
+
+target/release/spacetime start --listen-addr 0.0.0.0:3000 "$OUT_DIR/.spacetime" >"$OUT_DIR/spacetime.log" 2>&1 &
+SPACETIME_PROCESS=$!
+sleep 5
+
+target/release/spacetime publish -c bitcraft --wasm-file "$BOTS_DIR/bitcraft_spacetimedb_with_wasm_opt.wasm"
+
+python3 "$BOTS_DIR/deploy_world.py" -H http://127.0.0.1:3000 -m bitcraft -f "$BOTS_DIR/Spacetime128x128.snapshot"
+
+echo ------- RUNNING BOTS AND COLLECTING TRACE -------
+
+kill "$SPACETIME_PROCESS"
+
+target/release/spacetime start --listen-addr 0.0.0.0:3000 --enable-tracy "$OUT_DIR/.spacetime" >>"$OUT_DIR/spacetime.log" 2>&1 &
+SPACETIME_PROCESS=$!
+sleep 5
+
+target/release/spacetime call bitcraft nonexistent_reducer || echo "Call failed, as expected, but bitcraft module should be loaded"
+
+$TRACY_CAPTURE_BIN -a ::1 -o "$OUT_DIR/output.tracy" >"$OUT_DIR/tracy-capture.log" 2>&1 &
+TRACY_PROCESS=$!
+
+REPLICAS=30 docker-compose -f "$BOTS_DIR/docker-compose.yml" up &
+sleep 60
+
+docker-compose -f "$BOTS_DIR/docker-compose.yml" down
+
+zip "$OUT_DIR/bottest.zip" "$OUT_DIR/spacetime.log" "$OUT_DIR/tracy-capture.log" "$OUT_DIR/output.tracy" -r