diff --git a/.github/workflows/bottest.yml b/.github/workflows/bottest.yml
new file mode 100644
index 0000000000..a698ba5ef7
--- /dev/null
+++ b/.github/workflows/bottest.yml
@@ -0,0 +1,125 @@
+on:
+ push:
+ branches:
+ - master
+ - jgilles/bottest2
+
+ workflow_dispatch:
+ inputs:
+ pr_number:
+ description: 'Pull Request Number'
+ required: false
+ default: ''
+
+ issue_comment:
+ types: [created]
+
+name: Benchmarks
+
+env:
+ GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ GH_REPO: ${{ github.repository }}
+
+jobs:
+ benchmark:
+ name: run 30-bot bot test
+ runs-on: bots-runner
+ # filter for a comment containing 'benchmarks please'
+ if: ${{ github.event_name != 'issue_comment' || (github.event.issue.pull_request && contains(github.event.comment.body, 'bots please')) }}
+ env:
+ PR_NUMBER: ${{ github.event.inputs.pr_number || github.event.issue.number || null }}
+ steps:
+ - name: Check membership
+ if: ${{ github.event_name == 'issue_comment' }}
+ env:
+ CONTRIB_ORG: clockworklabs
+ COMMENT_AUTHOR: ${{ github.event.comment.user.login }}
+ ORG_READ_TOKEN: ${{ secrets.ORG_READ_TOKEN }}
+ run: |
+ curl -OL https://github.com/cli/cli/releases/download/v2.37.0/gh_2.37.0_linux_amd64.deb && sudo dpkg -i gh_2.37.0_linux_amd64.deb
+ if [[ $(GH_TOKEN=$ORG_READ_TOKEN gh api --paginate /orgs/{owner}/members --jq 'any(.login == env.COMMENT_AUTHOR)') != true ]]; then
+ gh pr comment $PR_NUMBER -b "Sorry, you don't have permission to run benchmarks."
+ exit 1
+ fi
+
+ - name: Checkout sources
+ uses: actions/checkout@v4
+
+ - name: Post initial comment
+ run: |
+ if [[ $PR_NUMBER ]]; then
+ comment_parent=issues/$PR_NUMBER
+ comment_update=issues/comments
+ else
+ comment_parent=commits/$GITHUB_SHA
+ comment_update=comments
+ fi
+ comment_body="Bot test in progress..."
+ comment_id=$(gh api "/repos/{owner}/{repo}/$comment_parent/comments" -f body="$comment_body" --jq .id)
+ echo "COMMENT_UPDATE_URL=/repos/{owner}/{repo}/$comment_update/$comment_id" >>$GITHUB_ENV
+
+ - name: find PR branch
+ if: ${{ env.PR_NUMBER }}
+ run: echo "PR_REF=$(gh pr view $PR_NUMBER --json headRefName --jq .headRefName)" >>"$GITHUB_ENV"
+
+ - name: Checkout sources
+ uses: actions/checkout@v4
+ with:
+ ref: ${{ env.PR_REF || github.ref }}
+ # if we're on master we want to know what the sha of HEAD~1 is so
+ # that we can compare results from it to HEAD (in the "Fetch markdown
+ # summary PR" step). otherwise, we can use a fully shallow checkout
+ fetch-depth: ${{ env.PR_NUMBER && 1 || 2 }}
+
+ - name: Install stable toolchain
+ uses: actions-rs/toolchain@v1
+ with:
+ profile: minimal
+ components: clippy
+ toolchain: stable
+ target: wasm32-unknown-unknown
+ override: true
+
+ - name: Run bot test script
+ run: |
+ echo "We're on a bots runner, so expecting to find bots and tracy in their expected locations"
+ export BOTS_DIR="$HOME/bots"
+ export TRACY_CAPTURE_BIN="$HOME/tracy/capture/build/unix/capture-release"
+ crates/bench/bottest.sh
+ mkdir bottest-results
+ cp crates/bench/bottest/bottest.zip bottest-results/$RESULTS_NAME.zip
+
+ # this will work for both PR and master
+ - name: Upload bot test results to DO spaces
+ uses: shallwefootball/s3-upload-action@master
+ with:
+ aws_key_id: ${{ secrets.AWS_KEY_ID }}
+ aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY}}
+ aws_bucket: "spacetimedb-ci-benchmarks"
+ source_dir: bottest-results
+ endpoint: https://nyc3.digitaloceanspaces.com
+ destination_dir: bottests
+
+ - name: Post comment
+ run: |
+ BODY="Bot test results
+
+ Click [here](https://spacetimedb-ci-benchmarks.nyc3.digitaloceanspaces.com/bottests/$RESULTS_NAME.zip) to download a ZIP file with the tracy trace
+ from this test.
+
+ "
+
+ gh api "$COMMENT_UPDATE_URL" -X PATCH -f body="$BODY"
+
+ - name: Post failure comment
+ if: ${{ failure() && env.COMMENT_UPDATE_URL }}
+ run: |
+ BODY="Bot test failed. Please check [the workflow run](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details."
+ gh api "$COMMENT_UPDATE_URL" -X PATCH -f body="$BODY"
+
+ - name: Clean up
+ if: always()
+ run: |
+ rm -fr /stdb/*
+ rm -fr bottest-results/
+ rm -fr crates/bench/bottest/
\ No newline at end of file
diff --git a/crates/bench/.gitignore b/crates/bench/.gitignore
index 43e5543187..61af247942 100644
--- a/crates/bench/.gitignore
+++ b/crates/bench/.gitignore
@@ -1,2 +1,3 @@
.spacetime/
target/
+bottest/
diff --git a/crates/bench/bottest.sh b/crates/bench/bottest.sh
new file mode 100755
index 0000000000..27c23dc10e
--- /dev/null
+++ b/crates/bench/bottest.sh
@@ -0,0 +1,128 @@
+#!/bin/bash
+
+# script to run a bot test. This is its own script for 2 reasons:
+# 1. you can run it off CI
+# 2. github actions can't run things in parallel anyway, so we need a shell script to run
+# the spacetime server, tracy capture, and the bots in parallel.
+
+# move into project root, keep paths stable
+cd "$(dirname "$0")/../.."
+
+# color constants
+RED='\033[0;31m'
+NC='\033[0m'
+
+function assert_dir {
+ if [ ! -d "$1" ]; then
+ echo -e "${RED}$1 does not exist${NC}"
+ exit 1
+ fi
+}
+function assert_file {
+ if [ ! -f "$1" ]; then
+ echo -e "${RED}$1 does not exist${NC}"
+ exit 1
+ fi
+}
+# we need to ensure everything dies in case of an early exit to avoid polluting
+# the bot test machine with processes
+function cleanup {
+ kill $SPACETIME_PROCESS || echo "already dead"
+ kill $TRACY_PROCESS || echo "already dead"
+ git checkout -- .
+}
+
+BOTS_DIR="${BOTS_DIR:-$HOME/bots}"
+TRACY_CAPTURE_BIN="${TRACY_CAPTURE_BIN:-$HOME/tracy/capture/build/unix/capture-release}"
+OUT_DIR="$PWD/crates/bench/bottest"
+
+echo "Expecting to find an unzipped bots directory at $BOTS_DIR (set BOTS_DIR env var to change)"
+echo "(Ask John for a copy of the bots directory if you need one)"
+assert_dir "$BOTS_DIR"
+assert_file "$BOTS_DIR/bitcraft_spacetimedb_with_wasm_opt.wasm"
+assert_file "$BOTS_DIR/bitcraft-bots.tar"
+assert_file "$BOTS_DIR/deploy_world.py"
+assert_file "$BOTS_DIR/docker-compose.yml"
+echo "Expecting to find tracy capture executable $TRACY_CAPTURE_BIN (set TRACY_CAPTURE_BIN env var to change) (tracy should be on git tag v0.10)"
+assert_file "$TRACY_CAPTURE_BIN"
+if [ -z "$(git status --porcelain)" ]; then
+ # Working directory clean
+ echo "Git is clean, can apply patch"
+else
+ echo -e "${RED}Git has changes, cannot apply bot test patch. Please commit or stash your uncommitted changes${NC}"
+ exit 1
+fi
+
+set -exo pipefail
+
+echo ------- PRELIMINARIES -------
+
+docker --version
+docker-compose --version
+rustc --version
+
+# run cleanup when script terminates
+trap cleanup EXIT
+
+rm -rf "$OUT_DIR"
+mkdir -p "$OUT_DIR/.spacetime"
+
+# load bot docker image
+docker load --input "$BOTS_DIR/bitcraft-bots.tar"
+
+git apply << EOF
+diff --git a/Cargo.toml b/Cargo.toml
+index 7caba831..8863989b 100644
+--- a/Cargo.toml
++++ b/Cargo.toml
+@@ -210,7 +210,7 @@ tokio-tungstenite = { version = "0.21", features = ["native-tls"] }
+ tokio-util = { version = "0.7.4", features = ["time"] }
+ toml = "0.8"
+ tower-http = { version = "0.5", features = ["cors"] }
+-tracing = { version = "0.1.37", features = ["release_max_level_off"] }
++tracing = { version = "0.1.37" } #, features = ["release_max_level_off"] }
+ tracing-appender = "0.2.2"
+ tracing-core = "0.1.31"
+ tracing-flame = "0.2.0"
+EOF
+
+cargo build --bin spacetime --release
+
+echo ------- PREPARING WORLD -------
+
+target/release/spacetime start --listen-addr 0.0.0.0:3000 "$OUT_DIR/.spacetime" >"$OUT_DIR/spacetime_deploy_world.log" 2>&1 &
+SPACETIME_PROCESS=$!
+sleep 5
+
+target/release/spacetime publish -c bitcraft --wasm-file "$BOTS_DIR/bitcraft_spacetimedb_with_wasm_opt.wasm"
+
+python3 "$BOTS_DIR/deploy_world.py" -H http://127.0.0.1:3000 -m bitcraft -f "$BOTS_DIR/Spacetime128x128.snapshot" >"$OUT_DIR/deploy_world.log" 2>&1
+
+echo ------- RUNNING BOTS AND COLLECTING TRACE -------
+
+kill -INT "$SPACETIME_PROCESS"
+sleep 5
+
+target/release/spacetime start --listen-addr 0.0.0.0:3000 --enable-tracy "$OUT_DIR/.spacetime" >"$OUT_DIR/spacetime_bots.log" 2>&1 &
+SPACETIME_PROCESS=$!
+sleep 5
+
+target/release/spacetime call bitcraft nonexistent_reducer || echo "Call failed, as expected, but bitcraft module should be loaded"
+
+$TRACY_CAPTURE_BIN -a ::1 -o "$OUT_DIR/output.tracy" >"$OUT_DIR/tracy-capture.log" 2>&1 &
+TRACY_PROCESS=$!
+
+REPLICAS=30 docker-compose -f "$BOTS_DIR/docker-compose.yml" up -d
+echo "Letting bots run around a while"
+sleep 300
+
+docker-compose -f "$BOTS_DIR/docker-compose.yml" down
+
+kill -INT "$SPACETIME_PROCESS" || echo "missing spacetime process?"
+kill -INT "$TRACY_PROCESS" || echo "missing tracy process?"
+sleep 30
+
+ls -al "$OUT_DIR"
+cat "$OUT_DIR/spacetime_bots.log"
+
+zip -r "$OUT_DIR/bottest.zip" "$OUT_DIR/*.log" "$OUT_DIR/output.tracy"
diff --git a/crates/standalone/src/control_db.rs b/crates/standalone/src/control_db.rs
index 44837b98f5..58f89b52b8 100644
--- a/crates/standalone/src/control_db.rs
+++ b/crates/standalone/src/control_db.rs
@@ -25,9 +25,9 @@ pub type Result = core::result::Result;
#[derive(thiserror::Error, Debug)]
pub enum Error {
- #[error("collection not found")]
+ #[error("collection not found: {0}")]
CollectionNotFound(sled::Error),
- #[error("database error")]
+ #[error("database error: {0}")]
Database(sled::Error),
#[error("record with the name {0} already exists")]
RecordAlreadyExists(DomainName),