add workflow to run NoSQLBench performance tests (#279)

stargate · Mar 22, 2023 · 3ecbfd6 · 3ecbfd6
1 parent e84193b
commit 3ecbfd6
Show file tree

Hide file tree

Showing 14 changed files with 222 additions and 56 deletions.
diff --git a/.github/workflows/performance-testing.yaml b/.github/workflows/performance-testing.yaml
@@ -5,3 +5,98 @@ name: Run Performance Tests
 # * manual trigger
 on:
   workflow_dispatch:
+    inputs:
+      docscount:
+        description: 'Number of documents to use for each test'
+        required: true
+        type: string
+        default: '20000'
+
+# global env vars, available in all jobs and steps
+env:
+  MAVEN_OPTS: '-Xmx4g'
+
+jobs:
+  # runs performance tests
+  performance-tests:
+    name: Performance tests
+    runs-on: ubuntu-latest
+
+    strategy:
+
+      # run all tests to completion
+      fail-fast: false
+
+      # matrix props:
+      matrix:
+        type: [ java, native ]
+
+        test: [ http-jsonapi-crud-basic, http-jsonapi-keyvalue ]
+
+        include:
+          - type: java
+            build-ops: '-Dquarkus.container-image.build=true'
+            docker-flags: ''
+
+          - type: native
+            build-ops: '-Pnative -Dquarkus.native.container-build=true -Dquarkus.container-image.build=true'
+            docker-flags: '-n'
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up JDK 17
+        uses: actions/setup-java@v3
+        with:
+          distribution: 'temurin'
+          java-version: '17'
+          cache: maven
+
+      # login to ECR to we can pull coord image from there
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v2
+        with:
+          aws-access-key-id: ${{ secrets.ECR_ACCESS_KEY }}
+          aws-secret-access-key: ${{ secrets.ECR_SECRET_KEY }}
+          aws-region: us-east-1
+
+      - name: Login to Amazon ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@v1
+
+      # Install NB and required library
+      # See: https://github.com/AppImage/AppImageKit/wiki/FUSE
+      # Print version to affirm correct install
+      - name: Download NoSQLBench
+        run: |
+          sudo add-apt-repository universe
+          sudo apt install libfuse2
+          curl -L -O https://github.com/nosqlbench/nosqlbench/releases/latest/download/nb5
+          chmod +x nb5
+          ./nb5 --version
+
+      - name: Build JSON API Docker image
+        run: |
+          ./mvnw clean package -DskipTests ${{ matrix.build-ops }} 
+
+      # Use docker-compose to start required stack
+      # Print version of images used for verification
+      - name: Start Docker Compose
+        run: |
+          cd docker-compose
+          ./start_dse_68_dev_mode.sh ${{ matrix.docker-flags }}
+          docker-compose images
+
+      # Run test after first obtaining an auth token
+      - name: Run NoSQLBench Test
+        run: |
+          cd nosqlbench
+          export AUTH_TOKEN=$(curl -s -X POST 'http://localhost:8081/v1/auth' -H 'Content-Type: application/json' --data-raw '{"username": "cassandra", "password": "cassandra"}' | jq -r '.authToken')
+          ../nb5 -v --report-csv-to logs --log-histograms logs/hdr_metrics.log ${{ matrix.test }} jsonapi_host=localhost auth_token=$AUTH_TOKEN docscount=${{ inputs.docscount }}
+
+      - name: Save Results
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ matrix.test }}-${{ matrix.type }}
+          path: nosqlbench/logs
+
diff --git a/.gitignore b/.gitignore
@@ -39,3 +39,6 @@ nb-configuration.xml
 # Local environment
 .env
 !/docker-compose/.env
+
+# NoSQLBench
+logs/
diff --git a/README.md b/README.md
@@ -210,7 +210,7 @@ You can create a Docker image named `io.stargate/jsonapi` using:
 
 Or, if you want to create a native-runnable Docker image named `io.stargate/jsonapi-native` using:
 ```shell script
-./mvnw clean package -Pnative -Dquarkus.native.container-build=true
+./mvnw clean package -Pnative -Dquarkus.native.container-build=true -Dquarkus.container-image.build=true
 ```
 
 If you want to learn more about building container images, please consult [Container images](https://quarkus.io/guides/container-image).

diff --git a/docker-compose/.env b/docker-compose/.env
@@ -4,3 +4,4 @@ LOGLEVEL=INFO
 REQUESTLOG=false
 SGTAG=v2
 JSONTAG=v1
+JSONIMAGE=stargateio/jsonapi
diff --git a/docker-compose/README.md b/docker-compose/README.md
@@ -50,6 +50,8 @@ Both convenience scripts support the following options:
 
 * You can specify an image tag (version) of the JSON API using `-t [VERSION]`. 
 
+* The scripts default to using the Java-based image for JSON API, you can specify to use the native GraalVM based variant using `-n`.
+
 * You can change the default root log level using `-r [LEVEL]` (default `INFO`). Valid values: `ERROR`, `WARN`, `INFO`, `DEBUG`
 
 * You can enable reguest logging using `-q`: if so, each request is logged under category `io.quarkus.http.access-log`

diff --git a/docker-compose/docker-compose-dev-mode.yml b/docker-compose/docker-compose-dev-mode.yml
@@ -8,8 +8,6 @@ services:
     ports:
       - "9042:9042"
       - "8081:8081"
-      - "8084:8084"
-      - "8090:8090"
     mem_limit: 2G
     environment:
       - JAVA_OPTS="-Xmx2G"
@@ -27,7 +25,7 @@ services:
       retries: 10
 
   jsonapi:
-    image: stargateio/jsonapi:${JSONTAG}
+    image: ${JSONIMAGE}:${JSONTAG}
     depends_on:
       coordinator:
         condition: service_healthy

diff --git a/docker-compose/docker-compose.yml b/docker-compose/docker-compose.yml
@@ -72,8 +72,6 @@ services:
     ports:
       - "9042:9042"
       - "8081:8081"
-      - "8084:8084"
-      - "8090:8090"
     mem_limit: 2G
     environment:
       - JAVA_OPTS="-Xmx2G"
@@ -91,7 +89,7 @@ services:
       retries: 10
 
   jsonapi:
-    image: stargateio/jsonapi:${JSONTAG}
+    image: ${JSONIMAGE}:${JSONTAG}
     depends_on:
       coordinator:
         condition: service_healthy

diff --git a/docker-compose/start_dse_68.sh b/docker-compose/start_dse_68.sh
@@ -7,9 +7,16 @@ LOGLEVEL=INFO
 DSETAG="$(../mvnw -f .. help:evaluate -Dexpression=stargate.int-test.cassandra.image-tag -q -DforceStdout)"
 SGTAG="$(../mvnw -f .. help:evaluate -Dexpression=stargate.int-test.coordinator.image-tag -q -DforceStdout)"
 JSONTAG="v$(../mvnw -f .. help:evaluate -Dexpression=project.version -q -DforceStdout)"
+JSONIMAGE="stargateio/jsonapi"
 
-while getopts "qr:t:" opt; do
+while getopts "qnr:t:j:" opt; do
   case $opt in
+    j)
+      JSONTAG=$OPTARG
+      ;;
+    n)
+      JSONIMAGE="stargateio/jsonapi-native"
+      ;;
     q)
       REQUESTLOG=true
       ;;
@@ -21,7 +28,9 @@ while getopts "qr:t:" opt; do
       ;;
     \?)
       echo "Valid options:"
-      echo "  -t <tag> - use Docker images tagged with specified Stargate version (will pull images from Docker Hub if needed)"
+      echo "  -j <tag> - use JSON API Docker image tagged with specified JSON API version (will pull images from Docker Hub if needed)"
+      echo "  -n <tag> - use JSON API native image instead of default Java-based image"
+      echo "  -t <tag> - use Stargate coordinator Docker image tagged with specified  version (will pull images from Docker Hub if needed)"
       echo "  -q - enable request logging for APIs in 'io.quarkus.http.access-log' (default: disabled)"
       echo "  -r - specify root log level for APIs (defaults to INFO); usually DEBUG, WARN or ERROR"
       exit 1
@@ -34,7 +43,8 @@ export REQUESTLOG
 export DSETAG
 export SGTAG
 export JSONTAG
+export JSONIMAGE
 
-echo "Running with DSE $DSETAG, Stargate $SGTAG, JSON API $JSONTAG"
+echo "Running with DSE $DSETAG, Stargate $SGTAG, JSON API $JSONIMAGE:$JSONTAG"
 
 docker-compose up -d
diff --git a/docker-compose/start_dse_68_dev_mode.sh b/docker-compose/start_dse_68_dev_mode.sh
@@ -7,21 +7,30 @@ LOGLEVEL=INFO
 DSETAG="$(../mvnw -f .. help:evaluate -Dexpression=stargate.int-test.cassandra.image-tag -q -DforceStdout)"
 SGTAG="$(../mvnw -f .. help:evaluate -Dexpression=stargate.int-test.coordinator.image-tag -q -DforceStdout)"
 JSONTAG="v$(../mvnw -f .. help:evaluate -Dexpression=project.version -q -DforceStdout)"
+JSONIMAGE="stargateio/jsonapi"
 
-while getopts "qr:t:" opt; do
+while getopts "qnr:t:j:" opt; do
   case $opt in
+    j)
+      JSONTAG=$OPTARG
+      ;;
+    n)
+      JSONIMAGE="stargateio/jsonapi-native"
+      ;;
     q)
       REQUESTLOG=true
       ;;
     r)
       LOGLEVEL=$OPTARG
       ;;
     t)
-      SGTAG=$OPTARG
+      JSONTAG=$OPTARG
       ;;
     \?)
       echo "Valid options:"
-      echo "  -t <tag> - use Docker images tagged with specified Stargate version (will pull images from Docker Hub if needed)"
+      echo "  -j <tag> - use JSON API Docker image tagged with specified JSON API version (will pull images from Docker Hub if needed)"
+      echo "  -n <tag> - use JSON API native image instead of default Java-based image"
+      echo "  -t <tag> - use Stargate coordinator Docker image tagged with specified  version (will pull images from Docker Hub if needed)"
       echo "  -q - enable request logging for APIs in 'io.quarkus.http.access-log' (default: disabled)"
       echo "  -r - specify root log level for APIs (defaults to INFO); usually DEBUG, WARN or ERROR"
       exit 1
@@ -34,7 +43,8 @@ export REQUESTLOG
 export DSETAG
 export SGTAG
 export JSONTAG
+export JSONIMAGE
 
-echo "Running with DSE $DSETAG, Stargate $SGTAG, JSON API $JSONTAG"
+echo "Running with DSE $DSETAG, Stargate $SGTAG, JSON API $JSONIMAGE:$JSONTAG"
 
 docker-compose -f docker-compose-dev-mode.yml up -d
diff --git a/nosqlbench/http-jsonapi-crud-basic.yaml b/nosqlbench/http-jsonapi-crud-basic.yaml
@@ -49,6 +49,7 @@ blocks:
         X-Cassandra-Request-Id: "{request_id}"
         X-Cassandra-Token: "<<auth_token:my_auth_token>>"
         Content-Type: "application/json"
+        ok-body: ".*\"ok\":1.*"
         body: |
           {
             "createNamespace": {
@@ -63,6 +64,7 @@ blocks:
         X-Cassandra-Request-Id: "{request_id}"
         X-Cassandra-Token: "<<auth_token:my_auth_token>>"
         Content-Type: "application/json"
+        ok-body: ".*\"ok\":1.*"
         body: |
           {
             "deleteCollection": {
@@ -77,6 +79,7 @@ blocks:
         X-Cassandra-Request-Id: "{request_id}"
         X-Cassandra-Token: "<<auth_token:my_auth_token>>"
         Content-Type: "application/json"
+        ok-body: ".*\"ok\":1.*"
         body: |
           {
             "createCollection": {
@@ -94,6 +97,7 @@ blocks:
         X-Cassandra-Request-Id: "{request_id}"
         X-Cassandra-Token: "<<auth_token:my_auth_token>>"
         Content-Type: "application/json"
+        ok-body: ".*\"insertedIds\":\"{seq_key}\".*"
         body: |
           {
             "insertOne" : {
@@ -133,6 +137,7 @@ blocks:
         X-Cassandra-Request-Id: "{request_id}"
         X-Cassandra-Token: "<<auth_token:my_auth_token>>"
         Content-Type: "application/json"
+        ok-body: ".*\"data\".*"
         body: |
           {
             "findOne" : {
@@ -144,7 +149,7 @@ blocks:
 
   update:
     ops:
-      # aka findOneAndUpdate
+      # aka updateOne
       # for parity with other tests this only uses set, not unset, no return value
       update-document:
         method: POST
@@ -153,11 +158,12 @@ blocks:
         X-Cassandra-Request-Id: "{request_id}"
         X-Cassandra-Token: "<<auth_token:my_auth_token>>"
         Content-Type: "application/json"
+        ok-body: ".*\"modifiedCount\":1.*"
         body: |
           {
-            "findOneAndUpdate" : {
+            "updateOne" : {
               "filter": {
-                "_id" : "{random_key}
+                "_id" : "{random_key}"
               },
               "update": {
                 "$set": {
@@ -196,9 +202,10 @@ blocks:
         X-Cassandra-Request-Id: "{request_id}"
         X-Cassandra-Token: "<<auth_token:my_auth_token>>"
         Content-Type: "application/json"
+        ok-body: ".*\"deletedCount\":[0,1].*"
         body: |
           {
-            "findOne" : {
+            "deleteOne" : {
               "filter" : {
                 "_id" : "{random_key}"
               }

diff --git a/nosqlbench/http-jsonapi-crud-dataset.yaml b/nosqlbench/http-jsonapi-crud-dataset.yaml
@@ -27,7 +27,6 @@ bindings:
   random_key: Uniform(0,<<docscount:10000000>>); ToString() -> String
 
 blocks:
-  # TODO schema creation not yet addressed by JSON API Spec
   schema:
     ops:
       create-namespace:
@@ -37,6 +36,7 @@ blocks:
         X-Cassandra-Request-Id: "{request_id}"
         X-Cassandra-Token: "<<auth_token:my_auth_token>>"
         Content-Type: "application/json"
+        ok-body: ".*\"ok\":1.*"
         body: |
           {
             "createNamespace": {
@@ -51,6 +51,7 @@ blocks:
         X-Cassandra-Request-Id: "{request_id}"
         X-Cassandra-Token: "<<auth_token:my_auth_token>>"
         Content-Type: "application/json"
+        ok-body: ".*\"ok\":1.*"
         body: |
           {
             "deleteCollection": {
@@ -65,6 +66,7 @@ blocks:
         X-Cassandra-Request-Id: "{request_id}"
         X-Cassandra-Token: "<<auth_token:my_auth_token>>"
         Content-Type: "application/json"
+        ok-body: ".*\"ok\":1.*"
         body: |
           {
             "createCollection": {
@@ -84,6 +86,7 @@ blocks:
         X-Cassandra-Request-Id: "{request_id}"
         X-Cassandra-Token: "<<auth_token:my_auth_token>>"
         Content-Type: "application/json"
+        ok-body: ".*\"insertedIds\":\"{seq_key}\".*"
         body: "{document_json}"
 
       read-document:
@@ -93,7 +96,7 @@ blocks:
         X-Cassandra-Request-Id: "{request_id}"
         X-Cassandra-Token: "<<auth_token:my_auth_token>>"
         Content-Type: "application/json"
-        ok-status: "[2-4][0-9][0-9]"
+        ok-body: ".*\"data\".*"
 
       update-document:
         method: PUT
@@ -111,4 +114,3 @@ blocks:
         X-Cassandra-Request-Id: "{request_id}"
         X-Cassandra-Token: "<<auth_token:my_auth_token>>"
         Content-Type: "application/json"
-        ok-status: "[2-4][0-9][0-9]"