separate nosqlbench execution in spaces to have more connections (#282)

Co-authored-by: Jeffrey Carpenter <jeffrey.carpenter@datastax.com>

.github/workflows/performance-testing.yaml

@@ -10,7 +10,12 @@ on:
         description: 'Number of documents to use for each test'
         required: true
         type: string
-        default: '20000'
+        default: '1000000'
+      connections:
+        description: 'Number of HTTP connections to use for each test'
+        required: true
+        type: string
+        default: '20'
 
 # global env vars, available in all jobs and steps
 env:
@@ -92,10 +97,11 @@ jobs:
         run: |
           cd nosqlbench
           export AUTH_TOKEN=$(curl -s -X POST 'http://localhost:8081/v1/auth' -H 'Content-Type: application/json' --data-raw '{"username": "cassandra", "password": "cassandra"}' | jq -r '.authToken')
-          ../nb5 -v --report-csv-to logs --log-histograms logs/hdr_metrics.log ${{ matrix.test }} jsonapi_host=localhost auth_token=$AUTH_TOKEN docscount=${{ inputs.docscount }}
+          ../nb5 -v --report-csv-to logs --log-histograms logs/hdr_metrics.log ${{ matrix.test }} jsonapi_host=localhost auth_token=$AUTH_TOKEN docscount=${{ inputs.docscount }} connections=${{ inputs.connections }}
 
       - name: Save Results
         uses: actions/upload-artifact@v3
+        if: ${{ always() }}
         with:
           name: ${{ matrix.test }}-${{ matrix.type }}
           path: nosqlbench/logs

docker-compose/start_dse_68.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 
 # Default to INFO as root log level
 LOGLEVEL=INFO
@@ -47,4 +47,11 @@ export JSONIMAGE
 
 echo "Running with DSE $DSETAG, Stargate $SGTAG, JSON API $JSONIMAGE:$JSONTAG"
 
-docker-compose up -d
+COMPOSE_ARGS=("-d")
+
+# only use --wait flag if Docker Compose is v2
+if [[ $(docker-compose version) =~ "v2" ]]; then
+   COMPOSE_ARGS+=("--wait")
+fi
+
+docker-compose up "${COMPOSE_ARGS[@]}"

docker-compose/start_dse_68_dev_mode.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 
 # Default to INFO as root log level
 LOGLEVEL=INFO
@@ -47,4 +47,11 @@ export JSONIMAGE
 
 echo "Running with DSE $DSETAG, Stargate $SGTAG, JSON API $JSONIMAGE:$JSONTAG"
 
-docker-compose -f docker-compose-dev-mode.yml up -d
+COMPOSE_ARGS=("-d")
+
+# only use --wait flag if Docker Compose is v2
+if [[ $(docker-compose version) =~ "v2" ]]; then
+   COMPOSE_ARGS+=("--wait")
+fi
+
+docker-compose -f docker-compose-dev-mode.yml up "${COMPOSE_ARGS[@]}"

nosqlbench/http-jsonapi-crud-basic.md

@@ -58,6 +58,7 @@ Note that error handling is set to `errors=timer,warn`, which means that in case
 ## Workload Parameters
 
 - `docscount` - the number of documents to process in each step of a scenario (default: `10_000_000`)
+- `connections` - number of HTTP2 connections to be shared between the threads (default: `20`) 
 
 Note that if number of documents is higher than `read-cycles` you would experience misses, which will result in `HTTP 404` and smaller latencies.
 

nosqlbench/http-jsonapi-crud-basic.yaml

@@ -22,6 +22,10 @@ bindings:
   #   multiple hosts: jsonapi_host=host1,host2,host3
   #   multiple weighted hosts: jsonapi_host=host1:3,host2:7
   weighted_hosts: WeightedStrings('<<jsonapi_host:jsonapi>>')
+
+  # spread into different spaces to use multiple connections
+  space: HashRange(1,<<connections:20>>); ToString();
+
   # http request id
   request_id: ToHashedUUID(); ToString();
 
@@ -91,6 +95,7 @@ blocks:
     ops:
       # aka insertOne
       write-document:
+        space: "{space}"
         method: POST
         uri: <<protocol:http>>://{weighted_hosts}:<<jsonapi_port:8080>><<path_prefix:>>/v1/<<namespace:jsonapi_crud_basic>>/<<collection:docs_collection>>
         Accept: "application/json"
@@ -131,6 +136,7 @@ blocks:
     ops:
       # aka findOne with _id as filter
       read-document:
+        space: "{space}"
         method: POST
         uri: <<protocol:http>>://{weighted_hosts}:<<jsonapi_port:8080>><<path_prefix:>>/v1/<<namespace:jsonapi_crud_basic>>/<<collection:docs_collection>>
         Accept: "application/json"
@@ -152,12 +158,16 @@ blocks:
       # aka updateOne
       # for parity with other tests this only uses set, not unset, no return value
       update-document:
+        space: "{space}"
         method: POST
         uri: <<protocol:http>>://{weighted_hosts}:<<jsonapi_port:8080>><<path_prefix:>>/v1/<<namespace:jsonapi_crud_basic>>/<<collection:docs_collection>>
         Accept: "application/json"
         X-Cassandra-Request-Id: "{request_id}"
         X-Cassandra-Token: "<<auth_token:my_auth_token>>"
         Content-Type: "application/json"
+        # because this is not an upsert, modified count could technically be 0 or 1,
+        # but since we are fixing the _ids to be sequential over the docscount range during the write phase,
+        # every update during this phase should update exactly 1 document.
         ok-body: ".*\"modifiedCount\":1.*"
         body: |
           {
@@ -196,6 +206,7 @@ blocks:
   delete:
     ops:
       delete-document:
+        space: "{space}"
         method: POST
         uri: <<protocol:http>>://{weighted_hosts}:<<jsonapi_port:8080>><<path_prefix:>>/v1/<<namespace:jsonapi_crud_basic>>/<<collection:docs_collection>>
         Accept: "application/json"

nosqlbench/http-jsonapi-crud-dataset.md

@@ -39,3 +39,4 @@ Above command creates a dataset with 5.000 latest unconfirmed transactions.
 
 - `docscount` - the number of documents to process in each step of a scenario (default: `10_000_000`)
 - `dataset_file` - the file to read the JSON documents from (note that if number of documents in a file is smaller than the `docscount` parameter, the documents will be reused)
+- `connections` - number of HTTP2 connections to be shared between the threads (default: `20`) 

nosqlbench/http-jsonapi-crud-dataset.yaml

@@ -20,6 +20,10 @@ bindings:
   #   multiple hosts: jsonapi_host=host1,host2,host3
   #   multiple weighted hosts: jsonapi_host=host1:3,host2:7
   weighted_hosts: WeightedStrings('<<jsonapi_host:jsonapi>>')
+
+  # spread into different spaces to use multiple connections
+  space: HashRange(1,<<connections:20>>); ToString();
+
   # http request id
   request_id: ToHashedUUID(); ToString();
 
@@ -80,6 +84,7 @@ blocks:
     ops:
       # TODO expect the document have the ID in it?
       write-document:
+        space: "{space}"
         method: POST
         uri: <<protocol:http>>://{weighted_hosts}:<<jsonapi_port:8180>><<path_prefix:>>/v1/<<namespace:jsonapi_crud_basic>>/<<collection:docs_collection>>
         Accept: "application/json"
@@ -90,6 +95,7 @@ blocks:
         body: "{document_json}"
 
       read-document:
+        space: "{space}"
         method: GET
         uri: <<protocol:http>>://{weighted_hosts}:<<jsonapi_port:8080>><<path_prefix:>>/v2/namespaces/<<namespace:jsonapi_crud_dataset>>/collections/<<table:docs_collection>>/{random_key}
         Accept: "application/json"
@@ -99,6 +105,7 @@ blocks:
         ok-body: ".*\"data\".*"
 
       update-document:
+        space: "{space}"
         method: PUT
         uri: <<protocol:http>>://{weighted_hosts}:<<jsonapi_port:8080>><<path_prefix:>>/v2/namespaces/<<namespace:jsonapi_crud_dataset>>/collections/<<table:docs_collection>>/{random_key}
         Accept: "application/json"
@@ -108,6 +115,7 @@ blocks:
         body: "{document_json}"
 
       delete-document:
+        space: "{space}"
         method: DELETE
         uri: <<protocol:http>>://{weighted_hosts}:<<jsonapi_port:8080>><<path_prefix:>>/v2/namespaces/<<namespace:jsonapi_crud_dataset>>/collections/<<table:docs_collection>>/{seq_key}
         Accept: "application/json"

nosqlbench/http-jsonapi-keyvalue.yaml

@@ -21,6 +21,10 @@ bindings:
   #   multiple hosts: jsonapi_host=host1,host2,host3
   #   multiple weighted hosts: jsonapi_host=host1:3,host2:7
   weighted_hosts: WeightedStrings('<<jsonapi_host:jsonapi>>')
+
+  # spread into different spaces to use multiple connections
+  space: HashRange(1,<<connections:20>>); ToString();
+
   # http request id
   request_id: ToHashedUUID(); ToString();
 
@@ -80,6 +84,7 @@ blocks:
   rampup:
     ops:
       rampup-insert:
+        space: "{space}"
         method: POST
         uri: <<protocol:http>>://{weighted_hosts}:<<jsonapi_port:8080>><<path_prefix:>>/v1/<<namespace:jsonapi_keyvalue>>/<<collection:docs_collection>>
         Accept: "application/json"
@@ -102,6 +107,7 @@ blocks:
       ratio: <<read_ratio:5>>
     ops:
       main-select:
+        space: "{space}"
         method: POST
         uri: <<protocol:http>>://{weighted_hosts}:<<jsonapi_port:8080>><<path_prefix:>>/v1/<<namespace:jsonapi_keyvalue>>/<<collection:docs_collection>>
         Accept: "application/json"
@@ -119,13 +125,15 @@ blocks:
           }
           
       main-write:
+        space: "{space}"
         method: POST
         uri: <<protocol:http>>://{weighted_hosts}:<<jsonapi_port:8080>><<path_prefix:>>/v1/<<namespace:jsonapi_keyvalue>>/<<collection:docs_collection>>
         Accept: "application/json"
         X-Cassandra-Request-Id: "{request_id}"
         X-Cassandra-Token: "<<auth_token:my_auth_token>>"
         Content-Type: "application/json"
-        ok-body: ".*\"modifiedCount\":1.*"
+        # because this is not an upsert, modified count could be 0 or 1
+        ok-body: ".*\"modifiedCount\":[0,1].*"
         body: |          
           {
             "updateOne" : {

nosqlbench/http-jsonapi-search-advanced.md

@@ -61,5 +61,6 @@ The advanced search workload can test the following `where` clauses:
 - `docpadding` - the number of fields to add to each document; useful for writing larger documents. A value of e.g. `5` would make each document have 20 leaf values, instead of 15. (default: `0`)
 - `match-ratio` - a value between 0 and 1 detailing what ratio of the documents written should match the search parameters. If match-ratio is e.g. `0.1` then approximately one-tenth of the documents will have `match1`, `match2`, and `match3` values that are `0`, `"true"`, and `true`, respectively. (default: `0.01`)
 - `fields` - the URL-encoded value for `fields` that you would send to the Docs API. This restricts the fields returned during benchmarking.
+- `connections` - number of HTTP2 connections to be shared between the threads (default: `20`) 
 
 

nosqlbench/http-jsonapi-search-advanced.yaml

@@ -18,8 +18,8 @@ scenarios:
   default:
     schema:             run driver=http tags==phase:schema threads==<<threads:1>> cycles==UNDEF
     rampup:
-      write:            run driver=http tags==name:"rampup-put.*" cycles===TEMPLATE(rampup-cycles, TEMPLATE(docscount,10000000)) docpadding=TEMPLATE(docpadding,0) match-ratio=TEMPLATE(match-ratio,0.01) threads=<<threads:auto>> errors=timer,warn
-      read:             run driver=http tags==phase:"rampup-get.*" cycles===TEMPLATE(rampup-cycles, TEMPLATE(docscount,10000000)) page-size=TEMPLATE(page-size,3) fields=TEMPLATE(fields,%5b%5d) threads=<<threads:auto>> errors=timer,warn
+      write:            run driver=http tags==name:"rampup-put.*" cycles===TEMPLATE(rampup-cycles,TEMPLATE(docscount,10000000)) docpadding=TEMPLATE(docpadding,0) match-ratio=TEMPLATE(match-ratio,0.01) threads=<<threads:auto>> errors=timer,warn
+      read:             run driver=http tags==phase:"rampup-get.*" cycles===TEMPLATE(rampup-cycles,TEMPLATE(docscount,10000000)) page-size=TEMPLATE(page-size,3) fields=TEMPLATE(fields,%5b%5d) threads=<<threads:auto>> errors=timer,warn
     main:
       all:               run driver=http tags==block:main cycles===TEMPLATE(read-cycles,TEMPLATE(docscount,10000000)) page-size=TEMPLATE(page-size,3) fields=TEMPLATE(fields,%5b%5d) threads=<<threads:auto>> errors=timer,warn
       get-in:            run driver=http tags==name:main-get-in,filter:in cycles===TEMPLATE(read-cycles,TEMPLATE(docscount,10000000)) page-size=TEMPLATE(page-size,3) fields=TEMPLATE(fields,%5b%5d) threads=<<threads:auto>> errors=timer,warn
@@ -37,6 +37,10 @@ bindings:
   #   multiple hosts: jsonapi_host=host1,host2,host3
   #   multiple weighted hosts: jsonapi_host=host1:3,host2:7
   weighted_hosts: WeightedStrings('<<jsonapi_host:jsonapi>>')
+
+  # spread into different spaces to use multiple connections
+  space: HashRange(1,<<connections:20>>); ToString();
+
   # http request id
   request_id: ToHashedUUID(); ToString();
 
@@ -110,6 +114,7 @@ blocks:
   rampup:
     ops:
       rampup-put:
+        space: "{space}"
         method: POST
         uri: <<protocol:http>>://{weighted_hosts}:<<jsonapi_port:8080>><<path_prefix:>>/v1/<<namespace:jsonapi_search_advanced>>/<<collection:docs_collection>>
         Accept: "application/json"
@@ -152,6 +157,7 @@ blocks:
 
       # where={"match1":{"$in":[0]}}
       rampup-get-in:
+        space: "{space}"
         method: POST
         uri: <<protocol:http>>://{weighted_hosts}:<<jsonapi_port:8080>><<path_prefix:>>/v1/<<namespace:jsonapi_search_advanced>>/<<collection:docs_collection>>
         Accept: "application/json"
@@ -175,6 +181,7 @@ blocks:
 
       # where={"match2":{"$nin":["false"]}}
       rampup-get-not-in:
+        space: "{space}"
         method: POST
         uri: <<protocol:http>>://{weighted_hosts}:<<jsonapi_port:8080>><<path_prefix:>>/v1/<<namespace:jsonapi_search_advanced>>/<<collection:docs_collection>>
         Accept: "application/json"
@@ -198,6 +205,7 @@ blocks:
 
       # where={"match2":{"$eq":"true"},"match3":{"$ne": false}}
       rampup-get-mem-and:
+        space: "{space}"
         method: POST
         uri: <<protocol:http>>://{weighted_hosts}:<<jsonapi_port:8080>><<path_prefix:>>/v1/<<namespace:jsonapi_search_advanced>>/<<collection:docs_collection>>
         Accept: "application/json"
@@ -222,8 +230,9 @@ blocks:
             }
           }
 
-      rampup-get-mem-or:
       # where={"$or":[{"match1":{"$lt":1}},{"match3":{"$exists":true}}]}
+      rampup-get-mem-or:
+        space: "{space}"
         method: POST
         uri: <<protocol:http>>://{weighted_hosts}:<<jsonapi_port:8080>><<path_prefix:>>/v1/<<namespace:jsonapi_search_advanced>>/<<collection:docs_collection>>
         Accept: "application/json"
@@ -256,6 +265,7 @@ blocks:
 
       # where={"$and":[{"match1":{"$eq":0}},{"$or":[{"match2":{"$eq":"true"}},{"match3":{"$eq":false}}]}]}
       rampup-get-complex1:
+        space: "{space}"
         method: POST
         uri: <<protocol:http>>://{weighted_hosts}:<<jsonapi_port:8080>><<path_prefix:>>/v1/<<namespace:jsonapi_search_advanced>>/<<collection:docs_collection>>
         Accept: "application/json"
@@ -295,8 +305,9 @@ blocks:
             }
           }
 
-      rampup-get-complex2:
       # where={"$and":[{"$or":[{"match1":{"$lte":0}},{"match2":{"$eq":"false"}}]},{"$or":[{"match2":{"$eq":"false"}},{"match3":{"$eq":true}}]}]}
+      rampup-get-complex2:
+        space: "{space}"
         method: POST
         uri: <<protocol:http>>://{weighted_hosts}:<<jsonapi_port:8080>><<path_prefix:>>/v1/<<namespace:jsonapi_search_advanced>>/<<collection:docs_collection>>
         Accept: "application/json"
@@ -347,6 +358,7 @@ blocks:
 
       # where={"$or":[{"$and":[{"match1":{"$lte":0}},{"match2":{"$eq":"true"}}]},{"$and":[{"match2":{"$eq":"false"}},{"match3":{"$eq":true}}]}]}
       rampup-get-complex3:
+        space: "{space}"
         method: POST
         uri: <<protocol:http>>://{weighted_hosts}:<<jsonapi_port:8080>><<path_prefix:>>/v1/<<namespace:jsonapi_search_advanced>>/<<collection:docs_collection>>
         Accept: "application/json"
@@ -399,6 +411,7 @@ blocks:
     ops:
       # where={"match1":{"$in":[0]}}
       main-get-in:
+        space: "{space}"
         method: POST
         uri: <<protocol:http>>://{weighted_hosts}:<<jsonapi_port:8080>><<path_prefix:>>/v1/<<namespace:jsonapi_search_advanced>>/<<collection:docs_collection>>
         Accept: "application/json"
@@ -422,6 +435,7 @@ blocks:
 
       # where={"match2":{"$nin":["false"]}}
       main-get-not-in:
+        space: "{space}"
         method: POST
         uri: <<protocol:http>>://{weighted_hosts}:<<jsonapi_port:8080>><<path_prefix:>>/v1/<<namespace:jsonapi_search_advanced>>/<<collection:docs_collection>>
         Accept: "application/json"
@@ -445,6 +459,7 @@ blocks:
 
       # where={"match2":{"$eq":"true"},"match3":{"$ne": false}}
       main-get-mem-and:
+        space: "{space}"
         method: POST
         uri: <<protocol:http>>://{weighted_hosts}:<<jsonapi_port:8080>><<path_prefix:>>/v1/<<namespace:jsonapi_search_advanced>>/<<collection:docs_collection>>
         Accept: "application/json"
@@ -471,6 +486,7 @@ blocks:
 
       # where={"$or":[{"match1":{"$lt":1}},{"match3":{"$exists":true}}]}
       main-get-mem-or:
+        space: "{space}"
         method: POST
         uri: <<protocol:http>>://{weighted_hosts}:<<jsonapi_port:8080>><<path_prefix:>>/v1/<<namespace:jsonapi_search_advanced>>/<<collection:docs_collection>>
         Accept: "application/json"
@@ -503,6 +519,7 @@ blocks:
 
       # where={"$and":[{"match1":{"$eq":0}},{"$or":[{"match2":{"$eq":"true"}},{"match3":{"$eq":false}}]}]}
       main-get-complex1:
+        space: "{space}"
         method: POST
         uri: <<protocol:http>>://{weighted_hosts}:<<jsonapi_port:8080>><<path_prefix:>>/v1/<<namespace:jsonapi_search_advanced>>/<<collection:docs_collection>>
         Accept: "application/json"
@@ -544,6 +561,7 @@ blocks:
 
       # where={"$and":[{"$or":[{"match1":{"$lte":0}},{"match2":{"$eq":"false"}}]},{"$or":[{"match2":{"$eq":"false"}},{"match3":{"$eq":true}}]}]}
       main-get-complex2:
+        space: "{space}"
         method: POST
         uri: <<protocol:http>>://{weighted_hosts}:<<jsonapi_port:8080>><<path_prefix:>>/v1/<<namespace:jsonapi_search_advanced>>/<<collection:docs_collection>>
         Accept: "application/json"
@@ -594,6 +612,7 @@ blocks:
 
       # where={"$or":[{"$and":[{"match1":{"$lte":0}},{"match2":{"$eq":"true"}}]},{"$and":[{"match2":{"$eq":"false"}},{"match3":{"$eq":true}}]}]}
       main-get-complex3:
+        space: "{space}"
         method: POST
         uri: <<protocol:http>>://{weighted_hosts}:<<jsonapi_port:8080>><<path_prefix:>>/v1/<<namespace:jsonapi_search_advanced>>/<<table:docs_collection>>
         Accept: "application/json"

nosqlbench/http-jsonapi-search-basic.md

@@ -59,5 +59,6 @@ The basic search workload can test the following `where` clauses:
 - `docpadding` - the number of fields to add to each document; useful for writing larger documents. A value of e.g. `5` would make each document have 20 leaf values, instead of 15. (default: `0`)
 - `match-ratio` - a value between 0 and 1 detailing what ratio of the documents written should match the search parameters. If match-ratio is e.g. `0.1` then approximately one-tenth of the documents will have `match1`, `match2`, and `match3` values that are `0`, `"true"`, and `true`, respectively. (default: `0.01`)
 - `fields` - the URL-encoded value for `fields` that you would send to the Docs API. This restricts the fields returned during benchmarking.
+- `connections` - number of HTTP2 connections to be shared between the threads (default: `20`)