Api tables perf

docker-compose/docker-compose-hcd.yml

@@ -82,12 +82,13 @@ services:
       - JAVA_MAX_MEM_RATIO=75
       - JAVA_INITIAL_MEM_RATIO=50
       - GC_CONTAINER_OPTIONS=-XX:+UseG1GC
-      - STARGATE_JSONAPI_OPERATIONS_DATABASE_CONFIG_CASSANDRA_END_POINTS=hcd
+      - STARGATE_JSONAPI_OPERATIONS_DATABASE_CONFIG_CASSANDRA_END_POINTS=hcd-1
       - STARGATE_JSONAPI_OPERATIONS_DATABASE_CONFIG_LOCAL_DATACENTER=dc1
       - QUARKUS_HTTP_ACCESS_LOG_ENABLED=${REQUESTLOG}
       - QUARKUS_LOG_LEVEL=${LOGLEVEL}
       - STARGATE_JSONAPI_OPERATIONS_VECTORIZE_ENABLED=true
       - JAVA_OPTS_APPEND=-Dquarkus.http.host=0.0.0.0 -Djava.util.logging.manager=org.jboss.logmanager.LogManager
+      - STARGATE_FEATURE_FLAGS_TABLES=true
     healthcheck:
       test: curl -f http://localhost:8181/stargate/health || exit 1
       interval: 5s

docker-compose/start_dse69.sh

@@ -11,7 +11,7 @@ LOGLEVEL=INFO
 
 # Default to latest released version
 DATAAPITAG="v1"
-DATAAPIIMAGE="stargateio/data-api"
+DATAAPIIMAGE="stargateio/jsonapi"
 
 DSETAG="6.9.4"
 DSEIMAGE="cr.dtsx.io/datastax/dse-server"

docker-compose/start_hcd.sh

@@ -11,7 +11,7 @@ LOGLEVEL=INFO
 
 # Default to latest released version
 DATAAPITAG="v1"
-DATAAPIIMAGE="stargateio/data-api"
+DATAAPIIMAGE="stargateio/jsonapi"
 
 HCDTAG="1.0.0"
 HCDIMAGE="cr.dtsx.io/datastax/hcd"
@@ -80,4 +80,4 @@ else
   else
     docker compose -f docker-compose-hcd.yml up -d --wait
   fi
-fi
+fi

nosqlbench/cql_keyvalue2.yaml

@@ -0,0 +1,110 @@
+min_version: "5.17.1"
+
+description: |
+  A workload with only text keys and text values.
+  The CQL Key-Value workload demonstrates the simplest possible schema with payload data. This is useful for measuring
+  system capacity most directly in terms of raw operations. As a reference point, it provides some insight around types of
+  workloads that are constrained around messaging, threading, and tasking, rather than bulk throughput.
+  During preload, all keys are set with a value. During the main phase of the workload, random keys from the known
+  population are replaced with new values which never repeat. During the main phase, random partitions are selected for
+  upsert, with row values never repeating.
+
+  TEMPLATE(batchsize,100)
+
+scenarios:
+  default:
+    schema: run driver=cql tags==block:schema threads==1 cycles==UNDEF
+    rampup: run driver=cql tags==block:rampup cycles===TEMPLATE(rampup-cycles,10000000) threads=auto
+    main: run driver=cql tags==block:"main.*" cycles===TEMPLATE(main-cycles,10000000) threads=auto
+  batch:
+    schema: run driver=cql tags==block:schema threads==1 cycles==UNDEF
+    rampup: run driver=cql tags==block:rampup_batch cycles===TEMPLATE(rampup-cycles,10000000) threads=auto
+    main: run driver=cql tags==block:"main.*" cycles===TEMPLATE(main-cycles,10000000) threads=auto
+  astra:
+    schema: run driver=cql tags==block:schema_astra threads==1 cycles==UNDEF
+    rampup: run driver=cql tags==block:rampup cycles===TEMPLATE(rampup-cycles,10000000) threads=auto
+    main: run driver=cql tags==block:"main.*" cycles===TEMPLATE(main-cycles,10000000) threads=auto
+  astra_read_only:
+    schema: run driver=cql tags==block:schema_astra threads==1 cycles==UNDEF
+    rampup: run driver=cql tags==block:rampup cycles===TEMPLATE(rampup-cycles,10000000) threads=auto
+    main: run driver=cql tags==block:main_read cycles===TEMPLATE(main-cycles,10000000) threads=auto
+  asta_write_only:
+    schema: run driver=cql tags==block:schema_astra threads==1 cycles==UNDEF
+    main: run driver=cql tags==block:main_write cycles===TEMPLATE(main-cycles,10000000) threads=auto
+  basic_check:
+    schema: run driver=cql tags==block:schema threads==1 cycles==UNDEF
+    rampup: run driver=cql tags==block:rampup cycles===TEMPLATE(rampup-cycles,10) threads=auto
+    main: run driver=cql tags==block:"main.*" cycles===TEMPLATE(main-cycles,10) threads=auto
+
+bindings:
+  seq_key: Mod(TEMPLATE(keycount,1000000000)); ToString() -> String
+  seq_value: Hash(); Mod(TEMPLATE(valuecount,1000000000)); ToString() -> String
+  batch_seq_value: Mul(TEMPLATE(batchsize,100)L); Hash(); Mod(TEMPLATE(valuecount,1000000000)); ToString() -> String
+  rw_key: TEMPLATE(keydist,Uniform(0,1000000000)); ToString() -> String
+  rw_value: Hash(); TEMPLATE(valdist,Uniform(0,1000000000)); ToString() -> String
+
+blocks:
+  schema:
+    params:
+      prepared: false
+    ops:
+      create_keyspace: |
+        create keyspace if not exists TEMPLATE(keyspace,baselines)
+        WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 'TEMPLATE(rf:1)'}
+        AND durable_writes = true;
+      create_table: |
+        create table if not exists TEMPLATE(keyspace,baselines).TEMPLATE(table,keyvalue) (
+        key text,
+         value text,
+         PRIMARY KEY (key)
+        );
+  schema_astra:
+    params:
+      prepared: false
+    statements:
+      create_table: |
+        create table if not exists TEMPLATE(keyspace,baselines).TEMPLATE(table,keyvalue) (
+        key text,
+         value text,
+         PRIMARY KEY (key)
+        );
+  rampup:
+    params:
+      cl: TEMPLATE(write_cl,LOCAL_QUORUM)
+    ops:
+      rampup_insert: |
+        insert into TEMPLATE(keyspace,baselines).TEMPLATE(table,keyvalue)
+        (key, value)
+        values ({seq_key},{seq_value});
+  rampup_batch:
+    params:
+      cl: TEMPLATE(write_cl,LOCAL_QUORUM)
+    ops:
+      rampup_insert:
+        batch: testing
+        repeat: 100
+        op_template:
+          prepared: |
+            insert into TEMPLATE(keyspace,baselines).TEMPLATE(table,keyvalue)
+            (key, value)
+            values ({seq_key},{seq_value});
+  verify:
+    params:
+      cl: TEMPLATE(read_cl,LOCAL_QUORUM)
+    ops:
+      verify_select: |
+        select * from TEMPLATE(keyspace,baselines).TEMPLATE(table,keyvalue) where key={seq_key};
+      verify-fields: key->seq_key, value->seq_value
+  main_read:
+    params:
+      cl: TEMPLATE(read_cl,LOCAL_QUORUM)
+    statements:
+      main_select: |
+        select * from TEMPLATE(keyspace,baselines).TEMPLATE(table,keyvalue) where key={rw_key};
+  main_write:
+    params:
+      cl: TEMPLATE(write_cl,LOCAL_QUORUM)
+    statements:
+      main_insert: |
+        insert into TEMPLATE(keyspace,baselines).TEMPLATE(table,keyvalue)
+        (key, value) values ({rw_key}, {rw_value});

nosqlbench/http-dataapi-tables-keyvalue.yaml

@@ -0,0 +1,173 @@
+min_version: "5.17.3"
+
+# Example command line (when Stargate is running on localhost):
+# nb5 -v http-dataapi-keyvalue dataapi_host=localhost rowscount=20000 threads=20
+
+description: >2
+  This workload emulates a key-value data model and access patterns.
+  This should be identical to the cql variant except for:
+  - Schema creation with the Data API
+  - There is no instrumentation with the http driver.
+  - There is no async mode with the http driver.
+  Note that dataapi_port should reflect the port where the Data API is exposed (defaults to 8181).
+
+scenarios:
+  default:
+    schema: run driver=http tags==block:schema* threads==1 cycles==UNDEF
+    rampup: run driver=http tags==block:rampup cycles===TEMPLATE(rampup-cycles,TEMPLATE(rowscount,10000000)) threads=auto
+    main: run driver=http tags==block:"main.*" cycles===TEMPLATE(main-cycles,TEMPLATE(rowscount,10000000)) threads=auto
+  astra:
+    schema: run driver=http tags==block:schema threads==1 cycles==UNDEF
+    rampup: run driver=http tags==block:rampup cycles===TEMPLATE(rampup-cycles,TEMPLATE(rowscount,10000000)) threads=auto
+    main: run driver=http tags==block:"main.*" cycles===TEMPLATE(main-cycles,TEMPLATE(rowscount,10000000)) threads=auto
+  astra_findone_only:
+    schema: run driver=http tags==block:schema threads==1 cycles==UNDEF
+    rampup: run driver=http tags==block:rampup cycles===TEMPLATE(rampup-cycles,TEMPLATE(rowscount,10000000)) threads=auto
+    main: run driver=http tags==block:main-findone cycles===TEMPLATE(main-cycles,TEMPLATE(rowscount,10000000)) threads=auto
+  astra_insertone_only:
+    schema: run driver=http tags==block:schema threads==1 cycles==UNDEF
+    main: run driver=http tags==block:main-insertone cycles===TEMPLATE(main-cycles,TEMPLATE(rowscount,10000000)) threads=auto
+
+bindings:
+  # To enable an optional weighted set of hosts in place of a load balancer
+  # Examples
+  #   single host: dataapi_host=host1
+  #   multiple hosts: dataapi_host=host1,host2,host3
+  #   multiple weighted hosts: dataapi_host=host1:3,host2:7
+  weighted_hosts: WeightedStrings('<<dataapi_host:dataapi>>')
+
+  # spread into different spaces to use multiple connections
+  space: HashRange(1,<<connections:20>>); ToString();
+
+  # http request id
+  request_id: ToHashedUUID(); ToString();
+
+  # autogenerate auth token to use on API calls using configured uri/uid/password, unless one is provided
+  token: Discard(); Token('<<auth_token:>>','<<uri:http://localhost:8081/v1/auth>>', '<<uid:cassandra>>', '<<pswd:cassandra>>');
+
+  seq_key: Mod(<<keycount:10000000>>); ToString() -> String
+  seq_value: Hash(); Mod(<<valuecount:10000000>>); ToString() -> String
+  rw_key: <<keydist:Uniform(0,<<keycount:10000000>>)->int>>; ToString() -> String
+  rw_value: Hash(); <<valdist:Uniform(0,<<keycount:10000000>>)->int>>; ToString() -> String
+
+blocks:
+  schema_local:
+    ops:
+      create-namespace:
+        method: POST
+        uri: <<protocol:http>>://{weighted_hosts}:<<dataapi_port:8181>><<path_prefix:>>/v1
+        Accept: "application/json"
+        X-Cassandra-Request-Id: "{request_id}"
+        Token: "{token}"
+        Content-Type: "application/json"
+        ok-body: ".*\"ok\":1.*"
+        body: >2
+          {
+            "createNamespace": {
+              "name": "<<namespace:dataapi_keyvalue>>"
+            }
+          }
+  schema:
+    ops:
+      drop-table:
+        method: POST
+        uri: <<protocol:http>>://{weighted_hosts}:<<dataapi_port:8181>><<path_prefix:>>/v1/<<namespace:dataapi_keyvalue>>
+        Accept: "application/json"
+        X-Cassandra-Request-Id: "{request_id}"
+        Token: "{token}"
+        Content-Type: "application/json"
+        ok-body: ".*\"ok\":1.*"
+        body: >2
+          {
+            "dropTable": {
+              "name": "<<table:api_table>>"
+            }
+          }
+
+      create-table:
+        method: POST
+        uri: <<protocol:http>>://{weighted_hosts}:<<dataapi_port:8181>><<path_prefix:>>/v1/<<namespace:dataapi_keyvalue>>
+        Accept: "application/json"
+        X-Cassandra-Request-Id: "{request_id}"
+        Token: "{token}"
+        Content-Type: "application/json"
+        ok-body: ".*\"ok\":1.*"
+        body: >2
+          {
+            "createTable": {
+              "name": "<<table:api_table>>",
+              "definition": {
+                "columns": {
+                  "key": {
+                    "type": "text"
+                  },
+                  "value": {
+                    "type": "text"
+                  }
+                },
+                "primaryKey": "key"
+              }
+            }
+          }
+
+  rampup:
+    ops:
+      rampup-insert:
+        space: "{space}"
+        method: POST
+        uri: <<protocol:http>>://{weighted_hosts}:<<dataapi_port:8181>><<path_prefix:>>/v1/<<namespace:dataapi_keyvalue>>/<<table:api_table>>
+        Accept: "application/json"
+        X-Cassandra-Request-Id: "{request_id}"
+        Token: "{token}"
+        Content-Type: "application/json"
+        ok-body: '.*\"insertedIds\":\[.*\].*'
+        body: >2
+          {
+            "insertOne" : {
+              "document" : {
+                "key"   :  "{seq_key}",
+                "value" :  "{seq_value}"
+              }
+            }
+          }
+
+  main-findone:
+    ops:
+      main-select:
+        space: "{space}"
+        method: POST
+        uri: <<protocol:http>>://{weighted_hosts}:<<dataapi_port:8181>><<path_prefix:>>/v1/<<namespace:dataapi_keyvalue>>/<<table:api_table>>
+        Accept: "application/json"
+        X-Cassandra-Request-Id: "{request_id}"
+        Token: "{token}"
+        Content-Type: "application/json"
+        ok-body: ".*\"data\".*"
+        body: >2
+          {
+            "findOne" : {
+              "filter" : {
+                "key" : "{rw_key}"
+              }
+            }
+          }
+  main-insertone:
+    ops:
+      main-write:
+        space: "{space}"
+        method: POST
+        uri: <<protocol:http>>://{weighted_hosts}:<<dataapi_port:8181>><<path_prefix:>>/v1/<<namespace:dataapi_keyvalue>>/<<table:api_table>>
+        Accept: "application/json"
+        X-Cassandra-Request-Id: "{request_id}"
+        Token: "{token}"
+        Content-Type: "application/json"
+        # because this is not an upsert, modified count could be 0 or 1
+        ok-body: '.*\"insertedIds\":\[.*\].*'
+        body: >2          
+          {
+            "insertOne" : {
+              "document" : {
+                "key"   :  "{seq_key}",
+                "value" :  "{seq_value}"
+              }
+            }
+          }