Updating track to work with Elasticsearch 6.x and Rally 0.10.x (elast…

…ic#16) * WIP * WIP * WIP * WIP * WIP * WIP * WIP * WIP * WIP * Updated rollover interval * Added delete elasticlogs_q index operation * Fixed field stats concurrency * WIP * Fixed max concurrency level
henningandersen · May 12, 2018 · 0974124 · 0974124
1 parent 3a96dac
commit 0974124
Show file tree

Hide file tree

Showing 20 changed files with 292 additions and 257 deletions.
diff --git a/eventdata/challenges/bulk-size-evaluation.json b/eventdata/challenges/bulk-size-evaluation.json
@@ -1,8 +1,12 @@
+{% set p_bulk_indexing_clients = (bulk_indexing_clients | default(8)) %}
+{% set p_replica_count = (replica_count | default(0)) %}
+{% set p_shard_count = (shard_count | default(2)) %}
+
 {
   "name": "bulk-size-evaluation",
   "description": "Indexes with different bulk sizes. IDs are autogenerated by Elasticsearch, meaning there are no conflicts.",
   "meta": {
-    "client_count": {{ clients }},
+    "client_count": {{ p_bulk_indexing_clients }},
     "benchmark_type": "indexing_bulksize"
   },
   "schedule": [
@@ -13,64 +17,64 @@
       "operation": {
         "operation-type": "create-index",
         "settings": {
-            "index.number_of_replicas": {{ replica_count }},
-            "index.number_of_shards": {{ shard_count }}
+            "index.number_of_replicas": {{ p_replica_count }},
+            "index.number_of_shards": {{ p_shard_count }}
         }
       }
     },
     {
       "operation": "index-append-125",
       "warmup-time-period": 0,
       "time-period": 600,
-      "clients": {{ clients }}
+      "clients": {{ p_bulk_indexing_clients }}
     },
     {
       "operation": "index-append-250",
       "warmup-time-period": 0,
       "time-period": 600,
-      "clients": {{ clients }}
+      "clients": {{ p_bulk_indexing_clients }}
     },
     {
       "operation": "index-append-500",
       "warmup-time-period": 0,
       "time-period": 600,
-      "clients": {{ clients }}
+      "clients": {{ p_bulk_indexing_clients }}
     },
     {
       "operation": "index-append-1000",
       "warmup-time-period": 0,
       "time-period": 600,
-      "clients": {{ clients }}
+      "clients": {{ p_bulk_indexing_clients }}
     },
     {
       "operation": "index-append-2000",
       "warmup-time-period": 0,
       "time-period": 600,
-      "clients": {{ clients }}
+      "clients": {{ p_bulk_indexing_clients }}
     },
     {
       "operation": "index-append-5000",
       "warmup-time-period": 0,
       "time-period": 600,
-      "clients": {{ clients }}
+      "clients": {{ p_bulk_indexing_clients }}
     },
     {
       "operation": "index-append-10000",
       "warmup-time-period": 0,
       "time-period": 600,
-      "clients": {{ clients }}
+      "clients": {{ p_bulk_indexing_clients }}
     },
     {
       "operation": "index-append-20000",
       "warmup-time-period": 0,
       "time-period": 600,
-      "clients": {{ clients }}
+      "clients": {{ p_bulk_indexing_clients }}
     },
     {
       "operation": "index-append-50000",
       "warmup-time-period": 0,
       "time-period": 600,
-      "clients": {{ clients }}
+      "clients": {{ p_bulk_indexing_clients }}
     }
   ]
 }
diff --git a/eventdata/challenges/combined-indexing-and-querying.json b/eventdata/challenges/combined-indexing-and-querying.json
@@ -1,3 +1,9 @@
+{% set p_bulk_indexing_clients = (bulk_indexing_clients | default(8)) %}
+{% set p_client_count = (8 + (bulk_indexing_clients | default(8))) %}
+{% set p_rate_limit_duration_secs = (rate_limit_duration_secs | default(1200)) %}
+{% set p_rate_limit_step = (rate_limit_step | default(2)) %}
+{% set p_rate_limit_max = (rate_limit_max | default(32)) %}
+
 {
   "name": "combined-indexing-and-querying",
   "description": "This challenge simulates a set of Kibana queries against historical data (elasticlogs_q-* indices) as well as against the most recent data currently being indexed. It combined this with rate-limited indexing at varying levels. It assumes one of the challenges creating elasticlogs_q-* indices has been run.",
@@ -10,7 +16,10 @@
       "operation": "deleteindex_elasticlogs_i-*"
     },
     {
-      "operation": "fieldstats_elasticlogs_q-*_ELASTICLOGS"
+      "operation": "fieldstats_elasticlogs_q-*_ELASTICLOGS",
+      "warmup-iterations": {{ p_client_count }},
+      "iterations": {{ p_client_count }},
+      "clients": {{ p_client_count }}
     },
     {
       "operation": "relative-kibana-content_issues-dashboard_75%",
@@ -24,7 +33,7 @@
       "operation": "relative-kibana-content_issues-dashboard_50%",
       "target-interval": 60,
       "warmup-time-period": 0,
-      "time-period": {{ rate_limit_duration_secs }},
+      "time-period": {{ p_rate_limit_duration_secs }},
       "meta": {
         "target_indexing_rate": 0,
         "query_type": "historic"
@@ -38,24 +47,25 @@
     },
     {# Add some data to index so it does not start empty #}
     {
-      "operation": "index-append-1000-elasticlogs_i_write",    
-      "warmup-time-period": {{ rate_limit_duration_secs }},
-      "time-period": {{ rate_limit_duration_secs }},
+      "operation": "index-append-1000-elasticlogs_i_write",
+      "time-period": {{ p_rate_limit_duration_secs }},
       "target-throughput": 10,
-      "clients": {{ clients }}
+      "clients": {{ p_bulk_indexing_clients }}
     },
-    {% for ops in rate_limit_ops %}
+    {% for ops in range(p_rate_limit_step, p_rate_limit_max, p_rate_limit_step) %}
+
+
     {% set rate = ops * 1000 %}
     {
       "parallel": {
         "warmup-time-period": 0,
-        "time-period": {{ rate_limit_duration_secs }},
+        "time-period": {{ p_rate_limit_duration_secs }},
         "tasks": [
           {
             "name": "index-append-1000-elasticlogs_i_write-{{rate}}",
             "operation": "index-append-1000-elasticlogs_i_write",
             "target-throughput": {{ ops }},
-            "clients": {{ clients }},
+            "clients": {{ p_bulk_indexing_clients }},
             "meta": {
               "target_indexing_rate": {{ rate }}
             }

diff --git a/eventdata/challenges/elasticlogs-1bn-load.json b/eventdata/challenges/elasticlogs-1bn-load.json
@@ -1,8 +1,10 @@
+{% set p_bulk_indexing_clients = (bulk_indexing_clients | default(8)) %}
+
 {
   "name": "elasticlogs-1bn-load",
-  "description": "Indexes 1bn documents (20 batches of 50M) into elasticlogs_q-* indices. IDs are autogenerated by Elasticsearch, meaning there are no conflicts.",
+  "description": "Indexes 1bn documents into elasticlogs_q-* indices. IDs are autogenerated by Elasticsearch, meaning there are no conflicts.",
   "meta": {
-    "client_count": {{ clients }},
+    "client_count": {{ p_bulk_indexing_clients }},
     "benchmark_type": "indexing"
   },
   "schedule": [
@@ -12,36 +14,28 @@
       "warmup-iterations": 0,
       "iterations": 1
     },
-    {% for n in range(19) %}
     {
-      "name": "index-append-1000-elasticlogs_q_write-{{n}}",
-      "operation": "index-append-1000-elasticlogs_q_write",
-      "iterations": 50000,
-      "clients": {{ clients }}
-    },
-    {
-      "name": "rollover_elasticlogs_q_write_10M-{{n}}",
-      "operation": "rollover_elasticlogs_q_write_10M",
-      "iterations": 1
-    },
-    {
-      "name": "node_storage-{{n}}",
-      "operation": "node_storage",
-      "iterations": 1
-    },
-    {% endfor %}
-    {
-      "operation": "index-append-1000-elasticlogs_q_write",    
-      "iterations": 50000,
-      "clients": {{ clients }}
-    },
-    {
-      "operation": "rollover_elasticlogs_q_write_10M",
-      "iterations": 1
+      "parallel": {
+        "completed-by": "index-append-1000-elasticlogs_q_write",
+        "tasks": [
+          {
+            "operation": "index-append-1000-elasticlogs_q_write",    
+            "iterations": 1000000,
+            "clients": {{ p_bulk_indexing_clients }}
+          },
+          {
+            "operation": "rollover_elasticlogs_q_write_100M",
+            "clients": 1,
+            "warmup-iterations": 100000,
+            "iterations": 100000,
+            "target-interval": 30
+          }
+        ]
+      }
     },
     {
       "operation": "node_storage",
       "iterations": 1
     }
   ]
-}
+}
diff --git a/eventdata/challenges/elasticlogs-querying.json b/eventdata/challenges/elasticlogs-querying.json
@@ -8,7 +8,8 @@
   "schedule": [
     {
       "operation": "fieldstats_elasticlogs_q-*_ELASTICLOGS",
-      "iterations": 1
+      "iterations": 1,
+      "clients": 3
     },
     {
       "parallel": {

diff --git a/eventdata/challenges/generate-historic-data.json b/eventdata/challenges/generate-historic-data.json
@@ -1,8 +1,10 @@
+{% set p_bulk_indexing_clients = (bulk_indexing_clients | default(8)) %}
+
 {
   "name": "generate-historic-data",
   "description": "Example of how to index 5 days worth of data at 6x speed (20 hours). IDs are autogenerated by Elasticsearch, meaning there are no conflicts.",
   "meta": {
-    "client_count": 8,
+    "client_count": {{ p_bulk_indexing_clients }},
     "benchmark_type": "generate-historic-data"
   },
   "schedule": [
@@ -17,7 +19,7 @@
       "target-throughput": 19,
       "warmup-time-period": 0,
       "time-period": 72000,
-      "clients": 8
+      "clients": {{ p_bulk_indexing_clients }}
     }
   ]
 }
diff --git a/eventdata/challenges/shard-size-on-disk.json b/eventdata/challenges/shard-size-on-disk.json
@@ -1,3 +1,5 @@
+{% set p_bulk_indexing_clients = (bulk_indexing_clients | default(8)) %}
+
 {
   "name": "shard-size-on-disk",
   "description": "Indexes sets of 2M events into Elasticsearch, followed by index statistics in order to track how index size depends on event count. IDs are autogenerated by Elasticsearch, meaning there are no conflicts. This process is repeatedly run until 100M events have been indexed into the shard.",
@@ -24,7 +26,7 @@
       "name": "index-append-1000-shard-sizing-{{n}}",
       "operation": "index-append-1000-shard-sizing",
       "iterations": 2000,
-      "clients": 8,
+      "clients": {{ p_bulk_indexing_clients }},
       "meta": {
         "iteration_number": {{ n }}
       }
@@ -41,7 +43,7 @@
     {
       "operation": "index-append-1000-shard-sizing",
       "iterations": 2000,
-      "clients": 8,
+      "clients": {{ p_bulk_indexing_clients }},
       "meta": {
         "iteration_number": {{ 50 }}
       }

diff --git a/eventdata/challenges/shard-sizing.json b/eventdata/challenges/shard-sizing.json
@@ -1,3 +1,7 @@
+{% set p_bulk_indexing_clients = (bulk_indexing_clients | default(8)) %}
+{% set p_shard_sizing_iterations = (shard_sizing_iterations | default(25)) %}
+{% set p_shard_sizing_queries = (shard_sizing_queries | default(20)) %}
+
 {
   "name": "shard-sizing",
   "description": "Indexes sets of 2M events into Elasticsearch, followed by index statistics and simulated Kibana queries. IDs are autogenerated by Elasticsearch, meaning there are no conflicts. This process is repeatedly run until 50M events have been indexed into the shard. This allows query latency to be evaluated as a function of shard size.",
@@ -19,12 +23,12 @@
         }
       }
     },
-    {% for n in range(1,shard_sizing_iterations) %}
+    {% for n in range(1, p_shard_sizing_iterations) %}
     {
       "name": "index-append-1000-shard-sizing-iteration-{{n}}",
       "operation": "index-append-1000-shard-sizing",
       "iterations": 2000,
-      "clients": 8,
+      "clients": {{ p_bulk_indexing_clients }},
       "meta": {
         "iteration_number": {{ n }}
       }
@@ -40,31 +44,31 @@
     {
       "name": "kibana-traffic-shard-sizing-50%-iteration-{{n}}",
       "operation": "kibana-traffic-shard-sizing-50%",
-      "iterations": {{ shard_sizing_queries }},
+      "iterations": {{ p_shard_sizing_queries }},
       "meta": {
         "iteration_number": {{ n }}
       }
     },
     {
       "name": "kibana-traffic-shard-sizing-90%-iteration-{{n}}",
       "operation": "kibana-traffic-shard-sizing-90%",
-      "iterations": {{ shard_sizing_queries }},
+      "iterations": {{ p_shard_sizing_queries }},
       "meta": {
         "iteration_number": {{ n }}
       }
     },
     {
       "name": "kibana-content_issues-shard-sizing-50%-iteration-{{n}}",
       "operation": "kibana-content_issues-shard-sizing-50%",
-      "iterations": {{ shard_sizing_queries }},
+      "iterations": {{ p_shard_sizing_queries }},
       "meta": {
         "iteration_number": {{ n }}
       }
     },
     {
       "name": "kibana-content_issues-shard-sizing-90%-iteration-{{n}}",
       "operation": "kibana-content_issues-shard-sizing-90%",
-      "iterations": {{ shard_sizing_queries }},
+      "iterations": {{ p_shard_sizing_queries }},
       "meta": {
         "iteration_number": {{ n }}
       }
@@ -73,44 +77,44 @@
     {
       "operation": "index-append-1000-shard-sizing",    
       "iterations": 2000,
-      "clients": 8,
+      "clients": {{ p_bulk_indexing_clients }},
       "meta": {
-        "iteration_number": {{ shard_sizing_iterations }}
+        "iteration_number": {{ p_shard_sizing_iterations }}
       }
     },
     {
       "operation": "indicesstats_elasticlogs",
       "iterations": 1,
       "meta": {
-        "iteration_number": {{ shard_sizing_iterations }}
+        "iteration_number": {{ p_shard_sizing_iterations }}
       }
     },
     {
       "operation": "kibana-traffic-shard-sizing-50%",
-      "iterations": {{ shard_sizing_queries }},
+      "iterations": {{ p_shard_sizing_queries }},
       "meta": {
-        "iteration_number": {{ shard_sizing_iterations }}
+        "iteration_number": {{ p_shard_sizing_iterations }}
       }
     },
     {
       "operation": "kibana-traffic-shard-sizing-90%",
-      "iterations": {{ shard_sizing_queries }},
+      "iterations": {{ p_shard_sizing_queries }},
       "meta": {
-        "iteration_number": {{ shard_sizing_iterations }}
+        "iteration_number": {{ p_shard_sizing_iterations }}
       }
     },
     {
       "operation": "kibana-content_issues-shard-sizing-50%",
-      "iterations": {{ shard_sizing_queries }},
+      "iterations": {{ p_shard_sizing_queries }},
       "meta": {
-        "iteration_number": {{ shard_sizing_iterations }}
+        "iteration_number": {{ p_shard_sizing_iterations }}
       }
     },
     {
       "operation": "kibana-content_issues-shard-sizing-90%",
-      "iterations": {{ shard_sizing_queries }},
+      "iterations": {{ p_shard_sizing_queries }},
       "meta": {
-        "iteration_number": {{ shard_sizing_iterations }}
+        "iteration_number": {{ p_shard_sizing_iterations }}
       }
     }
   ]