Configure persistent datastore index settings

docs/configuration.rst

@@ -69,6 +69,8 @@ The following settings are applicable only if ``datastore.type`` is set to "elas
 * ``datastore.user``: Sets the name of the Elasticsearch user for the metrics store.
 * ``datastore.password``: Sets the password of the Elasticsearch user for the metrics store.
 * ``datastore.probe.cluster_version`` (default: true): Enables automatic detection of the metric store's version.
+* ``datastore.number_of_shards`` (default: 1): The number of primary shards that the ``rally-*`` indices should have. Any updates to this setting after initial index creation will only be applied to new ``rally-*`` indices.
+* ``datastore.number_of_replicas`` (default: 0): The number of replicas each primary shard has. Defaults to 0. Any updates to this setting after initial index creation will only be applied to new ``rally-*`` indices.
 
 **Examples**
 

docs/migrate.rst

@@ -13,6 +13,14 @@ Migrating to Rally 2.3.0
 
 The deprecated property ``relative-time-ms`` has been removed in Rally 2.3.0. Use the property ``relative-time`` instead to retrieve the same metric.
 
+Primary and replica shard counts are now configurable for persistent Elasticsearch metrics stores
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. note::
+
+     Primary and replica shard counts are only configurable for persistent metrics stores (``datastore.type = elasticsearch``).
+
+With this release, the number of primary and replica shards are now configurable for ``rally-*`` indices. These can be set via the ``datastore.number_of_shards`` and ``datastore.number_of_replicas`` options in ``rally.ini``.
 
 Migrating to Rally 2.2.1
 ------------------------

esrally/metrics.py

@@ -237,7 +237,10 @@ class IndexTemplateProvider:
     """
 
     def __init__(self, cfg):
-        self.script_dir = cfg.opts("node", "rally.root")
+        self._config = cfg
+        self._number_of_shards = self._config.opts("reporting", "datastore.number_of_shards", default_value=1, mandatory=False)
+        self._number_of_replicas = self._config.opts("reporting", "datastore.number_of_replicas", default_value=0, mandatory=False)
+        self.script_dir = self._config.opts("node", "rally.root")
 
     def metrics_template(self):
         return self._read("metrics-template")
@@ -250,7 +253,10 @@ def results_template(self):
 
     def _read(self, template_name):
         with open("%s/resources/%s.json" % (self.script_dir, template_name), encoding="utf-8") as f:
-            return f.read()
+            template = json.load(f)
+            template["settings"]["index"]["number_of_shards"] = self._number_of_shards
+            template["settings"]["index"]["number_of_replicas"] = self._number_of_replicas
+            return json.dumps(template)
 
 
 class MetaInfoScope(Enum):

tests/metrics_test.py

@@ -18,6 +18,7 @@
 
 import collections
 import datetime
+import json
 import logging
 import os
 import random
@@ -2327,3 +2328,36 @@ def test_as_flat_list(self):
             {"node": "rally-node-1", "name": "bytes_written", "value": {"single": 833 * 1024 * 1024}},
             select(metric_list, "bytes_written", node="rally-node-1"),
         )
+
+
+class IndexTemplateProviderTests(TestCase):
+    def setUp(self):
+        self.cfg = config.Config()
+        self.cfg.add(config.Scope.application, "node", "root.dir", os.path.join(tempfile.gettempdir(), str(uuid.uuid4())))
+        self.cfg.add(config.Scope.application, "node", "rally.root", paths.rally_root())
+        self.cfg.add(config.Scope.application, "system", "env.name", "unittest-env")
+        self.cfg.add(config.Scope.application, "system", "list.races.max_results", 100)
+        self.cfg.add(config.Scope.application, "system", "time.start", FileRaceStoreTests.RACE_TIMESTAMP)
+        self.cfg.add(config.Scope.application, "system", "race.id", FileRaceStoreTests.RACE_ID)
+
+    def test_datastore_type_elasticsearch_index_template_update(self):
+        _datastore_type = "elasticsearch"
+        _datastore_number_of_shards = random.randint(1, 100)
+        _datastore_number_of_replicas = random.randint(1, 100)
+
+        self.cfg.add(config.Scope.applicationOverride, "reporting", "datastore.type", _datastore_type)
+        self.cfg.add(config.Scope.applicationOverride, "reporting", "datastore.number_of_shards", _datastore_number_of_shards)
+        self.cfg.add(config.Scope.applicationOverride, "reporting", "datastore.number_of_replicas", _datastore_number_of_replicas)
+
+        _index_template_provider = metrics.IndexTemplateProvider(self.cfg)
+
+        templates = [
+            _index_template_provider.metrics_template(),
+            _index_template_provider.races_template(),
+            _index_template_provider.results_template(),
+        ]
+
+        for template in templates:
+            t = json.loads(template)
+            assert t["settings"]["index"]["number_of_shards"] == _datastore_number_of_shards
+            assert t["settings"]["index"]["number_of_replicas"] == _datastore_number_of_replicas