diff --git a/docs/configuration.rst b/docs/configuration.rst index c0c12dd94..8a864c07e 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -69,8 +69,8 @@ The following settings are applicable only if ``datastore.type`` is set to "elas * ``datastore.user``: Sets the name of the Elasticsearch user for the metrics store. * ``datastore.password``: Sets the password of the Elasticsearch user for the metrics store. * ``datastore.probe.cluster_version`` (default: true): Enables automatic detection of the metric store's version. -* ``datastore.number_of_shards`` (default: 1): The number of primary shards that the ``rally-*`` indices should have. Any updates to this setting after initial index creation will only be applied to new ``rally-*`` indices. -* ``datastore.number_of_replicas`` (default: 0): The number of replicas each primary shard has. Defaults to 0. Any updates to this setting after initial index creation will only be applied to new ``rally-*`` indices. +* ``datastore.number_of_shards`` (default: `Elasticsearch default value `_): The number of primary shards that the ``rally-*`` indices should have. Any updates to this setting after initial index creation will only be applied to new ``rally-*`` indices. +* ``datastore.number_of_replicas`` (default: `Elasticsearch default value `_): The number of replicas each primary shard has. Defaults to . Any updates to this setting after initial index creation will only be applied to new ``rally-*`` indices. **Examples** diff --git a/esrally/metrics.py b/esrally/metrics.py index 220024828..f8ab93816 100644 --- a/esrally/metrics.py +++ b/esrally/metrics.py @@ -238,8 +238,8 @@ class IndexTemplateProvider: def __init__(self, cfg): self._config = cfg - self._number_of_shards = self._config.opts("reporting", "datastore.number_of_shards", default_value=1, mandatory=False) - self._number_of_replicas = self._config.opts("reporting", "datastore.number_of_replicas", default_value=0, mandatory=False) + self._number_of_shards = self._config.opts("reporting", "datastore.number_of_shards", default_value=None, mandatory=False) + self._number_of_replicas = self._config.opts("reporting", "datastore.number_of_replicas", default_value=None, mandatory=False) self.script_dir = self._config.opts("node", "rally.root") def metrics_template(self): @@ -254,8 +254,15 @@ def results_template(self): def _read(self, template_name): with open("%s/resources/%s.json" % (self.script_dir, template_name), encoding="utf-8") as f: template = json.load(f) - template["settings"]["index"]["number_of_shards"] = self._number_of_shards - template["settings"]["index"]["number_of_replicas"] = self._number_of_replicas + if self._number_of_shards is not None: + if int(self._number_of_shards) < 1: + raise exceptions.SystemSetupError( + f"The setting: datastore.number_of_shards must be >= 1. Please " + f"check the configuration in {self._config.config_file.location}" + ) + template["settings"]["index"]["number_of_shards"] = int(self._number_of_shards) + if self._number_of_replicas is not None: + template["settings"]["index"]["number_of_replicas"] = int(self._number_of_replicas) return json.dumps(template) diff --git a/esrally/resources/metrics-template.json b/esrally/resources/metrics-template.json index bb7881052..4564afc9d 100644 --- a/esrally/resources/metrics-template.json +++ b/esrally/resources/metrics-template.json @@ -2,8 +2,6 @@ "index_patterns": ["rally-metrics-*"], "settings": { "index": { - "refresh_interval": "5s", - "number_of_shards": 1 } }, "mappings": { diff --git a/esrally/resources/races-template.json b/esrally/resources/races-template.json index c27eb74b0..dec764cca 100644 --- a/esrally/resources/races-template.json +++ b/esrally/resources/races-template.json @@ -2,8 +2,6 @@ "index_patterns": ["rally-races-*"], "settings": { "index": { - "refresh_interval": "5s", - "number_of_shards": 1 } }, "mappings": { diff --git a/esrally/resources/results-template.json b/esrally/resources/results-template.json index f0ed8cc1c..6c39ca853 100644 --- a/esrally/resources/results-template.json +++ b/esrally/resources/results-template.json @@ -2,8 +2,6 @@ "index_patterns": ["rally-results-*"], "settings": { "index": { - "refresh_interval": "5s", - "number_of_shards": 1 } }, "mappings": { diff --git a/tests/metrics_test.py b/tests/metrics_test.py index 62853ad9c..db20afa0f 100644 --- a/tests/metrics_test.py +++ b/tests/metrics_test.py @@ -2340,10 +2340,10 @@ def setUp(self): self.cfg.add(config.Scope.application, "system", "time.start", FileRaceStoreTests.RACE_TIMESTAMP) self.cfg.add(config.Scope.application, "system", "race.id", FileRaceStoreTests.RACE_ID) - def test_datastore_type_elasticsearch_index_template_update(self): + def test_primary_and_replica_shard_count_specified_index_template_update(self): _datastore_type = "elasticsearch" _datastore_number_of_shards = random.randint(1, 100) - _datastore_number_of_replicas = random.randint(1, 100) + _datastore_number_of_replicas = random.randint(0, 100) self.cfg.add(config.Scope.applicationOverride, "reporting", "datastore.type", _datastore_type) self.cfg.add(config.Scope.applicationOverride, "reporting", "datastore.number_of_shards", _datastore_number_of_shards) @@ -2361,3 +2361,90 @@ def test_datastore_type_elasticsearch_index_template_update(self): t = json.loads(template) assert t["settings"]["index"]["number_of_shards"] == _datastore_number_of_shards assert t["settings"]["index"]["number_of_replicas"] == _datastore_number_of_replicas + + def test_primary_shard_count_specified_index_template_update(self): + _datastore_type = "elasticsearch" + _datastore_number_of_shards = random.randint(1, 100) + + self.cfg.add(config.Scope.applicationOverride, "reporting", "datastore.type", _datastore_type) + self.cfg.add(config.Scope.applicationOverride, "reporting", "datastore.number_of_shards", _datastore_number_of_shards) + + _index_template_provider = metrics.IndexTemplateProvider(self.cfg) + + templates = [ + _index_template_provider.metrics_template(), + _index_template_provider.races_template(), + _index_template_provider.results_template(), + ] + + for template in templates: + t = json.loads(template) + assert t["settings"]["index"]["number_of_shards"] == _datastore_number_of_shards + with self.assertRaises(KeyError): + # pylint: disable=unused-variable + number_of_replicas = t["settings"]["index"]["number_of_replicas"] + + def test_replica_shard_count_specified_index_template_update(self): + _datastore_type = "elasticsearch" + _datastore_number_of_replicas = random.randint(1, 100) + + self.cfg.add(config.Scope.applicationOverride, "reporting", "datastore.type", _datastore_type) + self.cfg.add(config.Scope.applicationOverride, "reporting", "datastore.number_of_replicas", _datastore_number_of_replicas) + + _index_template_provider = metrics.IndexTemplateProvider(self.cfg) + + templates = [ + _index_template_provider.metrics_template(), + _index_template_provider.races_template(), + _index_template_provider.results_template(), + ] + + for template in templates: + t = json.loads(template) + assert t["settings"]["index"]["number_of_replicas"] == _datastore_number_of_replicas + with self.assertRaises(KeyError): + # pylint: disable=unused-variable + number_of_shards = t["settings"]["index"]["number_of_shards"] + + def test_primary_shard_count_less_than_one(self): + _datastore_type = "elasticsearch" + _datastore_number_of_shards = 0 + + self.cfg.add(config.Scope.applicationOverride, "reporting", "datastore.type", _datastore_type) + self.cfg.add(config.Scope.applicationOverride, "reporting", "datastore.number_of_shards", _datastore_number_of_shards) + _index_template_provider = metrics.IndexTemplateProvider(self.cfg) + + with self.assertRaises(exceptions.SystemSetupError) as ctx: + # pylint: disable=unused-variable + templates = [ + _index_template_provider.metrics_template(), + _index_template_provider.races_template(), + _index_template_provider.results_template(), + ] + self.assertEqual( + "The setting: datastore.number_of_shards must be >= 1. Please check the configuration in " + f"{_index_template_provider._config.config_file.location}", + ctx.exception.args[0], + ) + + def test_primary_and_replica_shard_counts_passed_as_strings(self): + _datastore_type = "elasticsearch" + _datastore_number_of_shards = "200" + _datastore_number_of_replicas = "1" + + self.cfg.add(config.Scope.applicationOverride, "reporting", "datastore.type", _datastore_type) + self.cfg.add(config.Scope.applicationOverride, "reporting", "datastore.number_of_shards", _datastore_number_of_shards) + self.cfg.add(config.Scope.applicationOverride, "reporting", "datastore.number_of_replicas", _datastore_number_of_replicas) + + _index_template_provider = metrics.IndexTemplateProvider(self.cfg) + + templates = [ + _index_template_provider.metrics_template(), + _index_template_provider.races_template(), + _index_template_provider.results_template(), + ] + + for template in templates: + t = json.loads(template) + assert t["settings"]["index"]["number_of_shards"] == 200 + assert t["settings"]["index"]["number_of_replicas"] == 1