From 823b1e539e6705fcbbe201a4f135d30cd366727a Mon Sep 17 00:00:00 2001 From: Daniel Mitterdorfer Date: Mon, 18 Nov 2019 14:53:02 +0100 Subject: [PATCH 1/3] Store build.log in log directory With this commit we move Rally's build log from a race-specific directory to its log directory. This file captures Gradle's output when users are benchmarking a source build of Elasticsearch. By moving this file to the log directory it is better visible to users (already in the same location as other log files). --- docs/configuration.rst | 12 ++++++++++++ docs/migrate.rst | 15 +++++++++++++++ esrally/log.py | 14 ++++---------- esrally/mechanic/mechanic.py | 12 +++++------- esrally/mechanic/supplier.py | 6 +++--- esrally/paths.py | 6 ++++++ esrally/rally.py | 4 ++-- tests/mechanic/supplier_test.py | 10 +++++----- 8 files changed, 52 insertions(+), 27 deletions(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index 3b48e0146..25909ca95 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -135,6 +135,18 @@ The log file will not be rotated automatically as this is problematic due to Ral notifempty # don't attempt to rotate empty ones } +If you are benchmarking source builds of Elasticsearch, Rally will write the log output of the build to ``~/.rally/logs/build.log``. Similarly to the configuration for ``rally.log``, you can use the following ``logrotate`` configuration as a starting point:: + + /home/user/.rally/logs/build.log { + daily # rotate daily + rotate 7 # keep the last seven log files + maxage 14 # remove logs older than 14 days + compress # compress old logs ... + delaycompress # ... after moving them + missingok # ignore missing log files + notifempty # don't attempt to rotate empty ones + } + Example ~~~~~~~ diff --git a/docs/migrate.rst b/docs/migrate.rst index ede980485..f044f3438 100644 --- a/docs/migrate.rst +++ b/docs/migrate.rst @@ -4,6 +4,21 @@ Migration Guide Migrating to Rally 1.4.0 ------------------------ +Build logs are stored in Rally's log directory +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If you benchmark source builds of Elasticsearch, Rally has previously stored the build output log in a race-specific directory. With this release, Rally will store build logs in ``/home/user/.rally/logs/build.log``. We recommend to rotate logs with `logrotate `_. See the following example as a starting point for your own ``logrotate`` configuration and ensure to replace the path ``/home/user/.rally/logs/build.log`` with the proper one:: + + /home/user/.rally/logs/build.log { + daily # rotate daily + rotate 7 # keep the last seven log files + maxage 14 # remove logs older than 14 days + compress # compress old logs ... + delaycompress # ... after moving them + missingok # ignore missing log files + notifempty # don't attempt to rotate empty ones + } + Index size and Total Written are not included in the command line report ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/esrally/log.py b/esrally/log.py index c8f8db7e3..4016fb6de 100644 --- a/esrally/log.py +++ b/esrally/log.py @@ -22,6 +22,7 @@ import os import time +from esrally import paths from esrally.utils import io @@ -42,13 +43,6 @@ def log_config_path(): return os.path.join(os.path.expanduser("~"), ".rally", "logging.json") -def default_log_path(): - """ - :return: The absolute path to the directory that contains Rally's log file. - """ - return os.path.join(os.path.expanduser("~"), ".rally", "logs") - - def remove_obsolete_default_log_config(): """ Log rotation is problematic because Rally uses multiple processes and there is a lurking race condition when @@ -61,7 +55,7 @@ def remove_obsolete_default_log_config(): if io.exists(log_config): source_path = io.normalize_path(os.path.join(os.path.dirname(__file__), "resources", "logging_1_0_0.json")) with open(source_path, "r", encoding="UTF-8") as src: - contents = src.read().replace("${LOG_PATH}", default_log_path()) + contents = src.read().replace("${LOG_PATH}", paths.logs()) source_hash = hashlib.sha512(contents.encode()).hexdigest() with open(log_config, "r", encoding="UTF-8") as target: target_hash = hashlib.sha512(target.read().encode()).hexdigest() @@ -83,9 +77,9 @@ def install_default_log_config(): source_path = io.normalize_path(os.path.join(os.path.dirname(__file__), "resources", "logging.json")) with open(log_config, "w", encoding="UTF-8") as target: with open(source_path, "r", encoding="UTF-8") as src: - contents = src.read().replace("${LOG_PATH}", default_log_path()) + contents = src.read().replace("${LOG_PATH}", paths.logs()) target.write(contents) - io.ensure_dir(default_log_path()) + io.ensure_dir(paths.logs()) def load_configuration(): diff --git a/esrally/mechanic/mechanic.py b/esrally/mechanic/mechanic.py index 3b8be293e..05a76fa9e 100644 --- a/esrally/mechanic/mechanic.py +++ b/esrally/mechanic/mechanic.py @@ -30,11 +30,9 @@ def download(cfg): - challenge_root_path = paths.race_root(cfg) car, plugins = load_team(cfg, external=False) - s = supplier.create(cfg, sources=False, distribution=True, build=False, - challenge_root_path=challenge_root_path, car=car, plugins=plugins) + s = supplier.create(cfg, sources=False, distribution=True, build=False, car=car, plugins=plugins) binaries = s() console.println(json.dumps(binaries, indent=2), force=True) @@ -506,15 +504,15 @@ def load_team(cfg, external): def create(cfg, metrics_store, all_node_ips, all_node_ids, cluster_settings=None, sources=False, build=False, distribution=False, external=False, docker=False): races_root = paths.races_root(cfg) - challenge_root_path = paths.race_root(cfg) + race_root_path = paths.race_root(cfg) node_ids = cfg.opts("provisioning", "node.ids", mandatory=False) car, plugins = load_team(cfg, external) if sources or distribution: - s = supplier.create(cfg, sources, distribution, build, challenge_root_path, car, plugins) + s = supplier.create(cfg, sources, distribution, build, car, plugins) p = [] for node_id in node_ids: - p.append(provisioner.local_provisioner(cfg, car, plugins, cluster_settings, all_node_ips, all_node_ids, challenge_root_path, node_id)) + p.append(provisioner.local_provisioner(cfg, car, plugins, cluster_settings, all_node_ips, all_node_ids, race_root_path, node_id)) l = launcher.ProcessLauncher(cfg, metrics_store, races_root) elif external: raise exceptions.RallyAssertionError("Externally provisioned clusters should not need to be managed by Rally's mechanic") @@ -525,7 +523,7 @@ def create(cfg, metrics_store, all_node_ips, all_node_ids, cluster_settings=None s = lambda: None p = [] for node_id in node_ids: - p.append(provisioner.docker_provisioner(cfg, car, cluster_settings, challenge_root_path, node_id)) + p.append(provisioner.docker_provisioner(cfg, car, cluster_settings, race_root_path, node_id)) l = launcher.DockerLauncher(cfg, metrics_store) else: # It is a programmer error (and not a user error) if this function is called with wrong parameters diff --git a/esrally/mechanic/supplier.py b/esrally/mechanic/supplier.py index daa816a42..c642d4dff 100644 --- a/esrally/mechanic/supplier.py +++ b/esrally/mechanic/supplier.py @@ -21,7 +21,7 @@ import re import urllib.error -from esrally import exceptions, PROGRAM_NAME +from esrally import exceptions, paths, PROGRAM_NAME from esrally.exceptions import BuildError, SystemSetupError from esrally.utils import git, io, process, net, jvm, convert, sysstats @@ -29,7 +29,7 @@ REVISION_PATTERN = r"(\w.*?):(.*)" -def create(cfg, sources, distribution, build, challenge_root_path, car, plugins=None): +def create(cfg, sources, distribution, build, car, plugins=None): logger = logging.getLogger(__name__) if plugins is None: plugins = [] @@ -43,7 +43,7 @@ def create(cfg, sources, distribution, build, challenge_root_path, car, plugins= if build_needed: java_home = _java_home(car) es_src_dir = os.path.join(_src_dir(cfg), _config_value(src_config, "elasticsearch.src.subdir")) - builder = Builder(es_src_dir, java_home, challenge_root_path) + builder = Builder(es_src_dir, java_home, paths.logs()) else: builder = None diff --git a/esrally/paths.py b/esrally/paths.py index 80ab0744c..821b1049b 100644 --- a/esrally/paths.py +++ b/esrally/paths.py @@ -30,3 +30,9 @@ def race_root(cfg=None, race_id=None): race_id = cfg.opts("system", "race.id") return os.path.join(races_root(cfg), race_id) + +def logs(): + """ + :return: The absolute path to the directory that contains Rally's log file. + """ + return os.path.join(os.path.expanduser("~"), ".rally", "logs") diff --git a/esrally/rally.py b/esrally/rally.py index 9592c7250..8a8ac90b5 100644 --- a/esrally/rally.py +++ b/esrally/rally.py @@ -420,11 +420,11 @@ def print_help_on_errors(): heading = "Getting further help:" console.println(console.format.bold(heading)) console.println(console.format.underline_for(heading)) - console.println("* Check the log files in {} for errors.".format(log.default_log_path())) + console.println("* Check the log files in {} for errors.".format(paths.logs())) console.println("* Read the documentation at {}".format(console.format.link(doc_link()))) console.println("* Ask a question on the forum at {}".format(console.format.link("https://discuss.elastic.co/c/elasticsearch/rally"))) console.println("* Raise an issue at {} and include the log files in {}." - .format(console.format.link("https://github.com/elastic/rally/issues"), log.default_log_path())) + .format(console.format.link("https://github.com/elastic/rally/issues"), paths.logs())) def race(cfg): diff --git a/tests/mechanic/supplier_test.py b/tests/mechanic/supplier_test.py index 7b61bb3fe..0586e340c 100644 --- a/tests/mechanic/supplier_test.py +++ b/tests/mechanic/supplier_test.py @@ -392,7 +392,7 @@ def test_create_suppliers_for_es_only_config(self): car = team.Car("default", root_path=None, config_paths=[]) - composite_supplier = supplier.create(cfg, sources=False, distribution=True, build=False, challenge_root_path="/", car=car) + composite_supplier = supplier.create(cfg, sources=False, distribution=True, build=False, car=car) self.assertEqual(1, len(composite_supplier.suppliers)) self.assertIsInstance(composite_supplier.suppliers[0], supplier.ElasticsearchDistributionSupplier) @@ -414,7 +414,7 @@ def test_create_suppliers_for_es_distribution_plugin_source_skip(self): external_plugin = team.PluginDescriptor("community-plugin", core_plugin=False, variables={"enabled": True}) # --pipeline=from-sources-skip-build - composite_supplier = supplier.create(cfg, sources=True, distribution=False, build=False, challenge_root_path="/", car=car, plugins=[ + composite_supplier = supplier.create(cfg, sources=True, distribution=False, build=False, car=car, plugins=[ core_plugin, external_plugin ]) @@ -446,7 +446,7 @@ def test_create_suppliers_for_es_missing_distribution_plugin_source_skip(self): # --from-sources-skip-build --revision="community-plugin:current" (distribution version is missing!) with self.assertRaises(exceptions.SystemSetupError) as ctx: - supplier.create(cfg, sources=True, distribution=False, build=False, challenge_root_path="/", car=car, plugins=[ + supplier.create(cfg, sources=True, distribution=False, build=False, car=car, plugins=[ core_plugin, external_plugin ]) @@ -472,7 +472,7 @@ def test_create_suppliers_for_es_distribution_plugin_source_build(self): external_plugin = team.PluginDescriptor("community-plugin", core_plugin=False) # --revision="community-plugin:effab" --distribution-version="6.0.0" - composite_supplier = supplier.create(cfg, sources=False, distribution=True, build=False, challenge_root_path="/", car=car, plugins=[ + composite_supplier = supplier.create(cfg, sources=False, distribution=True, build=False, car=car, plugins=[ core_plugin, external_plugin ]) @@ -508,7 +508,7 @@ def test_create_suppliers_for_es_and_plugin_source_build(self): external_plugin = team.PluginDescriptor("community-plugin", core_plugin=False) # --revision="elasticsearch:abc,community-plugin:effab" - composite_supplier = supplier.create(cfg, sources=True, distribution=False, build=True, challenge_root_path="/", car=car, plugins=[ + composite_supplier = supplier.create(cfg, sources=True, distribution=False, build=True, car=car, plugins=[ core_plugin, external_plugin ]) From 679aef83ae2b12f85b7439f30b095c59728d1d13 Mon Sep 17 00:00:00 2001 From: Daniel Mitterdorfer Date: Mon, 18 Nov 2019 15:02:10 +0100 Subject: [PATCH 2/3] Fix existing pylint errors in code we touched --- esrally/log.py | 1 + esrally/mechanic/mechanic.py | 7 ++++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/esrally/log.py b/esrally/log.py index 4016fb6de..d2cddbf1d 100644 --- a/esrally/log.py +++ b/esrally/log.py @@ -26,6 +26,7 @@ from esrally.utils import io +# pylint: disable=unused-argument def configure_utc_formatter(*args, **kwargs): """ Logging formatter that renders timestamps in UTC to ensure consistent diff --git a/esrally/mechanic/mechanic.py b/esrally/mechanic/mechanic.py index 05a76fa9e..92a4b4878 100644 --- a/esrally/mechanic/mechanic.py +++ b/esrally/mechanic/mechanic.py @@ -501,8 +501,8 @@ def load_team(cfg, external): return car, plugins -def create(cfg, metrics_store, all_node_ips, all_node_ids, cluster_settings=None, sources=False, build=False, distribution=False, external=False, - docker=False): +def create(cfg, metrics_store, all_node_ips, all_node_ids, cluster_settings=None, sources=False, build=False, + distribution=False, external=False, docker=False): races_root = paths.races_root(cfg) race_root_path = paths.race_root(cfg) node_ids = cfg.opts("provisioning", "node.ids", mandatory=False) @@ -512,7 +512,8 @@ def create(cfg, metrics_store, all_node_ips, all_node_ids, cluster_settings=None s = supplier.create(cfg, sources, distribution, build, car, plugins) p = [] for node_id in node_ids: - p.append(provisioner.local_provisioner(cfg, car, plugins, cluster_settings, all_node_ips, all_node_ids, race_root_path, node_id)) + p.append( + provisioner.local_provisioner(cfg, car, plugins, cluster_settings, all_node_ips, all_node_ids, race_root_path, node_id)) l = launcher.ProcessLauncher(cfg, metrics_store, races_root) elif external: raise exceptions.RallyAssertionError("Externally provisioned clusters should not need to be managed by Rally's mechanic") From dd68f38f65058f1e9029ce8ade1e0d7071075de6 Mon Sep 17 00:00:00 2001 From: Daniel Mitterdorfer Date: Tue, 19 Nov 2019 11:24:38 +0100 Subject: [PATCH 3/3] Fix logrotate comments --- docs/configuration.rst | 21 ++++++++++++++------- docs/migrate.rst | 21 ++++++++++++++------- 2 files changed, 28 insertions(+), 14 deletions(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index 7f6b7a755..fa2a6dc0e 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -145,13 +145,20 @@ The log file will not be rotated automatically as this is problematic due to Ral If you are benchmarking source builds of Elasticsearch, Rally will write the log output of the build to ``~/.rally/logs/build.log``. Similarly to the configuration for ``rally.log``, you can use the following ``logrotate`` configuration as a starting point:: /home/user/.rally/logs/build.log { - daily # rotate daily - rotate 7 # keep the last seven log files - maxage 14 # remove logs older than 14 days - compress # compress old logs ... - delaycompress # ... after moving them - missingok # ignore missing log files - notifempty # don't attempt to rotate empty ones + # rotate daily + daily + # keep the last seven log files + rotate 7 + # remove logs older than 14 days + maxage 14 + # compress old logs ... + compress + # ... after moving them + delaycompress + # ignore missing log files + missingok + # don't attempt to rotate empty ones + notifempty } Example diff --git a/docs/migrate.rst b/docs/migrate.rst index f05a33176..870ca0139 100644 --- a/docs/migrate.rst +++ b/docs/migrate.rst @@ -10,13 +10,20 @@ Build logs are stored in Rally's log directory If you benchmark source builds of Elasticsearch, Rally has previously stored the build output log in a race-specific directory. With this release, Rally will store build logs in ``/home/user/.rally/logs/build.log``. We recommend to rotate logs with `logrotate `_. See the following example as a starting point for your own ``logrotate`` configuration and ensure to replace the path ``/home/user/.rally/logs/build.log`` with the proper one:: /home/user/.rally/logs/build.log { - daily # rotate daily - rotate 7 # keep the last seven log files - maxage 14 # remove logs older than 14 days - compress # compress old logs ... - delaycompress # ... after moving them - missingok # ignore missing log files - notifempty # don't attempt to rotate empty ones + # rotate daily + daily + # keep the last seven log files + rotate 7 + # remove logs older than 14 days + maxage 14 + # compress old logs ... + compress + # ... after moving them + delaycompress + # ignore missing log files + missingok + # don't attempt to rotate empty ones + notifempty } Index size and Total Written are not included in the command line report