From 97211a882db7433899c696586d47c3bc16b225c6 Mon Sep 17 00:00:00 2001 From: Manuel Giffels Date: Wed, 26 Feb 2020 11:37:57 +0100 Subject: [PATCH 1/4] Add yaml constructors and adjust documentation accordingly --- docs/source/changelog.rst | 4 ++-- docs/source/executors/executors.rst | 4 ++-- setup.py | 8 ++++++-- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst index ccd0c0be..30c5d28c 100644 --- a/docs/source/changelog.rst +++ b/docs/source/changelog.rst @@ -1,4 +1,4 @@ -.. Created by changelog.py at 2020-02-24, command +.. Created by changelog.py at 2020-02-26, command '/Users/giffler/.cache/pre-commit/repont7o94ca/py_env-default/bin/changelog docs/source/changes compile --output=docs/source/changelog.rst' based on the format of 'https://keepachangelog.com/' @@ -6,7 +6,7 @@ CHANGELOG ######### -[Unreleased] - 2020-02-24 +[Unreleased] - 2020-02-26 ========================= Added diff --git a/docs/source/executors/executors.rst b/docs/source/executors/executors.rst index 711d4442..f0a1398c 100644 --- a/docs/source/executors/executors.rst +++ b/docs/source/executors/executors.rst @@ -28,7 +28,7 @@ Shell Executor .. code-block:: yaml - !ShellExecutor + !TardisShellExecutor .. rubric:: Example configuration (`COBalD` legacy object initialisation) @@ -53,7 +53,7 @@ SSH Executor .. code-block:: yaml - !SSHExecutor + !TardisSSHExecutor host: login.dorie.somewherein.de username: clown client_keys: diff --git a/setup.py b/setup.py index b4005f50..20828acb 100644 --- a/setup.py +++ b/setup.py @@ -40,7 +40,11 @@ ], entry_points={ "cobald.config.yaml_constructors": [ - "TardisPoolFactory = tardis.resources.poolfactory:create_composite_pool" + "TardisPoolFactory = tardis.resources.poolfactory:create_composite_pool", + "TardisPeriodicValue = tardis.utilities.simulators.periodicvalue:PeriodicValue", + "TardisRandomGauss = tardis.utilities.simulators.randomgauss:RandomGauss", + "TardisSSHExecutor = tardis.utilities.executors.sshexecutor:SSHExecutor", + "TardisShellExecutor = tardis.utilities.executors.shellexecutor:ShellExecutor", ], "cobald.config.sections": [ "tardis = tardis.configuration.configuration:Configuration" @@ -53,7 +57,7 @@ "CloudStackAIO", "PyYAML", "AsyncOpenStackClient", - "cobald", + "cobald>=0.12.0", "asyncssh", "aiotelegraf", ], From 6cc71f52f0abb6044369f78accb0a1b9ced3a896 Mon Sep 17 00:00:00 2001 From: Manuel Giffels Date: Wed, 26 Feb 2020 11:38:56 +0100 Subject: [PATCH 2/4] Apply plugin constraint decorator to change order of loading entry points --- tardis/configuration/configuration.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tardis/configuration/configuration.py b/tardis/configuration/configuration.py index 76498b99..df8b64d2 100644 --- a/tardis/configuration/configuration.py +++ b/tardis/configuration/configuration.py @@ -7,6 +7,7 @@ from ..utilities.simulators import * # noqa: F403, F401 from cobald.daemon.config.mapping import Translator +from cobald.daemon.plugins import constraints as plugin_constraints from base64 import b64encode import os @@ -31,6 +32,7 @@ def translate_config(obj): return obj +@plugin_constraints(before={"pipeline"}) class Configuration(Borg): _shared_state = AttributeDict() From 21e36e4ec4998ad61d07e5bae1bebca6a4761a7b Mon Sep 17 00:00:00 2001 From: Manuel Giffels Date: Wed, 26 Feb 2020 16:22:02 +0100 Subject: [PATCH 3/4] Bump version to 0.3.0 --- docs/source/changelog.rst | 4 ++-- .../100.fix_transition_within_cleanupstate.yaml | 8 +++++--- docs/source/changes/116.adding_drone_life_time.yaml | 11 ++++++----- .../changes/119.add_legacy_object_initialisation.yaml | 11 ++++++----- docs/source/changes/120.add_python38_support.yaml | 7 ++++--- .../122.fix_state_transitions_for_retried_jobs.yaml | 5 ++--- .../changes/123.fix_draining_slots_w_startd_name.yaml | 5 +++-- .../source/changes/125.add_unified_configuration.yaml | 7 ++++--- docs/source/changes/126.fix_config_translation.yaml | 10 +++++----- ..._use_utilisation_as_weight_in_composite_pools.yaml | 7 ++++--- .../128.fix_state_transitions_in_slurm_adapter.yaml | 7 ++++--- .../changes/76.allow_removal_of_booting_drones.yaml | 7 ++++--- docs/source/changes/77.telegraf_config_update.yaml | 8 +++++--- .../78.improve_logging_of_htcondor_batch_adapter.yaml | 8 +++++--- .../92.register_pool_factory_as_yaml_plugin.yaml | 11 ++++++----- .../96.fix_handling_of_vanished_resources.yaml | 7 ++++--- docs/source/changes/versions.yaml | 2 ++ tardis/__about__.py | 2 +- 18 files changed, 72 insertions(+), 55 deletions(-) create mode 100644 docs/source/changes/versions.yaml diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst index 30c5d28c..3f2e5409 100644 --- a/docs/source/changelog.rst +++ b/docs/source/changelog.rst @@ -6,8 +6,8 @@ CHANGELOG ######### -[Unreleased] - 2020-02-26 -========================= +[0.3.0] - 2020-02-26 +==================== Added ----- diff --git a/docs/source/changes/100.fix_transition_within_cleanupstate.yaml b/docs/source/changes/100.fix_transition_within_cleanupstate.yaml index 5f824ef1..5a9a879d 100644 --- a/docs/source/changes/100.fix_transition_within_cleanupstate.yaml +++ b/docs/source/changes/100.fix_transition_within_cleanupstate.yaml @@ -1,7 +1,9 @@ category: fixed -summary: The `CleanupState` is now taking into account the status of the resource for state transitions -pull requests: - - 100 +summary: "The `CleanupState` is now taking into account the status of the resource\ + \ for state transitions" description: | The `CleanupState` is now taking into account the status of the resource and two missing state transitions have been added to the transistion dictionary. +pull requests: +- 100 +version: 0.3.0 diff --git a/docs/source/changes/116.adding_drone_life_time.yaml b/docs/source/changes/116.adding_drone_life_time.yaml index 453de816..3d271659 100644 --- a/docs/source/changes/116.adding_drone_life_time.yaml +++ b/docs/source/changes/116.adding_drone_life_time.yaml @@ -1,9 +1,10 @@ category: added -summary: An optional and per site configurable drone minimum lifetime has been added -pull requests: - - 116 -issues: - - 114 +summary: "An optional and per site configurable drone minimum lifetime has been added" description: | An optionally and per site configurable drone minimum life time to TARDIS. The minimum life time is defined as the time the drone is being in `AvailableState`. At the end of life, the drone is set to `DrainState`. +issues: +- 114 +pull requests: +- 116 +version: 0.3.0 diff --git a/docs/source/changes/119.add_legacy_object_initialisation.yaml b/docs/source/changes/119.add_legacy_object_initialisation.yaml index 1fb244d5..7f46ce3f 100644 --- a/docs/source/changes/119.add_legacy_object_initialisation.yaml +++ b/docs/source/changes/119.add_legacy_object_initialisation.yaml @@ -1,9 +1,10 @@ category: added -summary: Add support for COBalD legacy object initialisation -pull requests: - - 119 -issues: - - 118 +summary: "Add support for COBalD legacy object initialisation" description: | Support for the `COBalD` object initialisation syntax to construct python objects instead of using YAML tags has been enabled. +issues: +- 118 +pull requests: +- 119 +version: 0.3.0 diff --git a/docs/source/changes/120.add_python38_support.yaml b/docs/source/changes/120.add_python38_support.yaml index 5ec81026..977c7861 100644 --- a/docs/source/changes/120.add_python38_support.yaml +++ b/docs/source/changes/120.add_python38_support.yaml @@ -1,6 +1,7 @@ category: added -summary: Add support for Python 3.8 -pull requests: - - 120 +summary: "Add support for Python 3.8" description: | Support for Python3.8 added to setup.py classifiers and travis uniitests for Python3.8 have been enabled. +pull requests: +- 120 +version: 0.3.0 diff --git a/docs/source/changes/122.fix_state_transitions_for_retried_jobs.yaml b/docs/source/changes/122.fix_state_transitions_for_retried_jobs.yaml index 2faa2829..930b2524 100644 --- a/docs/source/changes/122.fix_state_transitions_for_retried_jobs.yaml +++ b/docs/source/changes/122.fix_state_transitions_for_retried_jobs.yaml @@ -1,9 +1,8 @@ category: fixed -summary: Fix state transitions for jobs retried by HTCondor -pull_requests: - - 122 +summary: "Fix state transitions for jobs retried by HTCondor" description: | In case a job is in `DrainingState`, `ShuttingDownState` or `ShutDownState` a transition back to BootingState is currently not covered by `TARDIS`. However, a currently running job can potentially be retried by HTCondor in case of an internal error. Now, retried jobs previously in `DrainingState`, `ShuttingDownState` or `ShutDownState` are removed entirely from the batch queue. +version: 0.3.0 diff --git a/docs/source/changes/123.fix_draining_slots_w_startd_name.yaml b/docs/source/changes/123.fix_draining_slots_w_startd_name.yaml index 02e3fa02..091930e9 100644 --- a/docs/source/changes/123.fix_draining_slots_w_startd_name.yaml +++ b/docs/source/changes/123.fix_draining_slots_w_startd_name.yaml @@ -1,8 +1,9 @@ category: fixed summary: "Fix draining of slots having a startd name" -pull requests: - - 123 description: | `TARDIS` supports running more than one Drone on the same host. To differentiate the drones, the `TardisDroneUuid` has been introduced and will be set as `STARTD_NAME` on the corresponding sites. `TARDIS` used to use only the `hostname` when draining resources. This change will take into account the provided `STARTD_NAME` as well. +pull requests: +- 123 +version: 0.3.0 diff --git a/docs/source/changes/125.add_unified_configuration.yaml b/docs/source/changes/125.add_unified_configuration.yaml index 74847ac0..725152f0 100644 --- a/docs/source/changes/125.add_unified_configuration.yaml +++ b/docs/source/changes/125.add_unified_configuration.yaml @@ -1,6 +1,7 @@ category: added -summary: Add the possibility to use an unified `COBalD` and `TARDIS` configuration -pull requests: - - 125 +summary: "Add the possibility to use an unified `COBalD` and `TARDIS` configuration" description: | The possibility to combine the `COBalD` and the `TARDIS` configuration in one single `yaml` has been added. +pull requests: +- 125 +version: 0.3.0 diff --git a/docs/source/changes/126.fix_config_translation.yaml b/docs/source/changes/126.fix_config_translation.yaml index d25fd884..000eac96 100644 --- a/docs/source/changes/126.fix_config_translation.yaml +++ b/docs/source/changes/126.fix_config_translation.yaml @@ -1,7 +1,7 @@ category: fixed -summary: Fix the translation of cloud init scripts into base64 encoded strings +summary: "Fix the translation of cloud init scripts into base64 encoded strings" +description: "The translation of cloud init scripts into base64 encoded strings has\ + \ been fixed. Reason was changing a data structure\nwhile interating over it. \n" pull requests: - - 126 -description: | - The translation of cloud init scripts into base64 encoded strings has been fixed. Reason was changing a data structure - while interating over it. +- 126 +version: 0.3.0 diff --git a/docs/source/changes/127.fix_use_utilisation_as_weight_in_composite_pools.yaml b/docs/source/changes/127.fix_use_utilisation_as_weight_in_composite_pools.yaml index 252e0340..d171a815 100644 --- a/docs/source/changes/127.fix_use_utilisation_as_weight_in_composite_pools.yaml +++ b/docs/source/changes/127.fix_use_utilisation_as_weight_in_composite_pools.yaml @@ -1,8 +1,9 @@ category: fixed -summary: Use utilisation as weight in composite pools -pull requests: - - 127 +summary: "Use utilisation as weight in composite pools" description: | The experience of using `WeightedComposite` pools in production shows that weighting the `demand` of resources by their `supply` is not always the best choice. Weighting the `demand` of resources by `utilisation` is instead more desired, since `COBalD` should increase the best utilised resource. +pull requests: +- 127 +version: 0.3.0 diff --git a/docs/source/changes/128.fix_state_transitions_in_slurm_adapter.yaml b/docs/source/changes/128.fix_state_transitions_in_slurm_adapter.yaml index 0050b73a..d279d086 100644 --- a/docs/source/changes/128.fix_state_transitions_in_slurm_adapter.yaml +++ b/docs/source/changes/128.fix_state_transitions_in_slurm_adapter.yaml @@ -1,9 +1,10 @@ category: fixed -summary: Fix state transitions and refactoring of the SLURM site adapter -pull requests: - - 128 +summary: "Fix state transitions and refactoring of the SLURM site adapter" description: | The SLURM site adapter contains now state transition for all possible job states in the SLURM batch system. For some states the transition to `TARDIS` `resources_states` has been corrected leading to an infinite execution of `scancel` to jobs not in the queue anymore. In addtion, `squeue` (jobs in batch queue) instead of the `sacct` (Accounting tool listing also history of jobs) is used in order to minize the number of jobs to keep track of. +pull requests: +- 128 +version: 0.3.0 diff --git a/docs/source/changes/76.allow_removal_of_booting_drones.yaml b/docs/source/changes/76.allow_removal_of_booting_drones.yaml index 70a587f7..6a2924f6 100644 --- a/docs/source/changes/76.allow_removal_of_booting_drones.yaml +++ b/docs/source/changes/76.allow_removal_of_booting_drones.yaml @@ -1,8 +1,9 @@ category: fixed -summary: Allow removal of booting drones if demand drops to zero -pull requests: - - 76 +summary: "Allow removal of booting drones if demand drops to zero" description: | So far draining of drones in `AvailableState` was supported by `TARDIS`, which works well for cloud providers. Since the queueing time in batch systems can be long, the deletion of queued drones (idle batch jobs) has been enabled as well. +pull requests: +- 76 +version: 0.3.0 diff --git a/docs/source/changes/77.telegraf_config_update.yaml b/docs/source/changes/77.telegraf_config_update.yaml index edc44f77..501e3017 100644 --- a/docs/source/changes/77.telegraf_config_update.yaml +++ b/docs/source/changes/77.telegraf_config_update.yaml @@ -1,7 +1,9 @@ category: added -summary: The machine name has been added as a default tag in the telegraf monitoring plugin, can be overwritten. -pull requests: - - 77 +summary: "The machine name has been added as a default tag in the telegraf monitoring\ + \ plugin, can be overwritten." description: | The machine name has been added as a default tag in the telegraf monitoring plugin. Can be overwritten by adding `tardis_machine_name` as default tag in the plugin configuration. +pull requests: +- 77 +version: 0.3.0 diff --git a/docs/source/changes/78.improve_logging_of_htcondor_batch_adapter.yaml b/docs/source/changes/78.improve_logging_of_htcondor_batch_adapter.yaml index 5e0d830f..fc1653c1 100644 --- a/docs/source/changes/78.improve_logging_of_htcondor_batch_adapter.yaml +++ b/docs/source/changes/78.improve_logging_of_htcondor_batch_adapter.yaml @@ -1,7 +1,9 @@ category: fixed -summary: Improved logging of the `HTCondor` batch system adapter and the status changes of the drones -pull requests: - - 78 +summary: "Improved logging of the `HTCondor` batch system adapter and the status changes\ + \ of the drones" description: | Drone resource attributes to have been addedd to the debug logging of drone status changes. The `condor_status` command has been added to the debug logging of the htcondor batch system adapter. +pull requests: +- 78 +version: 0.3.0 diff --git a/docs/source/changes/92.register_pool_factory_as_yaml_plugin.yaml b/docs/source/changes/92.register_pool_factory_as_yaml_plugin.yaml index 5526f47c..65efd1c4 100644 --- a/docs/source/changes/92.register_pool_factory_as_yaml_plugin.yaml +++ b/docs/source/changes/92.register_pool_factory_as_yaml_plugin.yaml @@ -1,9 +1,10 @@ category: added -summary: Register pool factory as `COBalD` yaml plugin -pull requests: - - 92 -issues: - - 91 +summary: "Register pool factory as `COBalD` yaml plugin" description: | Register the composite pool factory function `create_composite_pool` as `COBalD` yaml plugin, so that it can be loaded via yaml tags in the `COBalD` configuration. +issues: +- 91 +pull requests: +- 92 +version: 0.3.0 diff --git a/docs/source/changes/96.fix_handling_of_vanished_resources.yaml b/docs/source/changes/96.fix_handling_of_vanished_resources.yaml index 9807d51d..a47b86b3 100644 --- a/docs/source/changes/96.fix_handling_of_vanished_resources.yaml +++ b/docs/source/changes/96.fix_handling_of_vanished_resources.yaml @@ -1,8 +1,9 @@ category: fixed -summary: Fix the handling of the termination of vanished resources -pull requests: - - 96 +summary: "Fix the handling of the termination of vanished resources" description: | Fix a bug with resources that shut down itself while being idle for 20 minutes, which remained in `CleanUpState` forever, since the `teminate_resource` call keept raising `TardisResourceStatusUpdateFailed`, which led to infinte retries of the `terminate_resource` call. +pull requests: +- 96 +version: 0.3.0 diff --git a/docs/source/changes/versions.yaml b/docs/source/changes/versions.yaml new file mode 100644 index 00000000..c3b8059f --- /dev/null +++ b/docs/source/changes/versions.yaml @@ -0,0 +1,2 @@ +- semver: 0.3.0 + date: '2020-02-26' diff --git a/tardis/__about__.py b/tardis/__about__.py index 70ca7797..896682aa 100644 --- a/tardis/__about__.py +++ b/tardis/__about__.py @@ -3,7 +3,7 @@ __summary__ = "Transparent Adaptive Resource Dynamic Integration System" __url__ = "https://github.com/matterminers/tardis" -__version__ = "0.2.0" +__version__ = "0.3.0" __author__ = "Manuel Giffels, Matthias Schnepf" __email__ = "giffels@gmail.com" __copyright__ = "2019 %s" % __author__ From 446a9efdd63449723762decfc9d9ebac579491a3 Mon Sep 17 00:00:00 2001 From: Manuel Giffels Date: Wed, 26 Feb 2020 17:01:10 +0100 Subject: [PATCH 4/4] Make flake8 happy --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 20828acb..a0de4ea3 100644 --- a/setup.py +++ b/setup.py @@ -41,10 +41,10 @@ entry_points={ "cobald.config.yaml_constructors": [ "TardisPoolFactory = tardis.resources.poolfactory:create_composite_pool", - "TardisPeriodicValue = tardis.utilities.simulators.periodicvalue:PeriodicValue", + "TardisPeriodicValue = tardis.utilities.simulators.periodicvalue:PeriodicValue", # noqa: B950 "TardisRandomGauss = tardis.utilities.simulators.randomgauss:RandomGauss", "TardisSSHExecutor = tardis.utilities.executors.sshexecutor:SSHExecutor", - "TardisShellExecutor = tardis.utilities.executors.shellexecutor:ShellExecutor", + "TardisShellExecutor = tardis.utilities.executors.shellexecutor:ShellExecutor", # noqa: B950 ], "cobald.config.sections": [ "tardis = tardis.configuration.configuration:Configuration"