Skip to content

Commit

Permalink
✨Meet the clusters-keeper service (⚠️ devops) (#4591)
Browse files Browse the repository at this point in the history
  • Loading branch information
@sanderegg
sanderegg authored Aug 9, 2023
1 parent 330fa2f commit 74636e0
Show file tree
Hide file tree
Showing 61 changed files with 3,399 additions and 51 deletions.
1 change: 1 addition & 0 deletions .github/CODEOWNERS
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ Makefile @pcrespov @sanderegg
/services/api-server/ @pcrespov
/services/autoscaling/ @sanderegg @pcrespov
/services/catalog/ @pcrespov @sanderegg
/services/clusters-keeper/ @sanderegg
/services/datcore-adapter/ @sanderegg
/services/director*/ @sanderegg @pcrespov @GitHK
/services/docker-compose*.yml @sanderegg @mrnicegyu11 @YuryHrytsuk
Expand Down
63 changes: 63 additions & 0 deletions .github/workflows/ci-testing-deploy.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@ jobs:
api-server: ${{ steps.filter.outputs.api-server }}
autoscaling: ${{ steps.filter.outputs.autoscaling }}
catalog: ${{ steps.filter.outputs.catalog }}
clusters-keeper: ${{ steps.filter.outputs.clusters-keeper }}
dask-sidecar: ${{ steps.filter.outputs.dask-sidecar }}
datcore-adapter: ${{ steps.filter.outputs.datcore-adapter }}
director: ${{ steps.filter.outputs.director }}
Expand Down Expand Up @@ -147,6 +148,12 @@ jobs:
- 'services/docker-compose*'
- 'scripts/mypy/*'
- 'mypy.ini'
clusters-keeper:
- 'packages/**'
- 'services/clusters-keeper/**'
- 'services/docker-compose*'
- 'scripts/mypy/*'
- 'mypy.ini'
dask-sidecar:
- 'packages/**'
- 'services/dask-sidecar/**'
Expand Down Expand Up @@ -666,6 +673,61 @@ jobs:
with:
flags: unittests #optional

unit-test-clusters-keeper:
needs: changes
if: ${{ needs.changes.outputs.clusters-keeper == 'true' || github.event_name == 'push' }}
timeout-minutes: 18 # if this timeout gets too small, then split the tests
name: "[unit] clusters-keeper"
runs-on: ${{ matrix.os }}
strategy:
matrix:
python: ["3.10"]
os: [ubuntu-22.04]
docker_buildx: [v0.10.4]
docker_compose: [1.29.1]
include:
- docker_compose: 1.29.1
docker_compose_sha: 8097769d32e34314125847333593c8edb0dfc4a5b350e4839bef8c2fe8d09de7
fail-fast: false
steps:
- uses: actions/checkout@v3
- name: setup docker buildx
id: buildx
uses: docker/setup-buildx-action@v2
with:
version: ${{ matrix.docker_buildx }}
driver: docker-container
- name: setup docker-compose
run: sudo ./ci/github/helpers/setup_docker_compose.bash ${{ matrix.docker_compose }} ${{ matrix.docker_compose_sha }}
- name: setup python environment
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python }}
cache: "pip"
cache-dependency-path: "services/clusters-keeper/requirements/ci.txt"
- name: show system version
run: ./ci/helpers/show_system_versions.bash
- name: install
run: |
make devenv
source .venv/bin/activate && \
pushd services/clusters-keeper && \
make install-ci
- name: typecheck
run: |
source .venv/bin/activate && \
pushd services/clusters-keeper && \
make mypy
- name: test
if: always()
run: |
source .venv/bin/activate && \
pushd services/clusters-keeper && \
make test-ci-unit
- uses: codecov/codecov-action@v3.1.4
with:
flags: unittests #optional

unit-test-datcore-adapter:
needs: changes
if: ${{ needs.changes.outputs.datcore-adapter == 'true' || github.event_name == 'push' }}
Expand Down Expand Up @@ -1458,6 +1520,7 @@ jobs:
unit-test-api-server,
unit-test-autoscaling,
unit-test-catalog,
unit-test-clusters-keeper,
unit-test-dask-sidecar,
unit-test-dask-task-models-library,
unit-test-datcore-adapter,
Expand Down
102 changes: 51 additions & 51 deletions .ruff.toml
View file
Original file line number Diff line number Diff line change
@@ -1,73 +1,73 @@

select = [
"A", # [https://pypi.org/project/flake8-builtins/]
"ARG", # [https://pypi.org/project/flake8-unused-arguments/]
"ASYNC", # [https://pypi.org/project/flake8-async/]
"B", # [https://pypi.org/project/flake8-bugbear/]
"C4", # [https://pypi.org/project/flake8-comprehensions/]
"C90", # [https://pypi.org/project/mccabe/] (complexity)
"DTZ", # [https://pypi.org/project/flake8-datetimez/]
"E", # [https://pypi.org/project/pycodestyle/] errors
"EM", # [https://pypi.org/project/flake8-errmsg/]
"ERA", # [https://pypi.org/project/eradicate/]
"F", # [https://pypi.org/project/pyflakes/]
"FBT", # [https://pypi.org/project/flake8-boolean-trap/]
"FIX", # [https://github.com/tommilligan/flake8-fixme]
"G", # [https://pypi.org/project/flake8-logging-format/0.9.0/]
"I", # [https://pypi.org/project/isort/]
"ICN", # [https://github.com/joaopalmeiro/flake8-import-conventions]
"ISC", # [https://pypi.org/project/flake8-implicit-str-concat/]
"N", # [https://pypi.org/project/pep8-naming/]
"NPY", # NumPy-speficic rules
"PERF", # [https://pypi.org/project/perflint/]
"PIE", # [https://pypi.org/project/flake8-pie/]
"PL", # [https://pypi.org/project/pylint/]
"PT", # [https://pypi.org/project/flake8-pytest-style/]
"PTH", # [https://pypi.org/project/flake8-use-pathlib/]
"PYI", # [https://pypi.org/project/flake8-pyi/]
"RET", # [https://pypi.org/project/flake8-return/]
"RSE", # [https://pypi.org/project/flake8-raise/]
"RUF", # RUFF-specific rules
"S", # [https://pypi.org/project/flake8-bandit/] (Automated security testing)
"SIM", # [https://pypi.org/project/flake8-simplify/]
"SLF", # [https://pypi.org/project/flake8-self/]
"SLOT", # [https://pypi.org/project/flake8-slots/]
"T10", # https://pypi.org/project/flake8-debugger/
"T20", # [https://pypi.org/project/flake8-print/]
"TID", # [https://pypi.org/project/flake8-tidy-imports/]
"TRY", # [https://pypi.org/project/tryceratops/1.1.0/] (exception anti-patterns)
"UP", # [https://pypi.org/project/pyupgrade/]
"W", # [https://pypi.org/project/pycodestyle/] warnings
"YTT", # [https://pypi.org/project/flake8-2020/]
"A", # [https://pypi.org/project/flake8-builtins/]
"ARG", # [https://pypi.org/project/flake8-unused-arguments/]
"ASYNC", # [https://pypi.org/project/flake8-async/]
"B", # [https://pypi.org/project/flake8-bugbear/]
"C4", # [https://pypi.org/project/flake8-comprehensions/]
"C90", # [https://pypi.org/project/mccabe/] (complexity)
"DTZ", # [https://pypi.org/project/flake8-datetimez/]
"E", # [https://pypi.org/project/pycodestyle/] errors
"EM", # [https://pypi.org/project/flake8-errmsg/]
"ERA", # [https://pypi.org/project/eradicate/]
"F", # [https://pypi.org/project/pyflakes/]
"FBT", # [https://pypi.org/project/flake8-boolean-trap/]
"FIX", # [https://github.com/tommilligan/flake8-fixme]
"G", # [https://pypi.org/project/flake8-logging-format/0.9.0/]
"I", # [https://pypi.org/project/isort/]
"ICN", # [https://github.com/joaopalmeiro/flake8-import-conventions]
"ISC", # [https://pypi.org/project/flake8-implicit-str-concat/]
"N", # [https://pypi.org/project/pep8-naming/]
"NPY", # NumPy-speficic rules
"PERF", # [https://pypi.org/project/perflint/]
"PIE", # [https://pypi.org/project/flake8-pie/]
"PL", # [https://pypi.org/project/pylint/]
"PT", # [https://pypi.org/project/flake8-pytest-style/]
"PTH", # [https://pypi.org/project/flake8-use-pathlib/]
"PYI", # [https://pypi.org/project/flake8-pyi/]
"RET", # [https://pypi.org/project/flake8-return/]
"RSE", # [https://pypi.org/project/flake8-raise/]
"RUF", # RUFF-specific rules
"S", # [https://pypi.org/project/flake8-bandit/] (Automated security testing)
"SIM", # [https://pypi.org/project/flake8-simplify/]
"SLF", # [https://pypi.org/project/flake8-self/]
"SLOT", # [https://pypi.org/project/flake8-slots/]
"T10", # https://pypi.org/project/flake8-debugger/
"T20", # [https://pypi.org/project/flake8-print/]
"TID", # [https://pypi.org/project/flake8-tidy-imports/]
"TRY", # [https://pypi.org/project/tryceratops/1.1.0/] (exception anti-patterns)
"UP", # [https://pypi.org/project/pyupgrade/]
"W", # [https://pypi.org/project/pycodestyle/] warnings
"YTT", # [https://pypi.org/project/flake8-2020/]
]
ignore = [
"E501", # line too long, handled by black
"S101", # use of `assert` detected hanbled by pylance, does not support noseq
"TID252", # [*] Relative imports from parent modules are banned
"TRY300", # Checks for return statements in try blocks. SEE https://beta.ruff.rs/docs/rules/try-consider-else/
"E501", # line too long, handled by black
"S101", # use of `assert` detected hanbled by pylance, does not support noseq
"TID252", # [*] Relative imports from parent modules are banned
"TRY300", # Checks for return statements in try blocks. SEE https://beta.ruff.rs/docs/rules/try-consider-else/
]

target-version = "py310"


[per-file-ignores]
"**/tests/**" = [
"T201", # print found
"ARG001", # unused function argument
"PT019", # user pytest.mark.usefixture
"T201", # print found
"ARG001", # unused function argument
"PT019", # user pytest.mark.usefixture
"PLR2004", # use of magic values
"PLR0913", # too many arguments
"N806", # Uppercase variables in functions
"PT001", # use pytest.fixture over pytest.fixture() whatsoever
"ERA001", # found commented out code
"FBT001", # Boolean positional arg in function definition
"N806", # Uppercase variables in functions
"PT001", # use pytest.fixture over pytest.fixture() whatsoever
"PT004", # does not return anythin, add leading underscore
"ERA001", # found commented out code
"FBT001", # Boolean positional arg in function definition
]

[flake8-pytest-style]
fixture-parentheses = false
parametrize-names-type = "csv"



[pylint]
max-args = 10
1 change: 1 addition & 0 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ SERVICES_NAMES_TO_BUILD := \
api-server \
autoscaling \
catalog \
clusters-keeper \
dask-sidecar \
datcore-adapter \
director \
Expand Down
1 change: 1 addition & 0 deletions packages/pytest-simcore/src/pytest_simcore/simcore_services.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
DEFAULT_SERVICE_HEALTHCHECK_ENTRYPOINT = "/v0/"
MAP_SERVICE_HEALTHCHECK_ENTRYPOINT = {
"autoscaling": "/",
"clusters-keeper": "/",
"dask-scheduler": "/health",
"datcore-adapter": "/v0/live",
"director-v2": "/",
Expand Down
Loading

0 comments on commit 74636e0

Please sign in to comment.