From 44161d6ea56331cd3eaf653823b60ec5e6c47f85 Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Tue, 16 Jan 2024 17:12:02 +0100 Subject: [PATCH 1/6] start migration docs --- docs/MIGRATION.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 docs/MIGRATION.md diff --git a/docs/MIGRATION.md b/docs/MIGRATION.md new file mode 100644 index 00000000..f6935f77 --- /dev/null +++ b/docs/MIGRATION.md @@ -0,0 +1,20 @@ + +Start from DIRAC v8 + +Edit the CS: +* DiracX/DisabledVO is all of them +* DiracX/LegacyClientEnabled is empty + +Add in the CS: +* conversion info + + +Update to v9 + +Create a DiracX cluster + +Run the CS Sync + +slowly work through the Disabled VO + +Once it's empty, start enabling LegacyClient From 81a60cf8517f18acb150fcb44d30e5d049b9eb3b Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Tue, 16 Jan 2024 17:12:24 +0100 Subject: [PATCH 2/6] feat (cs_sync): DisabledVOs cannot be set if any Legacy clients are enabled --- diracx-cli/src/diracx/cli/internal/legacy.py | 12 ++++++++++ .../legacy/cs_sync/integration_test_buggy.cfg | 17 ++++++++++++++ .../tests/legacy/cs_sync/test_cssync.py | 23 +++++++++++++++++++ 3 files changed, 52 insertions(+) create mode 100644 diracx-cli/tests/legacy/cs_sync/integration_test_buggy.cfg diff --git a/diracx-cli/src/diracx/cli/internal/legacy.py b/diracx-cli/src/diracx/cli/internal/legacy.py index c3f091fe..21b3be1a 100644 --- a/diracx-cli/src/diracx/cli/internal/legacy.py +++ b/diracx-cli/src/diracx/cli/internal/legacy.py @@ -59,6 +59,18 @@ def cs_sync(old_file: Path, conversion_config: Path, new_file: Path): cfg = diraccfg.CFG().loadFromBuffer(old_data) raw = cfg.getAsDict() + diracx_section = raw["DiracX"] + # DisabledVOs cannot be set if any Legacy clients are enabled + disabled_vos = diracx_section.get("DisabledVOs") + enabled_clients = [] + for _, client_status in diracx_section.get("LegacyClientEnabled", {}).items(): + for _, str_status in client_status.items(): + enabled_clients.append(str_status == "True") + if disabled_vos and any(enabled_clients): + raise RuntimeError( + "DisabledVOs cannot be set if any Legacy clients are enabled" + ) + _apply_fixes(raw, conversion_config) config = Config.parse_obj(raw) diff --git a/diracx-cli/tests/legacy/cs_sync/integration_test_buggy.cfg b/diracx-cli/tests/legacy/cs_sync/integration_test_buggy.cfg new file mode 100644 index 00000000..861d4660 --- /dev/null +++ b/diracx-cli/tests/legacy/cs_sync/integration_test_buggy.cfg @@ -0,0 +1,17 @@ + +DiracX +{ + URL = https://dirac-integration-tests.invalid:1234 + LegacyExchangeApiKey = diracx:legacy:ChangeME + + # # DisabledVOs cannot be set if any Legacy clients are enabled + DisabledVOs = AnyVO + LegacyClientEnabled + { + WorkloadManagement + { + JobStateUpdate = True + } + } + +} diff --git a/diracx-cli/tests/legacy/cs_sync/test_cssync.py b/diracx-cli/tests/legacy/cs_sync/test_cssync.py index 3c3c5a22..4dc109a4 100644 --- a/diracx-cli/tests/legacy/cs_sync/test_cssync.py +++ b/diracx-cli/tests/legacy/cs_sync/test_cssync.py @@ -33,3 +33,26 @@ def test_cs_sync(tmp_path, monkeypatch): expected_output = yaml.safe_load((file_path / "integration_test.yaml").read_text()) assert actual_output == expected_output Config.parse_obj(actual_output) + + +def test_disabled_vos_empty(tmp_path, monkeypatch): + # # DisabledVOs cannot be set if any Legacy clients are enabled + monkeypatch.setenv("DIRAC_COMPAT_ENABLE_CS_CONVERSION", "Yes") + + output_file = tmp_path / "default.yaml" + + result = runner.invoke( + app, + [ + "internal", + "legacy", + "cs-sync", + f"{file_path / 'integration_test_buggy.cfg'}", + f"{file_path / 'convert_integration_test.yaml'}", + str(output_file), + ], + ) + assert result.exit_code == 1 + assert not output_file.is_file() + assert isinstance(result.exception, RuntimeError) + assert "DisabledVOs cannot be set" in str(result.exception) From 421bacf6426dfef70b3cbd9cd52a96e874d472ce Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Tue, 16 Jan 2024 17:33:33 +0100 Subject: [PATCH 3/6] feat (cs_sync): use old CS to store sync config --- diracx-cli/src/diracx/cli/internal/legacy.py | 10 ++--- .../cs_sync/convert_integration_test.yaml | 23 ---------- .../tests/legacy/cs_sync/integration_test.cfg | 43 +++++++++++++++++++ .../tests/legacy/cs_sync/test_cssync.py | 2 - 4 files changed, 47 insertions(+), 31 deletions(-) delete mode 100644 diracx-cli/tests/legacy/cs_sync/convert_integration_test.yaml diff --git a/diracx-cli/src/diracx/cli/internal/legacy.py b/diracx-cli/src/diracx/cli/internal/legacy.py index 21b3be1a..f7d6544b 100644 --- a/diracx-cli/src/diracx/cli/internal/legacy.py +++ b/diracx-cli/src/diracx/cli/internal/legacy.py @@ -48,7 +48,7 @@ class ConversionConfig(BaseModel): @app.command() -def cs_sync(old_file: Path, conversion_config: Path, new_file: Path): +def cs_sync(old_file: Path, new_file: Path): """Load the old CS and convert it to the new YAML format""" if not os.environ.get("DIRAC_COMPAT_ENABLE_CS_CONVERSION"): raise RuntimeError( @@ -71,18 +71,16 @@ def cs_sync(old_file: Path, conversion_config: Path, new_file: Path): "DisabledVOs cannot be set if any Legacy clients are enabled" ) - _apply_fixes(raw, conversion_config) + _apply_fixes(raw) config = Config.parse_obj(raw) new_file.write_text(yaml.safe_dump(config.dict(exclude_unset=True))) -def _apply_fixes(raw, conversion_config: Path): +def _apply_fixes(raw): """Modify raw in place to make any layout changes between the old and new structure""" - conv_config = ConversionConfig.parse_obj( - yaml.safe_load(conversion_config.read_text()) - ) + conv_config = ConversionConfig.parse_obj(raw["DiracX"]["CsSync"]) raw.pop("DiracX", None) # Remove dips specific parts from the CS diff --git a/diracx-cli/tests/legacy/cs_sync/convert_integration_test.yaml b/diracx-cli/tests/legacy/cs_sync/convert_integration_test.yaml deleted file mode 100644 index 3da51e61..00000000 --- a/diracx-cli/tests/legacy/cs_sync/convert_integration_test.yaml +++ /dev/null @@ -1,23 +0,0 @@ -VOs: - Jenkins: - DefaultGroup: jenkins_user - IdP: - ClientID: 995ed3b9-d5bd-49d3-a7f4-7fc7dbd5a0cd - URL: https://jenkins.invalid/ - UserSubjects: - adminusername: e2cb28ec-1a1e-40ee-a56d-d899b79879ce - ciuser: 26dbe36e-cf5c-4c52-a834-29a1c904ef74 - trialUser: a95ab678-3fa4-41b9-b863-fe62ce8064ce - Support: - Message: "Contact the help desk" - Email: "helpdesk@example.invalid" - Webpage: "https://helpdesk.vo.invalid" - vo: - DefaultGroup: dirac_user - IdP: - ClientID: 072afab5-ed92-46e0-a61d-4ecbc96e0770 - URL: https://vo.invalid/ - UserSubjects: - adminusername: 26b14fc9-6d40-4ca5-b014-6234eaf0fb6e - ciuser: d3adc733-6588-4d6f-8581-5986b02d0c87 - trialUser: ff2152ff-34f4-4739-b106-3def37e291e3 diff --git a/diracx-cli/tests/legacy/cs_sync/integration_test.cfg b/diracx-cli/tests/legacy/cs_sync/integration_test.cfg index d1da5b17..cd094a61 100644 --- a/diracx-cli/tests/legacy/cs_sync/integration_test.cfg +++ b/diracx-cli/tests/legacy/cs_sync/integration_test.cfg @@ -30,6 +30,49 @@ DiracX { URL = https://dirac-integration-tests.invalid:1234 LegacyExchangeApiKey = diracx:legacy:ChangeME + + CsSync + { + VOs + { + Jenkins + { + DefaultGroup = jenkins_user + IdP + { + ClientID = 995ed3b9-d5bd-49d3-a7f4-7fc7dbd5a0cd + URL = https://jenkins.invalid/ + } + UserSubjects + { + adminusername = e2cb28ec-1a1e-40ee-a56d-d899b79879ce + ciuser = 26dbe36e-cf5c-4c52-a834-29a1c904ef74 + trialUser = a95ab678-3fa4-41b9-b863-fe62ce8064ce + } + Support + { + Message = Contact the help desk + Email = helpdesk@example.invalid + Webpage = https://helpdesk.vo.invalid + } + } + vo + { + DefaultGroup = dirac_user + IdP + { + ClientID = 072afab5-ed92-46e0-a61d-4ecbc96e0770 + URL = https://vo.invalid/ + } + UserSubjects + { + adminusername = 26b14fc9-6d40-4ca5-b014-6234eaf0fb6e + ciuser = d3adc733-6588-4d6f-8581-5986b02d0c87 + trialUser = ff2152ff-34f4-4739-b106-3def37e291e3 + } + } + } + } } Systems { diff --git a/diracx-cli/tests/legacy/cs_sync/test_cssync.py b/diracx-cli/tests/legacy/cs_sync/test_cssync.py index 4dc109a4..12ac8bff 100644 --- a/diracx-cli/tests/legacy/cs_sync/test_cssync.py +++ b/diracx-cli/tests/legacy/cs_sync/test_cssync.py @@ -23,7 +23,6 @@ def test_cs_sync(tmp_path, monkeypatch): "legacy", "cs-sync", f"{file_path / 'integration_test.cfg'}", - f"{file_path / 'convert_integration_test.yaml'}", str(output_file), ], ) @@ -48,7 +47,6 @@ def test_disabled_vos_empty(tmp_path, monkeypatch): "legacy", "cs-sync", f"{file_path / 'integration_test_buggy.cfg'}", - f"{file_path / 'convert_integration_test.yaml'}", str(output_file), ], ) From 6ac8ff5977781c42164df3b7ac9cd1725710b0bb Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Fri, 19 Jan 2024 09:35:55 +0100 Subject: [PATCH 4/6] cast for mypy --- diracx-cli/src/diracx/cli/internal/legacy.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/diracx-cli/src/diracx/cli/internal/legacy.py b/diracx-cli/src/diracx/cli/internal/legacy.py index f7d6544b..1a004174 100644 --- a/diracx-cli/src/diracx/cli/internal/legacy.py +++ b/diracx-cli/src/diracx/cli/internal/legacy.py @@ -3,12 +3,16 @@ import json import os from pathlib import Path -from typing import cast +from typing import TYPE_CHECKING, cast from urllib.parse import urljoin, urlparse import diraccfg import typer import yaml + +if TYPE_CHECKING: + from diraccfg.cfg import CFGAsDict + from pydantic import BaseModel from typer import Option @@ -59,12 +63,14 @@ def cs_sync(old_file: Path, new_file: Path): cfg = diraccfg.CFG().loadFromBuffer(old_data) raw = cfg.getAsDict() - diracx_section = raw["DiracX"] + diracx_section = cast("CFGAsDict", raw["DiracX"]) # DisabledVOs cannot be set if any Legacy clients are enabled disabled_vos = diracx_section.get("DisabledVOs") enabled_clients = [] - for _, client_status in diracx_section.get("LegacyClientEnabled", {}).items(): - for _, str_status in client_status.items(): + for _, client_status in cast( + "CFGAsDict", diracx_section.get("LegacyClientEnabled", {}) + ).items(): + for _, str_status in cast("CFGAsDict", client_status).items(): enabled_clients.append(str_status == "True") if disabled_vos and any(enabled_clients): raise RuntimeError( From bd7fbd91697df200251908ac1c4d3d71c69d2205 Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Tue, 23 Jan 2024 17:29:16 +0100 Subject: [PATCH 5/6] test: modify login test to approve dex confirmation page --- diracx-testing/src/diracx/testing/__init__.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/diracx-testing/src/diracx/testing/__init__.py b/diracx-testing/src/diracx/testing/__init__.py index 02c5a0a0..99a04468 100644 --- a/diracx-testing/src/diracx/testing/__init__.py +++ b/diracx-testing/src/diracx/testing/__init__.py @@ -9,7 +9,7 @@ from html.parser import HTMLParser from pathlib import Path from typing import TYPE_CHECKING -from urllib.parse import urljoin +from urllib.parse import parse_qs, urljoin, urlparse from uuid import uuid4 import pytest @@ -551,5 +551,13 @@ def handle_starttag(self, tag, attrs): verify=ca_path, ) r.raise_for_status() + approval_url = r.url # This is not the same as URL as we redirect to dex + # Do the actual approval + r = requests.post( + approval_url, + {"approval": "approve", "req": parse_qs(urlparse(r.url).query)["req"][0]}, + verify=ca_path, + ) + # This should have redirected to the DiracX page that shows the login is complete assert "Please close the window" in r.text From edaa7c03f8b0309feb10b6e0e10bf5a9aaec60a3 Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Wed, 24 Jan 2024 17:22:03 +0100 Subject: [PATCH 6/6] fix run_local.sh --- run_local.sh | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/run_local.sh b/run_local.sh index 09324eb3..94fe4db8 100755 --- a/run_local.sh +++ b/run_local.sh @@ -19,12 +19,8 @@ dirac internal add-vo "${tmp_dir}/cs_store/initialRepo" \ --idp-client-id="idp-client-id" \ --default-group=admin -dirac internal add-group "${tmp_dir}/cs_store/initialRepo" \ - --vo=diracAdmin \ - --group=admin - dirac internal add-user "${tmp_dir}/cs_store/initialRepo" \ - --vo=diracAdmin --user-group=admin \ + --vo=diracAdmin --group=admin \ --sub=75212b23-14c2-47be-9374-eb0113b0575e \ --preferred-username=localuser