From 4503cc5f29f831016084c2289f3b78c53daabb71 Mon Sep 17 00:00:00 2001 From: Marcelo Villa Date: Wed, 21 Feb 2024 08:26:39 -0500 Subject: [PATCH 01/10] Modify regular expression to match both github and gitlab repos. --- src/nebari/schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nebari/schema.py b/src/nebari/schema.py index ab90f8ebc7..3c9998b8c2 100644 --- a/src/nebari/schema.py +++ b/src/nebari/schema.py @@ -17,7 +17,7 @@ email_regex = "^[^ @]+@[^ @]+\\.[^ @]+$" email_pydantic = pydantic.constr(regex=email_regex) -github_url_regex = "^(https://)?github.com/([^/]+)/([^/]+)/?$" +github_url_regex = "^(https://)?(github\.com|gitlab\.com)/([^/]+)/([^/]+)/?$" github_url_pydantic = pydantic.constr(regex=github_url_regex) From a1b59cab966e88000249aa9134f93b0687bd1311 Mon Sep 17 00:00:00 2001 From: Marcelo Villa Date: Wed, 21 Feb 2024 09:48:37 -0500 Subject: [PATCH 02/10] Avoid escaping dots. --- src/nebari/schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nebari/schema.py b/src/nebari/schema.py index 3c9998b8c2..d533ad55ed 100644 --- a/src/nebari/schema.py +++ b/src/nebari/schema.py @@ -17,7 +17,7 @@ email_regex = "^[^ @]+@[^ @]+\\.[^ @]+$" email_pydantic = pydantic.constr(regex=email_regex) -github_url_regex = "^(https://)?(github\.com|gitlab\.com)/([^/]+)/([^/]+)/?$" +github_url_regex = "^(https://)?(github.com|gitlab.com)/([^/]+)/([^/]+)/?$" github_url_pydantic = pydantic.constr(regex=github_url_regex) From e341baef88c1831d0da9937cb90666d8423fc150 Mon Sep 17 00:00:00 2001 From: Marcelo Villa Date: Wed, 21 Feb 2024 11:50:35 -0500 Subject: [PATCH 03/10] Improve regex and rename variable name. --- src/_nebari/subcommands/init.py | 4 ++-- src/nebari/schema.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/_nebari/subcommands/init.py b/src/_nebari/subcommands/init.py index f519b97f8f..8feaed6cd9 100644 --- a/src/_nebari/subcommands/init.py +++ b/src/_nebari/subcommands/init.py @@ -88,7 +88,7 @@ class InitInputs(schema.Base): namespace: Optional[schema.namespace_pydantic] = "dev" auth_provider: AuthenticationEnum = AuthenticationEnum.password auth_auto_provision: bool = False - repository: Optional[schema.github_url_pydantic] = None + repository: Optional[schema.git_url_pydantic] = None repository_auto_provision: bool = False ci_provider: CiEnum = CiEnum.none terraform_state: TerraformStateEnum = TerraformStateEnum.remote @@ -519,7 +519,7 @@ def init( None, help="Github repository URL to be initialized with --repository-auto-provision", callback=typer_validate_regex( - schema.github_url_regex, + schema.git_url_regex, "Must be a fully qualified GitHub repository URL.", ), ), diff --git a/src/nebari/schema.py b/src/nebari/schema.py index d533ad55ed..29de77249d 100644 --- a/src/nebari/schema.py +++ b/src/nebari/schema.py @@ -17,8 +17,8 @@ email_regex = "^[^ @]+@[^ @]+\\.[^ @]+$" email_pydantic = pydantic.constr(regex=email_regex) -github_url_regex = "^(https://)?(github.com|gitlab.com)/([^/]+)/([^/]+)/?$" -github_url_pydantic = pydantic.constr(regex=github_url_regex) +git_url_regex = "^(https://)?(git)(hub|lab)(.com)/([^/]+)/([^/]+)/?$" +git_url_pydantic = pydantic.constr(regex=git_url_regex) class Base(pydantic.BaseModel): From 9f73184f1c7a24615d63356e8dc2302d11f278d1 Mon Sep 17 00:00:00 2001 From: Marcelo Villa Date: Wed, 21 Feb 2024 11:51:19 -0500 Subject: [PATCH 04/10] Change match group numbers to account for changes in the regular expression. --- src/_nebari/initialize.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/_nebari/initialize.py b/src/_nebari/initialize.py index 2f07647521..8f93145453 100644 --- a/src/_nebari/initialize.py +++ b/src/_nebari/initialize.py @@ -23,7 +23,7 @@ from _nebari.stages.terraform_state import TerraformStateEnum from _nebari.utils import get_latest_kubernetes_version, random_secure_string from _nebari.version import __version__ -from nebari.schema import ProviderEnum, github_url_regex +from nebari.schema import ProviderEnum, git_url_regex logger = logging.getLogger(__name__) @@ -194,10 +194,10 @@ def render_config( print(str(e)) if repository_auto_provision: - match = re.search(github_url_regex, repository) + match = re.search(git_url_regex, repository) if match: git_repository = github_auto_provision( - config_model, match.group(2), match.group(3) + config_model, match.group(5), match.group(6) ) git_repository_initialize(git_repository) else: From 5371bcdfb40a02a7d0ecce14cbf8ca652db9ad4c Mon Sep 17 00:00:00 2001 From: Marcelo Villa Date: Wed, 21 Feb 2024 12:24:04 -0500 Subject: [PATCH 05/10] Adjust regex to complain about urls with just the org/user and a trailing slash. --- src/nebari/schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nebari/schema.py b/src/nebari/schema.py index 29de77249d..f8464e9ece 100644 --- a/src/nebari/schema.py +++ b/src/nebari/schema.py @@ -17,7 +17,7 @@ email_regex = "^[^ @]+@[^ @]+\\.[^ @]+$" email_pydantic = pydantic.constr(regex=email_regex) -git_url_regex = "^(https://)?(git)(hub|lab)(.com)/([^/]+)/([^/]+)/?$" +git_url_regex = "^(https://)?(git)(hub|lab)(.com)/([^/]+)/([^/]+)/*$" git_url_pydantic = pydantic.constr(regex=git_url_regex) From c16b09435dbf7151101f538c07eab4963854d38d Mon Sep 17 00:00:00 2001 From: Marcelo Villa Date: Wed, 21 Feb 2024 12:24:24 -0500 Subject: [PATCH 06/10] Add new test cases based on regex modification. --- tests/tests_unit/test_cli_init_repository.py | 48 +++++++++++++++----- 1 file changed, 36 insertions(+), 12 deletions(-) diff --git a/tests/tests_unit/test_cli_init_repository.py b/tests/tests_unit/test_cli_init_repository.py index 6bc0d4e7d4..c61f2fa8b2 100644 --- a/tests/tests_unit/test_cli_init_repository.py +++ b/tests/tests_unit/test_cli_init_repository.py @@ -174,21 +174,45 @@ def test_cli_init_error_repository_missing_env(monkeypatch: pytest.MonkeyPatch): assert tmp_file.exists() is False -def test_cli_init_error_invalid_repo(monkeypatch: pytest.MonkeyPatch): - monkeypatch.setenv("GITHUB_USERNAME", TEST_GITHUB_USERNAME) - monkeypatch.setenv("GITHUB_TOKEN", TEST_GITHUB_TOKEN) +@pytest.mark.parametrize( + "url", + [ + "https://github.com/user/repository", + "https://gitlab.com/user/repository/", + ], +) +def test_cli_init_valid_repo(url): + app = create_cli() + + args = ["init", "local", "--project-name", "test", "--repository", url] + + with tempfile.TemporaryDirectory() as tmp: + tmp_file = Path(tmp).resolve() / "nebari-config.yaml" + assert tmp_file.exists() is False + result = runner.invoke(app, args + ["--output", tmp_file.resolve()]) + + assert 0 == result.exit_code + assert not result.exception + assert tmp_file.exists() is True + + +@pytest.mark.parametrize( + "url", + [ + "https://github.com", + "https://gitlab.com/user/", + "https://gitlab.com/user", + "http://github.com/user/", + "gitlab.com/user/", + "github.com", + "https://notgithub.com/user/repository", + ], +) +def test_cli_init_error_invalid_repo(url): app = create_cli() - args = [ - "init", - "local", - "--project-name", - "test", - "--repository-auto-provision", - "--repository", - "https://notgithub.com", - ] + args = ["init", "local", "--project-name", "test", "--repository", url] with tempfile.TemporaryDirectory() as tmp: tmp_file = Path(tmp).resolve() / "nebari-config.yaml" From 90bf9364cbc7cb759389585a1cf5ee9ee8af854f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 12 Mar 2024 17:39:19 +0000 Subject: [PATCH 07/10] [pre-commit.ci] Apply automatic pre-commit fixes --- src/_nebari/stages/kubernetes_services/__init__.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/_nebari/stages/kubernetes_services/__init__.py b/src/_nebari/stages/kubernetes_services/__init__.py index a9124f41ac..9c47fee6ec 100644 --- a/src/_nebari/stages/kubernetes_services/__init__.py +++ b/src/_nebari/stages/kubernetes_services/__init__.py @@ -51,9 +51,15 @@ class Storage(schema.Base): class JupyterHubTheme(schema.Base): hub_title: str = "Nebari" hub_subtitle: str = "Your open source data science platform" - welcome: str = """Welcome! Learn about Nebari's features and configurations in the documentation. If you have any questions or feedback, reach the team on Nebari's support forums.""" - logo: str = "https://raw.githubusercontent.com/nebari-dev/nebari-design/main/logo-mark/horizontal/Nebari-Logo-Horizontal-Lockup-White-text.svg" - favicon: str = "https://raw.githubusercontent.com/nebari-dev/nebari-design/main/symbol/favicon.ico" + welcome: str = ( + """Welcome! Learn about Nebari's features and configurations in the documentation. If you have any questions or feedback, reach the team on Nebari's support forums.""" + ) + logo: str = ( + "https://raw.githubusercontent.com/nebari-dev/nebari-design/main/logo-mark/horizontal/Nebari-Logo-Horizontal-Lockup-White-text.svg" + ) + favicon: str = ( + "https://raw.githubusercontent.com/nebari-dev/nebari-design/main/symbol/favicon.ico" + ) primary_color: str = "#4f4173" primary_color_dark: str = "#4f4173" secondary_color: str = "#957da6" From 978eb80b70d3a41f771748b7dfe5a871f26346ac Mon Sep 17 00:00:00 2001 From: Marcelo Villa Date: Tue, 13 Aug 2024 17:40:09 -0500 Subject: [PATCH 08/10] Refactor guided init flow for gitops approach --- src/_nebari/initialize.py | 6 ++--- src/_nebari/subcommands/init.py | 43 ++++++++++++++++++--------------- 2 files changed, 26 insertions(+), 23 deletions(-) diff --git a/src/_nebari/initialize.py b/src/_nebari/initialize.py index b7c5a7a8b6..df693ca8f0 100644 --- a/src/_nebari/initialize.py +++ b/src/_nebari/initialize.py @@ -31,7 +31,7 @@ from _nebari.stages.terraform_state import TerraformStateEnum from _nebari.utils import get_latest_kubernetes_version, random_secure_string from _nebari.version import __version__ -from nebari.schema import ProviderEnum, git_url_regex +from nebari.schema import ProviderEnum, github_url_regex logger = logging.getLogger(__name__) @@ -206,10 +206,10 @@ def render_config( print(str(e)) if repository_auto_provision: - match = re.search(git_url_regex, repository) + match = re.search(github_url_regex, repository) if match: git_repository = github_auto_provision( - config_model, match.group(5), match.group(6) + config_model, match.group(2), match.group(3) ) git_repository_initialize(git_repository) else: diff --git a/src/_nebari/subcommands/init.py b/src/_nebari/subcommands/init.py index 4befdf0a65..4cb925fd9a 100644 --- a/src/_nebari/subcommands/init.py +++ b/src/_nebari/subcommands/init.py @@ -97,7 +97,7 @@ class InitInputs(schema.Base): namespace: Optional[schema.namespace_pydantic] = "dev" auth_provider: AuthenticationEnum = AuthenticationEnum.password auth_auto_provision: bool = False - repository: Optional[schema.git_url_pydantic] = None + repository: Optional[schema.github_url_pydantic] = None repository_auto_provision: bool = False ci_provider: CiEnum = CiEnum.none terraform_state: TerraformStateEnum = TerraformStateEnum.remote @@ -529,7 +529,7 @@ def init( None, help="Github repository URL to be initialized with --repository-auto-provision", callback=typer_validate_regex( - schema.git_url_regex, + schema.github_url_regex, "Must be a fully qualified GitHub repository URL.", ), ), @@ -812,21 +812,9 @@ def guided_init_wizard(ctx: typer.Context, guided_init: str): qmark=qmark, ).unsafe_ask() - org_name = questionary.text( - f"Which user or organization will this repository live under? ({repo_url.format(git_provider=git_provider, org_name='', repo_name='')})", - qmark=qmark, - ).unsafe_ask() - - repo_name = questionary.text( - f"And what will the name of this repository be? ({repo_url.format(git_provider=git_provider, org_name=org_name, repo_name='')})", - qmark=qmark, - ).unsafe_ask() - - inputs.repository = repo_url.format( - git_provider=git_provider, org_name=org_name, repo_name=repo_name - ) - if git_provider == GitRepoEnum.github.value.lower(): + inputs.ci_provider = CiEnum.github_actions.value.lower() + inputs.repository_auto_provision = questionary.confirm( f"Would you like nebari to create a remote repository on {git_provider}?", default=False, @@ -834,11 +822,26 @@ def guided_init_wizard(ctx: typer.Context, guided_init: str): auto_enter=False, ).unsafe_ask() - if not disable_checks and inputs.repository_auto_provision: - check_repository_creds(ctx, git_provider) + if inputs.repository_auto_provision: + org_name = questionary.text( + f"Which user or organization will this repository live under? ({repo_url.format(git_provider=git_provider, org_name='', repo_name='')})", + qmark=qmark, + ).unsafe_ask() + + repo_name = questionary.text( + f"And what will the name of this repository be? ({repo_url.format(git_provider=git_provider, org_name=org_name, repo_name='')})", + qmark=qmark, + ).unsafe_ask() + + inputs.repository = repo_url.format( + git_provider=git_provider, + org_name=org_name, + repo_name=repo_name, + ) + + if not disable_checks: + check_repository_creds(ctx, git_provider) - if git_provider == GitRepoEnum.github.value.lower(): - inputs.ci_provider = CiEnum.github_actions.value.lower() elif git_provider == GitRepoEnum.gitlab.value.lower(): inputs.ci_provider = CiEnum.gitlab_ci.value.lower() From a382d1c56efafe9b40f8c33a698174b7112c6993 Mon Sep 17 00:00:00 2001 From: Marcelo Villa Date: Tue, 13 Aug 2024 17:40:53 -0500 Subject: [PATCH 09/10] Keed github-oriented regex and fix edge case --- src/nebari/schema.py | 4 +-- tests/tests_unit/test_cli_init_repository.py | 30 ++------------------ 2 files changed, 4 insertions(+), 30 deletions(-) diff --git a/src/nebari/schema.py b/src/nebari/schema.py index c4773ce62a..49c8ad9804 100644 --- a/src/nebari/schema.py +++ b/src/nebari/schema.py @@ -19,8 +19,8 @@ email_regex = "^[^ @]+@[^ @]+\\.[^ @]+$" email_pydantic = Annotated[str, StringConstraints(pattern=email_regex)] -git_url_regex = "^(https://)?(git)(hub|lab)(.com)/([^/]+)/([^/]+)/*$" -git_url_pydantic = Annotated[str, StringConstraints(pattern=git_url_regex)] +github_url_regex = r"^(https://)?github\.com/([^/]+)/([^/]+)/?$" +github_url_pydantic = Annotated[str, StringConstraints(pattern=github_url_regex)] class Base(pydantic.BaseModel): diff --git a/tests/tests_unit/test_cli_init_repository.py b/tests/tests_unit/test_cli_init_repository.py index c61f2fa8b2..34746722ba 100644 --- a/tests/tests_unit/test_cli_init_repository.py +++ b/tests/tests_unit/test_cli_init_repository.py @@ -174,38 +174,12 @@ def test_cli_init_error_repository_missing_env(monkeypatch: pytest.MonkeyPatch): assert tmp_file.exists() is False -@pytest.mark.parametrize( - "url", - [ - "https://github.com/user/repository", - "https://gitlab.com/user/repository/", - ], -) -def test_cli_init_valid_repo(url): - app = create_cli() - - args = ["init", "local", "--project-name", "test", "--repository", url] - - with tempfile.TemporaryDirectory() as tmp: - tmp_file = Path(tmp).resolve() / "nebari-config.yaml" - assert tmp_file.exists() is False - - result = runner.invoke(app, args + ["--output", tmp_file.resolve()]) - - assert 0 == result.exit_code - assert not result.exception - assert tmp_file.exists() is True - - @pytest.mark.parametrize( "url", [ "https://github.com", - "https://gitlab.com/user/", - "https://gitlab.com/user", - "http://github.com/user/", - "gitlab.com/user/", - "github.com", + "http://github.com/user/repo", + "https://github.com/user/" "github.com/user/repo", "https://notgithub.com/user/repository", ], ) From 7136815135718d857a2e19fa904c5eb04dc8799e Mon Sep 17 00:00:00 2001 From: Marcelo Villa Date: Tue, 13 Aug 2024 17:41:35 -0500 Subject: [PATCH 10/10] Add gitlab parameter to test render --- tests/tests_unit/conftest.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/tests_unit/conftest.py b/tests/tests_unit/conftest.py index d78dfdf1ec..ce60e44799 100644 --- a/tests/tests_unit/conftest.py +++ b/tests/tests_unit/conftest.py @@ -125,7 +125,7 @@ def _mock_return_value(return_value): "gcp.nebari.dev", schema.ProviderEnum.gcp, GCP_DEFAULT_REGION, - CiEnum.github_actions, + CiEnum.gitlab_ci, AuthenticationEnum.password, ), ( @@ -154,6 +154,11 @@ def nebari_config_options(request) -> schema.Main: auth_provider, ) = request.param + if ci_provider == CiEnum.github_actions: + repo = DEFAULT_GH_REPO + else: + repo = None + return dict( project_name=project, namespace=namespace, @@ -162,7 +167,7 @@ def nebari_config_options(request) -> schema.Main: region=region, ci_provider=ci_provider, auth_provider=auth_provider, - repository=DEFAULT_GH_REPO, + repository=repo, repository_auto_provision=False, auth_auto_provision=False, terraform_state=DEFAULT_TERRAFORM_STATE,