diff --git a/server/kraken/server/bg/jobs.py b/server/kraken/server/bg/jobs.py index 76032eb0..85b63f0c 100644 --- a/server/kraken/server/bg/jobs.py +++ b/server/kraken/server/bg/jobs.py @@ -29,7 +29,7 @@ from ..models import db, Run, Job, TestCaseResult, Branch, Flow, Stage, Project, get_setting from ..models import AgentsGroup, Agent, System, TestCaseComment, Tool -from ..models import RepoChanges +from ..models import RepoChanges, Secret from ..schema import prepare_new_planner_triggers from ..schema import check_and_correct_stage_schema from ..cloud import cloud @@ -943,7 +943,7 @@ def refresh_schema_repo(stage_id, complete_starting_run_id=None): log.error('got unknown stage: %s', stage_id) return - log.info('refresh schema repo for stage: %d, run: %d', + log.info('refresh schema repo for stage: %d, run: %s', stage_id, complete_starting_run_id) planner_url = os.environ.get('KRAKEN_PLANNER_URL', consts.DEFAULT_PLANNER_URL) @@ -957,7 +957,13 @@ def refresh_schema_repo(stage_id, complete_starting_run_id=None): try: # get schema from repo - schema_code, version = gitops.get_schema_from_repo(stage.repo_url, stage.repo_branch, stage.repo_access_token, + if stage.repo_access_token: + secret = Secret.query.filter_by(project=stage.branch.project, name=stage.repo_access_token).one() + repo_access_token = secret.data['secret'] + else: + repo_access_token = None + + schema_code, version = gitops.get_schema_from_repo(stage.repo_url, stage.repo_branch, repo_access_token, stage.schema_file, stage.git_clone_params) # check schema @@ -966,7 +972,7 @@ def refresh_schema_repo(stage_id, complete_starting_run_id=None): stage.repo_error = str(e) stage.repo_state = consts.REPO_STATE_ERROR db.session.commit() - log.exception('problem with schema, stage: %d, run: %d', + log.exception('problem with schema, stage: %d, run: %s', stage_id, complete_starting_run_id) return diff --git a/server/kraken/server/gitops.py b/server/kraken/server/gitops.py index c87e7c88..c52a051d 100644 --- a/server/kraken/server/gitops.py +++ b/server/kraken/server/gitops.py @@ -18,13 +18,19 @@ import tempfile import subprocess +import furl + from . import minioops log = logging.getLogger(__name__) -def _run(cmd, check=True, cwd=None, capture_output=False, text=None): - log.info("execute '%s' in '%s'", cmd, cwd) +def _run(cmd, check=True, cwd=None, capture_output=False, text=None, secret=None): + if secret: + cmd2 = cmd.replace(secret, '******') + else: + cmd2 = cmd + log.info("execute '%s' in '%s'", cmd2, cwd) p = subprocess.run(cmd, shell=True, check=check, cwd=cwd, capture_output=capture_output, text=text) return p @@ -186,9 +192,17 @@ def get_schema_from_repo(repo_url, repo_branch, repo_access_token, schema_file, # clone repo if not git_clone_params: git_clone_params = '' + + if repo_access_token: + url = furl.furl(repo_url) + url.username = repo_access_token + repo_url = url.tostr() + cmd = "git clone --depth 1 --single-branch --branch %s %s '%s' repo" % (repo_branch, git_clone_params, repo_url) - p = _run(cmd, check=False, cwd=tmpdir, capture_output=True, text=True) + p = _run(cmd, check=False, cwd=tmpdir, capture_output=True, text=True, secret=repo_access_token) if p.returncode != 0: + if repo_access_token: + cmd = cmd.replace(repo_access_token, '******') err = "command '%s' returned non-zero exit status %d\n" % (cmd, p.returncode) err += p.stdout.strip()[:140] + '\n' err += p.stderr.strip()[:140] diff --git a/server/kraken/server/management.py b/server/kraken/server/management.py index f5c40434..7799621b 100644 --- a/server/kraken/server/management.py +++ b/server/kraken/server/management.py @@ -533,7 +533,14 @@ def update_stage(stage_id, body, token_info=None): if 'repo_branch' in body: stage.repo_branch = body['repo_branch'] if 'repo_access_token' in body: - stage.repo_access_token = body['repo_access_token'] + repo_access_token = body['repo_access_token'] + secret = Secret.query.filter_by(project=stage.branch.project, name=repo_access_token).one_or_none() + if secret is None: + abort(400, "Secret '%s' for access token does not exist" % repo_access_token) + if secret.kind != consts.SECRET_KIND_SIMPLE: + abort(400, "Type of '%s' access token secret should be Simple Secret" % repo_access_token) + stage.repo_access_token = repo_access_token + if 'schema_file' in body: stage.schema_file = body['schema_file'] if 'repo_refresh_interval' in body: diff --git a/server/poetry.lock b/server/poetry.lock index 7ff123b4..b40d3813 100644 --- a/server/poetry.lock +++ b/server/poetry.lock @@ -444,6 +444,18 @@ python-versions = ">= 2.7, != 3.0.*, != 3.1.*, != 3.2.*, != 3.3.*" Flask = ">=0.10" SQLAlchemy = ">=0.8.0" +[[package]] +name = "furl" +version = "2.1.3" +description = "URL manipulation made simple." +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +orderedmultidict = ">=1.0.1" +six = ">=1.8.0" + [[package]] name = "giturlparse" version = "0.10.0" @@ -786,6 +798,17 @@ rsa = ["cryptography (>=3.0.0)"] signals = ["blinker (>=1.4.0)"] signedtoken = ["cryptography (>=3.0.0)", "pyjwt (>=2.0.0,<3)"] +[[package]] +name = "orderedmultidict" +version = "1.0.1" +description = "Ordered Multivalue Dictionary" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +six = ">=1.8.0" + [[package]] name = "packaging" version = "22.0" @@ -1389,7 +1412,7 @@ testing = ["flake8 (<5)", "func-timeout", "jaraco.functools", "jaraco.itertools" [metadata] lock-version = "1.1" python-versions = "^3.7" -content-hash = "568edcce99ef15fd0dd2be3bb4f3f426a0ebe6390080b05b68b35140a318d3be" +content-hash = "9d79d73eb85c4116dfe184daddaabdc8c4268f3672e30a207bad74a75806cb32" [metadata.files] alembic = [ @@ -1702,6 +1725,10 @@ flask-sqlalchemy = [ {file = "Flask-SQLAlchemy-2.5.1.tar.gz", hash = "sha256:2bda44b43e7cacb15d4e05ff3cc1f8bc97936cc464623424102bfc2c35e95912"}, {file = "Flask_SQLAlchemy-2.5.1-py2.py3-none-any.whl", hash = "sha256:f12c3d4cc5cc7fdcc148b9527ea05671718c3ea45d50c7e732cceb33f574b390"}, ] +furl = [ + {file = "furl-2.1.3-py2.py3-none-any.whl", hash = "sha256:9ab425062c4217f9802508e45feb4a83e54324273ac4b202f1850363309666c0"}, + {file = "furl-2.1.3.tar.gz", hash = "sha256:5a6188fe2666c484a12159c18be97a1977a71d632ef5bb867ef15f54af39cc4e"}, +] giturlparse = [ {file = "giturlparse-0.10.0-py2.py3-none-any.whl", hash = "sha256:04ba1a3a099c3093fa8d24a422913c6a9b2c2cd22bcffc939cf72e3e98f672d7"}, {file = "giturlparse-0.10.0.tar.gz", hash = "sha256:2595ab291d30717cda8474b874c9fd509f1b9802ad7f6968c36a45e4b13eb337"}, @@ -1948,6 +1975,10 @@ oauthlib = [ {file = "oauthlib-3.2.2-py3-none-any.whl", hash = "sha256:8139f29aac13e25d502680e9e19963e83f16838d48a0d71c287fe40e7067fbca"}, {file = "oauthlib-3.2.2.tar.gz", hash = "sha256:9859c40929662bec5d64f34d01c99e093149682a3f38915dc0655d5a633dd918"}, ] +orderedmultidict = [ + {file = "orderedmultidict-1.0.1-py2.py3-none-any.whl", hash = "sha256:43c839a17ee3cdd62234c47deca1a8508a3f2ca1d0678a3bf791c87cf84adbf3"}, + {file = "orderedmultidict-1.0.1.tar.gz", hash = "sha256:04070bbb5e87291cc9bfa51df413677faf2141c73c61d2a5f7b26bea3cd882ad"}, +] packaging = [ {file = "packaging-22.0-py3-none-any.whl", hash = "sha256:957e2148ba0e1a3b282772e791ef1d8083648bc131c8ab0c1feba110ce1146c3"}, {file = "packaging-22.0.tar.gz", hash = "sha256:2198ec20bd4c017b8f9717e00f0c8714076fc2fd93816750ab48e2c41de2cfd3"}, diff --git a/server/pyproject.toml b/server/pyproject.toml index 98024676..79e0359f 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -62,6 +62,7 @@ casbin = "^1.17.1" python-ldap = "^3.4.3" Authlib = "^1.1.0" setuptools = "^66.0.0" # required by apscheduler +furl = "^2.1.3" [tool.poetry.dev-dependencies] pytest = "^7.1.2"