From ff2d9872e08bbb4e5963242584664f752e0ad17d Mon Sep 17 00:00:00 2001 From: "Pedro S. Lopez" Date: Wed, 2 Aug 2023 16:57:54 -0400 Subject: [PATCH] airbyte-ci: add dagger run url (#28947) * add dagger run url * version bump * making a change so tests are auto triggered * fix dagger cloud url * fix no args * naming / formatting * try nowrap * try soft_wrap * just log it * move log * revert fake change * rename * nits * Update airbyte-ci/connectors/pipelines/pipelines/tests/templates/test_report.html.j2 Co-authored-by: Augustin * Update airbyte-ci/connectors/pipelines/pipelines/contexts.py Co-authored-by: Augustin --------- Co-authored-by: Augustin --- airbyte-ci/connectors/pipelines/README.md | 7 ++++--- airbyte-ci/connectors/pipelines/pipelines/bases.py | 13 +++++++++++++ .../connectors/pipelines/pipelines/contexts.py | 12 ++++++++++++ .../connectors/pipelines/pipelines/dagger_run.py | 2 +- .../pipelines/tests/templates/test_report.html.j2 | 5 ++++- airbyte-ci/connectors/pipelines/pyproject.toml | 2 +- 6 files changed, 35 insertions(+), 6 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/README.md b/airbyte-ci/connectors/pipelines/README.md index 58c63e8e7e9f..d2e7e5b28b51 100644 --- a/airbyte-ci/connectors/pipelines/README.md +++ b/airbyte-ci/connectors/pipelines/README.md @@ -96,7 +96,7 @@ At this point you can run `airbyte-ci` commands from the root of the repository. #### Options | Option | Default value | Mapped environment variable | Description | -| --------------------------------------- | ------------------------------- | ----------------------------- | ------------------------------------------------------------------------------------------- | +|-----------------------------------------|---------------------------------|-------------------------------|---------------------------------------------------------------------------------------------| | `--no-tui` | | | Disables the Dagger terminal UI. | | `--is-local/--is-ci` | `--is-local` | | Determines the environment in which the CLI runs: local environment or CI environment. | | `--git-branch` | The checked out git branch name | `CI_GIT_BRANCH` | The git branch on which the pipelines will run. | @@ -378,8 +378,9 @@ This command runs the Python tests for a airbyte-ci poetry package. | Version | PR | Description | |---------|-----------------------------------------------------------|----------------------------------------------------------------------------------------------| -| 0.3.2 | [#28789](https://github.com/airbytehq/airbyte/pull/28789) | Do not consider empty reports as successfull. | -| 0.3.1 | [#28938](https://github.com/airbytehq/airbyte/pull/28938) | Handle 5 status code on MetadataUpload as skipped | +| 0.4.0 | [#28947](https://github.com/airbytehq/airbyte/pull/28947) | Show Dagger Cloud run URLs in CI | +| 0.3.2 | [#28789](https://github.com/airbytehq/airbyte/pull/28789) | Do not consider empty reports as successfull. | +| 0.3.1 | [#28938](https://github.com/airbytehq/airbyte/pull/28938) | Handle 5 status code on MetadataUpload as skipped | | 0.3.0 | [#28869](https://github.com/airbytehq/airbyte/pull/28869) | Enable the Dagger terminal UI on local `airbyte-ci` execution | | 0.2.3 | [#28907](https://github.com/airbytehq/airbyte/pull/28907) | Make dagger-in-dagger work for `airbyte-ci tests` command | | 0.2.2 | [#28897](https://github.com/airbytehq/airbyte/pull/28897) | Sentry: Ignore error logs without exceptions from reporting | diff --git a/airbyte-ci/connectors/pipelines/pipelines/bases.py b/airbyte-ci/connectors/pipelines/pipelines/bases.py index bd0f0084d617..b3e446fadbb5 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/bases.py +++ b/airbyte-ci/connectors/pipelines/pipelines/bases.py @@ -439,6 +439,7 @@ def to_json(self) -> str: "git_revision": self.pipeline_context.git_revision, "ci_context": self.pipeline_context.ci_context, "pull_request_url": self.pipeline_context.pull_request.html_url if self.pipeline_context.pull_request else None, + "dagger_cloud_url": self.pipeline_context.dagger_cloud_url, } ) @@ -477,6 +478,9 @@ def print(self): failures_group = Group(*sub_panels) to_render.append(failures_group) + if self.pipeline_context.dagger_cloud_url: + self.pipeline_context.logger.info(f"🔗 View runs for commit in Dagger Cloud: {self.pipeline_context.dagger_cloud_url}") + main_panel = Panel(Group(*to_render), title=main_panel_title, subtitle=duration_subtitle) console.print(main_panel) @@ -535,6 +539,7 @@ def to_json(self) -> str: "ci_context": self.pipeline_context.ci_context, "cdk_version": self.pipeline_context.cdk_version, "html_report_url": self.html_report_url, + "dagger_cloud_url": self.pipeline_context.dagger_cloud_url, } ) @@ -551,6 +556,10 @@ def post_comment_on_pr(self) -> None: ] markdown_comment += tabulate(report_data, headers=["Step", "Result"], tablefmt="pipe") + "\n\n" markdown_comment += f"🔗 [View the logs here]({self.html_report_url})\n\n" + + if self.pipeline_context.dagger_cloud_url: + markdown_comment += f"☁ī¸ [View runs for commit in Dagger Cloud]({self.pipeline_context.dagger_cloud_url})\n\n" + markdown_comment += "*Please note that tests are only run on PR ready for review. Please set your PR to draft mode to not flood the CI engine and upstream service on following commits.*\n" markdown_comment += "**You can run the same pipeline locally on this branch with the [airbyte-ci](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connector_ops/connector_ops/pipelines/README.md) tool with the following command**\n" markdown_comment += f"```bash\nairbyte-ci connectors --name={self.pipeline_context.connector.technical_name} test\n```\n\n" @@ -580,6 +589,7 @@ async def to_html(self) -> str: template_context["commit_url"] = f"https://github.com/airbytehq/airbyte/commit/{self.pipeline_context.git_revision}" template_context["gha_workflow_run_url"] = self.pipeline_context.gha_workflow_run_url template_context["dagger_logs_url"] = self.pipeline_context.dagger_logs_url + template_context["dagger_cloud_url"] = self.pipeline_context.dagger_cloud_url template_context[ "icon_url" ] = f"https://raw.githubusercontent.com/airbytehq/airbyte/{self.pipeline_context.git_revision}/{self.pipeline_context.connector.code_directory}/icon.svg" @@ -618,5 +628,8 @@ def print(self): details_instructions = Text("ℹī¸ You can find more details with step executions logs in the saved HTML report.") to_render = [step_results_table, details_instructions] + if self.pipeline_context.dagger_cloud_url: + self.pipeline_context.logger.info(f"🔗 View runs for commit in Dagger Cloud: {self.pipeline_context.dagger_cloud_url}") + main_panel = Panel(Group(*to_render), title=main_panel_title, subtitle=duration_subtitle) console.print(main_panel) diff --git a/airbyte-ci/connectors/pipelines/pipelines/contexts.py b/airbyte-ci/connectors/pipelines/pipelines/contexts.py index f54dda5482e9..50dc20b5bfed 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/contexts.py +++ b/airbyte-ci/connectors/pipelines/pipelines/contexts.py @@ -172,6 +172,18 @@ def github_commit_status(self) -> dict: def should_send_slack_message(self) -> bool: return self.slack_webhook is not None and self.reporting_slack_channel is not None + @property + def has_dagger_cloud_token(self) -> bool: + return "_EXPERIMENTAL_DAGGER_CLOUD_TOKEN" in os.environ + + @property + def dagger_cloud_url(self) -> str: + """Gets the link to the Dagger Cloud runs page for the current commit.""" + if self.is_local or not self.has_dagger_cloud_token: + return None + + return f"https://alpha.dagger.cloud/changeByPipelines?filter=dagger.io/git.ref:{self.git_revision}" + def get_repo_dir(self, subdir: str = ".", exclude: Optional[List[str]] = None, include: Optional[List[str]] = None) -> Directory: """Get a directory from the current repository. diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger_run.py b/airbyte-ci/connectors/pipelines/pipelines/dagger_run.py index 8fc6096fe0a6..60b18724eff8 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/dagger_run.py +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger_run.py @@ -89,7 +89,7 @@ def check_dagger_cli_install() -> str: def main(): os.environ[DAGGER_CLOUD_TOKEN_ENV_VAR_NAME_VALUE[0]] = DAGGER_CLOUD_TOKEN_ENV_VAR_NAME_VALUE[1] exit_code = 0 - if sys.argv[1] == "--no-tui": + if len(sys.argv) > 1 and sys.argv[1] == "--no-tui": command = ["airbyte-ci-internal"] + sys.argv[2:] else: dagger_path = check_dagger_cli_install() diff --git a/airbyte-ci/connectors/pipelines/pipelines/tests/templates/test_report.html.j2 b/airbyte-ci/connectors/pipelines/pipelines/tests/templates/test_report.html.j2 index 5c6531184349..5ac9282ac5bd 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/tests/templates/test_report.html.j2 +++ b/airbyte-ci/connectors/pipelines/pipelines/tests/templates/test_report.html.j2 @@ -128,6 +128,9 @@ {% if dagger_logs_url %}
  • Dagger logs
    • {% endif %} + {% if dagger_cloud_url %} +
  • Dagger Cloud UI
    • + {% endif %}

    Summary

    @@ -169,6 +172,6 @@ {% endfor %} -

    These reports are generated from this code, please reach out to the Connector Operations team for support.

    +

    These reports are generated from this code, please reach out to the Connector Operations team for support.

    diff --git a/airbyte-ci/connectors/pipelines/pyproject.toml b/airbyte-ci/connectors/pipelines/pyproject.toml index 3ead2144c4fa..9e773e1ffddc 100644 --- a/airbyte-ci/connectors/pipelines/pyproject.toml +++ b/airbyte-ci/connectors/pipelines/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "pipelines" -version = "0.3.2" +version = "0.4.0" description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines" authors = ["Airbyte "]