-
Notifications
You must be signed in to change notification settings - Fork 4.1k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Airbyte-ci: Add skippable connector test steps #32188
Changes from 22 commits
f1732af
866bc56
5a1c1cc
27267a6
3c0b584
bab78ab
2de8d43
f90f002
dd4d275
ab950d3
258b66d
3a518ca
be10290
3505170
e59dc3c
3e54bd4
ef6f26a
5a61d50
bba7057
0abd1eb
45a7e23
fbcf237
1821d91
5361936
7b61277
1a6e3d5
7f293f3
31d98c9
8a5bbb9
5ac61e3
f6ed2a5
8f57adb
fa233f9
cddd975
4ffa61d
13349dd
b99e39c
ffae7e2
485d0c4
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -6,9 +6,11 @@ | |
|
||
from datetime import datetime | ||
from types import TracebackType | ||
from typing import Optional | ||
from typing import List, Optional | ||
|
||
import yaml | ||
import functools | ||
|
||
from anyio import Path | ||
from asyncer import asyncify | ||
from dagger import Directory, Secret | ||
|
@@ -18,6 +20,7 @@ | |
from pipelines.helpers.connectors.modifed import ConnectorWithModifiedFiles | ||
from pipelines.helpers.github import update_commit_status_check | ||
from pipelines.helpers.slack import send_message_to_webhook | ||
from pipelines.helpers.run_steps import RunStepOptions | ||
from pipelines.helpers.utils import METADATA_FILE_NAME | ||
from pipelines.models.contexts import PipelineContext | ||
|
||
|
@@ -49,8 +52,6 @@ def __init__( | |
reporting_slack_channel: Optional[str] = None, | ||
pull_request: PullRequest = None, | ||
should_save_report: bool = True, | ||
fail_fast: bool = False, | ||
fast_tests_only: bool = False, | ||
code_tests_only: bool = False, | ||
use_local_cdk: bool = False, | ||
use_host_gradle_dist_tar: bool = False, | ||
|
@@ -60,6 +61,7 @@ def __init__( | |
s3_build_cache_access_key_id: Optional[str] = None, | ||
s3_build_cache_secret_key: Optional[str] = None, | ||
concurrent_cat: Optional[bool] = False, | ||
run_step_options: RunStepOptions = RunStepOptions(), | ||
): | ||
"""Initialize a connector context. | ||
|
||
|
@@ -78,8 +80,6 @@ def __init__( | |
slack_webhook (Optional[str], optional): The slack webhook to send messages to. Defaults to None. | ||
reporting_slack_channel (Optional[str], optional): The slack channel to send messages to. Defaults to None. | ||
pull_request (PullRequest, optional): The pull request object if the pipeline was triggered by a pull request. Defaults to None. | ||
fail_fast (bool, optional): Whether to fail fast. Defaults to False. | ||
fast_tests_only (bool, optional): Whether to run only fast tests. Defaults to False. | ||
code_tests_only (bool, optional): Whether to ignore non-code tests like QA and metadata checks. Defaults to False. | ||
use_host_gradle_dist_tar (bool, optional): Used when developing java connectors with gradle. Defaults to False. | ||
open_report_in_browser (bool, optional): Open HTML report in browser window. Defaults to True. | ||
|
@@ -99,8 +99,6 @@ def __init__( | |
self._updated_secrets_dir = None | ||
self.cdk_version = None | ||
self.should_save_report = should_save_report | ||
self.fail_fast = fail_fast | ||
self.fast_tests_only = fast_tests_only | ||
self.code_tests_only = code_tests_only | ||
self.use_local_cdk = use_local_cdk | ||
self.use_host_gradle_dist_tar = use_host_gradle_dist_tar | ||
|
@@ -110,6 +108,7 @@ def __init__( | |
self.s3_build_cache_access_key_id = s3_build_cache_access_key_id | ||
self.s3_build_cache_secret_key = s3_build_cache_secret_key | ||
self.concurrent_cat = concurrent_cat | ||
self._connector_secrets = None | ||
|
||
super().__init__( | ||
pipeline_name=pipeline_name, | ||
|
@@ -128,6 +127,7 @@ def __init__( | |
ci_git_user=ci_git_user, | ||
ci_github_access_token=ci_github_access_token, | ||
open_report_in_browser=open_report_in_browser, | ||
run_step_options=run_step_options, | ||
) | ||
|
||
@property | ||
|
@@ -206,6 +206,11 @@ def docker_hub_password_secret(self) -> Optional[Secret]: | |
return None | ||
return self.dagger_client.set_secret("docker_hub_password", self.docker_hub_password) | ||
|
||
async def connector_secrets(self): | ||
if self._connector_secrets is None: | ||
self._connector_secrets = await secrets.get_connector_secrets(self) | ||
return self._connector_secrets | ||
|
||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Is this related to your changes? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. It is! We were mounting connector secrets in between steps. So I hoisted that logic into the context so when you request There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Could we rename the function to There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Im torn on this On one hand it belongs in On the other hand connector secrets is a foundational concept that deserves a separate file to hold onto the logic. I think we gain more from it being separate. WDYT? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I suggest renaming it to
And we rename I'd be confused to have a method named There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Sounds good! Though, lets leave the internal field with the underscore. We want devs to always reference the getter and feel bad otherwise |
||
async def get_connector_dir(self, exclude=None, include=None) -> Directory: | ||
"""Get the connector under test source code directory. | ||
|
||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -12,6 +12,7 @@ | |
from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand | ||
from pipelines.consts import ContextState | ||
from pipelines.helpers.github import update_global_commit_status_check_for_tests | ||
from pipelines.helpers.run_steps import RunStepOptions | ||
from pipelines.helpers.utils import fail_if_missing_docker_hub_creds | ||
|
||
|
||
|
@@ -30,27 +31,27 @@ | |
type=bool, | ||
is_flag=True, | ||
) | ||
@click.option( | ||
"--fast-tests-only", | ||
help="When enabled, slow tests are skipped.", | ||
default=False, | ||
type=bool, | ||
is_flag=True, | ||
) | ||
@click.option( | ||
"--concurrent-cat", | ||
help="When enabled, the CAT tests will run concurrently. Be careful about rate limits", | ||
default=False, | ||
type=bool, | ||
is_flag=True, | ||
) | ||
@click.option( | ||
'--skip-step', | ||
'-x', | ||
multiple=True, | ||
type=str, | ||
bnchrch marked this conversation as resolved.
Show resolved
Hide resolved
|
||
help="Skip a step by name. Can be used multiple times to skip multiple steps.", | ||
) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. How big of a lift would it be to have an There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Not much at all! I can do that in another PR Issue here: https://github.com/airbytehq/airbyte/issues/33309 |
||
@click.pass_context | ||
async def test( | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Shall we add a There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Once I make this change you suggested Then --help should allow them to discover the step names |
||
ctx: click.Context, | ||
code_tests_only: bool, | ||
fail_fast: bool, | ||
fast_tests_only: bool, | ||
concurrent_cat: bool, | ||
skip_step: str, | ||
bnchrch marked this conversation as resolved.
Show resolved
Hide resolved
|
||
) -> bool: | ||
"""Runs a test pipeline for the selected connectors. | ||
|
||
|
@@ -70,6 +71,11 @@ async def test( | |
update_global_commit_status_check_for_tests(ctx.obj, "success") | ||
return True | ||
|
||
run_step_options = RunStepOptions( | ||
fail_fast=fail_fast, | ||
skip_steps=skip_step, | ||
) | ||
|
||
connectors_tests_contexts = [ | ||
ConnectorContext( | ||
pipeline_name=f"Testing connector {connector.technical_name}", | ||
|
@@ -86,18 +92,18 @@ async def test( | |
ci_context=ctx.obj.get("ci_context"), | ||
pull_request=ctx.obj.get("pull_request"), | ||
ci_gcs_credentials=ctx.obj["ci_gcs_credentials"], | ||
fail_fast=fail_fast, | ||
fast_tests_only=fast_tests_only, | ||
code_tests_only=code_tests_only, | ||
use_local_cdk=ctx.obj.get("use_local_cdk"), | ||
s3_build_cache_access_key_id=ctx.obj.get("s3_build_cache_access_key_id"), | ||
s3_build_cache_secret_key=ctx.obj.get("s3_build_cache_secret_key"), | ||
docker_hub_username=ctx.obj.get("docker_hub_username"), | ||
docker_hub_password=ctx.obj.get("docker_hub_password"), | ||
concurrent_cat=concurrent_cat, | ||
run_step_options=run_step_options, | ||
) | ||
for connector in ctx.obj["selected_connectors_with_modified_files"] | ||
] | ||
|
||
try: | ||
await run_connectors_pipelines( | ||
[connector_context for connector_context in connectors_tests_contexts], | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -3,125 +3,67 @@ | |
# | ||
"""This module groups factory like functions to dispatch tests steps according to the connector under test language.""" | ||
|
||
import itertools | ||
from typing import List | ||
|
||
import anyio | ||
import asyncer | ||
from connector_ops.utils import METADATA_FILE_NAME, ConnectorLanguage | ||
from pipelines.airbyte_ci.connectors.context import ConnectorContext | ||
from pipelines.airbyte_ci.connectors.reports import ConnectorReport | ||
from pipelines.airbyte_ci.connectors.test.steps import java_connectors, python_connectors | ||
from pipelines.airbyte_ci.connectors.test.steps.common import QaChecks, VersionFollowsSemverCheck, VersionIncrementCheck | ||
from pipelines.airbyte_ci.metadata.pipeline import MetadataValidation | ||
from pipelines.models.steps import StepResult | ||
from pipelines.helpers.run_steps import StepToRun, run_steps | ||
|
||
LANGUAGE_MAPPING = { | ||
"run_all_tests": { | ||
ConnectorLanguage.PYTHON: python_connectors.run_all_tests, | ||
ConnectorLanguage.LOW_CODE: python_connectors.run_all_tests, | ||
ConnectorLanguage.JAVA: java_connectors.run_all_tests, | ||
}, | ||
"run_code_format_checks": { | ||
ConnectorLanguage.PYTHON: python_connectors.run_code_format_checks, | ||
ConnectorLanguage.LOW_CODE: python_connectors.run_code_format_checks, | ||
# ConnectorLanguage.JAVA: java_connectors.run_code_format_checks | ||
"get_test_steps": { | ||
ConnectorLanguage.PYTHON: python_connectors.get_test_steps, | ||
ConnectorLanguage.LOW_CODE: python_connectors.get_test_steps, | ||
ConnectorLanguage.JAVA: java_connectors.get_test_steps, | ||
}, | ||
} | ||
|
||
|
||
async def run_metadata_validation(context: ConnectorContext) -> List[StepResult]: | ||
"""Run the metadata validation on a connector. | ||
Args: | ||
context (ConnectorContext): The current connector context. | ||
|
||
Returns: | ||
List[StepResult]: The results of the metadata validation steps. | ||
""" | ||
return [await MetadataValidation(context).run()] | ||
|
||
|
||
async def run_version_checks(context: ConnectorContext) -> List[StepResult]: | ||
"""Run the version checks on a connector. | ||
|
||
Args: | ||
context (ConnectorContext): The current connector context. | ||
|
||
Returns: | ||
List[StepResult]: The results of the version checks steps. | ||
""" | ||
return [await VersionFollowsSemverCheck(context).run(), await VersionIncrementCheck(context).run()] | ||
|
||
|
||
async def run_qa_checks(context: ConnectorContext) -> List[StepResult]: | ||
"""Run the QA checks on a connector. | ||
|
||
Args: | ||
context (ConnectorContext): The current connector context. | ||
|
||
Returns: | ||
List[StepResult]: The results of the QA checks steps. | ||
""" | ||
return [await QaChecks(context).run()] | ||
|
||
|
||
async def run_code_format_checks(context: ConnectorContext) -> List[StepResult]: | ||
"""Run the code format checks according to the connector language. | ||
|
||
Args: | ||
context (ConnectorContext): The current connector context. | ||
|
||
Returns: | ||
List[StepResult]: The results of the code format checks steps. | ||
""" | ||
if _run_code_format_checks := LANGUAGE_MAPPING["run_code_format_checks"].get(context.connector.language): | ||
return await _run_code_format_checks(context) | ||
else: | ||
context.logger.warning(f"No code format checks defined for connector language {context.connector.language}!") | ||
return [] | ||
|
||
|
||
async def run_all_tests(context: ConnectorContext) -> List[StepResult]: | ||
"""Run all the tests steps according to the connector language. | ||
def get_test_steps(context: ConnectorContext) -> List[StepToRun]: | ||
"""Get all the tests steps according to the connector language. | ||
|
||
Args: | ||
context (ConnectorContext): The current connector context. | ||
|
||
Returns: | ||
List[StepResult]: The results of the tests steps. | ||
List[StepResult]: The list of tests steps. | ||
""" | ||
if _run_all_tests := LANGUAGE_MAPPING["run_all_tests"].get(context.connector.language): | ||
return await _run_all_tests(context) | ||
if _get_test_steps := LANGUAGE_MAPPING["get_test_steps"].get(context.connector.language): | ||
return _get_test_steps(context) | ||
else: | ||
context.logger.warning(f"No tests defined for connector language {context.connector.language}!") | ||
return [] | ||
|
||
|
||
async def run_connector_test_pipeline(context: ConnectorContext, semaphore: anyio.Semaphore) -> ConnectorReport: | ||
"""Run a test pipeline for a single connector. | ||
async def run_connector_test_pipeline(context: ConnectorContext, semaphore: anyio.Semaphore): | ||
""" | ||
Compute the steps to run for a connector test pipeline. | ||
""" | ||
|
||
A visual DAG can be found on the README.md file of the pipelines modules. | ||
steps_to_run = get_test_steps(context) | ||
|
||
Args: | ||
context (ConnectorContext): The initialized connector context. | ||
if not context.code_tests_only: | ||
alafanechere marked this conversation as resolved.
Show resolved
Hide resolved
|
||
steps_to_run += [ | ||
[ | ||
StepToRun(id="metadata_validation", step=MetadataValidation(context)), | ||
StepToRun(id="version_follow_check", step=VersionFollowsSemverCheck(context)), | ||
StepToRun(id="version_inc_check", step=VersionIncrementCheck(context)), | ||
StepToRun(id="qa_checks", step=QaChecks(context)), | ||
] | ||
] | ||
|
||
Returns: | ||
ConnectorReport: The test reports holding tests results. | ||
""" | ||
async with semaphore: | ||
async with context: | ||
async with asyncer.create_task_group() as task_group: | ||
tasks = [ | ||
task_group.soonify(run_all_tests)(context), | ||
task_group.soonify(run_code_format_checks)(context), | ||
] | ||
if not context.code_tests_only: | ||
tasks += [ | ||
task_group.soonify(run_metadata_validation)(context), | ||
task_group.soonify(run_version_checks)(context), | ||
task_group.soonify(run_qa_checks)(context), | ||
] | ||
results = list(itertools.chain(*(task.value for task in tasks))) | ||
result_dict = await run_steps( | ||
runnables=steps_to_run, | ||
options=context.run_step_options, | ||
) | ||
Comment on lines
+62
to
+65
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 💎 |
||
|
||
results = list(result_dict.values()) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. nit: I'm wondering if casting to list is mandatory here, does directly passing There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Good call out, I think your right, let me check |
||
context.report = ConnectorReport(context, steps_results=results, name="TEST RESULTS") | ||
|
||
return context.report |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I think it would make sense to expose this parameter at the
PipelineContext
level for reusability. Wdyt?There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
We do! Im just passing it through to the pipeline context here