Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

🐛Source GitLab: Fix syncing branches for the project with Repository feature disabled #26422

Merged
merged 5 commits into from
May 25, 2023
Merged

🐛Source GitLab: Fix syncing branches for the project with Repository feature disabled #26422

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
ed92c6e
Add handling blocked repository for branches stream
tolik0 May 23, 2023
00bcec4
Update changelog
tolik0 May 23, 2023
dd5493d
Reused error handler in branches stream
tolik0 May 24, 2023
1edc7a4
Fix using constant
tolik0 May 24, 2023
daf2689
Merge branch 'master' into 23130-source-gitlab-error-404-not-found-wh…
tolik0 May 24, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion airbyte-integrations/connectors/source-gitlab/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,5 +13,5 @@ COPY main.py ./

ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]

LABEL io.airbyte.version=1.1.0
LABEL io.airbyte.version=1.1.1
LABEL io.airbyte.name=airbyte/source-gitlab
2 changes: 1 addition & 1 deletion ...e-integrations/connectors/source-gitlab/integration_tests/expected_records_with_ids.jsonl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{"stream": "groups", "data": {"id": 11329647, "web_url": "https://gitlab.com/groups/new-group-airbute", "name": "New Group Airbute", "path": "new-group-airbute", "description": "", "visibility": "public", "share_with_group_lock": false, "require_two_factor_authentication": false, "two_factor_grace_period": 48, "project_creation_level": "developer", "auto_devops_enabled": null, "subgroup_creation_level": "maintainer", "emails_disabled": null, "mentions_disabled": null, "lfs_enabled": true, "default_branch_protection": 2, "avatar_url": null, "request_access_enabled": true, "full_name": "New Group Airbute", "full_path": "new-group-airbute", "created_at": "2021-03-15T15:55:53.613Z", "parent_id": null, "ldap_cn": null, "ldap_access": null, "wiki_access_level": "enabled", "shared_with_groups": [], "runners_token": "GR1348941-PhosPap-Sf1UxL1g6m4", "prevent_sharing_groups_outside_hierarchy": false, "shared_projects": [], "shared_runners_minutes_limit": null, "extra_shared_runners_minutes_limit": null, "prevent_forking_outside_group": null, "membership_lock": false, "projects": [{"id": 25157276, "path_with_namespace": "new-group-airbute/new-ci-test-project"}]}, "emitted_at": 1683114590269}
{"stream": "groups", "data": {"id": 61014882, "web_url": "https://gitlab.com/groups/new-group-airbute/test-subgroup-airbyte/test-private-sg", "name": "Test Private SG", "path": "test-private-sg", "description": "", "visibility": "private", "share_with_group_lock": false, "require_two_factor_authentication": false, "two_factor_grace_period": 48, "project_creation_level": "developer", "auto_devops_enabled": null, "subgroup_creation_level": "maintainer", "emails_disabled": null, "mentions_disabled": null, "lfs_enabled": true, "default_branch_protection": 2, "avatar_url": null, "request_access_enabled": true, "full_name": "New Group Airbute / Test Subgroup Airbyte / Test Private SG", "full_path": "new-group-airbute/test-subgroup-airbyte/test-private-sg", "created_at": "2022-12-02T08:46:22.648Z", "parent_id": 61014863, "ldap_cn": null, "ldap_access": null, "wiki_access_level": "enabled", "shared_with_groups": [], "runners_token": "GR1348941bjUaJQy2zzar-JmNBjfq", "shared_projects": [], "shared_runners_minutes_limit": null, "extra_shared_runners_minutes_limit": null, "prevent_forking_outside_group": null, "membership_lock": false, "projects": []}, "emitted_at": 1683114590768}
{"stream": "groups", "data": {"id": 61015181, "web_url": "https://gitlab.com/groups/new-group-airbute/test-public-sg/test-sg-public-2/test-private-subsubg-1", "name": "Test Private SubSubG 1", "path": "test-private-subsubg-1", "description": "", "visibility": "private", "share_with_group_lock": false, "require_two_factor_authentication": false, "two_factor_grace_period": 48, "project_creation_level": "developer", "auto_devops_enabled": null, "subgroup_creation_level": "maintainer", "emails_disabled": null, "mentions_disabled": null, "lfs_enabled": true, "default_branch_protection": 2, "avatar_url": null, "request_access_enabled": true, "full_name": "New Group Airbute / Test Public SG / Test SG Public 2 / Test Private SubSubG 1", "full_path": "new-group-airbute/test-public-sg/test-sg-public-2/test-private-subsubg-1", "created_at": "2022-12-02T08:54:42.252Z", "parent_id": 61014943, "ldap_cn": null, "ldap_access": null, "wiki_access_level": "enabled", "shared_with_groups": [], "runners_token": "GR1348941x8xQf6K-UvnnyJ-bcut4", "shared_projects": [], "shared_runners_minutes_limit": null, "extra_shared_runners_minutes_limit": null, "prevent_forking_outside_group": null, "membership_lock": false, "projects": [{"id": 41551658, "path_with_namespace": "new-group-airbute/test-public-sg/test-sg-public-2/test-private-subsubg-1/test_project_in_nested_subgroup"}]}, "emitted_at": 1683114591226}
{"stream": "projects", "data": {"id": 25157276, "description": "", "name": "New CI Test Project ", "name_with_namespace": "New Group Airbute / New CI Test Project ", "path": "new-ci-test-project", "path_with_namespace": "new-group-airbute/new-ci-test-project", "created_at": "2021-03-15T15:08:36.498Z", "default_branch": "master", "tag_list": [], "topics": [], "ssh_url_to_repo": "git@gitlab.com:new-group-airbute/new-ci-test-project.git", "http_url_to_repo": "https://gitlab.com/new-group-airbute/new-ci-test-project.git", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project", "readme_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/blob/master/README.md", "forks_count": 0, "avatar_url": null, "star_count": 0, "last_activity_at": "2022-12-13T09:39:47.235Z", "namespace": {"id": 11329647, "name": "New Group Airbute", "path": "new-group-airbute", "kind": "group", "full_path": "new-group-airbute", "parent_id": null, "avatar_url": null, "web_url": "https://gitlab.com/groups/new-group-airbute"}, "container_registry_image_prefix": "registry.gitlab.com/new-group-airbute/new-ci-test-project", "_links": {"self": "https://gitlab.com/api/v4/projects/25157276", "issues": "https://gitlab.com/api/v4/projects/25157276/issues", "merge_requests": "https://gitlab.com/api/v4/projects/25157276/merge_requests", "repo_branches": "https://gitlab.com/api/v4/projects/25157276/repository/branches", "labels": "https://gitlab.com/api/v4/projects/25157276/labels", "events": "https://gitlab.com/api/v4/projects/25157276/events", "members": "https://gitlab.com/api/v4/projects/25157276/members", "cluster_agents": "https://gitlab.com/api/v4/projects/25157276/cluster_agents"}, "packages_enabled": true, "empty_repo": false, "archived": false, "visibility": "private", "resolve_outdated_diff_discussions": false, "container_expiration_policy": {"cadence": "1d", "enabled": false, "keep_n": 10, "older_than": "90d", "name_regex": ".*", "name_regex_keep": null, "next_run_at": "2021-03-16T15:08:36.518Z"}, "issues_enabled": true, "merge_requests_enabled": true, "wiki_enabled": true, "jobs_enabled": true, "snippets_enabled": true, "container_registry_enabled": true, "service_desk_enabled": true, "service_desk_address": "contact-project+new-group-airbute-new-ci-test-project-25157276-issue-@incoming.gitlab.com", "can_create_merge_request_in": true, "issues_access_level": "enabled", "repository_access_level": "enabled", "merge_requests_access_level": "enabled", "forking_access_level": "enabled", "wiki_access_level": "enabled", "builds_access_level": "enabled", "snippets_access_level": "enabled", "pages_access_level": "private", "analytics_access_level": "enabled", "container_registry_access_level": "enabled", "security_and_compliance_access_level": "private", "releases_access_level": "enabled", "environments_access_level": "enabled", "feature_flags_access_level": "enabled", "infrastructure_access_level": "enabled", "monitor_access_level": "enabled", "emails_disabled": null, "shared_runners_enabled": true, "lfs_enabled": true, "creator_id": 8375961, "import_url": null, "import_type": null, "import_status": "none", "import_error": null, "open_issues_count": 31, "description_html": "", "updated_at": "2022-12-13T09:39:47.235Z", "ci_default_git_depth": 50, "ci_forward_deployment_enabled": true, "ci_job_token_scope_enabled": false, "ci_separated_caches": true, "ci_allow_fork_pipelines_to_run_in_parent_project": true, "build_git_strategy": "fetch", "keep_latest_artifact": true, "restrict_user_defined_variables": false, "runners_token": "GR1348941eMJgWDU69xyyshaNsaTZ", "runner_token_expiration_interval": null, "group_runners_enabled": true, "auto_cancel_pending_pipelines": "enabled", "build_timeout": 3600, "auto_devops_enabled": false, "auto_devops_deploy_strategy": "continuous", "ci_config_path": "", "public_jobs": true, "shared_with_groups": [], "only_allow_merge_if_pipeline_succeeds": false, "allow_merge_on_skipped_pipeline": null, "request_access_enabled": true, "only_allow_merge_if_all_discussions_are_resolved": false, "remove_source_branch_after_merge": true, "printing_merge_request_link_enabled": true, "merge_method": "merge", "squash_option": "default_off", "enforce_auth_checks_on_uploads": true, "suggestion_commit_message": null, "merge_commit_template": null, "squash_commit_template": null, "issue_branch_template": null, "statistics": {"commit_count": 3, "storage_size": 291925, "repository_size": 283115, "wiki_size": 0, "lfs_objects_size": 0, "job_artifacts_size": 8810, "pipeline_artifacts_size": 0, "packages_size": 0, "snippets_size": 0, "uploads_size": 0}, "autoclose_referenced_issues": true, "external_authorization_classification_label": "", "requirements_enabled": false, "requirements_access_level": "enabled", "security_and_compliance_enabled": true, "compliance_frameworks": [], "permissions": {"project_access": {"access_level": 40, "notification_level": 3}, "group_access": {"access_level": 50, "notification_level": 3}}}, "emitted_at": 1684137553535}
{"stream": "projects", "data": {"id": 25157276, "description": "", "name": "New CI Test Project ", "name_with_namespace": "New Group Airbute / New CI Test Project ", "path": "new-ci-test-project", "path_with_namespace": "new-group-airbute/new-ci-test-project", "created_at": "2021-03-15T15:08:36.498Z", "default_branch": "master", "tag_list": [], "topics": [], "ssh_url_to_repo": "git@gitlab.com:new-group-airbute/new-ci-test-project.git", "http_url_to_repo": "https://gitlab.com/new-group-airbute/new-ci-test-project.git", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project", "readme_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/blob/master/README.md", "forks_count": 0, "avatar_url": null, "star_count": 0, "last_activity_at": "2022-12-13T09:39:47.235Z", "namespace": {"id": 11329647, "name": "New Group Airbute", "path": "new-group-airbute", "kind": "group", "full_path": "new-group-airbute", "parent_id": null, "avatar_url": null, "web_url": "https://gitlab.com/groups/new-group-airbute"}, "container_registry_image_prefix": "registry.gitlab.com/new-group-airbute/new-ci-test-project", "_links": {"self": "https://gitlab.com/api/v4/projects/25157276", "issues": "https://gitlab.com/api/v4/projects/25157276/issues", "merge_requests": "https://gitlab.com/api/v4/projects/25157276/merge_requests", "repo_branches": "https://gitlab.com/api/v4/projects/25157276/repository/branches", "labels": "https://gitlab.com/api/v4/projects/25157276/labels", "events": "https://gitlab.com/api/v4/projects/25157276/events", "members": "https://gitlab.com/api/v4/projects/25157276/members", "cluster_agents": "https://gitlab.com/api/v4/projects/25157276/cluster_agents"}, "packages_enabled": true, "empty_repo": false, "archived": false, "visibility": "private", "resolve_outdated_diff_discussions": false, "container_expiration_policy": {"cadence": "1d", "enabled": false, "keep_n": 10, "older_than": "90d", "name_regex": ".*", "name_regex_keep": null, "next_run_at": "2021-03-16T15:08:36.518Z"}, "issues_enabled": true, "merge_requests_enabled": true, "wiki_enabled": true, "jobs_enabled": true, "snippets_enabled": true, "container_registry_enabled": true, "service_desk_enabled": true, "service_desk_address": "contact-project+new-group-airbute-new-ci-test-project-25157276-issue-@incoming.gitlab.com", "can_create_merge_request_in": true, "issues_access_level": "enabled", "repository_access_level": "private", "merge_requests_access_level": "private", "forking_access_level": "enabled", "wiki_access_level": "enabled", "builds_access_level": "private", "snippets_access_level": "enabled", "pages_access_level": "private", "analytics_access_level": "enabled", "container_registry_access_level": "enabled", "security_and_compliance_access_level": "private", "releases_access_level": "enabled", "environments_access_level": "enabled", "feature_flags_access_level": "enabled", "infrastructure_access_level": "enabled", "monitor_access_level": "enabled", "emails_disabled": false, "shared_runners_enabled": true, "lfs_enabled": true, "creator_id": 8375961, "import_url": null, "import_type": null, "import_status": "none", "import_error": null, "open_issues_count": 31, "description_html": "", "updated_at": "2023-05-23T12:12:18.623Z", "ci_default_git_depth": 50, "ci_forward_deployment_enabled": true, "ci_job_token_scope_enabled": false, "ci_separated_caches": true, "ci_allow_fork_pipelines_to_run_in_parent_project": true, "build_git_strategy": "fetch", "keep_latest_artifact": true, "restrict_user_defined_variables": false, "runners_token": "GR1348941eMJgWDU69xyyshaNsaTZ", "runner_token_expiration_interval": null, "group_runners_enabled": true, "auto_cancel_pending_pipelines": "enabled", "build_timeout": 3600, "auto_devops_enabled": false, "auto_devops_deploy_strategy": "continuous", "ci_config_path": "", "public_jobs": true, "shared_with_groups": [], "only_allow_merge_if_pipeline_succeeds": false, "allow_merge_on_skipped_pipeline": null, "request_access_enabled": true, "only_allow_merge_if_all_discussions_are_resolved": false, "remove_source_branch_after_merge": true, "printing_merge_request_link_enabled": true, "merge_method": "merge", "squash_option": "default_off", "enforce_auth_checks_on_uploads": true, "suggestion_commit_message": null, "merge_commit_template": null, "squash_commit_template": null, "issue_branch_template": null, "statistics": {"commit_count": 3, "storage_size": 291925, "repository_size": 283115, "wiki_size": 0, "lfs_objects_size": 0, "job_artifacts_size": 8810, "pipeline_artifacts_size": 0, "packages_size": 0, "snippets_size": 0, "uploads_size": 0}, "autoclose_referenced_issues": true, "external_authorization_classification_label": "", "requirements_enabled": false, "requirements_access_level": "enabled", "security_and_compliance_enabled": true, "compliance_frameworks": [], "permissions": {"project_access": {"access_level": 40, "notification_level": 3}, "group_access": {"access_level": 50, "notification_level": 3}}}, "emitted_at": 1684137553535}
{"stream": "branches", "data": {"name": "31-fake-issue-30", "merged": true, "protected": false, "developers_can_push": false, "developers_can_merge": false, "can_push": true, "default": false, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/tree/31-fake-issue-30", "commit_id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "project_id": 25157276}, "emitted_at": 1683114594890}
{"stream": "branches", "data": {"name": "master", "merged": false, "protected": true, "developers_can_push": false, "developers_can_merge": false, "can_push": true, "default": true, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/tree/master", "commit_id": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "project_id": 25157276}, "emitted_at": 1683114594892}
{"stream": "branches", "data": {"name": "new-test-branch", "merged": true, "protected": false, "developers_can_push": false, "developers_can_merge": false, "can_push": true, "default": false, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/tree/new-test-branch", "commit_id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "project_id": 25157276}, "emitted_at": 1683114594892}
Expand Down
2 changes: 1 addition & 1 deletion airbyte-integrations/connectors/source-gitlab/metadata.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ data:
connectorSubtype: api
connectorType: source
definitionId: 5e6175e5-68e1-4c17-bff9-56103bbb0d80
dockerImageTag: 1.1.0
dockerImageTag: 1.1.1
dockerRepository: airbyte/source-gitlab
githubIssueLabel: source-gitlab
icon: gitlab.svg
Expand Down
20 changes: 20 additions & 0 deletions airbyte-integrations/connectors/source-gitlab/source_gitlab/streams.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -311,6 +311,26 @@ class Branches(GitlabChildStream):
flatten_id_keys = ["commit"]
flatten_parent_id = True

def read_records(
self,
sync_mode: SyncMode,
cursor_field: List[str] = None,
stream_slice: Mapping[str, Any] = None,
stream_state: Mapping[str, Any] = None,
) -> Iterable[StreamData]:
try:
yield from super().read_records(sync_mode, cursor_field, stream_slice, stream_state)
except requests.exceptions.HTTPError as error:
status = error.response.status_code
if status == 404:
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we already do have handling of 403 error, could it be extended to also cope with 404?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No, because this fix is done only for branches stream.

Copy link
Collaborator

@davydov-d davydov-d May 24, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We could make it customizable per stream, for instance:

class GitlabStream:
    non_retriable_codes: List[int] = [403]
    ...
    ...
    if response.status_code in self.non_retriable_codes:
        setattr(self, "raise_on_http_errors", False)
        self.logger.warning(
            f"Got 403 error when accessing URL {response.request.url}."
            f" Very likely the feature is disabled for this project and/or group. Please double check it, or report a bug otherwise.")
        return False
    return super().should_retry(response)

class Branches:
    non_retriable_codes = [403, 404]

self.logger.warning(
"Got 404 error when accessing branches."
" Very likely the feature is disabled for this project and/or group. Please double check it, or report a bug otherwise."
)
yield from []
else:
raise error


class Commits(IncrementalGitlabChildStream):
cursor_field = "created_at"
Expand Down
11 changes: 10 additions & 1 deletion airbyte-integrations/connectors/source-gitlab/unit_tests/test_streams.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

import pytest
from airbyte_cdk.sources.streams.http.auth import NoAuth
from source_gitlab.streams import Commits, Jobs, MergeRequestCommits, MergeRequests, Pipelines, Projects, Releases, Tags
from source_gitlab.streams import Branches, Commits, Jobs, MergeRequestCommits, MergeRequests, Pipelines, Projects, Releases, Tags

auth_params = {"authenticator": NoAuth(), "api_url": "gitlab.com"}

Expand All @@ -19,6 +19,7 @@
releases = Releases(parent_stream=projects, **auth_params)
jobs = Jobs(parent_stream=pipelines, **auth_params)
merge_request_commits = MergeRequestCommits(parent_stream=merge_requests, **auth_params)
branches = Branches(parent_stream=projects, **auth_params)
commits = Commits(parent_stream=projects, repository_part=True, start_date=str(start_date), **auth_params)


Expand Down Expand Up @@ -136,3 +137,11 @@ def test_transform(requests_mock, stream, response_mocks, expected_records):
)
def test_updated_state(stream, current_state, latest_record, new_state):
assert stream.get_updated_state(current_state, latest_record) == new_state


def test_blocked_branches(requests_mock):
requests_mock.get("https://gitlab.com/api/v4/projects/p_1/branches?per_page=50", status_code=404)
for stream_slice in branches.stream_slices(sync_mode="full_refresh"):
records = list(branches.read_records(sync_mode="full_refresh", stream_slice=stream_slice))
assert records == []
assert requests_mock.call_count == 1
1 change: 1 addition & 0 deletions docs/integrations/sources/gitlab.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,7 @@ Gitlab has the [rate limits](https://docs.gitlab.com/ee/user/gitlab_com/index.ht

| Version | Date | Pull Request | Subject |
|:--------|:-----------|:---------------------------------------------------------|:-------------------------------------------------------------------------------------------|
| 1.1.1 | 2023-05-23 | [26422](https://github.com/airbytehq/airbyte/pull/26422) | Fix error `404 Repository Not Found` when syncing project with Repository feature disabled |
| 1.1.0 | 2023-05-10 | [25948](https://github.com/airbytehq/airbyte/pull/25948) | Introduce two new fields in the `Projects` stream schema |
| 1.0.4 | 2023-04-20 | [21373](https://github.com/airbytehq/airbyte/pull/21373) | Accept api_url with or without scheme |
| 1.0.3 | 2023-02-14 | [22992](https://github.com/airbytehq/airbyte/pull/22992) | Specified date formatting in specification |
Expand Down