diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 7e2f6492874a..6294f35bd084 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -258,7 +258,7 @@ - name: GitHub sourceDefinitionId: ef69ef6e-aa7f-4af1-a01d-ef775033524e dockerRepository: airbyte/source-github - dockerImageTag: 0.2.27 + dockerImageTag: 0.2.28 documentationUrl: https://docs.airbyte.io/integrations/sources/github icon: github.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 188311e67478..29615c5c7d2a 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -2459,7 +2459,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-github:0.2.27" +- dockerImage: "airbyte/source-github:0.2.28" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/github" connectionSpecification: @@ -2485,9 +2485,6 @@ option_title: type: "string" const: "OAuth Credentials" - enum: - - "OAuth Credentials" - default: "OAuth Credentials" order: 0 access_token: type: "string" @@ -2502,9 +2499,6 @@ option_title: type: "string" const: "PAT Credentials" - enum: - - "PAT Credentials" - default: "PAT Credentials" order: 0 personal_access_token: type: "string" diff --git a/airbyte-integrations/connectors/source-github/Dockerfile b/airbyte-integrations/connectors/source-github/Dockerfile index d94aaa62a337..fdbd94465d7b 100644 --- a/airbyte-integrations/connectors/source-github/Dockerfile +++ b/airbyte-integrations/connectors/source-github/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.27 +LABEL io.airbyte.version=0.2.28 LABEL io.airbyte.name=airbyte/source-github diff --git a/airbyte-integrations/connectors/source-github/README.md b/airbyte-integrations/connectors/source-github/README.md index f9d231971317..7d82446bb141 100644 --- a/airbyte-integrations/connectors/source-github/README.md +++ b/airbyte-integrations/connectors/source-github/README.md @@ -21,6 +21,7 @@ development environment of choice. To activate it from the terminal, run: ``` source .venv/bin/activate pip install -r requirements.txt +pip install '.[tests]' ``` If you are in an IDE, follow your IDE's instructions to activate the virtualenv. diff --git a/airbyte-integrations/connectors/source-github/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-github/integration_tests/configured_catalog.json index 4b42ed1d5818..6998d9fd36cb 100644 --- a/airbyte-integrations/connectors/source-github/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-github/integration_tests/configured_catalog.json @@ -380,6 +380,26 @@ }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "team_members", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"], ["team_slug"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "team_memberships", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["url"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" } ] } diff --git a/airbyte-integrations/connectors/source-github/source_github/schemas/team_members.json b/airbyte-integrations/connectors/source-github/source_github/schemas/team_members.json new file mode 100644 index 000000000000..ed0fd963a872 --- /dev/null +++ b/airbyte-integrations/connectors/source-github/source_github/schemas/team_members.json @@ -0,0 +1,66 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "login": { + "type": ["null", "string"] + }, + "id": { + "type": "integer" + }, + "node_id": { + "type": ["null", "string"] + }, + "avatar_url": { + "type": ["null", "string"] + }, + "gravatar_id": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + }, + "html_url": { + "type": ["null", "string"] + }, + "followers_url": { + "type": ["null", "string"] + }, + "following_url": { + "type": ["null", "string"] + }, + "gists_url": { + "type": ["null", "string"] + }, + "starred_url": { + "type": ["null", "string"] + }, + "subscriptions_url": { + "type": ["null", "string"] + }, + "organizations_url": { + "type": ["null", "string"] + }, + "repos_url": { + "type": ["null", "string"] + }, + "events_url": { + "type": ["null", "string"] + }, + "received_events_url": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "site_admin": { + "type": ["null", "boolean"] + }, + "organization": { + "type": "string" + }, + "team_slug": { + "type": "string" + } + } +} diff --git a/airbyte-integrations/connectors/source-github/source_github/schemas/team_memberships.json b/airbyte-integrations/connectors/source-github/source_github/schemas/team_memberships.json new file mode 100644 index 000000000000..45797b2c7f3b --- /dev/null +++ b/airbyte-integrations/connectors/source-github/source_github/schemas/team_memberships.json @@ -0,0 +1,24 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "state": { + "type": ["null", "string"] + }, + "role": { + "type": ["null", "string"] + }, + "url": { + "type": "string" + }, + "organization": { + "type": "string" + }, + "team_slug": { + "type": "string" + }, + "username": { + "type": "string" + } + } +} diff --git a/airbyte-integrations/connectors/source-github/source_github/source.py b/airbyte-integrations/connectors/source-github/source_github/source.py index 65b9e913df4c..68c6a56f4e21 100644 --- a/airbyte-integrations/connectors/source-github/source_github/source.py +++ b/airbyte-integrations/connectors/source-github/source_github/source.py @@ -43,6 +43,8 @@ Reviews, Stargazers, Tags, + TeamMembers, + TeamMemberships, Teams, Users, WorkflowRuns, @@ -184,6 +186,8 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: pull_requests_stream = PullRequests(**repository_args_with_start_date) projects_stream = Projects(**repository_args_with_start_date) project_columns_stream = ProjectColumns(projects_stream, **repository_args_with_start_date) + teams_stream = Teams(**organization_args) + team_members_stream = TeamMembers(parent=teams_stream, **repository_args) return [ Assignees(**repository_args), @@ -215,8 +219,10 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: Reviews(parent=pull_requests_stream, **repository_args_with_start_date), Stargazers(**repository_args_with_start_date), Tags(**repository_args), - Teams(**organization_args), + teams_stream, + team_members_stream, Users(**organization_args), Workflows(**repository_args), WorkflowRuns(**repository_args), + TeamMemberships(parent=team_members_stream, **repository_args), ] diff --git a/airbyte-integrations/connectors/source-github/source_github/spec.json b/airbyte-integrations/connectors/source-github/source_github/spec.json index de44ea1bc51a..37e0f8bedde2 100644 --- a/airbyte-integrations/connectors/source-github/source_github/spec.json +++ b/airbyte-integrations/connectors/source-github/source_github/spec.json @@ -21,8 +21,6 @@ "option_title": { "type": "string", "const": "OAuth Credentials", - "enum": ["OAuth Credentials"], - "default": "OAuth Credentials", "order": 0 }, "access_token": { @@ -41,8 +39,6 @@ "option_title": { "type": "string", "const": "PAT Credentials", - "enum": ["PAT Credentials"], - "default": "PAT Credentials", "order": 0 }, "personal_access_token": { diff --git a/airbyte-integrations/connectors/source-github/source_github/streams.py b/airbyte-integrations/connectors/source-github/source_github/streams.py index 79368893d407..7f2dfcc2d1ba 100644 --- a/airbyte-integrations/connectors/source-github/source_github/streams.py +++ b/airbyte-integrations/connectors/source-github/source_github/streams.py @@ -1003,3 +1003,73 @@ def parse_response(self, response: requests.Response, stream_slice: Mapping[str, response = response.json().get("workflow_runs") for record in response: yield record + + +class TeamMembers(GithubStream): + """ + API docs: https://docs.github.com/en/rest/reference/teams#list-team-members + """ + + primary_key = ["id", "team_slug"] + + def __init__(self, parent: Teams, **kwargs): + super().__init__(**kwargs) + self.parent = parent + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + return f"orgs/{stream_slice['organization']}/teams/{stream_slice['team_slug']}/members" + + def stream_slices( + self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None + ) -> Iterable[Optional[Mapping[str, Any]]]: + parent_stream_slices = self.parent.stream_slices( + sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_state=stream_state + ) + for stream_slice in parent_stream_slices: + parent_records = self.parent.read_records( + sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_slice=stream_slice, stream_state=stream_state + ) + for record in parent_records: + yield {"organization": record["organization"], "team_slug": record["slug"]} + + def transform(self, record: MutableMapping[str, Any], stream_slice: Mapping[str, Any]) -> MutableMapping[str, Any]: + record["organization"] = stream_slice["organization"] + record["team_slug"] = stream_slice["team_slug"] + return record + + +class TeamMemberships(GithubStream): + """ + API docs: https://docs.github.com/en/rest/reference/teams#get-team-membership-for-a-user + """ + + primary_key = ["url"] + + def __init__(self, parent: TeamMembers, **kwargs): + super().__init__(**kwargs) + self.parent = parent + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + return f"orgs/{stream_slice['organization']}/teams/{stream_slice['team_slug']}/memberships/{stream_slice['username']}" + + def stream_slices( + self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None + ) -> Iterable[Optional[Mapping[str, Any]]]: + parent_stream_slices = self.parent.stream_slices( + sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_state=stream_state + ) + for stream_slice in parent_stream_slices: + parent_records = self.parent.read_records( + sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_slice=stream_slice, stream_state=stream_state + ) + for record in parent_records: + yield {"organization": record["organization"], "team_slug": record["team_slug"], "username": record["login"]} + + def parse_response(self, response: requests.Response, stream_slice: Mapping[str, Any], **kwargs) -> Iterable[Mapping]: + yield self.transform(response.json(), stream_slice=stream_slice) + + def transform(self, record: MutableMapping[str, Any], stream_slice: Mapping[str, Any]) -> MutableMapping[str, Any]: + record["organization"] = stream_slice["organization"] + record["team_slug"] = stream_slice["team_slug"] + record["username"] = stream_slice["username"] + return record diff --git a/airbyte-integrations/connectors/source-github/unit_tests/test_stream.py b/airbyte-integrations/connectors/source-github/unit_tests/test_stream.py index 8656a49368ca..1d4c6d11a2c7 100644 --- a/airbyte-integrations/connectors/source-github/unit_tests/test_stream.py +++ b/airbyte-integrations/connectors/source-github/unit_tests/test_stream.py @@ -32,6 +32,8 @@ Reviews, Stargazers, Tags, + TeamMembers, + TeamMemberships, Teams, Users, ) @@ -744,3 +746,34 @@ def test_stream_reviews_incremental_read(): assert responses.calls[0].request.params["direction"] == "asc" assert urlbase(responses.calls[3].request.url) == url_pulls assert responses.calls[3].request.params["direction"] == "asc" + + +@responses.activate +def test_stream_team_members_full_refresh(): + organization_args = {"organizations": ["org1"]} + repository_args = {"repositories": [], "page_size_for_large_streams": 100} + + responses.add("GET", "https://api.github.com/orgs/org1/teams", json=[{"slug": "team1"}, {"slug": "team2"}]) + responses.add("GET", "https://api.github.com/orgs/org1/teams/team1/members", json=[{"login": "login1"}, {"login": "login2"}]) + responses.add("GET", "https://api.github.com/orgs/org1/teams/team1/memberships/login1", json={"username": "login1"}) + responses.add("GET", "https://api.github.com/orgs/org1/teams/team1/memberships/login2", json={"username": "login2"}) + responses.add("GET", "https://api.github.com/orgs/org1/teams/team2/members", json=[{"login": "login2"}]) + responses.add("GET", "https://api.github.com/orgs/org1/teams/team2/memberships/login2", json={"username": "login2"}) + + stream = TeamMembers(parent=Teams(**organization_args), **repository_args) + records = read_full_refresh(stream) + + assert records == [ + {"login": "login1", "organization": "org1", "team_slug": "team1"}, + {"login": "login2", "organization": "org1", "team_slug": "team1"}, + {"login": "login2", "organization": "org1", "team_slug": "team2"}, + ] + + stream = TeamMemberships(parent=stream, **repository_args) + records = read_full_refresh(stream) + + assert records == [ + {"username": "login1", "organization": "org1", "team_slug": "team1"}, + {"username": "login2", "organization": "org1", "team_slug": "team1"}, + {"username": "login2", "organization": "org1", "team_slug": "team2"}, + ] diff --git a/docs/integrations/sources/github.md b/docs/integrations/sources/github.md index 6d07072567ce..76ecabbe9b80 100644 --- a/docs/integrations/sources/github.md +++ b/docs/integrations/sources/github.md @@ -20,6 +20,8 @@ This connector outputs the following full refresh streams: * [Pull request commits](https://docs.github.com/en/rest/reference/pulls#list-commits-on-a-pull-request) * [Repositories](https://docs.github.com/en/rest/reference/repos#list-organization-repositories) * [Tags](https://docs.github.com/en/rest/reference/repos#list-repository-tags) +* [TeamMembers](https://docs.github.com/en/rest/teams/members#list-team-members) +* [TeamMemberships](https://docs.github.com/en/rest/reference/teams#get-team-membership-for-a-user) * [Teams](https://docs.github.com/en/rest/reference/teams#list-teams) * [Users](https://docs.github.com/en/rest/reference/orgs#list-organization-members) * [Workflows](https://docs.github.com/en/rest/reference/actions#workflows) @@ -111,6 +113,7 @@ Your token should have at least the `repo` scope. Depending on which streams you | Version | Date | Pull Request | Subject | |:--------|:-----------| :--- |:-------------------------------------------------------------------------------------------------------------| +| 0.2.28 | 2022-04-21 | [11893](https://github.com/airbytehq/airbyte/pull/11893) | Add new streams `TeamMembers`, `TeamMemberships` | | 0.2.27 | 2022-04-02 | [11678](https://github.com/airbytehq/airbyte/pull/11678) | Fix "PAT Credentials" in spec | | 0.2.26 | 2022-03-31 | [11623](https://github.com/airbytehq/airbyte/pull/11623) | Re-factored incremental sync for `Reviews` stream | | 0.2.25 | 2022-03-31 | [11567](https://github.com/airbytehq/airbyte/pull/11567) | Improve code for better error handling |