🐛 Source Github: add new streams TeamMembers, TeamMemberships (#11893)

Signed-off-by: Sergey Chvalyuk <grubberr@gmail.com>

airbyte-config/init/src/main/resources/seed/source_definitions.yaml

@@ -258,7 +258,7 @@
 - name: GitHub
   sourceDefinitionId: ef69ef6e-aa7f-4af1-a01d-ef775033524e
   dockerRepository: airbyte/source-github
-  dockerImageTag: 0.2.27
+  dockerImageTag: 0.2.28
   documentationUrl: https://docs.airbyte.io/integrations/sources/github
   icon: github.svg
   sourceType: api

airbyte-config/init/src/main/resources/seed/source_specs.yaml

@@ -2459,7 +2459,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-github:0.2.27"
+- dockerImage: "airbyte/source-github:0.2.28"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/github"
     connectionSpecification:
@@ -2485,9 +2485,6 @@
               option_title:
                 type: "string"
                 const: "OAuth Credentials"
-                enum:
-                - "OAuth Credentials"
-                default: "OAuth Credentials"
                 order: 0
               access_token:
                 type: "string"
@@ -2502,9 +2499,6 @@
               option_title:
                 type: "string"
                 const: "PAT Credentials"
-                enum:
-                - "PAT Credentials"
-                default: "PAT Credentials"
                 order: 0
               personal_access_token:
                 type: "string"

airbyte-integrations/connectors/source-github/Dockerfile

@@ -12,5 +12,5 @@ RUN pip install .
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.2.27
+LABEL io.airbyte.version=0.2.28
 LABEL io.airbyte.name=airbyte/source-github

airbyte-integrations/connectors/source-github/README.md

@@ -21,6 +21,7 @@ development environment of choice. To activate it from the terminal, run:
 ```
 source .venv/bin/activate
 pip install -r requirements.txt
+pip install '.[tests]'
 ```
 If you are in an IDE, follow your IDE's instructions to activate the virtualenv.
 

airbyte-integrations/connectors/source-github/integration_tests/configured_catalog.json

@@ -380,6 +380,26 @@
       },
       "sync_mode": "full_refresh",
       "destination_sync_mode": "overwrite"
+    },
+    {
+      "stream": {
+        "name": "team_members",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"],
+        "source_defined_primary_key": [["id"], ["team_slug"]]
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
+    },
+    {
+      "stream": {
+        "name": "team_memberships",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"],
+        "source_defined_primary_key": [["url"]]
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
     }
   ]
 }

airbyte-integrations/connectors/source-github/source_github/schemas/team_members.json

@@ -0,0 +1,66 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "type": "object",
+  "properties": {
+    "login": {
+      "type": ["null", "string"]
+    },
+    "id": {
+      "type": "integer"
+    },
+    "node_id": {
+      "type": ["null", "string"]
+    },
+    "avatar_url": {
+      "type": ["null", "string"]
+    },
+    "gravatar_id": {
+      "type": ["null", "string"]
+    },
+    "url": {
+      "type": ["null", "string"]
+    },
+    "html_url": {
+      "type": ["null", "string"]
+    },
+    "followers_url": {
+      "type": ["null", "string"]
+    },
+    "following_url": {
+      "type": ["null", "string"]
+    },
+    "gists_url": {
+      "type": ["null", "string"]
+    },
+    "starred_url": {
+      "type": ["null", "string"]
+    },
+    "subscriptions_url": {
+      "type": ["null", "string"]
+    },
+    "organizations_url": {
+      "type": ["null", "string"]
+    },
+    "repos_url": {
+      "type": ["null", "string"]
+    },
+    "events_url": {
+      "type": ["null", "string"]
+    },
+    "received_events_url": {
+      "type": ["null", "string"]
+    },
+    "type": {
+      "type": ["null", "string"]
+    },
+    "site_admin": {
+      "type": ["null", "boolean"]
+    },
+    "organization": {
+      "type": "string"
+    },
+    "team_slug": {
+      "type": "string"
+    }
+  }
+}

airbyte-integrations/connectors/source-github/source_github/schemas/team_memberships.json

@@ -0,0 +1,24 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "type": "object",
+  "properties": {
+    "state": {
+      "type": ["null", "string"]
+    },
+    "role": {
+      "type": ["null", "string"]
+    },
+    "url": {
+      "type": "string"
+    },
+    "organization": {
+      "type": "string"
+    },
+    "team_slug": {
+      "type": "string"
+    },
+    "username": {
+      "type": "string"
+    }
+  }
+}

airbyte-integrations/connectors/source-github/source_github/source.py

@@ -43,6 +43,8 @@
     Reviews,
     Stargazers,
     Tags,
+    TeamMembers,
+    TeamMemberships,
     Teams,
     Users,
     WorkflowRuns,
@@ -184,6 +186,8 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]:
         pull_requests_stream = PullRequests(**repository_args_with_start_date)
         projects_stream = Projects(**repository_args_with_start_date)
         project_columns_stream = ProjectColumns(projects_stream, **repository_args_with_start_date)
+        teams_stream = Teams(**organization_args)
+        team_members_stream = TeamMembers(parent=teams_stream, **repository_args)
 
         return [
             Assignees(**repository_args),
@@ -215,8 +219,10 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]:
             Reviews(parent=pull_requests_stream, **repository_args_with_start_date),
             Stargazers(**repository_args_with_start_date),
             Tags(**repository_args),
-            Teams(**organization_args),
+            teams_stream,
+            team_members_stream,
             Users(**organization_args),
             Workflows(**repository_args),
             WorkflowRuns(**repository_args),
+            TeamMemberships(parent=team_members_stream, **repository_args),
         ]

airbyte-integrations/connectors/source-github/source_github/spec.json

@@ -21,8 +21,6 @@
               "option_title": {
                 "type": "string",
                 "const": "OAuth Credentials",
-                "enum": ["OAuth Credentials"],
-                "default": "OAuth Credentials",
                 "order": 0
               },
               "access_token": {
@@ -41,8 +39,6 @@
               "option_title": {
                 "type": "string",
                 "const": "PAT Credentials",
-                "enum": ["PAT Credentials"],
-                "default": "PAT Credentials",
                 "order": 0
               },
               "personal_access_token": {

airbyte-integrations/connectors/source-github/source_github/streams.py

@@ -1003,3 +1003,73 @@ def parse_response(self, response: requests.Response, stream_slice: Mapping[str,
         response = response.json().get("workflow_runs")
         for record in response:
             yield record
+
+
+class TeamMembers(GithubStream):
+    """
+    API docs: https://docs.github.com/en/rest/reference/teams#list-team-members
+    """
+
+    primary_key = ["id", "team_slug"]
+
+    def __init__(self, parent: Teams, **kwargs):
+        super().__init__(**kwargs)
+        self.parent = parent
+
+    def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str:
+        return f"orgs/{stream_slice['organization']}/teams/{stream_slice['team_slug']}/members"
+
+    def stream_slices(
+        self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None
+    ) -> Iterable[Optional[Mapping[str, Any]]]:
+        parent_stream_slices = self.parent.stream_slices(
+            sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_state=stream_state
+        )
+        for stream_slice in parent_stream_slices:
+            parent_records = self.parent.read_records(
+                sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_slice=stream_slice, stream_state=stream_state
+            )
+            for record in parent_records:
+                yield {"organization": record["organization"], "team_slug": record["slug"]}
+
+    def transform(self, record: MutableMapping[str, Any], stream_slice: Mapping[str, Any]) -> MutableMapping[str, Any]:
+        record["organization"] = stream_slice["organization"]
+        record["team_slug"] = stream_slice["team_slug"]
+        return record
+
+
+class TeamMemberships(GithubStream):
+    """
+    API docs: https://docs.github.com/en/rest/reference/teams#get-team-membership-for-a-user
+    """
+
+    primary_key = ["url"]
+
+    def __init__(self, parent: TeamMembers, **kwargs):
+        super().__init__(**kwargs)
+        self.parent = parent
+
+    def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str:
+        return f"orgs/{stream_slice['organization']}/teams/{stream_slice['team_slug']}/memberships/{stream_slice['username']}"
+
+    def stream_slices(
+        self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None
+    ) -> Iterable[Optional[Mapping[str, Any]]]:
+        parent_stream_slices = self.parent.stream_slices(
+            sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_state=stream_state
+        )
+        for stream_slice in parent_stream_slices:
+            parent_records = self.parent.read_records(
+                sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_slice=stream_slice, stream_state=stream_state
+            )
+            for record in parent_records:
+                yield {"organization": record["organization"], "team_slug": record["team_slug"], "username": record["login"]}
+
+    def parse_response(self, response: requests.Response, stream_slice: Mapping[str, Any], **kwargs) -> Iterable[Mapping]:
+        yield self.transform(response.json(), stream_slice=stream_slice)
+
+    def transform(self, record: MutableMapping[str, Any], stream_slice: Mapping[str, Any]) -> MutableMapping[str, Any]:
+        record["organization"] = stream_slice["organization"]
+        record["team_slug"] = stream_slice["team_slug"]
+        record["username"] = stream_slice["username"]
+        return record

airbyte-integrations/connectors/source-github/unit_tests/test_stream.py

@@ -32,6 +32,8 @@
     Reviews,
     Stargazers,
     Tags,
+    TeamMembers,
+    TeamMemberships,
     Teams,
     Users,
 )
@@ -744,3 +746,34 @@ def test_stream_reviews_incremental_read():
     assert responses.calls[0].request.params["direction"] == "asc"
     assert urlbase(responses.calls[3].request.url) == url_pulls
     assert responses.calls[3].request.params["direction"] == "asc"
+
+
+@responses.activate
+def test_stream_team_members_full_refresh():
+    organization_args = {"organizations": ["org1"]}
+    repository_args = {"repositories": [], "page_size_for_large_streams": 100}
+
+    responses.add("GET", "https://api.github.com/orgs/org1/teams", json=[{"slug": "team1"}, {"slug": "team2"}])
+    responses.add("GET", "https://api.github.com/orgs/org1/teams/team1/members", json=[{"login": "login1"}, {"login": "login2"}])
+    responses.add("GET", "https://api.github.com/orgs/org1/teams/team1/memberships/login1", json={"username": "login1"})
+    responses.add("GET", "https://api.github.com/orgs/org1/teams/team1/memberships/login2", json={"username": "login2"})
+    responses.add("GET", "https://api.github.com/orgs/org1/teams/team2/members", json=[{"login": "login2"}])
+    responses.add("GET", "https://api.github.com/orgs/org1/teams/team2/memberships/login2", json={"username": "login2"})
+
+    stream = TeamMembers(parent=Teams(**organization_args), **repository_args)
+    records = read_full_refresh(stream)
+
+    assert records == [
+        {"login": "login1", "organization": "org1", "team_slug": "team1"},
+        {"login": "login2", "organization": "org1", "team_slug": "team1"},
+        {"login": "login2", "organization": "org1", "team_slug": "team2"},
+    ]
+
+    stream = TeamMemberships(parent=stream, **repository_args)
+    records = read_full_refresh(stream)
+
+    assert records == [
+        {"username": "login1", "organization": "org1", "team_slug": "team1"},
+        {"username": "login2", "organization": "org1", "team_slug": "team1"},
+        {"username": "login2", "organization": "org1", "team_slug": "team2"},
+    ]

docs/integrations/sources/github.md

@@ -20,6 +20,8 @@ This connector outputs the following full refresh streams:
 * [Pull request commits](https://docs.github.com/en/rest/reference/pulls#list-commits-on-a-pull-request)
 * [Repositories](https://docs.github.com/en/rest/reference/repos#list-organization-repositories)
 * [Tags](https://docs.github.com/en/rest/reference/repos#list-repository-tags)
+* [TeamMembers](https://docs.github.com/en/rest/teams/members#list-team-members)
+* [TeamMemberships](https://docs.github.com/en/rest/reference/teams#get-team-membership-for-a-user)
 * [Teams](https://docs.github.com/en/rest/reference/teams#list-teams)
 * [Users](https://docs.github.com/en/rest/reference/orgs#list-organization-members)
 * [Workflows](https://docs.github.com/en/rest/reference/actions#workflows)
@@ -111,6 +113,7 @@ Your token should have at least the `repo` scope. Depending on which streams you
 
 | Version | Date       | Pull Request | Subject                                                                                                      |
 |:--------|:-----------| :--- |:-------------------------------------------------------------------------------------------------------------|
+| 0.2.28  | 2022-04-21 | [11893](https://github.com/airbytehq/airbyte/pull/11893) | Add new streams `TeamMembers`, `TeamMemberships`                                                           |
 | 0.2.27  | 2022-04-02 | [11678](https://github.com/airbytehq/airbyte/pull/11678) | Fix "PAT Credentials" in spec                                                                              |
 | 0.2.26  | 2022-03-31 | [11623](https://github.com/airbytehq/airbyte/pull/11623) | Re-factored incremental sync for `Reviews` stream                                                          |
 | 0.2.25  | 2022-03-31 | [11567](https://github.com/airbytehq/airbyte/pull/11567) | Improve code for better error handling                                                                     |