From 8fba0270c7d23fc31219682db03a3e7fc0da8a3a Mon Sep 17 00:00:00 2001 From: Yiyang Li Date: Sat, 9 Jul 2022 15:46:33 -0700 Subject: [PATCH 1/4] Add User_Role_Assignments and Group_Role_Assignments stream - it's all about okta source connector - these 2 API streams don't support the sync mode incremental, the api doesn's mention cursor or pagination for these 2 APIs - add $id to shared schema, rename shared schema --- .../integration_tests/catalog.json | 18 ++++ .../integration_tests/configured_catalog.json | 18 ++++ .../source-okta/sample_files/catalog.json | 20 +++++ .../source_okta/schemas/group_members.json | 2 +- .../schemas/group_role_assignments.json | 3 + .../schemas/shared/shared-role.json | 85 +++++++++++++++++++ .../shared/{users.json => shared-user.json} | 1 + .../schemas/user_role_assignments.json | 3 + .../source_okta/schemas/users.json | 2 +- .../source-okta/source_okta/source.py | 27 ++++++ docs/integrations/sources/okta.md | 3 + 11 files changed, 180 insertions(+), 2 deletions(-) create mode 100644 airbyte-integrations/connectors/source-okta/source_okta/schemas/group_role_assignments.json create mode 100644 airbyte-integrations/connectors/source-okta/source_okta/schemas/shared/shared-role.json rename airbyte-integrations/connectors/source-okta/source_okta/schemas/shared/{users.json => shared-user.json} (99%) create mode 100644 airbyte-integrations/connectors/source-okta/source_okta/schemas/user_role_assignments.json diff --git a/airbyte-integrations/connectors/source-okta/integration_tests/catalog.json b/airbyte-integrations/connectors/source-okta/integration_tests/catalog.json index 7f052c7b1c09..5d4472188523 100644 --- a/airbyte-integrations/connectors/source-okta/integration_tests/catalog.json +++ b/airbyte-integrations/connectors/source-okta/integration_tests/catalog.json @@ -48,6 +48,24 @@ "sync_mode": "full_refresh", "destination_sync_mode": "overwrite", "primary_key": [["id"]] + }, + { + "stream": { + "name": "user_role_assignments", + "json_schema": {} + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite", + "primary_key": [["id"]] + }, + { + "stream": { + "name": "group_role_assignments", + "json_schema": {} + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite", + "primary_key": [["id"]] } ] } diff --git a/airbyte-integrations/connectors/source-okta/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-okta/integration_tests/configured_catalog.json index b8aa901b4197..7e146a15260f 100644 --- a/airbyte-integrations/connectors/source-okta/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-okta/integration_tests/configured_catalog.json @@ -52,6 +52,24 @@ }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "user_role_assignments", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "group_role_assignments", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" } ] } diff --git a/airbyte-integrations/connectors/source-okta/sample_files/catalog.json b/airbyte-integrations/connectors/source-okta/sample_files/catalog.json index c7793f364181..468a300fac08 100644 --- a/airbyte-integrations/connectors/source-okta/sample_files/catalog.json +++ b/airbyte-integrations/connectors/source-okta/sample_files/catalog.json @@ -40,6 +40,26 @@ "destination_sync_mode": "overwrite", "cursor_field": ["id"], "primary_key": [["id"]] + }, + { + "stream": { + "name": "user_role_assignments", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite", + "primary_key": [["id"]] + }, + { + "stream": { + "name": "group_role_assignments", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite", + "primary_key": [["id"]] } ] } diff --git a/airbyte-integrations/connectors/source-okta/source_okta/schemas/group_members.json b/airbyte-integrations/connectors/source-okta/source_okta/schemas/group_members.json index d02387b101f1..f1dd1977e55a 100644 --- a/airbyte-integrations/connectors/source-okta/source_okta/schemas/group_members.json +++ b/airbyte-integrations/connectors/source-okta/source_okta/schemas/group_members.json @@ -1,3 +1,3 @@ { - "$ref": "users.json" + "$ref": "shared-user.json" } diff --git a/airbyte-integrations/connectors/source-okta/source_okta/schemas/group_role_assignments.json b/airbyte-integrations/connectors/source-okta/source_okta/schemas/group_role_assignments.json new file mode 100644 index 000000000000..49a6dfb4c85a --- /dev/null +++ b/airbyte-integrations/connectors/source-okta/source_okta/schemas/group_role_assignments.json @@ -0,0 +1,3 @@ +{ + "$ref": "shared-role.json" +} diff --git a/airbyte-integrations/connectors/source-okta/source_okta/schemas/shared/shared-role.json b/airbyte-integrations/connectors/source-okta/source_okta/schemas/shared/shared-role.json new file mode 100644 index 000000000000..be3e8fdcb0f8 --- /dev/null +++ b/airbyte-integrations/connectors/source-okta/source_okta/schemas/shared/shared-role.json @@ -0,0 +1,85 @@ +{ + "properties": { + "id": { + "type": "string" + }, + "label": { + "type": "string" + }, + "type": { + "enum": [ + "API_ACCESS_MANAGEMENT_ADMIN", + "APP_ADMIN", + "GROUP_MEMBERSHIP_ADMIN", + "HELP_DESK_ADMIN", + "MOBILE_ADMIN", + "ORG_ADMIN", + "READ_ONLY_ADMIN", + "REPORT_ADMIN", + "SUPER_ADMIN", + "USER_ADMIN", + "CUSTOM" + ], + "type": "string" + }, + "status": { + "enum": [ + "ACTIVE", + "DEPROVISIONED", + "LOCKED_OUT", + "PASSWORD_EXPIRED", + "PROVISIONED", + "RECOVERY", + "STAGED", + "SUSPENDED" + ], + "type": "string" + }, + "created": { + "format": "date-time", + "type": "string" + }, + "lastUpdated": { + "format": "date-time", + "type": "string" + }, + "assignmentType": { + "enum": ["USER", "GROUP"], + "type": "string" + }, + "resource-set": { + "description": "The Resource Set ID in which the Role is granted (only present for Custom Roles)", + "type": ["string", "null"] + }, + "_links": { + "properties": { + "assignee": { + "properties": { + "assignee": { + "$ref": "shared-link.json", + "description": "Gets the User or Group through which this Role is assigned" + }, + "role": { + "$ref": "shared-link.json", + "description": "(Only for Custom Roles) gets the Role that is granted through this assignment" + }, + "resource-set": { + "$ref": "shared-link.json", + "description": "(Only for Custom Roles) Gets the Resource Set that is targeted by this assignment" + }, + "permissions": { + "$ref": "shared-link.json", + "description": "(Only for Custom Roles) Gets a list of Permissions that is granted through this assignment" + }, + "member": { + "$ref": "shared-link.json", + "description": "(Only for Custom Roles) gets the Member object from the Binding that grants this Role" + } + } + } + }, + "type": ["object", "null"] + } + }, + "type": "object" +} diff --git a/airbyte-integrations/connectors/source-okta/source_okta/schemas/shared/users.json b/airbyte-integrations/connectors/source-okta/source_okta/schemas/shared/shared-user.json similarity index 99% rename from airbyte-integrations/connectors/source-okta/source_okta/schemas/shared/users.json rename to airbyte-integrations/connectors/source-okta/source_okta/schemas/shared/shared-user.json index eef92c8a7c4b..a1e5c2e841d0 100644 --- a/airbyte-integrations/connectors/source-okta/source_okta/schemas/shared/users.json +++ b/airbyte-integrations/connectors/source-okta/source_okta/schemas/shared/shared-user.json @@ -1,4 +1,5 @@ { + "$id": "shared-users.json", "properties": { "_links": { "additionalProperties": { diff --git a/airbyte-integrations/connectors/source-okta/source_okta/schemas/user_role_assignments.json b/airbyte-integrations/connectors/source-okta/source_okta/schemas/user_role_assignments.json new file mode 100644 index 000000000000..49a6dfb4c85a --- /dev/null +++ b/airbyte-integrations/connectors/source-okta/source_okta/schemas/user_role_assignments.json @@ -0,0 +1,3 @@ +{ + "$ref": "shared-role.json" +} diff --git a/airbyte-integrations/connectors/source-okta/source_okta/schemas/users.json b/airbyte-integrations/connectors/source-okta/source_okta/schemas/users.json index d02387b101f1..f1dd1977e55a 100644 --- a/airbyte-integrations/connectors/source-okta/source_okta/schemas/users.json +++ b/airbyte-integrations/connectors/source-okta/source_okta/schemas/users.json @@ -1,3 +1,3 @@ { - "$ref": "users.json" + "$ref": "shared-user.json" } diff --git a/airbyte-integrations/connectors/source-okta/source_okta/source.py b/airbyte-integrations/connectors/source-okta/source_okta/source.py index 6684c5a6c9b0..40d296a80557 100644 --- a/airbyte-integrations/connectors/source-okta/source_okta/source.py +++ b/airbyte-integrations/connectors/source-okta/source_okta/source.py @@ -149,6 +149,20 @@ def get_updated_state(self, current_stream_state: MutableMapping[str, Any], late } +class GroupRoleAssignments(OktaStream): + primary_key = "id" + use_cache = True + + def stream_slices(self, **kwargs): + group_stream = Groups(authenticator=self.authenticator, url_base=self.url_base) + for group in group_stream.read_records(sync_mode=SyncMode.full_refresh): + yield {"group_id": group["id"]} + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + group_id = stream_slice["group_id"] + return f"groups/{group_id}/roles" + + class Logs(IncrementalOktaStream): cursor_field = "published" @@ -202,7 +216,18 @@ def parse_response( **kwargs, ) -> Iterable[Mapping]: yield from response.json()["roles"] +class UserRoleAssignments(OktaStream): + primary_key = "id" + use_cache = True + + def stream_slices(self, **kwargs): + user_stream = Users(authenticator=self.authenticator, url_base=self.url_base) + for user in user_stream.read_records(sync_mode=SyncMode.full_refresh): + yield {"user_id": user["id"]} + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + user_id = stream_slice["user_id"] + return f"users/{user_id}/roles" class SourceOkta(AbstractSource): def initialize_authenticator(self, config: Mapping[str, Any]) -> TokenAuthenticator: @@ -245,4 +270,6 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: Users(**initialization_params), GroupMembers(**initialization_params), CustomRoles(**initialization_params), + UserRoleAssignments(**initialization_params), + GroupRoleAssignments(**initialization_params), ] diff --git a/docs/integrations/sources/okta.md b/docs/integrations/sources/okta.md index d8a12e8e29ba..046d43e52118 100644 --- a/docs/integrations/sources/okta.md +++ b/docs/integrations/sources/okta.md @@ -9,8 +9,10 @@ This source can sync data for the [Okta API](https://developer.okta.com/docs/ref This Source is capable of syncing the following core Streams: * [Users](https://developer.okta.com/docs/reference/api/users/#list-users) +* [User Role Assignments](https://developer.okta.com/docs/reference/api/roles/#list-roles-assigned-to-a-user) * [Groups](https://developer.okta.com/docs/reference/api/groups/#list-groups) * [Group Members](https://developer.okta.com/docs/reference/api/groups/#list-group-members) +* [Group Role Assignments](https://developer.okta.com/docs/reference/api/roles/#list-roles-assigned-to-a-group) * [System Log](https://developer.okta.com/docs/reference/api/system-log/#get-started) * [Custom Roles](https://developer.okta.com/docs/reference/api/roles/#list-roles) @@ -63,6 +65,7 @@ Different Okta APIs require different admin privilege levels. API tokens inherit | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.1.6 | 2022-07-13 | [14556](https://github.com/airbytehq/airbyte/pull/14556) | add User_Role_Assignments and Group_Role_Assignments streams (full fetch only) | | 0.1.6 | 2022-07-11 | [14610](https://github.com/airbytehq/airbyte/pull/14610) | add custom roles stream | | 0.1.5 | 2022-07-04 | [14380](https://github.com/airbytehq/airbyte/pull/14380) | add Group_Members stream to okta source | | 0.1.4 | 2021-11-02 | [7584](https://github.com/airbytehq/airbyte/pull/7584) | Fix incremental params for log stream | From 94fbebc9db437315048da56a068ad6a72bd3c2db Mon Sep 17 00:00:00 2001 From: Yiyang Li Date: Wed, 13 Jul 2022 08:36:14 -0700 Subject: [PATCH 2/4] formatted --- .../connectors/source-okta/source_okta/source.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/airbyte-integrations/connectors/source-okta/source_okta/source.py b/airbyte-integrations/connectors/source-okta/source_okta/source.py index 40d296a80557..81ec19a69197 100644 --- a/airbyte-integrations/connectors/source-okta/source_okta/source.py +++ b/airbyte-integrations/connectors/source-okta/source_okta/source.py @@ -216,6 +216,8 @@ def parse_response( **kwargs, ) -> Iterable[Mapping]: yield from response.json()["roles"] + + class UserRoleAssignments(OktaStream): primary_key = "id" use_cache = True @@ -229,6 +231,7 @@ def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: user_id = stream_slice["user_id"] return f"users/{user_id}/roles" + class SourceOkta(AbstractSource): def initialize_authenticator(self, config: Mapping[str, Any]) -> TokenAuthenticator: return TokenAuthenticator(config["token"], auth_method="SSWS") From f037ff06ff73a1c2053ead5aca90952148a02aaf Mon Sep 17 00:00:00 2001 From: Harshith Mullapudi Date: Tue, 19 Jul 2022 00:08:04 +0530 Subject: [PATCH 3/4] chore: added connector to definitions --- .../resources/seed/source_definitions.yaml | 2 +- .../connectors/source-okta/Dockerfile | 2 +- docs/integrations/sources/okta.md | 57 +++++++++---------- 3 files changed, 30 insertions(+), 31 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 8a7868f50196..c70e0254e28e 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -620,7 +620,7 @@ - name: Okta sourceDefinitionId: 1d4fdb25-64fc-4569-92da-fcdca79a8372 dockerRepository: airbyte/source-okta - dockerImageTag: 0.1.6 + dockerImageTag: 0.1.7 documentationUrl: https://docs.airbyte.io/integrations/sources/okta icon: okta.svg sourceType: api diff --git a/airbyte-integrations/connectors/source-okta/Dockerfile b/airbyte-integrations/connectors/source-okta/Dockerfile index 2ed3127ed797..ed4dd1de885d 100644 --- a/airbyte-integrations/connectors/source-okta/Dockerfile +++ b/airbyte-integrations/connectors/source-okta/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.6 +LABEL io.airbyte.version=0.1.7 LABEL io.airbyte.name=airbyte/source-okta diff --git a/docs/integrations/sources/okta.md b/docs/integrations/sources/okta.md index 046d43e52118..7ecf8d291fae 100644 --- a/docs/integrations/sources/okta.md +++ b/docs/integrations/sources/okta.md @@ -8,30 +8,30 @@ This source can sync data for the [Okta API](https://developer.okta.com/docs/ref This Source is capable of syncing the following core Streams: -* [Users](https://developer.okta.com/docs/reference/api/users/#list-users) -* [User Role Assignments](https://developer.okta.com/docs/reference/api/roles/#list-roles-assigned-to-a-user) -* [Groups](https://developer.okta.com/docs/reference/api/groups/#list-groups) -* [Group Members](https://developer.okta.com/docs/reference/api/groups/#list-group-members) -* [Group Role Assignments](https://developer.okta.com/docs/reference/api/roles/#list-roles-assigned-to-a-group) -* [System Log](https://developer.okta.com/docs/reference/api/system-log/#get-started) -* [Custom Roles](https://developer.okta.com/docs/reference/api/roles/#list-roles) +- [Users](https://developer.okta.com/docs/reference/api/users/#list-users) +- [User Role Assignments](https://developer.okta.com/docs/reference/api/roles/#list-roles-assigned-to-a-user) +- [Groups](https://developer.okta.com/docs/reference/api/groups/#list-groups) +- [Group Members](https://developer.okta.com/docs/reference/api/groups/#list-group-members) +- [Group Role Assignments](https://developer.okta.com/docs/reference/api/roles/#list-roles-assigned-to-a-group) +- [System Log](https://developer.okta.com/docs/reference/api/system-log/#get-started) +- [Custom Roles](https://developer.okta.com/docs/reference/api/roles/#list-roles) ### Data type mapping | Integration Type | Airbyte Type | Notes | -| :--- | :--- | :--- | -| `string` | `string` | | -| `number` | `number` | | -| `array` | `array` | | -| `object` | `object` | | +| :--------------- | :----------- | :---- | +| `string` | `string` | | +| `number` | `number` | | +| `array` | `array` | | +| `object` | `object` | | ### Features -| Feature | Supported?\(Yes/No\) | Notes | -| :--- | :--- | :--- | -| Full Refresh Sync | Yes | | -| Incremental Sync | Yes | | -| Namespaces | No | | +| Feature | Supported?\(Yes/No\) | Notes | +| :---------------- | :------------------- | :---- | +| Full Refresh Sync | Yes | | +| Incremental Sync | Yes | | +| Namespaces | No | | ### Performance considerations @@ -41,7 +41,7 @@ The connector is restricted by normal Okta [requests limitation](https://develop ### Requirements -* Okta API Token +- Okta API Token ### Setup guide @@ -63,14 +63,13 @@ Different Okta APIs require different admin privilege levels. API tokens inherit ## Changelog -| Version | Date | Pull Request | Subject | -| :--- | :--- | :--- | :--- | -| 0.1.6 | 2022-07-13 | [14556](https://github.com/airbytehq/airbyte/pull/14556) | add User_Role_Assignments and Group_Role_Assignments streams (full fetch only) | -| 0.1.6 | 2022-07-11 | [14610](https://github.com/airbytehq/airbyte/pull/14610) | add custom roles stream | -| 0.1.5 | 2022-07-04 | [14380](https://github.com/airbytehq/airbyte/pull/14380) | add Group_Members stream to okta source | -| 0.1.4 | 2021-11-02 | [7584](https://github.com/airbytehq/airbyte/pull/7584) | Fix incremental params for log stream | -| 0.1.3 | 2021-09-08 | [5905](https://github.com/airbytehq/airbyte/pull/5905) | Fix incremental stream defect | -| 0.1.2 | 2021-07-01 | [4456](https://github.com/airbytehq/airbyte/pull/4456) | Bugfix infinite pagination in logs stream | -| 0.1.1 | 2021-06-09 | [3937](https://github.com/airbytehq/airbyte/pull/3973) | Add `AIRBYTE_ENTRYPOINT` env variable for kubernetes support | -| 0.1.0 | 2021-05-30 | [3563](https://github.com/airbytehq/airbyte/pull/3563) | Initial Release | - +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :----------------------------------------------------------------------------- | +| 0.1.7 | 2022-07-13 | [14556](https://github.com/airbytehq/airbyte/pull/14556) | add User_Role_Assignments and Group_Role_Assignments streams (full fetch only) | +| 0.1.6 | 2022-07-11 | [14610](https://github.com/airbytehq/airbyte/pull/14610) | add custom roles stream | +| 0.1.5 | 2022-07-04 | [14380](https://github.com/airbytehq/airbyte/pull/14380) | add Group_Members stream to okta source | +| 0.1.4 | 2021-11-02 | [7584](https://github.com/airbytehq/airbyte/pull/7584) | Fix incremental params for log stream | +| 0.1.3 | 2021-09-08 | [5905](https://github.com/airbytehq/airbyte/pull/5905) | Fix incremental stream defect | +| 0.1.2 | 2021-07-01 | [4456](https://github.com/airbytehq/airbyte/pull/4456) | Bugfix infinite pagination in logs stream | +| 0.1.1 | 2021-06-09 | [3937](https://github.com/airbytehq/airbyte/pull/3973) | Add `AIRBYTE_ENTRYPOINT` env variable for kubernetes support | +| 0.1.0 | 2021-05-30 | [3563](https://github.com/airbytehq/airbyte/pull/3563) | Initial Release | From 5f9a24faaebb0b32317d147a4ff57f7b4450620d Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Mon, 18 Jul 2022 18:54:35 +0000 Subject: [PATCH 4/4] auto-bump connector version [ci skip] --- airbyte-config/init/src/main/resources/seed/source_specs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 7d2ef667263e..7cc8bdd68ce5 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -5955,7 +5955,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-okta:0.1.6" +- dockerImage: "airbyte/source-okta:0.1.7" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/okta" connectionSpecification: