🎉 Source Google Workspace Admin: use SAT (#6878)

* Add SAT * Upd changelog * Refactor * Upd acceptance-test-config * Upd schemas * Update docs Co-authored-by: Sherif A. Nada <snadalive@gmail.com> Co-authored-by: Sherif A. Nada <snadalive@gmail.com>
airbytehq · Oct 31, 2021 · ff2882d · ff2882d
1 parent 74c78b9
commit ff2882d
Show file tree

Hide file tree

Showing 21 changed files with 407 additions and 322 deletions.
diff --git a/...ain/resources/config/STANDARD_SOURCE_DEFINITION/ed9dfefa-1bbc-419d-8c5e-4d78f0ef6734.json b/...ain/resources/config/STANDARD_SOURCE_DEFINITION/ed9dfefa-1bbc-419d-8c5e-4d78f0ef6734.json
@@ -2,6 +2,6 @@
   "sourceDefinitionId": "ed9dfefa-1bbc-419d-8c5e-4d78f0ef6734",
   "name": "Google Workspace Admin Reports",
   "dockerRepository": "airbyte/source-google-workspace-admin-reports",
-  "dockerImageTag": "0.1.4",
+  "dockerImageTag": "0.1.5",
   "documentationUrl": "https://docs.airbyte.io/integrations/sources/google-workspace-admin-reports"
 }
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -205,7 +205,7 @@
 - name: Google Workspace Admin Reports
   sourceDefinitionId: ed9dfefa-1bbc-419d-8c5e-4d78f0ef6734
   dockerRepository: airbyte/source-google-workspace-admin-reports
-  dockerImageTag: 0.1.4
+  dockerImageTag: 0.1.5
   documentationUrl: https://docs.airbyte.io/integrations/sources/google-workspace-admin-reports
   sourceType: api
 - name: Greenhouse

diff --git a/airbyte-integrations/connectors/source-google-workspace-admin-reports/Dockerfile b/airbyte-integrations/connectors/source-google-workspace-admin-reports/Dockerfile
@@ -14,5 +14,5 @@ RUN pip install .
 
 ENV AIRBYTE_ENTRYPOINT "/airbyte/base.sh"
 
-LABEL io.airbyte.version=0.1.4
+LABEL io.airbyte.version=0.1.5
 LABEL io.airbyte.name=airbyte/source-google-workspace-admin-reports
diff --git a/...-integrations/connectors/source-google-workspace-admin-reports/acceptance-test-config.yml b/...-integrations/connectors/source-google-workspace-admin-reports/acceptance-test-config.yml
@@ -0,0 +1,24 @@
+# See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference)
+# for more information about how to configure these tests
+connector_image: airbyte/source-google-workspace-admin-reports:dev
+tests:
+  spec:
+    - spec_path: "source_google_workspace_admin_reports/spec.json"
+  connection:
+    - config_path: "secrets/config.json"
+      status: "succeed"
+    - config_path: "integration_tests/invalid_config.json"
+      status: "failed"
+  discovery:
+    - config_path: "secrets/config.json"
+  basic_read:
+    - config_path: "secrets/config.json"
+      configured_catalog_path: "integration_tests/configured_catalog.json"
+      empty_streams: ["admin"]
+  full_refresh:
+    - config_path: "secrets/config.json"
+      configured_catalog_path: "integration_tests/configured_catalog.json"
+  incremental:
+    - config_path: "secrets/config.json"
+      configured_catalog_path: "integration_tests/configured_catalog.json"
+      future_state_path: "integration_tests/abnormal_state.json"
diff --git a/...e-integrations/connectors/source-google-workspace-admin-reports/acceptance-test-docker.sh b/...e-integrations/connectors/source-google-workspace-admin-reports/acceptance-test-docker.sh
@@ -0,0 +1,16 @@
+#!/usr/bin/env sh
+
+# Build latest connector image
+docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2):dev
+
+# Pull latest acctest image
+docker pull airbyte/source-acceptance-test:latest
+
+# Run
+docker run --rm -it \
+    -v /var/run/docker.sock:/var/run/docker.sock \
+    -v /tmp:/tmp \
+    -v $(pwd):/test_input \
+    airbyte/source-acceptance-test \
+    --acceptance-test-config /test_input
+
diff --git a/airbyte-integrations/connectors/source-google-workspace-admin-reports/build.gradle b/airbyte-integrations/connectors/source-google-workspace-admin-reports/build.gradle
@@ -1,21 +1,14 @@
 plugins {
     id 'airbyte-python'
     id 'airbyte-docker'
-    id 'airbyte-standard-source-test-file'
+    id 'airbyte-source-acceptance-test'
 }
 
 airbytePython {
     moduleDirectory 'source_google_workspace_admin_reports'
 }
 
-airbyteStandardSourceTestFile {
-    specPath = "source_google_workspace_admin_reports/spec.json"
-    configPath = "secrets/config.json"
-    configuredCatalogPath = "sample_files/configured_catalog.json"
-}
-
-
 dependencies {
-    implementation files(project(':airbyte-integrations:bases:base-standard-source-test-file').airbyteDocker.outputs)
+    implementation files(project(':airbyte-integrations:bases:source-acceptance-test').airbyteDocker.outputs)
     implementation files(project(':airbyte-integrations:bases:base-python').airbyteDocker.outputs)
 }
diff --git a/...tegrations/connectors/source-google-workspace-admin-reports/integration_tests/__init__.py b/...tegrations/connectors/source-google-workspace-admin-reports/integration_tests/__init__.py
diff --git a/...ns/connectors/source-google-workspace-admin-reports/integration_tests/abnormal_state.json b/...ns/connectors/source-google-workspace-admin-reports/integration_tests/abnormal_state.json
@@ -0,0 +1,17 @@
+{
+  "drive": {
+    "time": "2121-04-04T22:01:22.313Z"
+  },
+  "oauth_tokens": {
+    "time": "2121-04-05T03:06:30.849Z"
+  },
+  "admin": {
+    "time": "2121-04-05T03:06:30.849Z"
+  },
+  "logins": {
+    "time": "2121-04-05T03:06:30.849Z"
+  },
+  "mobile": {
+    "time": "2121-04-05T03:06:30.849Z"
+  }
+}
diff --git a/...grations/connectors/source-google-workspace-admin-reports/integration_tests/acceptance.py b/...grations/connectors/source-google-workspace-admin-reports/integration_tests/acceptance.py
@@ -0,0 +1,14 @@
+#
+# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
+#
+
+
+import pytest
+
+pytest_plugins = ("source_acceptance_test.plugin",)
+
+
+@pytest.fixture(scope="session", autouse=True)
+def connector_setup():
+    """This fixture is a placeholder for external resources that acceptance test might require."""
+    yield
diff --git a/...onnectors/source-google-workspace-admin-reports/integration_tests/configured_catalog.json b/...onnectors/source-google-workspace-admin-reports/integration_tests/configured_catalog.json
@@ -0,0 +1,64 @@
+{
+  "streams": [
+    {
+      "stream": {
+        "name": "admin",
+        "json_schema": {},
+        "supported_sync_modes": ["incremental"],
+        "source_defined_cursor": true,
+        "default_cursor_field": ["time"]
+      },
+      "sync_mode": "incremental",
+      "cursor_field": ["time"],
+      "destination_sync_mode": "append"
+    },
+    {
+      "stream": {
+        "name": "drive",
+        "json_schema": {},
+        "supported_sync_modes": ["incremental"],
+        "source_defined_cursor": true,
+        "default_cursor_field": ["time"]
+      },
+      "sync_mode": "incremental",
+      "cursor_field": ["time"],
+      "destination_sync_mode": "append"
+    },
+    {
+      "stream": {
+        "name": "logins",
+        "json_schema": {},
+        "supported_sync_modes": ["incremental"],
+        "source_defined_cursor": true,
+        "default_cursor_field": ["time"]
+      },
+      "sync_mode": "incremental",
+      "cursor_field": ["time"],
+      "destination_sync_mode": "append"
+    },
+    {
+      "stream": {
+        "name": "mobile",
+        "json_schema": {},
+        "supported_sync_modes": ["incremental"],
+        "source_defined_cursor": true,
+        "default_cursor_field": ["time"]
+      },
+      "sync_mode": "incremental",
+      "cursor_field": ["time"],
+      "destination_sync_mode": "append"
+    },
+    {
+      "stream": {
+        "name": "oauth_tokens",
+        "json_schema": {},
+        "supported_sync_modes": ["incremental"],
+        "source_defined_cursor": true,
+        "default_cursor_field": ["time"]
+      },
+      "sync_mode": "incremental",
+      "cursor_field": ["time"],
+      "destination_sync_mode": "append"
+    }
+  ]
+}
diff --git a/...ns/connectors/source-google-workspace-admin-reports/integration_tests/invalid_config.json b/...ns/connectors/source-google-workspace-admin-reports/integration_tests/invalid_config.json
@@ -0,0 +1,5 @@
+{
+    "credentials_json": "{\n  \"type\": \"service_account\"}\n",
+    "email": "test_email",
+    "lookback": 0
+}
diff --git a/airbyte-integrations/connectors/source-google-workspace-admin-reports/requirements.txt b/airbyte-integrations/connectors/source-google-workspace-admin-reports/requirements.txt
@@ -1,4 +1,5 @@
 # This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies.
 -e ../../bases/airbyte-protocol
 -e ../../bases/base-python
+-e ../../bases/source-acceptance-test
 -e .
diff --git a/...egrations/connectors/source-google-workspace-admin-reports/sample_files/sample_state.json b/...egrations/connectors/source-google-workspace-admin-reports/sample_files/sample_state.json
diff --git a/airbyte-integrations/connectors/source-google-workspace-admin-reports/setup.py b/airbyte-integrations/connectors/source-google-workspace-admin-reports/setup.py
@@ -5,6 +5,11 @@
 
 from setuptools import find_packages, setup
 
+TEST_REQUIREMENTS = [
+    "pytest~=6.1",
+    "source-acceptance-test",
+]
+
 setup(
     name="source_google_workspace_admin_reports",
     description="Source implementation for Google Workspace Admin Reports.",
@@ -14,12 +19,14 @@
     install_requires=[
         "airbyte-protocol",
         "base-python",
-        "pytest==6.1.2",
         "google-api-python-client==2.0.2",
         "google-auth-httplib2==0.1.0",
         "google-auth-oauthlib==0.4.3",
         "backoff==1.10.0",
         "pendulum==2.1.2",
     ],
     package_data={"": ["*.json", "schemas/*.json"]},
+    extras_require={
+        "tests": TEST_REQUIREMENTS,
+    },
 )
diff --git a/...ectors/source-google-workspace-admin-reports/source_google_workspace_admin_reports/api.py b/...ectors/source-google-workspace-admin-reports/source_google_workspace_admin_reports/api.py
@@ -108,13 +108,13 @@ class IncrementalStreamAPI(StreamAPI, ABC):
     def state(self) -> Optional[Mapping[str, Any]]:
         """Current state, if wasn't set return None"""
         if self._state:
-            return {self.state_pk: str(self._state)}
+            return {self.state_pk: self._state.isoformat()}
         return None
 
     @state.setter
     def state(self, value):
         self._state = pendulum.parse(value[self.state_pk])
-        self._start_time = self._state.isoformat()
+        self._start_time = self._state.to_iso8601_string()
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
@@ -128,6 +128,7 @@ def read(self, getter: Callable, params: Mapping[str, Any] = None) -> Iterator:
             "Report API return records from newest to oldest"
             if not cursor:
                 cursor = pendulum.parse(record[self.state_pk])
+            record[self.state_pk] = pendulum.parse(record[self.state_pk]).isoformat()
             yield record
 
         if cursor:

diff --git a/...e-google-workspace-admin-reports/source_google_workspace_admin_reports/schemas/admin.json b/...e-google-workspace-admin-reports/source_google_workspace_admin_reports/schemas/admin.json
@@ -20,8 +20,7 @@
         "customerId": {
           "type": "string"
         }
-      },
-      "required": ["time", "uniqueQualifier", "applicationName", "customerId"]
+      }
     },
     "actor": {
       "type": "object",
@@ -38,8 +37,7 @@
         "key": {
           "type": "string"
         }
-      },
-      "required": ["callerType", "email", "profileId", "key"]
+      }
     },
     "ownerDomain": {
       "type": "string"
@@ -49,19 +47,20 @@
     },
     "events": {
       "type": "array",
-      "items": [
-        {
-          "type": "object",
-          "properties": {
-            "type": {
-              "type": "string"
-            },
-            "name": {
-              "type": "string"
-            },
-            "parameters": {
-              "type": "array",
-              "items": [
+      "items": {
+        "type": "object",
+        "properties": {
+          "type": {
+            "type": "string"
+          },
+          "name": {
+            "type": "string"
+          },
+          "parameters": {
+            "type": "array",
+            "items": {
+              "type": "object",
+              "anyOf": [
                 {
                   "type": "object",
                   "properties": {
@@ -70,22 +69,53 @@
                     },
                     "value": {
                       "type": "string"
+                    }
+                  }
+                },
+                {
+                  "type": "object",
+                  "properties": {
+                    "name": {
+                      "type": "string"
                     },
                     "intValue": {
                       "type": "string"
+                    }
+                  }
+                },
+                {
+                  "type": "object",
+                  "properties": {
+                    "name": {
+                      "type": "string"
                     },
                     "boolValue": {
+                      "type": "boolean"
+                    }
+                  }
+                },
+                {
+                  "type": "object",
+                  "properties": {
+                    "name": {
                       "type": "string"
+                    },
+                    "multiValue": {
+                      "type": "array",
+                      "items": {
+                        "type": "string"
+                      }
                     }
-                  },
-                  "required": ["name", "value", "intValue", "boolValue"]
+                  }
                 }
               ]
             }
-          },
-          "required": ["type", "name", "parameters"]
+          }
         }
-      ]
+      }
+    },
+    "time": {
+      "type": "string"
     }
   }
 }