Source Greenhouse: certify to GA (#17239)

* #16061 source GA: certify to GA * source greenhouse: upd changelog * #16061 source greenhouse: do not fail when encounter 403 error, skip instead * #16061 retry 429 * auto-bump connector version [ci skip] Co-authored-by: Octavia Squidington III <octavia-squidington-iii@users.noreply.github.com>
airbytehq · Sep 28, 2022 · 3f99a22 · 3f99a22
1 parent e8736ee
commit 3f99a22
Show file tree

Hide file tree

Showing 9 changed files with 587 additions and 541 deletions.
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -419,11 +419,11 @@
 - name: Greenhouse
   sourceDefinitionId: 59f1e50a-331f-4f09-b3e8-2e8d4d355f44
   dockerRepository: airbyte/source-greenhouse
-  dockerImageTag: 0.2.10
+  dockerImageTag: 0.2.11
   documentationUrl: https://docs.airbyte.io/integrations/sources/greenhouse
   icon: greenhouse.svg
   sourceType: api
-  releaseStage: beta
+  releaseStage: generally_available
 - name: Harness
   sourceDefinitionId: 6fe89830-d04d-401b-aad6-6552ffa5c4af
   dockerRepository: farosai/airbyte-harness-source

diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml
@@ -4308,7 +4308,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-greenhouse:0.2.10"
+- dockerImage: "airbyte/source-greenhouse:0.2.11"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/greenhouse"
     connectionSpecification:

diff --git a/airbyte-integrations/connectors/source-greenhouse/Dockerfile b/airbyte-integrations/connectors/source-greenhouse/Dockerfile
@@ -12,5 +12,5 @@ COPY main.py ./
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.2.10
+LABEL io.airbyte.version=0.2.11
 LABEL io.airbyte.name=airbyte/source-greenhouse
diff --git a/airbyte-integrations/connectors/source-greenhouse/acceptance-test-config.yml b/airbyte-integrations/connectors/source-greenhouse/acceptance-test-config.yml
@@ -5,8 +5,6 @@ tests:
   spec:
     - spec_path: "source_greenhouse/spec.json"
   connection:
-    - config_path: "secrets/config.json"
-      status: "succeed"
     - config_path: "secrets/config.json"
       status: "succeed"
     - config_path: "integration_tests/config_invalid.json"
@@ -19,8 +17,16 @@ tests:
       configured_catalog_path: "integration_tests/configured_catalog.json"
       expect_records:
         path: "integration_tests/expected_records.txt"
-    - config_path: "secrets/config.json"
-      configured_catalog_path: "integration_tests/configured_catalog_users_only.json"
+    - config_path: "secrets/config_users_only.json"
+      # test we do not fail when encounter 403 error
+      configured_catalog_path: "integration_tests/configured_catalog.json"
+      empty_streams: [
+        "applications", "applications_demographics_answers", "applications_interviews", "candidates", "close_reasons",
+        "custom_fields", "degrees", "demographics_answers", "demographics_answer_options", "questions",
+        "demographics_answers_answer_options", "demographics_question_sets", "demographics_question_sets_questions",
+        "departments", "jobs", "jobs_openings", "interviews", "job_posts", "job_stages", "jobs_stages", "offers",
+        "rejection_reasons", "scorecards", "sources", "demographics_questions"
+      ]
   full_refresh:
     - config_path: "secrets/config.json"
       configured_catalog_path: "integration_tests/configured_catalog.json"

diff --git a/airbyte-integrations/connectors/source-greenhouse/integration_tests/expected_records.txt b/airbyte-integrations/connectors/source-greenhouse/integration_tests/expected_records.txt
diff --git a/airbyte-integrations/connectors/source-greenhouse/setup.py b/airbyte-integrations/connectors/source-greenhouse/setup.py
@@ -16,7 +16,7 @@
     author="Airbyte",
     author_email="contact@airbyte.io",
     packages=find_packages(),
-    install_requires=["airbyte-cdk~=0.1.79", "dataclasses-jsonschema==2.15.1"],
+    install_requires=["airbyte-cdk", "dataclasses-jsonschema==2.15.1"],
     package_data={"": ["*.json", "*.yaml", "schemas/*.json"]},
     extras_require={
         "tests": TEST_REQUIREMENTS,

diff --git a/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/greenhouse.yaml b/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/greenhouse.yaml
@@ -13,6 +13,15 @@ definitions:
     name: "{{ options['name'] }}"
     url_base: "https://harvest.greenhouse.io/v1/"
     http_method: "GET"
+    error_handler:
+      type: CompositeErrorHandler
+      # ignore 403 error but retry default retriable http errors (429, 500 - 600)
+      error_handlers:
+        - type: DefaultErrorHandler
+          response_filters:
+            - http_codes: [ 403 ]
+              action: IGNORE
+        - type: DefaultErrorHandler
     authenticator:
       type: BasicHttpAuthenticator
       username: "{{ config['api_key'] }}"

diff --git a/airbyte-integrations/connectors/source-greenhouse/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-greenhouse/unit_tests/test_streams.py
@@ -152,3 +152,20 @@ def test_parse_response_empty_content(applications_stream):
     records = [record for record in parsed_response]
 
     assert records == []
+
+
+def test_ignore_403(applications_stream):
+    response = requests.Response()
+    response.status_code = 403
+    response._content = b""
+    parsed_response = applications_stream.retriever.parse_response(response, stream_state={})
+    records = [record for record in parsed_response]
+    assert records == []
+
+
+def test_retry_429(applications_stream):
+    response = requests.Response()
+    response.status_code = 429
+    response._content = b"{}"
+    should_retry = applications_stream.retriever.should_retry(response)
+    assert should_retry is True
diff --git a/docs/integrations/sources/greenhouse.md b/docs/integrations/sources/greenhouse.md
@@ -29,9 +29,9 @@ Please follow the [Greenhouse documentation for generating an API key](https://d
 The Greenhouse source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes):
 
 | Feature                       | Supported?  |
-| :---------------------------- | :---------- |
+|:------------------------------|:------------|
 | Full Refresh Sync             | Yes         |
-| Incremental Sync              | Coming soon |
+| Incremental Sync              | Yes         |
 | Replicate Incremental Deletes | Coming soon |
 | SSL connection                | Yes         |
 | Namespaces                    | No          |
@@ -57,6 +57,13 @@ The Greenhouse source connector supports the following [sync modes](https://docs
 * [Sources](https://developers.greenhouse.io/harvest.html#get-list-sources)
 * [Users](https://developers.greenhouse.io/harvest.html#get-list-users)
 
+## Setting permissions for API Keys
+You can specify which API endpoints your API keys have access to from the Greenhouse Dev Center. This will allow you to permit or deny access to each endpoint individually. Any API keys created before January 18th, 2017 will have full permissions to all API endpoints that existed at that time, but any new API keys created after that point will need to be explicitly granted the required endpoint permissions.
+To add or remove endpoint permissions on an API key, go to the Dev Center in Greenhouse, click “API Credential Management,” then click “Manage Permissions” next to your Harvest API Key. From there, check or uncheck permissions for any endpoints.
+
+**Important Note**: Users with Harvest API keys may access all the data in the endpoint. Access to data in Harvest is binary: everything or nothing. Harvest API keys should be given to internal developers with this understanding and to third parties with caution. Each key should only be allowed to access the endpoints it absolutely needs.
+See more on this [here](https://developers.greenhouse.io/harvest.html#authentication).
+
 ## Performance considerations
 
 The Greenhouse connector should not run into Greenhouse API limitations under normal usage. 
@@ -66,6 +73,7 @@ Please [create an issue](https://github.com/airbytehq/airbyte/issues) if you see
 
 | Version | Date       | Pull Request                                             | Subject                                                                        |
 |:--------|:-----------|:---------------------------------------------------------|:-------------------------------------------------------------------------------|
+| 0.2.11  | 2022-09-27 | [17239](https://github.com/airbytehq/airbyte/pull/17239) | Always install the latest version of Airbyte CDK                               |
 | 0.2.10  | 2022-09-05 | [16338](https://github.com/airbytehq/airbyte/pull/16338) | Implement incremental syncs & fix SATs                                         |
 | 0.2.9   | 2022-08-22 | [15800](https://github.com/airbytehq/airbyte/pull/15800) | Bugfix to allow reading sentry.yaml and schemas at runtime                     |
 | 0.2.8   | 2022-08-10 | [15344](https://github.com/airbytehq/airbyte/pull/15344) | Migrate connector to config-based framework                                    |