🐛 Source Tempo: Avoid infinite loop for non-paginated APIs (airbytehq…

…#16361) * 🐛 Source Tempo: Avoid infinite loop for APIs that do not support pagination and support more than DEFAULT_ITEMS_PER_PAGE results. * Source Tempo: Update changelog for version 0.2.6 * fix: update additionalProperties to true * fix: remove additionalProperties field in schema files Co-authored-by: Sajarin <sajarindider@gmail.com>
robbinhan · Sep 29, 2022 · ae8b2e5 · ae8b2e5
1 parent 77b32c4
commit ae8b2e5
Show file tree

Hide file tree

Showing 10 changed files with 15 additions and 29 deletions.
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -1004,7 +1004,7 @@
 - name: Tempo
   sourceDefinitionId: d1aa448b-7c54-498e-ad95-263cbebcd2db
   dockerRepository: airbyte/source-tempo
-  dockerImageTag: 0.2.5
+  dockerImageTag: 0.2.6
   documentationUrl: https://docs.airbyte.io/integrations/sources/tempo
   icon: tempo.svg
   sourceType: api

diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml
@@ -10427,7 +10427,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-tempo:0.2.5"
+- dockerImage: "airbyte/source-tempo:0.2.6"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/sources/"
     connectionSpecification:
@@ -10436,7 +10436,6 @@
       type: "object"
       required:
       - "api_token"
-      additionalProperties: false
       properties:
         api_token:
           type: "string"

diff --git a/airbyte-integrations/connectors/source-tempo/Dockerfile b/airbyte-integrations/connectors/source-tempo/Dockerfile
@@ -34,5 +34,5 @@ COPY source_tempo ./source_tempo
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.2.5
+LABEL io.airbyte.version=0.2.6
 LABEL io.airbyte.name=airbyte/source-tempo
diff --git a/airbyte-integrations/connectors/source-tempo/source_tempo/client.py b/airbyte-integrations/connectors/source-tempo/source_tempo/client.py
@@ -22,10 +22,10 @@ class Client(BaseClient):
 
     PARAMS = {"limit": DEFAULT_ITEMS_PER_PAGE, "offset": 0}
     ENTITIES_MAP = {
-        "accounts": {"url": "/accounts", "func": lambda v: v["results"], "params": PARAMS},
-        "customers": {"url": "/customers", "func": lambda v: v["results"], "params": PARAMS},
-        "worklogs": {"url": "/worklogs", "func": lambda v: v["results"], "params": PARAMS},
-        "workload-schemes": {"url": "/workload-schemes", "func": lambda v: v["results"], "params": PARAMS},
+        "accounts": {"url": "/accounts", "func": lambda v: v["results"], "paginated": False, "params": PARAMS},
+        "customers": {"url": "/customers", "func": lambda v: v["results"], "paginated": False, "params": PARAMS},
+        "worklogs": {"url": "/worklogs", "func": lambda v: v["results"], "paginated": True, "params": PARAMS},
+        "workload-schemes": {"url": "/workload-schemes", "func": lambda v: v["results"], "paginated": True, "params": PARAMS},
     }
 
     def __init__(self, api_token):
@@ -38,7 +38,7 @@ def lists(self, name, url, params, func, **kwargs):
             response = requests.get(f"{self.base_api_url}{url}?limit={params['limit']}&offset={params['offset']}", headers=self.headers)
             data = func(response.json())
             yield from data
-            if len(data) < self.DEFAULT_ITEMS_PER_PAGE:
+            if not self.ENTITIES_MAP[name]["paginated"] or len(data) < self.DEFAULT_ITEMS_PER_PAGE:
                 break
             params["offset"] += self.DEFAULT_ITEMS_PER_PAGE
 

diff --git a/airbyte-integrations/connectors/source-tempo/source_tempo/schemas/accounts.json b/airbyte-integrations/connectors/source-tempo/source_tempo/schemas/accounts.json
@@ -99,6 +99,5 @@
         }
       }
     }
-  },
-  "additionalProperties": false
+  }
 }
diff --git a/airbyte-integrations/connectors/source-tempo/source_tempo/schemas/customers.json b/airbyte-integrations/connectors/source-tempo/source_tempo/schemas/customers.json
@@ -14,6 +14,5 @@
     "name": {
       "type": "string"
     }
-  },
-  "additionalProperties": false
+  }
 }
diff --git a/airbyte-integrations/connectors/source-tempo/source_tempo/schemas/workload-schemes.json b/airbyte-integrations/connectors/source-tempo/source_tempo/schemas/workload-schemes.json
@@ -46,6 +46,5 @@
         }
       }
     }
-  },
-  "additionalProperties": false
+  }
 }
diff --git a/airbyte-integrations/connectors/source-tempo/source_tempo/schemas/worklogs.json b/airbyte-integrations/connectors/source-tempo/source_tempo/schemas/worklogs.json
@@ -21,10 +21,6 @@
     "issue": {
       "type": "object",
       "description": "Details of the issue",
-      "additionalProperties": {
-        "type": ["string", "integer"],
-        "readOnly": true
-      },
       "readOnly": true
     },
     "timeSpentSeconds": {
@@ -60,22 +56,16 @@
     "author": {
       "type": "object",
       "description": "Author of the worklog",
-      "readOnly": true,
-      "additionalProperties": {
-        "type": "string",
-        "readOnly": true
-      }
+      "readOnly": true
     },
     "attributes": {
       "type": "object",
       "description": "Additional attribute of the worklog",
-      "readOnly": true,
-      "additionalProperties": true
+      "readOnly": true
     },
     "billableSeconds": {
       "type": ["null", "integer"],
       "description": "Billable time spent working"
     }
-  },
-  "additionalProperties": false
+  }
 }
diff --git a/airbyte-integrations/connectors/source-tempo/source_tempo/spec.json b/airbyte-integrations/connectors/source-tempo/source_tempo/spec.json
@@ -5,7 +5,6 @@
     "title": "Tempo Spec",
     "type": "object",
     "required": ["api_token"],
-    "additionalProperties": false,
     "properties": {
       "api_token": {
         "type": "string",

diff --git a/docs/integrations/sources/tempo.md b/docs/integrations/sources/tempo.md
@@ -41,5 +41,6 @@ Go to **Tempo &gt; Settings**, scroll down to **Data Access** and select **API i
 
 | Version | Date       | Pull Request | Subject |
 | :------ | :--------  | :-----       | :------ |
+| 0.2.6   | 2022-09-08 | [16361](https://github.com/airbytehq/airbyte/pull/16361) | Avoid infinite loop for non-paginated APIs |
 | 0.2.4   | 2021-11-08 | [7649](https://github.com/airbytehq/airbyte/pull/7649) | Migrate to the CDK |