Harshith/test pr 12280 (#13079)

* full stream coverage

* update integration tests with new streams

* fix: tests

* chore: updated the source-typeform version

* chore: updated the source-typeform version

* chore: update seed file

Co-authored-by: itaseski <ivica.taseski94@gmail.com>

airbyte-config/init/src/main/resources/seed/source_definitions.yaml

@@ -938,7 +938,7 @@
 - name: Typeform
   sourceDefinitionId: e7eff203-90bf-43e5-a240-19ea3056c474
   dockerRepository: airbyte/source-typeform
-  dockerImageTag: 0.1.4
+  dockerImageTag: 0.1.6
   documentationUrl: https://docs.airbyte.io/integrations/sources/typeform
   icon: typeform.svg
   sourceType: api

airbyte-config/init/src/main/resources/seed/source_specs.yaml

@@ -9008,7 +9008,7 @@
     supportsDBT: false
     supported_destination_sync_modes:
     - "append"
-- dockerImage: "airbyte/source-typeform:0.1.4"
+- dockerImage: "airbyte/source-typeform:0.1.6"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/sources/typeform"
     connectionSpecification:

airbyte-integrations/connectors/source-typeform/Dockerfile

@@ -12,5 +12,5 @@ RUN pip install .
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.4
+LABEL io.airbyte.version=0.1.6
 LABEL io.airbyte.name=airbyte/source-typeform

airbyte-integrations/connectors/source-typeform/integration_tests/configured_catalog.json

@@ -19,6 +19,33 @@
       },
       "sync_mode": "incremental",
       "destination_sync_mode": "append"
+    },
+    {
+      "stream": {
+        "name": "workspaces",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"]
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "append"
+    },
+    {
+      "stream": {
+        "name": "images",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"]
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "append"
+    },
+    {
+      "stream": {
+        "name": "themes",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"]
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "append"
     }
   ]
 }

airbyte-integrations/connectors/source-typeform/source_typeform/schemas/images.json

@@ -0,0 +1,15 @@
+{
+  "type": "object",
+  "properties": {
+    "file_name": {
+      "type": ["null", "string"]
+    },
+    "id": {
+      "type": ["null", "string"]
+    },
+    "src": {
+      "type": ["null", "string"]
+    }
+  },
+  "$schema": "http://json-schema.org/draft-07/schema#"
+}

airbyte-integrations/connectors/source-typeform/source_typeform/schemas/themes.json

@@ -0,0 +1,74 @@
+{
+  "type": "object",
+  "properties": {
+    "background": {
+      "type": ["null", "object"],
+      "properties": {
+        "brightness": {
+          "type": ["null", "number"]
+        },
+        "href": {
+          "type": ["null", "string"]
+        },
+        "layout": {
+          "type": ["null", "string"]
+        }
+      }
+    },
+    "colors": {
+      "type": ["null", "object"],
+      "properties": {
+        "answer": {
+          "type": ["null", "string"]
+        },
+        "background": {
+          "type": ["null", "string"]
+        },
+        "button": {
+          "type": ["null", "string"]
+        },
+        "question": {
+          "type": ["null", "string"]
+        }
+      }
+    },
+    "fields": {
+      "type": ["null", "object"],
+      "properties": {
+        "alignment": {
+          "type": ["null", "string"]
+        },
+        "font_size": {
+          "type": ["null", "string"]
+        }
+      }
+    },
+    "font": {
+      "type": ["null", "string"]
+    },
+    "has_transparent_button": {
+      "type": ["null", "boolean"]
+    },
+    "id": {
+      "type": ["null", "string"]
+    },
+    "name": {
+      "type": ["null", "string"]
+    },
+    "screens": {
+      "type": ["null", "object"],
+      "properties": {
+        "alignment": {
+          "type": ["null", "string"]
+        },
+        "font_size": {
+          "type": ["null", "string"]
+        }
+      }
+    },
+    "visibility": {
+      "type": ["null", "string"]
+    }
+  },
+  "$schema": "http://json-schema.org/draft-07/schema#"
+}

airbyte-integrations/connectors/source-typeform/source_typeform/schemas/webhooks.json

@@ -0,0 +1,30 @@
+{
+  "type": "object",
+  "properties": {
+    "created_at": {
+      "type": ["null", "string"]
+    },
+    "enabled": {
+      "type": ["null", "boolean"]
+    },
+    "form_id": {
+      "type": ["null", "string"]
+    },
+    "id": {
+      "type": ["null", "string"]
+    },
+    "tag": {
+      "type": ["null", "string"]
+    },
+    "updated_at": {
+      "type": ["null", "string"]
+    },
+    "url": {
+      "type": ["null", "string"]
+    },
+    "verify_ssl": {
+      "type": ["null", "boolean"]
+    }
+  },
+  "$schema": "http://json-schema.org/draft-07/schema#"
+}

airbyte-integrations/connectors/source-typeform/source_typeform/schemas/workspaces.json

@@ -0,0 +1,34 @@
+{
+  "type": "object",
+  "properties": {
+    "forms": {
+      "type": ["null", "object"],
+      "properties": {
+        "count": {
+          "type": ["null", "number"]
+        },
+        "href": {
+          "type": ["null", "string"]
+        }
+      }
+    },
+    "id": {
+      "type": ["null", "string"]
+    },
+    "name": {
+      "type": ["null", "string"]
+    },
+    "self": {
+      "type": ["null", "object"],
+      "properties": {
+        "href": {
+          "type": ["null", "string"]
+        }
+      }
+    },
+    "shared": {
+      "type": ["null", "boolean"]
+    }
+  },
+  "$schema": "http://json-schema.org/draft-07/schema#"
+}

airbyte-integrations/connectors/source-typeform/source_typeform/source.py

@@ -41,22 +41,7 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp
         yield from response.json()["items"]
 
 
-class TrimForms(TypeformStream):
-    """
-    This stream is responsible for fetching list of from_id(s) which required to process data from Forms and Responses.
-    API doc: https://developer.typeform.com/create/reference/retrieve-forms/
-    """
-
-    primary_key = "id"
-
-    def path(
-        self,
-        stream_state: Mapping[str, Any] = None,
-        stream_slice: Mapping[str, Any] = None,
-        next_page_token: Optional[Any] = None,
-    ) -> str:
-        return "forms"
-
+class PaginatedStream(TypeformStream):
     def next_page_token(self, response: requests.Response) -> Optional[Any]:
         page = self.get_current_page_token(response.url)
         # stop pagination if current page equals to total pages
@@ -70,17 +55,24 @@ def get_current_page_token(self, url: str) -> Optional[int]:
         page = parse_qs(parsed.query).get("page")
         return int(page[0]) if page else None
 
-    def request_params(
-        self,
-        stream_state: Mapping[str, Any],
-        stream_slice: Mapping[str, any] = None,
-        next_page_token: Optional[Any] = None,
-    ) -> MutableMapping[str, Any]:
+    def request_params(self, next_page_token: Optional[Any] = None, **kwargs) -> MutableMapping[str, Any]:
         params = {"page_size": self.limit}
         params["page"] = next_page_token or 1
         return params
 
 
+class TrimForms(PaginatedStream):
+    """
+    This stream is responsible for fetching list of from_id(s) which required to process data from Forms and Responses.
+    API doc: https://developer.typeform.com/create/reference/retrieve-forms/
+    """
+
+    primary_key = "id"
+
+    def path(self, **kwargs) -> str:
+        return "forms"
+
+
 class TrimFormsMixin:
     def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, any]]]:
         form_ids = self.config.get("form_ids", [])
@@ -102,12 +94,7 @@ class Forms(TrimFormsMixin, TypeformStream):
 
     primary_key = "id"
 
-    def path(
-        self,
-        stream_state: Mapping[str, Any] = None,
-        stream_slice: Mapping[str, Any] = None,
-        next_page_token: Optional[Any] = None,
-    ) -> str:
+    def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str:
         return f"forms/{stream_slice['form_id']}"
 
     def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]:
@@ -198,6 +185,57 @@ def request_params(
         return params
 
 
+class Webhooks(TrimFormsMixin, TypeformStream):
+    """
+    This stream is responsible for fetching webhooks for particular form_id.
+    API doc: https://developer.typeform.com/webhooks/reference/retrieve-webhooks/
+    """
+
+    primary_key = "id"
+
+    def path(self, stream_slice: Optional[Mapping[str, Any]] = None, **kwargs) -> str:
+        return f"forms/{stream_slice['form_id']}/webhooks"
+
+
+class Workspaces(PaginatedStream):
+    """
+    This stream is responsible for fetching workspaces.
+    API doc: https://developer.typeform.com/create/reference/retrieve-workspaces/
+    """
+
+    primary_key = "id"
+
+    def path(self, **kwargs) -> str:
+        return "workspaces"
+
+
+class Images(TypeformStream):
+    """
+    This stream is responsible for fetching images.
+    API doc: https://developer.typeform.com/create/reference/retrieve-images-collection/
+    """
+
+    primary_key = "id"
+
+    def path(self, **kwargs) -> str:
+        return "images"
+
+    def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]:
+        return response.json()
+
+
+class Themes(PaginatedStream):
+    """
+    This stream is responsible for fetching themes.
+    API doc: https://developer.typeform.com/create/reference/retrieve-themes/
+    """
+
+    primary_key = "id"
+
+    def path(self, **kwargs) -> str:
+        return "themes"
+
+
 class SourceTypeform(AbstractSource):
     def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, any]:
         try:
@@ -231,4 +269,11 @@ def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) ->
 
     def streams(self, config: Mapping[str, Any]) -> List[Stream]:
         auth = TokenAuthenticator(token=config["token"])
-        return [Forms(authenticator=auth, **config), Responses(authenticator=auth, **config)]
+        return [
+            Forms(authenticator=auth, **config),
+            Responses(authenticator=auth, **config),
+            Webhooks(authenticator=auth, **config),
+            Workspaces(authenticator=auth, **config),
+            Images(authenticator=auth, **config),
+            Themes(authenticator=auth, **config),
+        ]