Update source-gnews manifest to use inline stream schemas (#20405)

airbytehq · Dec 19, 2022 · ddab3ce · ddab3ce
1 parent 9f0bfa2
commit ddab3ce
Show file tree

Hide file tree

Showing 8 changed files with 119 additions and 94 deletions.
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -578,7 +578,7 @@
 - name: GNews
   sourceDefinitionId: ce38aec4-5a77-439a-be29-9ca44fd4e811
   dockerRepository: airbyte/source-gnews
-  dockerImageTag: 0.1.0
+  dockerImageTag: 0.1.2
   documentationUrl: https://docs.airbyte.com/integrations/sources/gnews
   sourceType: api
   releaseStage: alpha

diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml
@@ -4746,7 +4746,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-gnews:0.1.0"
+- dockerImage: "airbyte/source-gnews:0.1.2"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/gnews"
     connectionSpecification:

diff --git a/airbyte-integrations/connectors/source-gnews/Dockerfile b/airbyte-integrations/connectors/source-gnews/Dockerfile
@@ -34,5 +34,5 @@ COPY source_gnews ./source_gnews
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.0
+LABEL io.airbyte.version=0.1.2
 LABEL io.airbyte.name=airbyte/source-gnews
diff --git a/airbyte-integrations/connectors/source-gnews/setup.py b/airbyte-integrations/connectors/source-gnews/setup.py
@@ -6,7 +6,7 @@
 from setuptools import find_packages, setup
 
 MAIN_REQUIREMENTS = [
-    "airbyte-cdk~=0.1",
+    "airbyte-cdk~=0.14",
 ]
 
 TEST_REQUIREMENTS = [

diff --git a/airbyte-integrations/connectors/source-gnews/source_gnews/gnews.yaml b/airbyte-integrations/connectors/source-gnews/source_gnews/gnews.yaml
@@ -1,5 +1,107 @@
 version: "0.1.0"
 
+schemas:
+  search_stream_schema:
+    type: object
+    properties:
+      title:
+        type:
+          - "null"
+          - string
+        description: The main title of the article.
+      description:
+        type:
+          - "null"
+          - string
+        description: The small paragraph under the title.
+      content:
+        type:
+          - "null"
+          - string
+        description: All the content of the article.
+      url:
+        type:
+          - "null"
+          - string
+        description: The URL of the article.
+      image:
+        type:
+          - "null"
+          - string
+        description: The main image of the article.
+      publishedAt:
+        type:
+          - "null"
+          - string
+        description:
+          The date of publication of the article. The date is always in the
+          UTC time zone.
+      source:
+        type:
+          - "null"
+          - object
+        properties:
+          name:
+            type:
+              - "null"
+              - string
+            description: The name of the source.
+          url:
+            type:
+              - "null"
+              - string
+            description: The home page of the source.
+  top_headlines_stream_schema:
+    type: object
+    properties:
+      title:
+        type:
+          - "null"
+          - string
+        description: The main title of the article.
+      description:
+        type:
+          - "null"
+          - string
+        description: The small paragraph under the title.
+      content:
+        type:
+          - "null"
+          - string
+        description: All the content of the article.
+      url:
+        type:
+          - "null"
+          - string
+        description: The URL of the article.
+      image:
+        type:
+          - "null"
+          - string
+        description: The main image of the article.
+      publishedAt:
+        type:
+          - "null"
+          - string
+        description:
+          The date of publication of the article. The date is always in the
+          UTC time zone.
+      source:
+        type:
+          - "null"
+          - object
+        properties:
+          name:
+            type:
+              - "null"
+              - string
+            description: The name of the source.
+          url:
+            type:
+              - "null"
+              - string
+            description: The home page of the source.
+
 definitions:
   selector:
     extractor:
@@ -44,7 +146,12 @@ definitions:
     nullable: "{{ ','.join(config['nullable']) }}"
     from: "{{ stream_slice['start_time'] }}"
     to: "{{ stream_slice['end_time'] }}"
+  schema_loader:
+    type: InlineSchemaLoader
   search_stream:
+    schema_loader:
+      $ref: "*ref(definitions.schema_loader)"
+      schema: "*ref(schemas.search_stream_schema)"
     $options:
       name: "search"
       primary_key: "url"
@@ -61,6 +168,9 @@ definitions:
             in: "{{ ','.join(config['in']) }}"
             sortby: "{{ config['sortby'] }}"
   top_headlines_stream:
+    schema_loader:
+      $ref: "*ref(definitions.schema_loader)"
+      schema: "*ref(schemas.top_headlines_stream_schema)"
     $options:
       name: "top_headlines"
       primary_key: "url"

diff --git a/airbyte-integrations/connectors/source-gnews/source_gnews/schemas/search.json b/airbyte-integrations/connectors/source-gnews/source_gnews/schemas/search.json
diff --git a/airbyte-integrations/connectors/source-gnews/source_gnews/schemas/top_headlines.json b/airbyte-integrations/connectors/source-gnews/source_gnews/schemas/top_headlines.json
diff --git a/docs/integrations/sources/gnews.md b/docs/integrations/sources/gnews.md
@@ -35,7 +35,8 @@ Rate Limiting is based on the API Key tier subscription, get more info [here](ht
 
 ## Changelog
 
-| Version | Date       | Pull Request                                             | Subject                              |
-|:--------|:-----------|:---------------------------------------------------------|:-------------------------------------|
-| 0.1.1   | 2022-12-13 | [20460](https://github.com/airbytehq/airbyte/pull/20460) | Update source acceptance test config |
-| 0.1.0   | 2022-11-01 | [18808](https://github.com/airbytehq/airbyte/pull/18808) | 🎉 New Source: GNews                 |
+| Version | Date       | Pull Request                                             | Subject                                          |
+|:--------|:-----------|:---------------------------------------------------------|:-------------------------------------------------|
+| 0.1.2   | 2022-12-16 | [20405](https://github.com/airbytehq/airbyte/pull/20405) | Update the manifest to use inline stream schemas |
+| 0.1.1   | 2022-12-13 | [20460](https://github.com/airbytehq/airbyte/pull/20460) | Update source acceptance test config             |
+| 0.1.0   | 2022-11-01 | [18808](https://github.com/airbytehq/airbyte/pull/18808) | 🎉 New Source: GNews                             |