Harshith/test pr 7063 (airbytehq#7382)

* Add location stream to Shopify connector Add intentory level stream Apply formatting * Update doc and version configs * Update version * Format after rebasing * Address review comments * fix: location stream has no data with the current cred * Add location stream to Shopify connector Add intentory level stream Apply formatting * Update doc and version configs * Update version * Format after rebasing * Address review comments * fix: location stream has no data with the current cred * fix: integration test * fix: integrationtests * fix: integrationtests * Fixed primary key for inventory levels stream Co-authored-by: Yuhui Shi <yuhui@convect.ai>
schlattk · Jan 4, 2022 · 0ed8e79 · 0ed8e79
1 parent 47c944c
commit 0ed8e79
Show file tree

Hide file tree

Showing 10 changed files with 158 additions and 3 deletions.
diff --git a/...ain/resources/config/STANDARD_SOURCE_DEFINITION/9da77001-af33-4bcd-be46-6252bf9342b9.json b/...ain/resources/config/STANDARD_SOURCE_DEFINITION/9da77001-af33-4bcd-be46-6252bf9342b9.json
@@ -2,6 +2,6 @@
   "sourceDefinitionId": "9da77001-af33-4bcd-be46-6252bf9342b9",
   "name": "Shopify",
   "dockerRepository": "airbyte/source-shopify",
-  "dockerImageTag": "0.1.19",
+  "dockerImageTag": "0.1.21",
   "documentationUrl": "https://docs.airbyte.io/integrations/sources/shopify"
 }
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -462,7 +462,7 @@
 - name: Shopify
   sourceDefinitionId: 9da77001-af33-4bcd-be46-6252bf9342b9
   dockerRepository: airbyte/source-shopify
-  dockerImageTag: 0.1.19
+  dockerImageTag: 0.1.21
   documentationUrl: https://docs.airbyte.io/integrations/sources/shopify
   sourceType: api
 - name: Short.io

diff --git a/airbyte-integrations/connectors/source-shopify/Dockerfile b/airbyte-integrations/connectors/source-shopify/Dockerfile
@@ -28,5 +28,5 @@ COPY source_shopify ./source_shopify
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.19
+LABEL io.airbyte.version=0.1.21
 LABEL io.airbyte.name=airbyte/source-shopify
diff --git a/airbyte-integrations/connectors/source-shopify/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-shopify/integration_tests/abnormal_state.json
@@ -40,5 +40,11 @@
   },
   "discount_codes": {
     "updated_at": "2024-07-08T05:40:38-07:00"
+  },
+  "locations": {
+    "updated_at": "2024-07-08T05:40:38-07:00"
+  },
+  "inventory_levels": {
+    "updated_at": "2024-07-08T05:40:38-07:00"
   }
 }
diff --git a/airbyte-integrations/connectors/source-shopify/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-shopify/integration_tests/configured_catalog.json
@@ -167,6 +167,30 @@
       "sync_mode": "incremental",
       "cursor_field": ["updated_at"],
       "destination_sync_mode": "append"
+    },
+    {
+      "stream": {
+        "name": "locations",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"],
+        "source_defined_cursor": true,
+        "default_cursor_field": ["id"]
+      },
+      "sync_mode": "full_refresh",
+      "cursor_field": ["id"],
+      "destination_sync_mode": "overwrite"
+    },
+    {
+      "stream": {
+        "name": "inventory_levels",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh", "incremental"],
+        "source_defined_cursor": true,
+        "default_cursor_field": ["updated_at"]
+      },
+      "sync_mode": "incremental",
+      "cursor_field": ["updated_at"],
+      "destination_sync_mode": "append"
     }
   ]
 }
diff --git a/airbyte-integrations/connectors/source-shopify/integration_tests/state.json b/airbyte-integrations/connectors/source-shopify/integration_tests/state.json
@@ -40,5 +40,11 @@
   },
   "discount_codes": {
     "updated_at": "2021-09-10T06:48:10-07:00"
+  },
+  "locations": {
+    "updated_at": "2021-09-10T06:48:10-07:00"
+  },
+  "inventory_levels": {
+    "updated_at": "2021-09-10T06:48:10-07:00"
   }
 }
diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/inventory_levels.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/inventory_levels.json
@@ -0,0 +1,18 @@
+{
+  "type": "object",
+  "properties": {
+    "available": {
+      "type": ["null", "integer"]
+    },
+    "inventory_item_id": {
+      "type": ["null", "integer"]
+    },
+    "location_id": {
+      "type": ["null", "integer"]
+    },
+    "updated_at": {
+      "type": ["null", "string"],
+      "format": "date-time"
+    }
+  }
+}
diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/locations.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/locations.json
@@ -0,0 +1,55 @@
+{
+  "type": "object",
+  "properties": {
+    "active": {
+      "type": ["null", "boolean"]
+    },
+    "address1": {
+      "type": ["null", "string"]
+    },
+    "address2": {
+      "type": ["null", "string"]
+    },
+    "city": {
+      "type": ["null", "string"]
+    },
+    "country": {
+      "type": ["null", "string"]
+    },
+    "country_code": {
+      "type": ["null", "string"]
+    },
+    "created_at": {
+      "type": ["null", "string"],
+      "format": "date-time"
+    },
+    "id": {
+      "type": ["null", "integer"]
+    },
+    "legacy": {
+      "type": ["null", "boolean"]
+    },
+    "name": {
+      "type": ["null", "string"]
+    },
+    "phone": {
+      "type": ["null", "string"]
+    },
+    "province": {
+      "type": ["null", "string"]
+    },
+    "updated_at": {
+      "type": ["null", "string"],
+      "format": "date-time"
+    },
+    "zip": {
+      "type": ["null", "string"]
+    },
+    "localized_country_name": {
+      "type": ["null", "string"]
+    },
+    "localized_province_name": {
+      "type": ["null", "string"]
+    }
+  }
+}
diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/source.py b/airbyte-integrations/connectors/source-shopify/source_shopify/source.py
@@ -325,6 +325,46 @@ def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str:
         return f"price_rules/{price_rule_id}/{self.data_field}.json"
 
 
+class Locations(ShopifyStream):
+
+    """
+    The location API does not support any form of filtering.
+    https://shopify.dev/api/admin-rest/2021-07/resources/location
+
+    Therefore, only FULL_REFRESH mode is supported.
+    """
+
+    data_field = "locations"
+
+    def path(self, **kwargs):
+        return f"{self.data_field}.json"
+
+
+class InventoryLevels(ChildSubstream):
+    parent_stream_class: object = Locations
+    slice_key = "location_id"
+    cursor_field = "updated_at"
+
+    data_field = "inventory_levels"
+
+    def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str:
+        location_id = stream_slice["location_id"]
+        return f"locations/{location_id}/{self.data_field}.json"
+
+    def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]:
+        records_stream = super().parse_response(response, **kwargs)
+
+        def generate_key(record):
+            record.update({"id": "|".join((str(record.get("location_id", "")), str(record.get("inventory_item_id", ""))))})
+            return record
+
+        # associate the surrogate key
+        yield from map(
+            generate_key,
+            records_stream,
+        )
+
+
 class SourceShopify(AbstractSource):
     def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, any]:
 
@@ -365,4 +405,6 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]:
             Pages(config),
             PriceRules(config),
             DiscountCodes(config),
+            Locations(config),
+            InventoryLevels(config),
         ]
diff --git a/docs/integrations/sources/shopify.md b/docs/integrations/sources/shopify.md
@@ -35,6 +35,8 @@ This Source is capable of syncing the following core Streams:
 * [Transactions](https://help.shopify.com/en/api/reference/orders/transaction)
 * [Pages](https://help.shopify.com/en/api/reference/online-store/page)
 * [Price Rules](https://help.shopify.com/en/api/reference/discounts/pricerule)
+* [Locations](https://shopify.dev/api/admin-rest/2021-10/resources/location)
+* [InventoryLevels](https://shopify.dev/api/admin-rest/2021-10/resources/inventorylevel)
 
 #### NOTE:
 
@@ -95,6 +97,8 @@ This connector support both: `OAuth 2.0` and `API PASSWORD` (for private applica
 
 | Version | Date | Pull Request | Subject |
 | :--- | :--- | :--- | :--- |
+| 0.1.21 | 2021-10-14 | [7382](https://github.com/airbytehq/airbyte/pull/7382) | Fixed `InventoryLevels` primary key |
+| 0.1.20 | 2021-10-14 | [7063](https://github.com/airbytehq/airbyte/pull/7063) | Added `Location` and `InventoryLevels` as streams |
 | 0.1.19 | 2021-10-11 | [6951](https://github.com/airbytehq/airbyte/pull/6951) | Added support of `OAuth 2.0` authorisation option |
 | 0.1.18 | 2021-09-21 | [6056](https://github.com/airbytehq/airbyte/pull/6056) | Added `pre_tax_price` to the `orders/line_items` schema |
 | 0.1.17 | 2021-09-17 | [5244](https://github.com/airbytehq/airbyte/pull/5244) | Created data type enforcer for converting prices into numbers |