diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/9da77001-af33-4bcd-be46-6252bf9342b9.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/9da77001-af33-4bcd-be46-6252bf9342b9.json index 71bd12225a49..90ce4d4c9d5e 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/9da77001-af33-4bcd-be46-6252bf9342b9.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/9da77001-af33-4bcd-be46-6252bf9342b9.json @@ -2,6 +2,6 @@ "sourceDefinitionId": "9da77001-af33-4bcd-be46-6252bf9342b9", "name": "Shopify", "dockerRepository": "airbyte/source-shopify", - "dockerImageTag": "0.1.19", + "dockerImageTag": "0.1.21", "documentationUrl": "https://docs.airbyte.io/integrations/sources/shopify" } diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 360d6b7696d9..ebfab5c137c2 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -450,7 +450,7 @@ - name: Shopify sourceDefinitionId: 9da77001-af33-4bcd-be46-6252bf9342b9 dockerRepository: airbyte/source-shopify - dockerImageTag: 0.1.19 + dockerImageTag: 0.1.21 documentationUrl: https://docs.airbyte.io/integrations/sources/shopify sourceType: api - name: Short.io diff --git a/airbyte-integrations/connectors/source-shopify/Dockerfile b/airbyte-integrations/connectors/source-shopify/Dockerfile index beb50a9f8999..b20a97b48340 100644 --- a/airbyte-integrations/connectors/source-shopify/Dockerfile +++ b/airbyte-integrations/connectors/source-shopify/Dockerfile @@ -28,5 +28,5 @@ COPY source_shopify ./source_shopify ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.19 +LABEL io.airbyte.version=0.1.21 LABEL io.airbyte.name=airbyte/source-shopify diff --git a/airbyte-integrations/connectors/source-shopify/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-shopify/integration_tests/abnormal_state.json index a6b169a7e2dd..a103ca3dd890 100644 --- a/airbyte-integrations/connectors/source-shopify/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-shopify/integration_tests/abnormal_state.json @@ -40,5 +40,11 @@ }, "discount_codes": { "updated_at": "2024-07-08T05:40:38-07:00" + }, + "locations": { + "updated_at": "2024-07-08T05:40:38-07:00" + }, + "inventory_levels": { + "updated_at": "2024-07-08T05:40:38-07:00" } } diff --git a/airbyte-integrations/connectors/source-shopify/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-shopify/integration_tests/configured_catalog.json index ef348454eb08..c9ebbd2ca0db 100644 --- a/airbyte-integrations/connectors/source-shopify/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-shopify/integration_tests/configured_catalog.json @@ -167,6 +167,30 @@ "sync_mode": "incremental", "cursor_field": ["updated_at"], "destination_sync_mode": "append" + }, + { + "stream": { + "name": "locations", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": true, + "default_cursor_field": ["id"] + }, + "sync_mode": "full_refresh", + "cursor_field": ["id"], + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "inventory_levels", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"] + }, + "sync_mode": "incremental", + "cursor_field": ["updated_at"], + "destination_sync_mode": "append" } ] } diff --git a/airbyte-integrations/connectors/source-shopify/integration_tests/state.json b/airbyte-integrations/connectors/source-shopify/integration_tests/state.json index 575f2171d4b3..65ca1ccdfed2 100644 --- a/airbyte-integrations/connectors/source-shopify/integration_tests/state.json +++ b/airbyte-integrations/connectors/source-shopify/integration_tests/state.json @@ -40,5 +40,11 @@ }, "discount_codes": { "updated_at": "2021-09-10T06:48:10-07:00" + }, + "locations": { + "updated_at": "2021-09-10T06:48:10-07:00" + }, + "inventory_levels": { + "updated_at": "2021-09-10T06:48:10-07:00" } } diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/inventory_levels.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/inventory_levels.json new file mode 100644 index 000000000000..89baedb11728 --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/inventory_levels.json @@ -0,0 +1,18 @@ +{ + "type": "object", + "properties": { + "available": { + "type": ["null", "integer"] + }, + "inventory_item_id": { + "type": ["null", "integer"] + }, + "location_id": { + "type": ["null", "integer"] + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + } + } +} diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/locations.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/locations.json new file mode 100644 index 000000000000..5f15a0aa956b --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/locations.json @@ -0,0 +1,55 @@ +{ + "type": "object", + "properties": { + "active": { + "type": ["null", "boolean"] + }, + "address1": { + "type": ["null", "string"] + }, + "address2": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "country_code": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "id": { + "type": ["null", "integer"] + }, + "legacy": { + "type": ["null", "boolean"] + }, + "name": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + }, + "province": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "zip": { + "type": ["null", "string"] + }, + "localized_country_name": { + "type": ["null", "string"] + }, + "localized_province_name": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/source.py b/airbyte-integrations/connectors/source-shopify/source_shopify/source.py index e7a4944b3671..17e348790ec6 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/source.py +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/source.py @@ -325,6 +325,46 @@ def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: return f"price_rules/{price_rule_id}/{self.data_field}.json" +class Locations(ShopifyStream): + + """ + The location API does not support any form of filtering. + https://shopify.dev/api/admin-rest/2021-07/resources/location + + Therefore, only FULL_REFRESH mode is supported. + """ + + data_field = "locations" + + def path(self, **kwargs): + return f"{self.data_field}.json" + + +class InventoryLevels(ChildSubstream): + parent_stream_class: object = Locations + slice_key = "location_id" + cursor_field = "updated_at" + + data_field = "inventory_levels" + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + location_id = stream_slice["location_id"] + return f"locations/{location_id}/{self.data_field}.json" + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + records_stream = super().parse_response(response, **kwargs) + + def generate_key(record): + record.update({"id": "|".join((str(record.get("location_id", "")), str(record.get("inventory_item_id", ""))))}) + return record + + # associate the surrogate key + yield from map( + generate_key, + records_stream, + ) + + class SourceShopify(AbstractSource): def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, any]: @@ -365,4 +405,6 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: Pages(config), PriceRules(config), DiscountCodes(config), + Locations(config), + InventoryLevels(config), ] diff --git a/docs/integrations/sources/shopify.md b/docs/integrations/sources/shopify.md index d104ebfb055e..db1b2d51560a 100644 --- a/docs/integrations/sources/shopify.md +++ b/docs/integrations/sources/shopify.md @@ -35,6 +35,8 @@ This Source is capable of syncing the following core Streams: * [Transactions](https://help.shopify.com/en/api/reference/orders/transaction) * [Pages](https://help.shopify.com/en/api/reference/online-store/page) * [Price Rules](https://help.shopify.com/en/api/reference/discounts/pricerule) +* [Locations](https://shopify.dev/api/admin-rest/2021-10/resources/location) +* [InventoryLevels](https://shopify.dev/api/admin-rest/2021-10/resources/inventorylevel) #### NOTE: @@ -95,6 +97,8 @@ This connector support both: `OAuth 2.0` and `API PASSWORD` (for private applica | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.1.21 | 2021-10-14 | [7382](https://github.com/airbytehq/airbyte/pull/7382) | Fixed `InventoryLevels` primary key | +| 0.1.20 | 2021-10-14 | [7063](https://github.com/airbytehq/airbyte/pull/7063) | Added `Location` and `InventoryLevels` as streams | | 0.1.19 | 2021-10-11 | [6951](https://github.com/airbytehq/airbyte/pull/6951) | Added support of `OAuth 2.0` authorisation option | | 0.1.18 | 2021-09-21 | [6056](https://github.com/airbytehq/airbyte/pull/6056) | Added `pre_tax_price` to the `orders/line_items` schema | | 0.1.17 | 2021-09-17 | [5244](https://github.com/airbytehq/airbyte/pull/5244) | Created data type enforcer for converting prices into numbers |