From b8a647270a84e2fead3890e8cca4c7ea07a24838 Mon Sep 17 00:00:00 2001 From: darynaishchenko <80129833+darynaishchenko@users.noreply.github.com> Date: Mon, 26 Sep 2022 11:16:16 +0300 Subject: [PATCH] Chargebee: Added "custom fields" to the relevant Chargebee source data streams. (#17056) * Chargebee: added custom fields * Chargebee: added value types json schema * Chargebee: added docs, updated docker label, refactored utils * auto-bump connector version [ci skip] Co-authored-by: Octavia Squidington III --- .../resources/seed/source_definitions.yaml | 2 +- .../src/main/resources/seed/source_specs.yaml | 2 +- .../connectors/source-chargebee/Dockerfile | 2 +- .../source_chargebee/schemas/addon.json | 14 +++++++ .../schemas/attached_item.json | 14 +++++++ .../source_chargebee/schemas/coupon.json | 14 +++++++ .../source_chargebee/schemas/credit_note.json | 14 +++++++ .../source_chargebee/schemas/customer.json | 14 +++++++ .../source_chargebee/schemas/event.json | 14 +++++++ .../source_chargebee/schemas/invoice.json | 14 +++++++ .../source_chargebee/schemas/item.json | 14 +++++++ .../source_chargebee/schemas/item_price.json | 14 +++++++ .../source_chargebee/schemas/order.json | 14 +++++++ .../source_chargebee/schemas/plan.json | 14 +++++++ .../schemas/subscription.json | 14 +++++++ .../source_chargebee/schemas/transaction.json | 14 +++++++ .../source_chargebee/streams.py | 7 ++-- .../source_chargebee/utils.py | 38 +++++++++++++++++++ docs/integrations/sources/chargebee.md | 33 ++++++++-------- 19 files changed, 244 insertions(+), 22 deletions(-) create mode 100644 airbyte-integrations/connectors/source-chargebee/source_chargebee/utils.py diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 9801c0fd7139..f6d3dbbc6929 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -144,7 +144,7 @@ - name: Chargebee sourceDefinitionId: 686473f1-76d9-4994-9cc7-9b13da46147c dockerRepository: airbyte/source-chargebee - dockerImageTag: 0.1.13 + dockerImageTag: 0.1.14 documentationUrl: https://docs.airbyte.io/integrations/sources/chargebee icon: chargebee.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 724791355505..d533d5de8c67 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -1557,7 +1557,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-chargebee:0.1.13" +- dockerImage: "airbyte/source-chargebee:0.1.14" spec: documentationUrl: "https://apidocs.chargebee.com/docs/api" connectionSpecification: diff --git a/airbyte-integrations/connectors/source-chargebee/Dockerfile b/airbyte-integrations/connectors/source-chargebee/Dockerfile index 90b26db6b2ea..c057552ff5f2 100644 --- a/airbyte-integrations/connectors/source-chargebee/Dockerfile +++ b/airbyte-integrations/connectors/source-chargebee/Dockerfile @@ -13,5 +13,5 @@ RUN pip install . ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.13 +LABEL io.airbyte.version=0.1.14 LABEL io.airbyte.name=airbyte/source-chargebee diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/addon.json b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/addon.json index 9e1324f1517b..222901b1558d 100644 --- a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/addon.json +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/addon.json @@ -174,6 +174,20 @@ } } } + }, + "custom_fields": { + "type": ["null", "array"], + "items" : { + "type" : ["null", "object"], + "properties": { + "name" : { + "type" : ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } + } + } } } } diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/attached_item.json b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/attached_item.json index ed5ecfae5e63..70f3cd47ed09 100644 --- a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/attached_item.json +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/attached_item.json @@ -61,5 +61,19 @@ "object": { "type": ["string", "null"] } + }, + "custom_fields": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties" : { + "name" : { + "type" : ["null", "string"] + }, + "value" : { + "type" : ["null", "string"] + } + } + } } } diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/coupon.json b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/coupon.json index 6bd23de93b6f..43cb11166825 100644 --- a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/coupon.json +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/coupon.json @@ -116,6 +116,20 @@ } } } + }, + "custom_fields": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "name": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } + } + } } } } diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/credit_note.json b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/credit_note.json index 0a9347e3b2f0..779667d4dac6 100644 --- a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/credit_note.json +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/credit_note.json @@ -507,6 +507,20 @@ } } } + }, + "custom_fields": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "name": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } + } + } } } } diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/customer.json b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/customer.json index 5a2f367c5fd3..269c0cbb99d9 100644 --- a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/customer.json +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/customer.json @@ -536,6 +536,20 @@ "meta_data": { "type": ["object", "null"], "properties": {} + }, + "custom_fields": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "name": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } + } + } } } } diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/event.json b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/event.json index a64b0dea32d0..522ecf844012 100644 --- a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/event.json +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/event.json @@ -75,6 +75,20 @@ } } } + }, + "custom_fields": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "name": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } + } + } } } } diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/invoice.json b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/invoice.json index 37b2fb106584..8450fce970d3 100644 --- a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/invoice.json +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/invoice.json @@ -867,6 +867,20 @@ "enum": ["not_validated", "valid", "partially_valid", "invalid"] } } + }, + "custom_fields": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "name": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } + } + } } } } diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/item.json b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/item.json index a719490ce4a0..2718200b1489 100644 --- a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/item.json +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/item.json @@ -95,6 +95,20 @@ }, "object": { "type": ["string", "null"] + }, + "custom_fields": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "name": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } + } + } } } } diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/item_price.json b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/item_price.json index 2926a085c3fd..71779783cbd0 100644 --- a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/item_price.json +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/item_price.json @@ -200,6 +200,20 @@ }, "object": { "type": ["string", "null"] + }, + "custom_fields": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "name": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } + } + } } } } diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/order.json b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/order.json index e607fa3cff67..4618f09c3b61 100644 --- a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/order.json +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/order.json @@ -582,6 +582,20 @@ } } } + }, + "custom_fields": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "name": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } + } + } } } } diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/plan.json b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/plan.json index 7c385cb18f48..aadf98eff1f6 100644 --- a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/plan.json +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/plan.json @@ -282,6 +282,20 @@ } } } + }, + "custom_fields": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "name": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } + } + } } } } diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/subscription.json b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/subscription.json index 2268637bce45..7c6de5a3acf3 100644 --- a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/subscription.json +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/subscription.json @@ -668,6 +668,20 @@ } } } + }, + "custom_fields": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "name": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } + } + } } } } diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/transaction.json b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/transaction.json index d46d3f2dba69..2f05fc094096 100644 --- a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/transaction.json +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/transaction.json @@ -326,6 +326,20 @@ } } } + }, + "custom_fields": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "name": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } + } + } } } } diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/streams.py b/airbyte-integrations/connectors/source-chargebee/source_chargebee/streams.py index 7d7782b90100..3cabd67ee4bb 100644 --- a/airbyte-integrations/connectors/source-chargebee/source_chargebee/streams.py +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/streams.py @@ -25,6 +25,7 @@ from chargebee.models import Transaction as TransactionModel from .rate_limiting import default_backoff_handler +from .utils import transform_custom_fields # Backoff params below according to Chargebee's guidance on rate limit. # https://apidocs.chargebee.com/docs/api?prod_cat_ver=2#api_rate_limits @@ -59,7 +60,7 @@ def request_params( def parse_response(self, list_result: ListResult, **kwargs) -> Iterable[Mapping]: for message in list_result: - yield message._response[self.name] + yield from transform_custom_fields(message._response[self.name]) @default_backoff_handler(max_tries=MAX_TRIES, factor=MAX_TIME) def _send_request(self, **kwargs) -> ListResult: @@ -133,7 +134,7 @@ def parse_response( for message in list_result: record = message._response[self.name] if record[self.cursor_field] > starting_point: - yield record + yield from transform_custom_fields(record) def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]): """ @@ -179,7 +180,7 @@ def request_params( def parse_response(self, list_result: ListResult, **kwargs) -> Iterable[Mapping]: for message in list_result: - yield message._response[self.name] + yield from transform_custom_fields(message._response[self.name]) def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]): """ diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/utils.py b/airbyte-integrations/connectors/source-chargebee/source_chargebee/utils.py new file mode 100644 index 000000000000..33d0d614f8c9 --- /dev/null +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/utils.py @@ -0,0 +1,38 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from typing import Any, Dict + + +def transform_custom_fields(record: Dict[str, Any]) -> Dict[str, Any]: + """ + Method to detect custom fields that start with 'cf_' from chargbee models. + Args: + record: + { + ... + 'cf_custom_fields': 'some_value', + ... + } + + Returns: + record: + { + ... + 'custom_fields': [{ + 'name': 'cf_custom_fields', + 'value': some_value' + }], + ... + } + """ + custom_fields = [] + + for key, value in record.items(): + if key.startswith("cf_"): + custom_fields.append({"name": key, "value": value}) + + record["custom_fields"] = custom_fields + + yield record diff --git a/docs/integrations/sources/chargebee.md b/docs/integrations/sources/chargebee.md index b272a7d9f4da..b1d8031a1828 100644 --- a/docs/integrations/sources/chargebee.md +++ b/docs/integrations/sources/chargebee.md @@ -100,20 +100,21 @@ regardless of how many AttachedItems were actually changed or synced in a partic ## Changelog -| Version | Date | Pull Request | Subject | -| :--- | :--- | :--- | :--- | -| 0.1.13 | 2022-08-18 | [15743](https://github.com/airbytehq/airbyte/pull/15743) | Fix transaction `exchange_rate` field type | -| 0.1.12 | 2022-07-13 | [14672](https://github.com/airbytehq/airbyte/pull/14672) | Fix transaction sort by | -| 0.1.11 | 2022-03-03 | [10827](https://github.com/airbytehq/airbyte/pull/10827) | Fix Credit Note stream | -| 0.1.10 | 2022-03-02 | [10795](https://github.com/airbytehq/airbyte/pull/10795) | Add support for Credit Note stream | -| 0.1.9 | 2022-0224 | [10312](https://github.com/airbytehq/airbyte/pull/10312) | Add support for Transaction Stream | -| 0.1.8 | 2022-02-22 | [10366](https://github.com/airbytehq/airbyte/pull/10366) | Fix broken `coupon` stream + add unit tests | -| 0.1.7 | 2022-02-14 | [10269](https://github.com/airbytehq/airbyte/pull/10269) | Add support for Coupon stream | -| 0.1.6 | 2022-02-10 | [10143](https://github.com/airbytehq/airbyte/pull/10143) | Add support for Event stream | -| 0.1.5 | 2021-12-23 | [8434](https://github.com/airbytehq/airbyte/pull/8434) | Update fields in source-connectors specifications | -| 0.1.4 | 2021-09-27 | [6454](https://github.com/airbytehq/airbyte/pull/6454) | Fix examples in spec file | -| 0.1.3 | 2021-08-17 | [5421](https://github.com/airbytehq/airbyte/pull/5421) | Add support for "Product Catalog 2.0" specific streams: `Items`, `Item prices` and `Attached Items` | -| 0.1.2 | 2021-07-30 | [5067](https://github.com/airbytehq/airbyte/pull/5067) | Prepare connector for publishing | -| 0.1.1 | 2021-07-07 | [4539](https://github.com/airbytehq/airbyte/pull/4539) | Add entrypoint and bump version for connector | -| 0.1.0 | 2021-06-30 | [3410](https://github.com/airbytehq/airbyte/pull/3410) | New Source: Chargebee | +| Version | Date | Pull Request | Subject | +|:--------|:-----------| :--- |:----------------------------------------------------------------------------------------------------| +| 0.1.14 | 2022-09-23 | [17056](https://github.com/airbytehq/airbyte/pull/17056) | Add "custom fields" to the relevant Chargebee source data streams | +| 0.1.13 | 2022-08-18 | [15743](https://github.com/airbytehq/airbyte/pull/15743) | Fix transaction `exchange_rate` field type | +| 0.1.12 | 2022-07-13 | [14672](https://github.com/airbytehq/airbyte/pull/14672) | Fix transaction sort by | +| 0.1.11 | 2022-03-03 | [10827](https://github.com/airbytehq/airbyte/pull/10827) | Fix Credit Note stream | +| 0.1.10 | 2022-03-02 | [10795](https://github.com/airbytehq/airbyte/pull/10795) | Add support for Credit Note stream | +| 0.1.9 | 2022-0224 | [10312](https://github.com/airbytehq/airbyte/pull/10312) | Add support for Transaction Stream | +| 0.1.8 | 2022-02-22 | [10366](https://github.com/airbytehq/airbyte/pull/10366) | Fix broken `coupon` stream + add unit tests | +| 0.1.7 | 2022-02-14 | [10269](https://github.com/airbytehq/airbyte/pull/10269) | Add support for Coupon stream | +| 0.1.6 | 2022-02-10 | [10143](https://github.com/airbytehq/airbyte/pull/10143) | Add support for Event stream | +| 0.1.5 | 2021-12-23 | [8434](https://github.com/airbytehq/airbyte/pull/8434) | Update fields in source-connectors specifications | +| 0.1.4 | 2021-09-27 | [6454](https://github.com/airbytehq/airbyte/pull/6454) | Fix examples in spec file | +| 0.1.3 | 2021-08-17 | [5421](https://github.com/airbytehq/airbyte/pull/5421) | Add support for "Product Catalog 2.0" specific streams: `Items`, `Item prices` and `Attached Items` | +| 0.1.2 | 2021-07-30 | [5067](https://github.com/airbytehq/airbyte/pull/5067) | Prepare connector for publishing | +| 0.1.1 | 2021-07-07 | [4539](https://github.com/airbytehq/airbyte/pull/4539) | Add entrypoint and bump version for connector | +| 0.1.0 | 2021-06-30 | [3410](https://github.com/airbytehq/airbyte/pull/3410) | New Source: Chargebee |