Chargebee: Added "custom fields" to the relevant Chargebee source dat…

…a streams. (#17056) * Chargebee: added custom fields * Chargebee: added value types json schema * Chargebee: added docs, updated docker label, refactored utils * auto-bump connector version [ci skip] Co-authored-by: Octavia Squidington III <octavia-squidington-iii@users.noreply.github.com>
airbytehq · Sep 26, 2022 · b8a6472 · b8a6472
1 parent d14b442
commit b8a6472
Show file tree

Hide file tree

Showing 19 changed files with 244 additions and 22 deletions.
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -144,7 +144,7 @@
 - name: Chargebee
   sourceDefinitionId: 686473f1-76d9-4994-9cc7-9b13da46147c
   dockerRepository: airbyte/source-chargebee
-  dockerImageTag: 0.1.13
+  dockerImageTag: 0.1.14
   documentationUrl: https://docs.airbyte.io/integrations/sources/chargebee
   icon: chargebee.svg
   sourceType: api

diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml
@@ -1557,7 +1557,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-chargebee:0.1.13"
+- dockerImage: "airbyte/source-chargebee:0.1.14"
   spec:
     documentationUrl: "https://apidocs.chargebee.com/docs/api"
     connectionSpecification:

diff --git a/airbyte-integrations/connectors/source-chargebee/Dockerfile b/airbyte-integrations/connectors/source-chargebee/Dockerfile
@@ -13,5 +13,5 @@ RUN pip install .
 
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.13
+LABEL io.airbyte.version=0.1.14
 LABEL io.airbyte.name=airbyte/source-chargebee
diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/addon.json b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/addon.json
@@ -174,6 +174,20 @@
           }
         }
       }
+    },
+    "custom_fields": {
+      "type": ["null", "array"],
+      "items" : {
+        "type" : ["null", "object"],
+        "properties": {
+          "name" : {
+            "type" : ["null", "string"]
+          },
+          "value": {
+            "type": ["null", "string"]
+          }
+        }
+      }
     }
   }
 }
diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/attached_item.json b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/attached_item.json
@@ -61,5 +61,19 @@
     "object": {
       "type": ["string", "null"]
     }
+  },
+  "custom_fields": {
+    "type": ["null", "array"],
+    "items": {
+      "type": ["null", "object"],
+      "properties" : {
+        "name" : {
+          "type" : ["null", "string"]
+        },
+        "value" : {
+          "type" : ["null", "string"]
+        }
+      }
+    }
   }
 }
diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/coupon.json b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/coupon.json
@@ -116,6 +116,20 @@
           }
         }
       }
+    },
+    "custom_fields": {
+      "type": ["null", "array"],
+      "items": {
+        "type": ["null", "object"],
+        "properties": {
+          "name": {
+            "type": ["null", "string"]
+          },
+          "value": {
+            "type": ["null", "string"]
+          }
+        }
+      }
     }
   }
 }
diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/credit_note.json b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/credit_note.json
@@ -507,6 +507,20 @@
           }
         }
       }
+    },
+    "custom_fields": {
+      "type": ["null", "array"],
+      "items": {
+        "type": ["null", "object"],
+        "properties": {
+          "name": {
+            "type": ["null", "string"]
+          },
+          "value": {
+            "type": ["null", "string"]
+          }
+        }
+      }
     }
   }
 }
diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/customer.json b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/customer.json
@@ -536,6 +536,20 @@
     "meta_data": {
       "type": ["object", "null"],
       "properties": {}
+    },
+    "custom_fields": {
+      "type": ["null", "array"],
+      "items": {
+        "type": ["null", "object"],
+        "properties": {
+          "name": {
+            "type": ["null", "string"]
+          },
+          "value": {
+            "type": ["null", "string"]
+          }
+        }
+      }
     }
   }
 }
diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/event.json b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/event.json
@@ -75,6 +75,20 @@
           }
         }
       }
+    },
+    "custom_fields": {
+      "type": ["null", "array"],
+      "items": {
+        "type": ["null", "object"],
+        "properties": {
+          "name": {
+            "type": ["null", "string"]
+          },
+          "value": {
+            "type": ["null", "string"]
+          }
+        }
+      }
     }
   }
 }
diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/invoice.json b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/invoice.json
@@ -867,6 +867,20 @@
           "enum": ["not_validated", "valid", "partially_valid", "invalid"]
         }
       }
+    },
+    "custom_fields": {
+      "type": ["null", "array"],
+      "items": {
+        "type": ["null", "object"],
+        "properties": {
+          "name": {
+            "type": ["null", "string"]
+          },
+          "value": {
+            "type": ["null", "string"]
+          }
+        }
+      }
     }
   }
 }
diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/item.json b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/item.json
@@ -95,6 +95,20 @@
     },
     "object": {
       "type": ["string", "null"]
+    },
+    "custom_fields": {
+      "type": ["null", "array"],
+      "items": {
+        "type": ["null", "object"],
+        "properties": {
+          "name": {
+            "type": ["null", "string"]
+          },
+          "value": {
+            "type": ["null", "string"]
+          }
+        }
+      }
     }
   }
 }
diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/item_price.json b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/item_price.json
@@ -200,6 +200,20 @@
     },
     "object": {
       "type": ["string", "null"]
+    },
+    "custom_fields": {
+      "type": ["null", "array"],
+      "items": {
+        "type": ["null", "object"],
+        "properties": {
+          "name": {
+            "type": ["null", "string"]
+          },
+          "value": {
+            "type": ["null", "string"]
+          }
+        }
+      }
     }
   }
 }
diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/order.json b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/order.json
@@ -582,6 +582,20 @@
           }
         }
       }
+    },
+    "custom_fields": {
+      "type": ["null", "array"],
+      "items": {
+        "type": ["null", "object"],
+        "properties": {
+          "name": {
+            "type": ["null", "string"]
+          },
+          "value": {
+            "type": ["null", "string"]
+          }
+        }
+      }
     }
   }
 }
diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/plan.json b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/plan.json
@@ -282,6 +282,20 @@
           }
         }
       }
+    },
+    "custom_fields": {
+      "type": ["null", "array"],
+      "items": {
+        "type": ["null", "object"],
+        "properties": {
+          "name": {
+            "type": ["null", "string"]
+          },
+          "value": {
+            "type": ["null", "string"]
+          }
+        }
+      }
     }
   }
 }
diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/subscription.json b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/subscription.json
@@ -668,6 +668,20 @@
           }
         }
       }
+    },
+    "custom_fields": {
+      "type": ["null", "array"],
+      "items": {
+        "type": ["null", "object"],
+        "properties": {
+          "name": {
+            "type": ["null", "string"]
+          },
+          "value": {
+            "type": ["null", "string"]
+          }
+        }
+      }
     }
   }
 }
diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/transaction.json b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/transaction.json
@@ -326,6 +326,20 @@
           }
         }
       }
+    },
+    "custom_fields": {
+      "type": ["null", "array"],
+      "items": {
+        "type": ["null", "object"],
+        "properties": {
+          "name": {
+            "type": ["null", "string"]
+          },
+          "value": {
+            "type": ["null", "string"]
+          }
+        }
+      }
     }
   }
 }
diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/streams.py b/airbyte-integrations/connectors/source-chargebee/source_chargebee/streams.py
@@ -25,6 +25,7 @@
 from chargebee.models import Transaction as TransactionModel
 
 from .rate_limiting import default_backoff_handler
+from .utils import transform_custom_fields
 
 # Backoff params below according to Chargebee's guidance on rate limit.
 # https://apidocs.chargebee.com/docs/api?prod_cat_ver=2#api_rate_limits
@@ -59,7 +60,7 @@ def request_params(
 
     def parse_response(self, list_result: ListResult, **kwargs) -> Iterable[Mapping]:
         for message in list_result:
-            yield message._response[self.name]
+            yield from transform_custom_fields(message._response[self.name])
 
     @default_backoff_handler(max_tries=MAX_TRIES, factor=MAX_TIME)
     def _send_request(self, **kwargs) -> ListResult:
@@ -133,7 +134,7 @@ def parse_response(
         for message in list_result:
             record = message._response[self.name]
             if record[self.cursor_field] > starting_point:
-                yield record
+                yield from transform_custom_fields(record)
 
     def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]):
         """
@@ -179,7 +180,7 @@ def request_params(
 
     def parse_response(self, list_result: ListResult, **kwargs) -> Iterable[Mapping]:
         for message in list_result:
-            yield message._response[self.name]
+            yield from transform_custom_fields(message._response[self.name])
 
     def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]):
         """

diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/utils.py b/airbyte-integrations/connectors/source-chargebee/source_chargebee/utils.py
@@ -0,0 +1,38 @@
+#
+# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
+#
+
+from typing import Any, Dict
+
+
+def transform_custom_fields(record: Dict[str, Any]) -> Dict[str, Any]:
+    """
+    Method to detect custom fields that start with 'cf_' from chargbee models.
+    Args:
+        record:
+        {
+            ...
+            'cf_custom_fields': 'some_value',
+            ...
+        }
+
+    Returns:
+        record:
+        {
+            ...
+            'custom_fields': [{
+                'name': 'cf_custom_fields',
+                'value': some_value'
+            }],
+            ...
+        }
+    """
+    custom_fields = []
+
+    for key, value in record.items():
+        if key.startswith("cf_"):
+            custom_fields.append({"name": key, "value": value})
+
+    record["custom_fields"] = custom_fields
+
+    yield record