Merge branch 'master' into leti/tools-add-posthog

* master:
  source-sentry: migrate to per-stream states (#17466)
  Greg/clickhouse polishing (#17483)
  upgrade debezium version to 1.9.6 (#17459)
  🐛 Source Twilio: Lookback_window config (#17478)
  hide S3 source connector from catalog (#17472)
  🪟 Migrate styles for Connection-related Components (#17339)
  Added new title (#17480)
  Refactor & convert `PageTitle` to SCSS (#17139)
  updated releaseStage for zendesk-talk (#17477)
  [low-code] Apply log level to stream loggers (#17284)
  🐛 Source Salesforce: filter out objects not supported by the Bulk API (#17453)
  Source Marketo: certify GA (#17445)
  Update greenhouse paginator (#17429)
  Add some services start validation to acceptance_tests.sh (#17425)
  📖 Removes $ from terminal commands to allow direct copying. (#17467)

airbyte-cdk/python/airbyte_cdk/sources/declarative/yaml_declarative_source.py

@@ -63,7 +63,11 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]:
             "parsed YAML into declarative source",
             extra={"path_to_yaml_file": self._path_to_yaml, "source_name": self.name, "parsed_config": json.dumps(self._source_config)},
         )
-        return [self._factory.create_component(stream_config, config, True)() for stream_config in self._stream_configs()]
+        source_streams = [self._factory.create_component(stream_config, config, True)() for stream_config in self._stream_configs()]
+        for stream in source_streams:
+            # make sure the log level is always appied to the stream's logger
+            self._apply_log_level_to_stream_logger(self.logger, stream)
+        return source_streams
 
     def _read_and_parse_yaml_file(self, path_to_yaml_file):
         package = self.__class__.__module__.split(".")[0]

airbyte-config/init/src/main/resources/seed/destination_definitions.yaml

@@ -67,7 +67,7 @@
 - name: Clickhouse
   destinationDefinitionId: ce0d828e-1dc4-496c-b122-2da42e637e48
   dockerRepository: airbyte/destination-clickhouse
-  dockerImageTag: 0.1.12
+  dockerImageTag: 0.2.0
   documentationUrl: https://docs.airbyte.io/integrations/destinations/clickhouse
   releaseStage: alpha
 - name: Cloudflare R2

airbyte-config/init/src/main/resources/seed/destination_specs.yaml

@@ -797,7 +797,7 @@
     supported_destination_sync_modes:
     - "overwrite"
     - "append"
-- dockerImage: "airbyte/destination-clickhouse:0.1.12"
+- dockerImage: "airbyte/destination-clickhouse:0.2.0"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/destinations/clickhouse"
     connectionSpecification:
@@ -818,53 +818,43 @@
           order: 0
         port:
           title: "Port"
-          description: "JDBC port (not the native port) of the database."
+          description: "HTTP port of the database."
           type: "integer"
           minimum: 0
           maximum: 65536
           default: 8123
           examples:
           - "8123"
           order: 1
-        tcp-port:
-          title: "Native Port"
-          description: "Native port (not the JDBC) of the database."
-          type: "integer"
-          minimum: 0
-          maximum: 65536
-          default: 9000
-          examples:
-          - "9000"
-          order: 2
         database:
           title: "DB Name"
           description: "Name of the database."
           type: "string"
-          order: 3
+          order: 2
         username:
           title: "User"
           description: "Username to use to access the database."
           type: "string"
-          order: 4
+          order: 3
         password:
           title: "Password"
           description: "Password associated with the username."
           type: "string"
           airbyte_secret: true
-          order: 5
+          order: 4
         jdbc_url_params:
           description: "Additional properties to pass to the JDBC URL string when\
             \ connecting to the database formatted as 'key=value' pairs separated\
             \ by the symbol '&'. (example: key1=value1&key2=value2&key3=value3)."
           title: "JDBC URL Params"
           type: "string"
-          order: 6
+          order: 5
         ssl:
           title: "SSL Connection"
           description: "Encrypt data using SSL."
           type: "boolean"
           default: false
-          order: 7
+          order: 6
         tunnel_method:
           type: "object"
           title: "SSH Tunnel Method"

airbyte-config/init/src/main/resources/seed/source_definitions.yaml

@@ -614,11 +614,11 @@
 - name: Marketo
   sourceDefinitionId: 9e0556f4-69df-4522-a3fb-03264d36b348
   dockerRepository: airbyte/source-marketo
-  dockerImageTag: 0.1.9
+  dockerImageTag: 0.1.11
   documentationUrl: https://docs.airbyte.io/integrations/sources/marketo
   icon: marketo.svg
   sourceType: api
-  releaseStage: beta
+  releaseStage: generally_available
 - name: Metabase
   sourceDefinitionId: c7cb421b-942e-4468-99ee-e369bcabaec5
   dockerRepository: airbyte/source-metabase
@@ -678,7 +678,7 @@
 - name: MySQL
   sourceDefinitionId: 435bb9a5-7887-4809-aa58-28c27df0d7ad
   dockerRepository: airbyte/source-mysql
-  dockerImageTag: 1.0.0
+  dockerImageTag: 1.0.1
   documentationUrl: https://docs.airbyte.io/integrations/sources/mysql
   icon: mysql.svg
   sourceType: database
@@ -836,7 +836,7 @@
 - name: Postgres
   sourceDefinitionId: decd338e-5647-4c0b-adf4-da0e75f5a750
   dockerRepository: airbyte/source-postgres
-  dockerImageTag: 1.0.12
+  dockerImageTag: 1.0.13
   documentationUrl: https://docs.airbyte.io/integrations/sources/postgres
   icon: postgresql.svg
   sourceType: database
@@ -931,7 +931,7 @@
 - name: Salesforce
   sourceDefinitionId: b117307c-14b6-41aa-9422-947e34922962
   dockerRepository: airbyte/source-salesforce
-  dockerImageTag: 1.0.19
+  dockerImageTag: 1.0.20
   documentationUrl: https://docs.airbyte.io/integrations/sources/salesforce
   icon: salesforce.svg
   sourceType: api
@@ -1083,7 +1083,7 @@
 - name: Twilio
   sourceDefinitionId: b9dc6155-672e-42ea-b10d-9f1f1fb95ab1
   dockerRepository: airbyte/source-twilio
-  dockerImageTag: 0.1.10
+  dockerImageTag: 0.1.11
   documentationUrl: https://docs.airbyte.io/integrations/sources/twilio
   icon: twilio.svg
   sourceType: api
@@ -1175,7 +1175,7 @@
   documentationUrl: https://docs.airbyte.io/integrations/sources/zendesk-talk
   icon: zendesk.svg
   sourceType: api
-  releaseStage: beta
+  releaseStage: generally_available
 - name: Zenefits
   sourceDefinitionId: 8baba53d-2fe3-4e33-bc85-210d0eb62884
   dockerRepository: airbyte/source-zenefits
@@ -1194,7 +1194,7 @@
 - sourceDefinitionId: cdaf146a-9b75-49fd-9dd2-9d64a0bb4781
   name: Sentry
   dockerRepository: airbyte/source-sentry
-  dockerImageTag: 0.1.6
+  dockerImageTag: 0.1.7
   documentationUrl: https://docs.airbyte.io/integrations/sources/sentry
   icon: sentry.svg
   sourceType: api

airbyte-config/init/src/main/resources/seed/source_specs.yaml

@@ -6046,7 +6046,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-marketo:0.1.9"
+- dockerImage: "airbyte/source-marketo:0.1.11"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/sources/marketo"
     connectionSpecification:
@@ -6965,7 +6965,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-mysql:1.0.0"
+- dockerImage: "airbyte/source-mysql:1.0.1"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/sources/mysql"
     connectionSpecification:
@@ -8578,7 +8578,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-postgres:1.0.12"
+- dockerImage: "airbyte/source-postgres:1.0.13"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/postgres"
     connectionSpecification:
@@ -9763,7 +9763,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-salesforce:1.0.19"
+- dockerImage: "airbyte/source-salesforce:1.0.20"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/salesforce"
     connectionSpecification:
@@ -11403,7 +11403,7 @@
         oauthFlowOutputParameters:
         - - "token"
         - - "key"
-- dockerImage: "airbyte/source-twilio:0.1.10"
+- dockerImage: "airbyte/source-twilio:0.1.11"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/sources/twilio"
     connectionSpecification:
@@ -11443,6 +11443,8 @@
           examples:
           - 60
           default: 0
+          minimum: 0
+          maximum: 576000
           type: "integer"
           order: 4
     supportsIncremental: true
@@ -12276,7 +12278,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-sentry:0.1.6"
+- dockerImage: "airbyte/source-sentry:0.1.7"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/sources/sentry"
     connectionSpecification:

airbyte-config/init/src/test/resources/connector_catalog.json

@@ -791,24 +791,14 @@
             },
             "port": {
               "title": "Port",
-              "description": "JDBC port (not the native port) of the database.",
+              "description": "HTTP port of the database.",
               "type": "integer",
               "minimum": 0,
               "maximum": 65536,
               "default": 8123,
               "examples": ["8123"],
               "order": 1
             },
-            "tcp-port": {
-              "title": "Native Port",
-              "description": "Native port (not the JDBC) of the database.",
-              "type": "integer",
-              "minimum": 0,
-              "maximum": 65536,
-              "default": 9000,
-              "examples": ["9000"],
-              "order": 2
-            },
             "database": {
               "title": "DB Name",
               "description": "Name of the database.",
@@ -6120,7 +6110,7 @@
       "dockerRepository": "airbyte/source-clickhouse-strict-encrypt",
       "dockerImageTag": "0.1.8",
       "documentationUrl": "https://docs.airbyte.io/integrations/sources/clickhouse",
-      "icon": "cliskhouse.svg",
+      "icon": "clickhouse.svg",
       "sourceType": "database",
       "spec": {
         "documentationUrl": "https://docs.airbyte.io/integrations/destinations/clickhouse",

airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java

@@ -41,7 +41,7 @@ public class JdbcUtils {
   public static final String JDBC_URL_PARAMS_KEY = "jdbc_url_params";
   public static final String PASSWORD_KEY = "password";
   public static final String PORT_KEY = "port";
-  public static final String TCP_PORT_KEY = "tcp-port";
+
   public static final List<String> PORT_LIST_KEY = List.of("port");
   public static final String SCHEMA_KEY = "schema";
   // NOTE: this is the plural version of SCHEMA_KEY

airbyte-integrations/bases/base-normalization/integration_tests/dbt_integration_test.py

@@ -27,7 +27,6 @@
 NORMALIZATION_TEST_MYSQL_DB_PORT = "NORMALIZATION_TEST_MYSQL_DB_PORT"
 NORMALIZATION_TEST_POSTGRES_DB_PORT = "NORMALIZATION_TEST_POSTGRES_DB_PORT"
 NORMALIZATION_TEST_CLICKHOUSE_DB_PORT = "NORMALIZATION_TEST_CLICKHOUSE_DB_PORT"
-NORMALIZATION_TEST_CLICKHOUSE_DB_TCP_PORT = "NORMALIZATION_TEST_CLICKHOUSE_DB_TCP_PORT"
 NORMALIZATION_TEST_TIDB_DB_PORT = "NORMALIZATION_TEST_TIDB_DB_PORT"
 
 
@@ -224,28 +223,20 @@ def setup_mssql_db(self):
 
     def setup_clickhouse_db(self):
         """
-        ClickHouse official JDBC driver use HTTP port 8123, while Python ClickHouse
-        driver uses native port 9000, so we need to open both ports for destination
-        connector and dbt container respectively.
+        ClickHouse official JDBC driver uses HTTP port 8123.
 
         Ref: https://altinity.com/blog/2019/3/15/clickhouse-networking-part-1
         """
         start_db = True
         port = 8123
-        tcp_port = 9000
         if os.getenv(NORMALIZATION_TEST_CLICKHOUSE_DB_PORT):
             port = int(os.getenv(NORMALIZATION_TEST_CLICKHOUSE_DB_PORT))
             start_db = False
-        if os.getenv(NORMALIZATION_TEST_CLICKHOUSE_DB_TCP_PORT):
-            tcp_port = int(os.getenv(NORMALIZATION_TEST_CLICKHOUSE_DB_TCP_PORT))
-            start_db = False
         if start_db:
             port = self.find_free_port()
-            tcp_port = self.find_free_port()
         config = {
             "host": "localhost",
             "port": port,
-            "tcp-port": tcp_port,
             "database": self.target_schema,
             "username": "default",
             "password": "",
@@ -263,8 +254,6 @@ def setup_clickhouse_db(self):
                 "--ulimit",
                 "nofile=262144:262144",
                 "-p",
-                f"{config['tcp-port']}:9000",  # Python clickhouse driver use native port
-                "-p",
                 f"{config['port']}:8123",  # clickhouse JDBC driver use HTTP port
                 "-d",
                 # so far, only the latest version ClickHouse server image turned on

airbyte-integrations/bases/base-normalization/normalization/transform_config/transform.py

@@ -318,7 +318,8 @@ def transform_clickhouse(config: Dict[str, Any]):
         # https://docs.getdbt.com/reference/warehouse-profiles/clickhouse-profile
         dbt_config = {
             "type": "clickhouse",
-            "driver": "native",
+            "driver": "http",
+            "verify": False,
             "host": config["host"],
             "port": config["port"],
             "schema": config["database"],
@@ -327,8 +328,6 @@ def transform_clickhouse(config: Dict[str, Any]):
         }
         if "password" in config:
             dbt_config["password"] = config["password"]
-        if "tcp-port" in config:
-            dbt_config["port"] = config["tcp-port"]
         return dbt_config
 
     @staticmethod

airbyte-integrations/bases/base-normalization/unit_tests/test_transform_config.py

@@ -440,7 +440,8 @@ def test_transform_clickhouse(self):
         actual = TransformConfig().transform_clickhouse(input)
         expected = {
             "type": "clickhouse",
-            "driver": "native",
+            "driver": "http",
+            "verify": False,
             "host": "airbyte.io",
             "port": 9440,
             "schema": "default",

airbyte-integrations/bases/debezium-v1-9-2/build.gradle → airbyte-integrations/bases/debezium-v1-9-6/build.gradle

@@ -9,11 +9,11 @@ dependencies {
     implementation project(':airbyte-protocol:protocol-models')
     implementation project(':airbyte-db:db-lib')
 
-    implementation 'io.debezium:debezium-api:1.9.2.Final'
-    implementation 'io.debezium:debezium-embedded:1.9.2.Final'
+    implementation 'io.debezium:debezium-api:1.9.6.Final'
+    implementation 'io.debezium:debezium-embedded:1.9.6.Final'
 //    implementation 'io.debezium:debezium-connector-sqlserver:1.9.2.Final'
-    implementation 'io.debezium:debezium-connector-mysql:1.9.2.Final'
-    implementation 'io.debezium:debezium-connector-postgres:1.9.2.Final'
+    implementation 'io.debezium:debezium-connector-mysql:1.9.6.Final'
+    implementation 'io.debezium:debezium-connector-postgres:1.9.6.Final'
     implementation 'org.codehaus.plexus:plexus-utils:3.4.2'
 
     testFixturesImplementation project(':airbyte-db:db-lib')

airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/Dockerfile

@@ -16,5 +16,5 @@ ENV APPLICATION destination-clickhouse-strict-encrypt
 
 COPY --from=build /airbyte /airbyte
 
-LABEL io.airbyte.version=0.1.12
+LABEL io.airbyte.version=0.2.0
 LABEL io.airbyte.name=airbyte/destination-clickhouse-strict-encrypt

airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationStrictEncryptAcceptanceTest.java

@@ -113,7 +113,6 @@ protected String getDefaultSchema(final JsonNode config) {
   protected JsonNode getConfig() {
     return Jsons.jsonNode(ImmutableMap.builder()
         .put(JdbcUtils.HOST_KEY, HostPortResolver.resolveIpAddress(db))
-        .put(JdbcUtils.TCP_PORT_KEY, NATIVE_SECURE_PORT)
         .put(JdbcUtils.PORT_KEY, HTTPS_PORT)
         .put(JdbcUtils.DATABASE_KEY, DB_NAME)
         .put(JdbcUtils.USERNAME_KEY, USER_NAME)