Merge branch 'master' into leti/fix-page-tracking-in-segment

* master: (47 commits)
  Add email to identify users analytics call (#16327)
  🎉 Source Amazon Ads: improve `config.start_date` validation (#16191)
  Add comments about intermediate state emission (#16262)
  MySQL Source : Standardize spec.json for DB connectors that support log-based CDC replication (#16216)
  MSSQL Source : Standardize spec.json for DB connectors that support log-based CDC replication (#16215)
  Hide a bunch more destination with potential unsecure API access (#16320)
  Skip unit tests when run-tests is false (#16267)
  Hide Destination connections from UI (#16310)
  Add scheduled task to clean up old files from workspace (#16247)
  Source Google Analytics v4: Re-name google analytics connector (#16306)
  🐛 Source Facebook Marketing: remove "end_date" from config if empty value (re-implement #16096) (#16222)
  Fix github action syntax (#16277)
  Re-name google analytics cionnectors (#16287)
  Bump Airbyte version from 0.40.3 to 0.40.4 (#16275)
  Hide ES and Redis destination connectors from Cloud (#16276)
  15700  add tests for PokeAPI (#15701)
  Add ProtocolVersion to StandardDefs (#16237)
  🪟 🔧 🧹 Migrate attempt `bytesSynced` to `totalStats.bytesEmitted` and cleanup `AttemptDetails` component (#16126)
  Improve behavior of password input field (#16011)
  Improve airbyte-metrics support in the Helm chart (#16166)
  ...

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.40.3
+current_version = 0.40.4
 commit = False
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-[a-z]+)?
@@ -14,6 +14,8 @@ serialize =
 
 [bumpversion:file:airbyte-container-orchestrator/Dockerfile]
 
+[bumpversion:file:airbyte-cron/Dockerfile]
+
 [bumpversion:file:airbyte-metrics/reporter/Dockerfile]
 
 [bumpversion:file:airbyte-server/Dockerfile]

.env

@@ -10,7 +10,7 @@
 
 
 ### SHARED ###
-VERSION=0.40.3
+VERSION=0.40.4
 
 # When using the airbyte-db via default docker image
 CONFIG_ROOT=/data

.github/workflows/project.yaml → .github/workflows/assign-issue-to-project.yaml

@@ -1,5 +1,5 @@
 # https://github.com/marketplace/actions/assign-to-one-project#:~:text=GitHub%20Action%20for%20Assign%20to,columns%20in%20your%20project%20dashboard.
-name: Auto Assign to Airbyte Github Project
+name: Assign Issue to Airbyte Github Project
 
 on:
   issues:
@@ -32,3 +32,9 @@ jobs:
         with:
           project: "https://github.com/airbytehq/airbyte/projects/15"
           column_name: "To do"
+      - name: Assign Onboarding Improvements PR to Project
+        uses: srggrs/assign-one-project-github-action@1.2.0
+        if: contains(github.event.pull_request.labels.*.name, 'project/onboarding-improvements')
+        with:
+          project: "https://github.com/airbytehq/airbyte/projects/15"
+          column_name: "To do"

.github/workflows/build-report.yml

@@ -1,4 +1,7 @@
-name: Build Report
+name: Generate Build Report
+
+# Uses Python to Generate a build report
+# and send it to Slack
 
 on:
   workflow_dispatch:

.github/workflows/connector_integration_tests.yml

@@ -1,5 +1,7 @@
 name: Connector Integration Tests
 
+# Launches the connector integration tests
+
 on:
   workflow_dispatch:
   schedule:

.github/workflows/create-release.yml

@@ -1,7 +1,8 @@
 # This is an action that runs when an Airbyte version bump is merged into master.
 # It fetches the changelog from the version bump PR and automatically creates a
 # Release for the version bump.
-name: Create Release
+
+name: Create Airbyte GH Release
 
 on:
   push:

.github/workflows/doc-link-check.yml

@@ -1,6 +1,7 @@
 # Perform link check on all markdown files
+# https://github.com/gaurav-nelson/github-action-markdown-link-check
 
-name: Doc Link Checker
+name: Check for broken links in docs
 
 on:
   workflow_dispatch:

.github/workflows/fe-validate-links.yml

@@ -1,4 +1,4 @@
-name: Frontend Link Validation
+name: Check for broken links in FE
 
 on:
   workflow_dispatch:

.github/workflows/gke-kube-test-command.yml

@@ -1,7 +1,7 @@
 name: GKE Kube Acceptance Test
 on:
   schedule:
-    - cron '0 0 * * 0' # runs at midnight UTC every Sunday
+    - cron: '0 0 * * 0' # runs at midnight UTC every Sunday
   workflow_dispatch:
     inputs:
       repo:

.github/workflows/labeler.yml

@@ -1,4 +1,7 @@
-name: "Pull Request Labeler"
+# the mapping from filepath to label
+# is defined in .github/labeler.yml
+
+name: "Label PR based on filepath"
 on:
   - pull_request_target
 

.github/workflows/notify-on-label.yml

@@ -1,4 +1,6 @@
-name: Notify team
+# Notify users/teams when labels are added to an issue.
+
+name: Notify when adding label to issue
 
 on:
   issues:

.github/workflows/run-specific-test-command.yml → .github/workflows/run-performance-test.yml

@@ -1,4 +1,4 @@
-name: performance-test
+name: Run Performance Test
 on:
   workflow_dispatch:
     inputs:

.github/workflows/shared-issues.yml

@@ -1,4 +1,7 @@
-name: "Shared Issues"
+# Adds metadata and labels to issues based on context
+# https://github.com/airbytehq/workflow-actions/blob/main/src/bin_issue.ts <-- this gets run
+
+name: "Add metadata and labels to issues based on context"
 on:
   issues:
     types: [opened, labeled, unlabeled]

.github/workflows/shared-pulls.yml

@@ -1,4 +1,6 @@
-name: "Shared Pull Requests"
+# Runs internal automation for pull requests
+
+name: "Add metadata and labels to PRs based on context"
 on:
   pull_request_target:
     types: [opened, labeled, unlabeled, ready_for_review, synchronize, reopened]

.github/workflows/slash-commands.yml

@@ -18,7 +18,7 @@ jobs:
         id: scd
         uses: peter-evans/slash-command-dispatch@v2
         with:
-          token: ${{ secrets.DAVINCHIA_PAT }}
+          token: ${{ secrets.SUPERTOPHER_PAT }}
           commands: |
             test
             test-performance

.github/workflows/test-command.yml

@@ -1,4 +1,4 @@
-name: integration-test
+name: Run Integration Test
 on:
   workflow_dispatch:
     inputs:

.github/workflows/test-performance-command.yml

@@ -1,4 +1,6 @@
-name: performance-test
+# runs ./tools/bin/ci_performance_test.sh
+# which is more or less ./gradlew performanceTest limited to connectors with changes
+name: Run Performance Test
 on:
   workflow_dispatch:
     inputs:

.github/workflows/workflow-cleanup.yml

@@ -1,4 +1,4 @@
-name: Cleanup old GH action workflow runs
+name: Cleanup old GH workflow runs
 
 on: 
   schedule:

airbyte-bootloader/Dockerfile

@@ -2,7 +2,7 @@ ARG JDK_VERSION=19-slim-bullseye
 ARG JDK_IMAGE=openjdk:${JDK_VERSION}
 FROM ${JDK_IMAGE}
 
-ARG VERSION=0.40.3
+ARG VERSION=0.40.4
 
 ENV APPLICATION airbyte-bootloader
 ENV VERSION ${VERSION}

airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderAppTest.java

@@ -133,7 +133,7 @@ void testBootloaderAppBlankDb() throws Exception {
       val configsMigrator = new ConfigsDatabaseMigrator(configDatabase, configsFlyway);
       // this line should change with every new migration
       // to show that you meant to make a new migration to the prod database
-      assertEquals("0.40.3.001", configsMigrator.getLatestMigration().getVersion().getVersion());
+      assertEquals("0.40.3.002", configsMigrator.getLatestMigration().getVersion().getVersion());
 
       val jobsPersistence = new DefaultJobPersistence(jobDatabase);
       assertEquals(VERSION_0330_ALPHA, jobsPersistence.getVersion().get());

airbyte-cdk/python/setup.py

@@ -44,7 +44,7 @@
     packages=find_packages(exclude=("unit_tests",)),
     install_requires=[
         "backoff",
-        "dataclasses-jsonschema~=2.15.1",
+        "dataclasses-jsonschema==2.15.1",  # pinned to the last working version for us temporarily while we fix
         "dpath~=2.0.1",
         "jsonschema~=3.2.0",
         "jsonref~=0.2",

airbyte-config/config-models/src/main/java/io/airbyte/config/Configs.java

@@ -7,6 +7,7 @@
 import io.airbyte.commons.version.AirbyteVersion;
 import io.airbyte.config.helpers.LogConfigs;
 import io.airbyte.config.storage.CloudStorageConfigs;
+import java.net.URI;
 import java.nio.file.Path;
 import java.util.List;
 import java.util.Map;
@@ -71,6 +72,11 @@ public interface Configs {
    */
   Path getWorkspaceRoot();
 
+  /**
+   * Defines the URL to pull the remote connector catalog from.
+   */
+  URI getRemoteConnectorCatalogUrl();
+
   // Docker Only
 
   /**

airbyte-config/config-models/src/main/java/io/airbyte/config/EnvConfigs.java

@@ -16,6 +16,7 @@
 import io.airbyte.config.storage.CloudStorageConfigs.GcsConfig;
 import io.airbyte.config.storage.CloudStorageConfigs.MinioConfig;
 import io.airbyte.config.storage.CloudStorageConfigs.S3Config;
+import java.net.URI;
 import java.nio.file.Path;
 import java.util.Arrays;
 import java.util.HashSet;
@@ -148,6 +149,8 @@ public class EnvConfigs implements Configs {
   public static final String METRIC_CLIENT = "METRIC_CLIENT";
   private static final String OTEL_COLLECTOR_ENDPOINT = "OTEL_COLLECTOR_ENDPOINT";
 
+  public static final String REMOTE_CONNECTOR_CATALOG_URL = "REMOTE_CONNECTOR_CATALOG_URL";
+
   // job-type-specific overrides
   public static final String SPEC_JOB_KUBE_NODE_SELECTORS = "SPEC_JOB_KUBE_NODE_SELECTORS";
   public static final String CHECK_JOB_KUBE_NODE_SELECTORS = "CHECK_JOB_KUBE_NODE_SELECTORS";
@@ -323,6 +326,12 @@ public Path getWorkspaceRoot() {
     return getPath(WORKSPACE_ROOT);
   }
 
+  @Override
+  public URI getRemoteConnectorCatalogUrl() {
+    // Default to reuse the job database
+    return URI.create(getEnsureEnv(REMOTE_CONNECTOR_CATALOG_URL));
+  }
+
   // Docker Only
   @Override
   public String getWorkspaceDockerMount() {

airbyte-config/config-models/src/main/resources/types/CombinedConnectorCatalog.yaml

@@ -0,0 +1,18 @@
+---
+"$schema": http://json-schema.org/draft-07/schema#
+"$id": https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/resources/types/CombinedConnectorCatalog.yaml
+title: CombinedConnectorCatalog
+description: Used to provide the connector catalog from a remote source
+type: object
+required:
+  - destinations
+  - sources
+properties:
+  destinations:
+    type: array
+    items:
+      existingJavaType: io.airbyte.config.StandardDestinationDefinition
+  sources:
+    type: array
+    items:
+      existingJavaType: io.airbyte.config.StandardSourceDefinition

airbyte-config/config-models/src/main/resources/types/StandardDestinationDefinition.yaml

@@ -55,3 +55,6 @@ properties:
     format: date
   resourceRequirements:
     "$ref": ActorDefinitionResourceRequirements.yaml
+  protocolVersion:
+    type: string
+    description: the Airbyte Protocol version supported by the connector

airbyte-config/config-models/src/main/resources/types/StandardSourceDefinition.yaml

@@ -62,3 +62,6 @@ properties:
     format: date
   resourceRequirements:
     "$ref": ActorDefinitionResourceRequirements.yaml
+  protocolVersion:
+    type: string
+    description: the Airbyte Protocol version supported by the connector

airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigWriter.java

@@ -50,6 +50,7 @@ static void writeStandardSourceDefinition(final List<StandardSourceDefinition> c
                     : Enums.toEnum(standardSourceDefinition.getSourceType().value(),
                         SourceType.class).orElseThrow())
             .set(Tables.ACTOR_DEFINITION.SPEC, JSONB.valueOf(Jsons.serialize(standardSourceDefinition.getSpec())))
+            .set(Tables.ACTOR_DEFINITION.PROTOCOL_VERSION, standardSourceDefinition.getProtocolVersion())
             .set(Tables.ACTOR_DEFINITION.TOMBSTONE, standardSourceDefinition.getTombstone())
             .set(Tables.ACTOR_DEFINITION.PUBLIC, standardSourceDefinition.getPublic())
             .set(Tables.ACTOR_DEFINITION.CUSTOM, standardSourceDefinition.getCustom())
@@ -79,6 +80,7 @@ static void writeStandardSourceDefinition(final List<StandardSourceDefinition> c
                     : Enums.toEnum(standardSourceDefinition.getSourceType().value(),
                         SourceType.class).orElseThrow())
             .set(Tables.ACTOR_DEFINITION.SPEC, JSONB.valueOf(Jsons.serialize(standardSourceDefinition.getSpec())))
+            .set(Tables.ACTOR_DEFINITION.PROTOCOL_VERSION, standardSourceDefinition.getProtocolVersion())
             .set(Tables.ACTOR_DEFINITION.TOMBSTONE, standardSourceDefinition.getTombstone() != null && standardSourceDefinition.getTombstone())
             .set(Tables.ACTOR_DEFINITION.PUBLIC, standardSourceDefinition.getPublic())
             .set(Tables.ACTOR_DEFINITION.CUSTOM, standardSourceDefinition.getCustom())
@@ -115,6 +117,7 @@ static void writeStandardDestinationDefinition(final List<StandardDestinationDef
             .set(Tables.ACTOR_DEFINITION.ICON, standardDestinationDefinition.getIcon())
             .set(Tables.ACTOR_DEFINITION.ACTOR_TYPE, ActorType.destination)
             .set(Tables.ACTOR_DEFINITION.SPEC, JSONB.valueOf(Jsons.serialize(standardDestinationDefinition.getSpec())))
+            .set(Tables.ACTOR_DEFINITION.PROTOCOL_VERSION, standardDestinationDefinition.getProtocolVersion())
             .set(Tables.ACTOR_DEFINITION.TOMBSTONE, standardDestinationDefinition.getTombstone())
             .set(Tables.ACTOR_DEFINITION.PUBLIC, standardDestinationDefinition.getPublic())
             .set(Tables.ACTOR_DEFINITION.CUSTOM, standardDestinationDefinition.getCustom())
@@ -140,6 +143,7 @@ static void writeStandardDestinationDefinition(final List<StandardDestinationDef
             .set(Tables.ACTOR_DEFINITION.ICON, standardDestinationDefinition.getIcon())
             .set(Tables.ACTOR_DEFINITION.ACTOR_TYPE, ActorType.destination)
             .set(Tables.ACTOR_DEFINITION.SPEC, JSONB.valueOf(Jsons.serialize(standardDestinationDefinition.getSpec())))
+            .set(Tables.ACTOR_DEFINITION.PROTOCOL_VERSION, standardDestinationDefinition.getProtocolVersion())
             .set(Tables.ACTOR_DEFINITION.TOMBSTONE,
                 standardDestinationDefinition.getTombstone() != null && standardDestinationDefinition.getTombstone())
             .set(Tables.ACTOR_DEFINITION.PUBLIC, standardDestinationDefinition.getPublic())

airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/DbConverter.java

@@ -100,6 +100,7 @@ public static StandardSourceDefinition buildStandardSourceDefinition(final Recor
         .withSourceType(record.get(ACTOR_DEFINITION.SOURCE_TYPE) == null ? null
             : Enums.toEnum(record.get(ACTOR_DEFINITION.SOURCE_TYPE, String.class), SourceType.class).orElseThrow())
         .withSpec(Jsons.deserialize(record.get(ACTOR_DEFINITION.SPEC).data(), ConnectorSpecification.class))
+        .withProtocolVersion(record.get(ACTOR_DEFINITION.PROTOCOL_VERSION, String.class))
         .withTombstone(record.get(ACTOR_DEFINITION.TOMBSTONE))
         .withPublic(record.get(ACTOR_DEFINITION.PUBLIC))
         .withCustom(record.get(ACTOR_DEFINITION.CUSTOM))
@@ -121,6 +122,7 @@ public static StandardDestinationDefinition buildStandardDestinationDefinition(f
         .withDocumentationUrl(record.get(ACTOR_DEFINITION.DOCUMENTATION_URL))
         .withName(record.get(ACTOR_DEFINITION.NAME))
         .withSpec(Jsons.deserialize(record.get(ACTOR_DEFINITION.SPEC).data(), ConnectorSpecification.class))
+        .withProtocolVersion(record.get(ACTOR_DEFINITION.PROTOCOL_VERSION, String.class))
         .withTombstone(record.get(ACTOR_DEFINITION.TOMBSTONE))
         .withPublic(record.get(ACTOR_DEFINITION.PUBLIC))
         .withCustom(record.get(ACTOR_DEFINITION.CUSTOM))

airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/MockData.java

@@ -178,6 +178,7 @@ public static StandardSourceDefinition publicSourceDefinition() {
         .withDocumentationUrl("documentation-url-1")
         .withIcon("icon-1")
         .withSpec(connectorSpecification())
+        .withProtocolVersion("0.2.1")
         .withTombstone(false)
         .withPublic(true)
         .withCustom(false)
@@ -206,6 +207,7 @@ public static StandardSourceDefinition grantableSourceDefinition2() {
         .withDockerImageTag("tag-3")
         .withDockerRepository("repository-3")
         .withDocumentationUrl("documentation-url-3")
+        .withProtocolVersion("0.2.2")
         .withIcon("icon-3")
         .withTombstone(false)
         .withPublic(false)
@@ -220,6 +222,7 @@ public static StandardSourceDefinition customSourceDefinition() {
         .withDockerImageTag("tag-4")
         .withDockerRepository("repository-4")
         .withDocumentationUrl("documentation-url-4")
+        .withProtocolVersion("0.2.4")
         .withIcon("icon-4")
         .withTombstone(false)
         .withPublic(false)
@@ -256,6 +259,7 @@ public static StandardDestinationDefinition publicDestinationDefinition() {
         .withDocumentationUrl("documentation-url-3")
         .withIcon("icon-3")
         .withSpec(connectorSpecification())
+        .withProtocolVersion("0.3.1")
         .withTombstone(false)
         .withPublic(true)
         .withCustom(false)
@@ -299,6 +303,7 @@ public static StandardDestinationDefinition cusstomDestinationDefinition() {
         .withDocumentationUrl("documentation-url-44")
         .withIcon("icon-4")
         .withSpec(connectorSpecification())
+        .withProtocolVersion("0.3.2")
         .withTombstone(false)
         .withPublic(false)
         .withCustom(true);

airbyte-config/init/build.gradle

@@ -10,6 +10,9 @@ dependencies {
     implementation project(':airbyte-protocol:protocol-models')
     implementation project(':airbyte-commons-docker')
     implementation project(':airbyte-json-validation')
+
+    testImplementation project(':airbyte-test-utils')
+    testImplementation 'com.squareup.okhttp3:mockwebserver:4.9.1'
 }
 
 task copyScripts(type: Copy) {