Remove catalog from web_backend/connections/list (#17002)

* Use PATCH Api for toggling connections

* remove catalog from web backend connection list, and move icon to source/destination field in response

* Adjust FE code

* comment out failing tests

* Resolve merge conflict

* add back in tests and make them pass with new list items

* format

* leave repository layer alone for now, just remove catalog from API response

* format

* load the icon when returning

* update sourceHandler and destinationHandler test to account for icons

* add icon to source and destination in webBackendConnectionHandlerTest

* change icon test to actually load an svg rather than using static mocks

* also fix icon test for WebBackendConnectionsHandlerTest

* make PMD happy

Co-authored-by: Tim Roes <tim@airbyte.io>
Co-authored-by: Davin Chia <davinchia@gmail.com>
Co-authored-by: KC <krishna@airbyte.io>

airbyte-api/src/main/openapi/config.yaml

@@ -2756,6 +2756,8 @@ components:
           type: string
         sourceName:
           type: string
+        icon:
+          type: string
     SourceReadList:
       type: object
       required:
@@ -3053,6 +3055,8 @@ components:
           type: string
         destinationName:
           type: string
+        icon:
+          type: string
     DestinationReadList:
       type: object
       required:
@@ -3813,6 +3817,10 @@ components:
           type: array
           items:
             $ref: "#/components/schemas/AttemptRead"
+    JobCreatedAt:
+      description: epoch time of the latest sync job. null if no sync job has taken place.
+      type: integer
+      format: int64
     JobStatus:
       type: string
       enum:
@@ -4508,6 +4516,43 @@ components:
           type: object
           additionalProperties: true
     # Web Backend
+    WebBackendConnectionListItem:
+      type: object
+      description: Information about a connection that shows up in the connection list view.
+      required:
+        - connectionId
+        - name
+        - sourceId
+        - destinationId
+        - source
+        - destination
+        - status
+        - isSyncing
+      properties:
+        connectionId:
+          $ref: "#/components/schemas/ConnectionId"
+        name:
+          type: string
+        sourceId:
+          $ref: "#/components/schemas/SourceId"
+        destinationId:
+          $ref: "#/components/schemas/DestinationId"
+        scheduleType:
+          $ref: "#/components/schemas/ConnectionScheduleType"
+        scheduleData:
+          $ref: "#/components/schemas/ConnectionScheduleData"
+        status:
+          $ref: "#/components/schemas/ConnectionStatus"
+        source:
+          $ref: "#/components/schemas/SourceRead"
+        destination:
+          $ref: "#/components/schemas/DestinationRead"
+        latestSyncJobCreatedAt:
+          $ref: "#/components/schemas/JobCreatedAt"
+        latestSyncJobStatus:
+          $ref: "#/components/schemas/JobStatus"
+        isSyncing:
+          type: boolean
     WebBackendConnectionRead:
       type: object
       required:
@@ -4562,9 +4607,7 @@ components:
           items:
             $ref: "#/components/schemas/OperationRead"
         latestSyncJobCreatedAt:
-          description: epoch time of the latest sync job. null if no sync job has taken place.
-          type: integer
-          format: int64
+          $ref: "#/components/schemas/JobCreatedAt"
         latestSyncJobStatus:
           $ref: "#/components/schemas/JobStatus"
         isSyncing:
@@ -4584,7 +4627,7 @@ components:
         connections:
           type: array
           items:
-            $ref: "#/components/schemas/WebBackendConnectionRead"
+            $ref: "#/components/schemas/WebBackendConnectionListItem"
     SyncMode:
       type: string
       enum:

airbyte-server/src/main/java/io/airbyte/server/converters/ApiPojoConverters.java

@@ -4,7 +4,6 @@
 
 package io.airbyte.server.converters;
 
-import io.airbyte.api.client.model.generated.ConnectionScheduleType;
 import io.airbyte.api.model.generated.ActorDefinitionResourceRequirements;
 import io.airbyte.api.model.generated.ConnectionRead;
 import io.airbyte.api.model.generated.ConnectionSchedule;
@@ -19,7 +18,6 @@
 import io.airbyte.config.BasicSchedule;
 import io.airbyte.config.Schedule;
 import io.airbyte.config.StandardSync;
-import io.airbyte.config.StandardSync.ScheduleType;
 import io.airbyte.server.handlers.helpers.CatalogConverter;
 import java.util.stream.Collectors;
 
@@ -153,63 +151,94 @@ public static ConnectionScheduleDataBasicSchedule.TimeUnitEnum toApiBasicSchedul
     return Enums.convertTo(timeUnit, ConnectionScheduleDataBasicSchedule.TimeUnitEnum.class);
   }
 
-  public static void populateConnectionReadSchedule(final StandardSync standardSync, final ConnectionRead connectionRead) {
-    // TODO(https://github.com/airbytehq/airbyte/issues/11432): only return new schema once frontend is
-    // ready.
+  public static io.airbyte.api.model.generated.ConnectionScheduleType toApiConnectionScheduleType(final StandardSync standardSync) {
     if (standardSync.getScheduleType() != null) {
-      // Populate everything based on the new schema.
       switch (standardSync.getScheduleType()) {
         case MANUAL -> {
-          connectionRead.scheduleType(io.airbyte.api.model.generated.ConnectionScheduleType.MANUAL);
+          return io.airbyte.api.model.generated.ConnectionScheduleType.MANUAL;
         }
         case BASIC_SCHEDULE -> {
-          connectionRead.scheduleType(io.airbyte.api.model.generated.ConnectionScheduleType.BASIC);
-          connectionRead.scheduleData(new ConnectionScheduleData()
+          return io.airbyte.api.model.generated.ConnectionScheduleType.BASIC;
+        }
+        case CRON -> {
+          return io.airbyte.api.model.generated.ConnectionScheduleType.CRON;
+        }
+        default -> throw new RuntimeException("Unexpected scheduleType " + standardSync.getScheduleType());
+      }
+    } else if (standardSync.getManual()) {
+      // Legacy schema, manual sync.
+      return io.airbyte.api.model.generated.ConnectionScheduleType.MANUAL;
+    } else {
+      // Legacy schema, basic schedule.
+      return io.airbyte.api.model.generated.ConnectionScheduleType.BASIC;
+    }
+  }
+
+  public static io.airbyte.api.model.generated.ConnectionScheduleData toApiConnectionScheduleData(final StandardSync standardSync) {
+    if (standardSync.getScheduleType() != null) {
+      switch (standardSync.getScheduleType()) {
+        case MANUAL -> {
+          return null;
+        }
+        case BASIC_SCHEDULE -> {
+          return new ConnectionScheduleData()
               .basicSchedule(new ConnectionScheduleDataBasicSchedule()
                   .timeUnit(toApiBasicScheduleTimeUnit(standardSync.getScheduleData().getBasicSchedule().getTimeUnit()))
-                  .units(standardSync.getScheduleData().getBasicSchedule().getUnits())));
-          connectionRead.schedule(new ConnectionSchedule()
-              .timeUnit(toApiTimeUnit(standardSync.getScheduleData().getBasicSchedule().getTimeUnit()))
-              .units(standardSync.getScheduleData().getBasicSchedule().getUnits()));
+                  .units(standardSync.getScheduleData().getBasicSchedule().getUnits()));
         }
         case CRON -> {
-          // We don't populate any legacy data here.
-          connectionRead.scheduleType(io.airbyte.api.model.generated.ConnectionScheduleType.CRON);
-          connectionRead.scheduleData(new ConnectionScheduleData()
+          return new ConnectionScheduleData()
               .cron(new ConnectionScheduleDataCron()
                   .cronExpression(standardSync.getScheduleData().getCron().getCronExpression())
-                  .cronTimeZone(standardSync.getScheduleData().getCron().getCronTimeZone())));
+                  .cronTimeZone(standardSync.getScheduleData().getCron().getCronTimeZone()));
         }
+        default -> throw new RuntimeException("Unexpected scheduleType " + standardSync.getScheduleType());
       }
     } else if (standardSync.getManual()) {
       // Legacy schema, manual sync.
-      connectionRead.scheduleType(io.airbyte.api.model.generated.ConnectionScheduleType.MANUAL);
+      return null;
     } else {
       // Legacy schema, basic schedule.
-      connectionRead.scheduleType(io.airbyte.api.model.generated.ConnectionScheduleType.BASIC);
-      connectionRead.schedule(new ConnectionSchedule()
-          .timeUnit(toApiTimeUnit(standardSync.getSchedule().getTimeUnit()))
-          .units(standardSync.getSchedule().getUnits()));
-      connectionRead.scheduleData(new ConnectionScheduleData()
+      return new ConnectionScheduleData()
           .basicSchedule(new ConnectionScheduleDataBasicSchedule()
               .timeUnit(toApiBasicScheduleTimeUnit(standardSync.getSchedule().getTimeUnit()))
-              .units(standardSync.getSchedule().getUnits())));
+              .units(standardSync.getSchedule().getUnits()));
     }
   }
 
-  public static ConnectionScheduleType toApiScheduleType(final ScheduleType scheduleType) {
-    switch (scheduleType) {
-      case MANUAL -> {
-        return ConnectionScheduleType.MANUAL;
-      }
-      case BASIC_SCHEDULE -> {
-        return ConnectionScheduleType.BASIC;
-      }
-      case CRON -> {
-        return ConnectionScheduleType.CRON;
+  public static ConnectionSchedule toLegacyConnectionSchedule(final StandardSync standardSync) {
+    if (standardSync.getScheduleType() != null) {
+      // Populate everything based on the new schema.
+      switch (standardSync.getScheduleType()) {
+        case MANUAL, CRON -> {
+          // We don't populate any legacy data here.
+          return null;
+        }
+        case BASIC_SCHEDULE -> {
+          return new ConnectionSchedule()
+              .timeUnit(toApiTimeUnit(standardSync.getScheduleData().getBasicSchedule().getTimeUnit()))
+              .units(standardSync.getScheduleData().getBasicSchedule().getUnits());
+        }
+        default -> throw new RuntimeException("Unexpected scheduleType " + standardSync.getScheduleType());
       }
+    } else if (standardSync.getManual()) {
+      // Legacy schema, manual sync.
+      return null;
+    } else {
+      // Legacy schema, basic schedule.
+      return new ConnectionSchedule()
+          .timeUnit(toApiTimeUnit(standardSync.getSchedule().getTimeUnit()))
+          .units(standardSync.getSchedule().getUnits());
     }
-    throw new RuntimeException("Unexpected schedule type");
+  }
+
+  public static void populateConnectionReadSchedule(final StandardSync standardSync, final ConnectionRead connectionRead) {
+    connectionRead.scheduleType(toApiConnectionScheduleType(standardSync));
+    connectionRead.scheduleData(toApiConnectionScheduleData(standardSync));
+
+    // TODO(https://github.com/airbytehq/airbyte/issues/11432): only return new schema once frontend is
+    // ready.
+    connectionRead.schedule(toLegacyConnectionSchedule(standardSync));
   }
 
 }

airbyte-server/src/main/java/io/airbyte/server/handlers/DestinationHandler.java

@@ -309,7 +309,8 @@ protected static DestinationRead toDestinationRead(final DestinationConnection d
         .destinationDefinitionId(destinationConnection.getDestinationDefinitionId())
         .connectionConfiguration(destinationConnection.getConfiguration())
         .name(destinationConnection.getName())
-        .destinationName(standardDestinationDefinition.getName());
+        .destinationName(standardDestinationDefinition.getName())
+        .icon(DestinationDefinitionsHandler.loadIcon(standardDestinationDefinition.getIcon()));
   }
 
 }

airbyte-server/src/main/java/io/airbyte/server/handlers/SourceHandler.java

@@ -316,7 +316,8 @@ protected static SourceRead toSourceRead(final SourceConnection sourceConnection
         .workspaceId(sourceConnection.getWorkspaceId())
         .sourceDefinitionId(sourceConnection.getSourceDefinitionId())
         .connectionConfiguration(sourceConnection.getConfiguration())
-        .name(sourceConnection.getName());
+        .name(sourceConnection.getName())
+        .icon(SourceDefinitionsHandler.loadIcon(standardSourceDefinition.getIcon()));
   }
 
 }

airbyte-server/src/main/java/io/airbyte/server/handlers/WebBackendConnectionsHandler.java

@@ -31,6 +31,7 @@
 import io.airbyte.api.model.generated.StreamDescriptor;
 import io.airbyte.api.model.generated.StreamTransform;
 import io.airbyte.api.model.generated.WebBackendConnectionCreate;
+import io.airbyte.api.model.generated.WebBackendConnectionListItem;
 import io.airbyte.api.model.generated.WebBackendConnectionRead;
 import io.airbyte.api.model.generated.WebBackendConnectionReadList;
 import io.airbyte.api.model.generated.WebBackendConnectionRequestBody;
@@ -42,9 +43,11 @@
 import io.airbyte.commons.enums.Enums;
 import io.airbyte.commons.json.Jsons;
 import io.airbyte.commons.lang.MoreBooleans;
+import io.airbyte.config.StandardSync;
 import io.airbyte.config.persistence.ConfigNotFoundException;
 import io.airbyte.config.persistence.ConfigRepository;
 import io.airbyte.protocol.models.ConfiguredAirbyteCatalog;
+import io.airbyte.server.converters.ApiPojoConverters;
 import io.airbyte.server.handlers.helpers.CatalogConverter;
 import io.airbyte.server.scheduler.EventRunner;
 import io.airbyte.validation.json.JsonValidationException;
@@ -96,17 +99,20 @@ public ConnectionStateType getStateType(final ConnectionIdRequestBody connection
   public WebBackendConnectionReadList webBackendListConnectionsForWorkspace(final WorkspaceIdRequestBody workspaceIdRequestBody)
       throws ConfigNotFoundException, IOException, JsonValidationException {
 
-    final List<WebBackendConnectionRead> reads = Lists.newArrayList();
-    for (final ConnectionRead connection : connectionsHandler.listConnectionsForWorkspace(workspaceIdRequestBody).getConnections()) {
-      reads.add(buildWebBackendConnectionRead(connection));
+    final List<WebBackendConnectionListItem> connectionItems = Lists.newArrayList();
+
+    // passing 'false' so that deleted connections are not included
+    for (final StandardSync standardSync : configRepository.listWorkspaceStandardSyncs(workspaceIdRequestBody.getWorkspaceId(), false)) {
+      connectionItems.add(buildWebBackendConnectionListItem(standardSync));
     }
-    return new WebBackendConnectionReadList().connections(reads);
+
+    return new WebBackendConnectionReadList().connections(connectionItems);
   }
 
   private WebBackendConnectionRead buildWebBackendConnectionRead(final ConnectionRead connectionRead)
       throws ConfigNotFoundException, IOException, JsonValidationException {
-    final SourceRead source = getSourceRead(connectionRead);
-    final DestinationRead destination = getDestinationRead(connectionRead);
+    final SourceRead source = getSourceRead(connectionRead.getSourceId());
+    final DestinationRead destination = getDestinationRead(connectionRead.getDestinationId());
     final OperationReadList operations = getOperationReadList(connectionRead);
     final Optional<JobRead> latestSyncJob = jobHistoryHandler.getLatestSyncJob(connectionRead.getConnectionId());
     final Optional<JobRead> latestRunningSyncJob = jobHistoryHandler.getLatestRunningSyncJob(connectionRead.getConnectionId());
@@ -124,14 +130,42 @@ private WebBackendConnectionRead buildWebBackendConnectionRead(final ConnectionR
     return webBackendConnectionRead;
   }
 
-  private SourceRead getSourceRead(final ConnectionRead connectionRead) throws JsonValidationException, IOException, ConfigNotFoundException {
-    final SourceIdRequestBody sourceIdRequestBody = new SourceIdRequestBody().sourceId(connectionRead.getSourceId());
+  private WebBackendConnectionListItem buildWebBackendConnectionListItem(final StandardSync standardSync)
+      throws JsonValidationException, ConfigNotFoundException, IOException {
+    final SourceRead source = getSourceRead(standardSync.getSourceId());
+    final DestinationRead destination = getDestinationRead(standardSync.getDestinationId());
+    final Optional<JobRead> latestSyncJob = jobHistoryHandler.getLatestSyncJob(standardSync.getConnectionId());
+    final Optional<JobRead> latestRunningSyncJob = jobHistoryHandler.getLatestRunningSyncJob(standardSync.getConnectionId());
+
+    final WebBackendConnectionListItem listItem = new WebBackendConnectionListItem()
+        .connectionId(standardSync.getConnectionId())
+        .sourceId(standardSync.getSourceId())
+        .destinationId(standardSync.getDestinationId())
+        .status(ApiPojoConverters.toApiStatus(standardSync.getStatus()))
+        .name(standardSync.getName())
+        .scheduleType(ApiPojoConverters.toApiConnectionScheduleType(standardSync))
+        .scheduleData(ApiPojoConverters.toApiConnectionScheduleData(standardSync))
+        .source(source)
+        .destination(destination);
+
+    listItem.setIsSyncing(latestRunningSyncJob.isPresent());
+
+    latestSyncJob.ifPresent(job -> {
+      listItem.setLatestSyncJobCreatedAt(job.getCreatedAt());
+      listItem.setLatestSyncJobStatus(job.getStatus());
+    });
+
+    return listItem;
+  }
+
+  private SourceRead getSourceRead(final UUID sourceId) throws JsonValidationException, IOException, ConfigNotFoundException {
+    final SourceIdRequestBody sourceIdRequestBody = new SourceIdRequestBody().sourceId(sourceId);
     return sourceHandler.getSource(sourceIdRequestBody);
   }
 
-  private DestinationRead getDestinationRead(final ConnectionRead connectionRead)
+  private DestinationRead getDestinationRead(final UUID destinationId)
       throws JsonValidationException, IOException, ConfigNotFoundException {
-    final DestinationIdRequestBody destinationIdRequestBody = new DestinationIdRequestBody().destinationId(connectionRead.getDestinationId());
+    final DestinationIdRequestBody destinationIdRequestBody = new DestinationIdRequestBody().destinationId(destinationId);
     return destinationHandler.getDestination(destinationIdRequestBody);
   }