Add /v1/web_backend/check_updates

airbyte-api/src/main/openapi/config.yaml

@@ -1801,6 +1801,19 @@ paths:
           $ref: "#/components/responses/ExceptionResponse"
         "404":
           $ref: "#/components/responses/NotFoundResponse"
+  /v1/web_backend/check_updates:
+    post:
+      tags:
+        - web_backend
+      summary: Returns a summary of source and destination definitions that could be updated.
+      operationId: webBackendCheckUpdates
+      responses:
+        "200":
+          description: Successful operation
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/WebBackendCheckUpdatesRead"
   /v1/web_backend/connections/list:
     post:
       tags:
@@ -4633,6 +4646,17 @@ components:
           type: object
           additionalProperties: true
     # Web Backend
+    WebBackendCheckUpdatesRead:
+      type: object
+      description: Summary of source and destination definitions that could be updated
+      required:
+        - destinationDefinitions
+        - sourceDefinitions
+      properties:
+        destinationDefinitions:
+          type: integer
+        sourceDefinitions:
+          type: integer
     WebBackendConnectionListItem:
       type: object
       description: Information about a connection that shows up in the connection list view.

airbyte-server/src/main/java/io/airbyte/server/ServerApp.java

@@ -64,12 +64,14 @@
 import io.airbyte.server.handlers.SourceDefinitionsHandler;
 import io.airbyte.server.handlers.SourceHandler;
 import io.airbyte.server.handlers.StateHandler;
+import io.airbyte.server.handlers.WebBackendCheckUpdatesHandler;
 import io.airbyte.server.handlers.WebBackendConnectionsHandler;
 import io.airbyte.server.handlers.WebBackendGeographiesHandler;
 import io.airbyte.server.handlers.WorkspacesHandler;
 import io.airbyte.server.scheduler.DefaultSynchronousSchedulerClient;
 import io.airbyte.server.scheduler.EventRunner;
 import io.airbyte.server.scheduler.TemporalEventRunner;
+import io.airbyte.server.services.AirbyteGithubStore;
 import io.airbyte.validation.json.JsonSchemaValidator;
 import io.airbyte.workers.helper.ConnectionHelper;
 import io.airbyte.workers.normalization.NormalizationRunnerFactory;
@@ -350,6 +352,9 @@ public static ServerRunnable getServer(final ServerFactory apiFactory,
 
     final WebBackendGeographiesHandler webBackendGeographiesHandler = new WebBackendGeographiesHandler();
 
+    final WebBackendCheckUpdatesHandler webBackendCheckUpdatesHandler =
+        new WebBackendCheckUpdatesHandler(configRepository, AirbyteGithubStore.production());
+
     LOGGER.info("Starting server...");
 
     return apiFactory.create(
@@ -386,7 +391,8 @@ public static ServerRunnable getServer(final ServerFactory apiFactory,
         stateHandler,
         workspacesHandler,
         webBackendConnectionsHandler,
-        webBackendGeographiesHandler);
+        webBackendGeographiesHandler,
+        webBackendCheckUpdatesHandler);
   }
 
   public static void main(final String[] args) {

airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java

@@ -91,6 +91,7 @@
 import io.airbyte.server.handlers.SourceDefinitionsHandler;
 import io.airbyte.server.handlers.SourceHandler;
 import io.airbyte.server.handlers.StateHandler;
+import io.airbyte.server.handlers.WebBackendCheckUpdatesHandler;
 import io.airbyte.server.handlers.WebBackendConnectionsHandler;
 import io.airbyte.server.handlers.WebBackendGeographiesHandler;
 import io.airbyte.server.handlers.WorkspacesHandler;
@@ -138,7 +139,8 @@ ServerRunnable create(final SynchronousSchedulerClient synchronousSchedulerClien
                         final StateHandler stateHandler,
                         final WorkspacesHandler workspacesHandler,
                         final WebBackendConnectionsHandler webBackendConnectionsHandler,
-                        final WebBackendGeographiesHandler webBackendGeographiesHandler);
+                        final WebBackendGeographiesHandler webBackendGeographiesHandler,
+                        final WebBackendCheckUpdatesHandler webBackendCheckUpdatesHandler);
 
   class Api implements ServerFactory {
 
@@ -176,7 +178,8 @@ public ServerRunnable create(final SynchronousSchedulerClient synchronousSchedul
                                  final StateHandler stateHandler,
                                  final WorkspacesHandler workspacesHandler,
                                  final WebBackendConnectionsHandler webBackendConnectionsHandler,
-                                 final WebBackendGeographiesHandler webBackendGeographiesHandler) {
+                                 final WebBackendGeographiesHandler webBackendGeographiesHandler,
+                                 final WebBackendCheckUpdatesHandler webBackendCheckUpdatesHandler) {
       final Map<String, String> mdc = MDC.getCopyOfContextMap();
 
       AttemptApiFactory.setValues(attemptHandler, mdc);
@@ -221,7 +224,7 @@ public ServerRunnable create(final SynchronousSchedulerClient synchronousSchedul
 
       StateApiFactory.setValues(stateHandler);
 
-      WebBackendApiFactory.setValues(webBackendConnectionsHandler, webBackendGeographiesHandler);
+      WebBackendApiFactory.setValues(webBackendConnectionsHandler, webBackendGeographiesHandler, webBackendCheckUpdatesHandler);
 
       WorkspaceApiFactory.setValues(workspacesHandler);
 

airbyte-server/src/main/java/io/airbyte/server/apis/WebBackendApiController.java

@@ -7,6 +7,7 @@
 import io.airbyte.api.generated.WebBackendApi;
 import io.airbyte.api.model.generated.ConnectionIdRequestBody;
 import io.airbyte.api.model.generated.ConnectionStateType;
+import io.airbyte.api.model.generated.WebBackendCheckUpdatesRead;
 import io.airbyte.api.model.generated.WebBackendConnectionCreate;
 import io.airbyte.api.model.generated.WebBackendConnectionRead;
 import io.airbyte.api.model.generated.WebBackendConnectionReadList;
@@ -16,6 +17,7 @@
 import io.airbyte.api.model.generated.WebBackendWorkspaceState;
 import io.airbyte.api.model.generated.WebBackendWorkspaceStateResult;
 import io.airbyte.api.model.generated.WorkspaceIdRequestBody;
+import io.airbyte.server.handlers.WebBackendCheckUpdatesHandler;
 import io.airbyte.server.handlers.WebBackendConnectionsHandler;
 import io.airbyte.server.handlers.WebBackendGeographiesHandler;
 import javax.ws.rs.Path;
@@ -27,12 +29,18 @@ public class WebBackendApiController implements WebBackendApi {
 
   private final WebBackendConnectionsHandler webBackendConnectionsHandler;
   private final WebBackendGeographiesHandler webBackendGeographiesHandler;
+  private final WebBackendCheckUpdatesHandler webBackendCheckUpdatesHandler;
 
   @Override
   public ConnectionStateType getStateType(final ConnectionIdRequestBody connectionIdRequestBody) {
     return ApiHelper.execute(() -> webBackendConnectionsHandler.getStateType(connectionIdRequestBody));
   }
 
+  @Override
+  public WebBackendCheckUpdatesRead webBackendCheckUpdates() {
+    return ApiHelper.execute(webBackendCheckUpdatesHandler::checkUpdates);
+  }
+
   @Override
   public WebBackendConnectionRead webBackendCreateConnection(final WebBackendConnectionCreate webBackendConnectionCreate) {
     return ApiHelper.execute(() -> webBackendConnectionsHandler.webBackendCreateConnection(webBackendConnectionCreate));

airbyte-server/src/main/java/io/airbyte/server/apis/factories/WebBackendApiFactory.java

@@ -5,6 +5,7 @@
 package io.airbyte.server.apis.factories;
 
 import io.airbyte.server.apis.WebBackendApiController;
+import io.airbyte.server.handlers.WebBackendCheckUpdatesHandler;
 import io.airbyte.server.handlers.WebBackendConnectionsHandler;
 import io.airbyte.server.handlers.WebBackendGeographiesHandler;
 import org.glassfish.hk2.api.Factory;
@@ -13,16 +14,20 @@ public class WebBackendApiFactory implements Factory<WebBackendApiController> {
 
   private static WebBackendConnectionsHandler webBackendConnectionsHandler;
   private static WebBackendGeographiesHandler webBackendGeographiesHandler;
+  private static WebBackendCheckUpdatesHandler webBackendCheckUpdatesHandler;
 
   public static void setValues(final WebBackendConnectionsHandler webBackendConnectionsHandler,
-                               final WebBackendGeographiesHandler webBackendGeographiesHandler) {
+                               final WebBackendGeographiesHandler webBackendGeographiesHandler,
+                               final WebBackendCheckUpdatesHandler webBackendCheckUpdatesHandler) {
     WebBackendApiFactory.webBackendConnectionsHandler = webBackendConnectionsHandler;
     WebBackendApiFactory.webBackendGeographiesHandler = webBackendGeographiesHandler;
+    WebBackendApiFactory.webBackendCheckUpdatesHandler = webBackendCheckUpdatesHandler;
   }
 
   @Override
   public WebBackendApiController provide() {
-    return new WebBackendApiController(WebBackendApiFactory.webBackendConnectionsHandler, WebBackendApiFactory.webBackendGeographiesHandler);
+    return new WebBackendApiController(WebBackendApiFactory.webBackendConnectionsHandler, WebBackendApiFactory.webBackendGeographiesHandler,
+        WebBackendApiFactory.webBackendCheckUpdatesHandler);
   }
 
   @Override

airbyte-server/src/main/java/io/airbyte/server/handlers/WebBackendCheckUpdatesHandler.java

@@ -0,0 +1,103 @@
+/*
+ * Copyright (c) 2022 Airbyte, Inc., all rights reserved.
+ */
+
+package io.airbyte.server.handlers;
+
+import io.airbyte.api.model.generated.WebBackendCheckUpdatesRead;
+import io.airbyte.config.StandardDestinationDefinition;
+import io.airbyte.config.StandardSourceDefinition;
+import io.airbyte.config.persistence.ConfigRepository;
+import io.airbyte.server.services.AirbyteGithubStore;
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.UUID;
+import java.util.stream.Collectors;
+import lombok.AllArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+@AllArgsConstructor
+@Slf4j
+public class WebBackendCheckUpdatesHandler {
+
+  private static final int NO_CHANGES_FOUND = 0;
+
+  final ConfigRepository configRepository;
+  final AirbyteGithubStore githubStore;
+
+  public WebBackendCheckUpdatesRead checkUpdates() {
+
+    final int destinationDiffCount = getDestinationDiffCount();
+    final int sourceDiffCount = getSourceDiffCount();
+
+    return new WebBackendCheckUpdatesRead()
+        .destinationDefinitions(destinationDiffCount)
+        .sourceDefinitions(sourceDiffCount);
+  }
+
+  private int getDestinationDiffCount() {
+    final List<Entry<UUID, String>> currentActorDefToDockerImageTag;
+    final Map<UUID, String> newActorDefToDockerImageTag;
+
+    try {
+      currentActorDefToDockerImageTag = configRepository.listStandardDestinationDefinitions(false)
+          .stream()
+          .map(def -> Map.entry(def.getDestinationDefinitionId(), def.getDockerImageTag()))
+          .toList();
+    } catch (final IOException e) {
+      log.error("Failed to get current list of standard destination definitions", e);
+      return NO_CHANGES_FOUND;
+    }
+
+    try {
+      newActorDefToDockerImageTag = githubStore.getLatestDestinations()
+          .stream()
+          .collect(Collectors.toMap(StandardDestinationDefinition::getDestinationDefinitionId, StandardDestinationDefinition::getDockerImageTag));
+    } catch (final InterruptedException e) {
+      log.error("Failed to get latest list of standard destination definitions", e);
+      return NO_CHANGES_FOUND;
+    }
+
+    return getDiffCount(currentActorDefToDockerImageTag, newActorDefToDockerImageTag);
+  }
+
+  private int getSourceDiffCount() {
+    final List<Entry<UUID, String>> currentActorDefToDockerImageTag;
+    final Map<UUID, String> newActorDefToDockerImageTag;
+
+    try {
+      currentActorDefToDockerImageTag = configRepository.listStandardSourceDefinitions(false)
+          .stream()
+          .map(def -> Map.entry(def.getSourceDefinitionId(), def.getDockerImageTag()))
+          .toList();
+    } catch (final IOException e) {
+      log.error("Failed to get current list of standard source definitions", e);
+      return NO_CHANGES_FOUND;
+    }
+
+    try {
+      newActorDefToDockerImageTag = githubStore.getLatestSources()
+          .stream()
+          .collect(Collectors.toMap(StandardSourceDefinition::getSourceDefinitionId, StandardSourceDefinition::getDockerImageTag));
+    } catch (final InterruptedException e) {
+      log.error("Failed to get latest list of standard source definitions", e);
+      return NO_CHANGES_FOUND;
+    }
+
+    return getDiffCount(currentActorDefToDockerImageTag, newActorDefToDockerImageTag);
+  }
+
+  private int getDiffCount(final List<Entry<UUID, String>> initialSet, final Map<UUID, String> newSet) {
+    int diffCount = 0;
+    for (final Entry<UUID, String> kvp : initialSet) {
+      final String newDockerImageTag = newSet.get(kvp.getKey());
+      if (newDockerImageTag != null && !kvp.getValue().equals(newDockerImageTag)) {
+        ++diffCount;
+      }
+    }
+    return diffCount;
+  }
+
+}