Make listing all connections for workspace faster. (#17004)

Instead of always listing all the connections in the workspace and filtering deleted connections in memory, we want to do this in the database.
airbytehq · Sep 23, 2022 · 4fe6a8b · 4fe6a8b
1 parent c1981e1
commit 4fe6a8b
Show file tree

Hide file tree

Showing 5 changed files with 20 additions and 13 deletions.
diff --git a/...nfig/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java b/...nfig/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java
@@ -15,6 +15,7 @@
 import static io.airbyte.db.instance.configs.jooq.generated.Tables.OPERATION;
 import static io.airbyte.db.instance.configs.jooq.generated.Tables.WORKSPACE;
 import static org.jooq.impl.DSL.asterisk;
+import static org.jooq.impl.DSL.noCondition;
 
 import com.fasterxml.jackson.databind.JsonNode;
 import com.google.common.base.Charsets;
@@ -622,11 +623,13 @@ public List<StandardSync> listStandardSyncsUsingOperation(final UUID operationId
     return getStandardSyncsFromResult(result);
   }
 
-  public List<StandardSync> listWorkspaceStandardSyncs(final UUID workspaceId) throws IOException {
+  public List<StandardSync> listWorkspaceStandardSyncs(final UUID workspaceId, final boolean includeDeleted) throws IOException {
     final Result<Record> result = database.query(ctx -> ctx.select(CONNECTION.asterisk())
         .from(CONNECTION)
         .join(ACTOR).on(CONNECTION.SOURCE_ID.eq(ACTOR.ID))
-        .where(ACTOR.WORKSPACE_ID.eq(workspaceId))).fetch();
+        .where(ACTOR.WORKSPACE_ID.eq(workspaceId)
+            .and(includeDeleted ? noCondition() : CONNECTION.STATUS.notEqual(StatusType.deprecated))))
+        .fetch();
     return getStandardSyncsFromResult(result);
   }
 

diff --git a/...istence/src/test/java/io/airbyte/config/persistence/ConfigRepositoryE2EReadWriteTest.java b/...istence/src/test/java/io/airbyte/config/persistence/ConfigRepositoryE2EReadWriteTest.java
@@ -139,7 +139,7 @@ public static void dbDown() {
   void testWorkspaceCountConnections() throws IOException {
 
     final UUID workspaceId = MockData.standardWorkspaces().get(0).getWorkspaceId();
-    assertEquals(4, configRepository.countConnectionsForWorkspace(workspaceId));
+    assertEquals(3, configRepository.countConnectionsForWorkspace(workspaceId));
     assertEquals(2, configRepository.countDestinationsForWorkspace(workspaceId));
     assertEquals(2, configRepository.countSourcesForWorkspace(workspaceId));
   }
@@ -199,12 +199,19 @@ void testSimpleInsertActorCatalog() throws IOException, JsonValidationException,
   }
 
   @Test
-  void testListWorkspaceStandardSync() throws IOException {
+  void testListWorkspaceStandardSyncAll() throws IOException {
 
-    final List<StandardSync> syncs = configRepository.listWorkspaceStandardSyncs(MockData.standardWorkspaces().get(0).getWorkspaceId());
+    final List<StandardSync> syncs = configRepository.listWorkspaceStandardSyncs(MockData.standardWorkspaces().get(0).getWorkspaceId(), true);
     assertThat(MockData.standardSyncs().subList(0, 4)).hasSameElementsAs(syncs);
   }
 
+  @Test
+  void testListWorkspaceStandardSyncExcludeDeleted() throws IOException {
+
+    final List<StandardSync> syncs = configRepository.listWorkspaceStandardSyncs(MockData.standardWorkspaces().get(0).getWorkspaceId(), false);
+    assertThat(MockData.standardSyncs().subList(0, 3)).hasSameElementsAs(syncs);
+  }
+
   @Test
   void testGetWorkspaceBySlug()
       throws IOException {

diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/MockData.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/MockData.java
@@ -492,7 +492,7 @@ public static List<StandardSync> standardSyncs() {
         .withNamespaceFormat("")
         .withPrefix("")
         .withResourceRequirements(resourceRequirements)
-        .withStatus(Status.INACTIVE)
+        .withStatus(Status.DEPRECATED)
         .withSchedule(schedule);
 
     final StandardSync standardSync5 = new StandardSync()

diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/ConnectionsHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/ConnectionsHandler.java
@@ -363,12 +363,7 @@ public ConnectionReadList listConnectionsForWorkspace(final WorkspaceIdRequestBo
       throws JsonValidationException, IOException, ConfigNotFoundException {
     final List<ConnectionRead> connectionReads = Lists.newArrayList();
 
-    // listing all of this is also bad
-    for (final StandardSync standardSync : configRepository.listWorkspaceStandardSyncs(workspaceIdRequestBody.getWorkspaceId())) {
-      if (standardSync.getStatus() == StandardSync.Status.DEPRECATED && !includeDeleted) {
-        continue;
-      }
-
+    for (final StandardSync standardSync : configRepository.listWorkspaceStandardSyncs(workspaceIdRequestBody.getWorkspaceId(), includeDeleted)) {
       connectionReads.add(ApiPojoConverters.internalToConnectionRead(standardSync));
     }
 

diff --git a/airbyte-server/src/test/java/io/airbyte/server/handlers/ConnectionsHandlerTest.java b/airbyte-server/src/test/java/io/airbyte/server/handlers/ConnectionsHandlerTest.java
@@ -642,7 +642,9 @@ void testGetConnection() throws JsonValidationException, ConfigNotFoundException
 
     @Test
     void testListConnectionsForWorkspace() throws JsonValidationException, ConfigNotFoundException, IOException {
-      when(configRepository.listWorkspaceStandardSyncs(source.getWorkspaceId()))
+      when(configRepository.listWorkspaceStandardSyncs(source.getWorkspaceId(), false))
+          .thenReturn(Lists.newArrayList(standardSync));
+      when(configRepository.listWorkspaceStandardSyncs(source.getWorkspaceId(), true))
           .thenReturn(Lists.newArrayList(standardSync, standardSyncDeleted));
       when(configRepository.getStandardSync(standardSync.getConnectionId()))
           .thenReturn(standardSync);