test to show how automatic migration handles deprecated definitions (#…

…4655)

* test to show definitions not present in latest seed would be deleted in automatic migration

* format

* add deprecated config being used scenario

airbyte-server/src/test/java/io/airbyte/server/migration/RunMigrationTest.java

@@ -70,6 +70,8 @@ public class RunMigrationTest {
 
   private static final String INITIAL_VERSION = "0.17.0-alpha";
   private static final String TARGET_VERSION = Migrations.MIGRATIONS.get(Migrations.MIGRATIONS.size() - 1).getVersion();
+  private static final String DEPRECATED_SOURCE_DEFINITION_NOT_BEING_USED = "d2147be5-fa36-4936-977e-f031affa5895";
+  private static final String DEPRECATED_SOURCE_DEFINITION_BEING_USED = "4eb22946-2a79-4d20-a3e6-effd234613c3";
   private List<File> resourceToBeCleanedUp;
 
   @BeforeEach
@@ -103,7 +105,7 @@ public void testRunMigration() {
       FileUtils.copyDirectory(dummyDataSource.toFile(), configRoot.toFile());
       resourceToBeCleanedUp.add(configRoot.toFile());
       final JobPersistence jobPersistence = getJobPersistence(stubAirbyteDB.getDatabase(), file, INITIAL_VERSION);
-      assertDatabaseVersion(jobPersistence, INITIAL_VERSION);
+      assertPreMigrationConfigs(configRoot, jobPersistence);
 
       runMigration(jobPersistence, configRoot);
 
@@ -115,6 +117,15 @@ public void testRunMigration() {
     }
   }
 
+  private void assertPreMigrationConfigs(Path configRoot, JobPersistence jobPersistence) throws IOException, JsonValidationException {
+    assertDatabaseVersion(jobPersistence, INITIAL_VERSION);
+    ConfigRepository configRepository = new ConfigRepository(FileSystemConfigPersistence.createWithValidation(configRoot));
+    Map<String, StandardSourceDefinition> sourceDefinitionsBeforeMigration = configRepository.listStandardSources().stream()
+        .collect(Collectors.toMap(c -> c.getSourceDefinitionId().toString(), c -> c));
+    assertTrue(sourceDefinitionsBeforeMigration.containsKey(DEPRECATED_SOURCE_DEFINITION_NOT_BEING_USED));
+    assertTrue(sourceDefinitionsBeforeMigration.containsKey(DEPRECATED_SOURCE_DEFINITION_BEING_USED));
+  }
+
   private void assertDatabaseVersion(JobPersistence jobPersistence, String version) throws IOException {
     final Optional<String> versionFromDb = jobPersistence.getVersion();
     assertTrue(versionFromDb.isPresent());
@@ -137,6 +148,12 @@ private void assertSourceDefinitions(ConfigRepository configRepository) throws J
         .stream()
         .collect(Collectors.toMap(c -> c.getSourceDefinitionId().toString(), c -> c));
     assertTrue(sourceDefinitions.size() >= 59);
+    // the definition is not present in latest seeds so it should be deleted
+    assertFalse(sourceDefinitions.containsKey(DEPRECATED_SOURCE_DEFINITION_NOT_BEING_USED));
+    // the definition is not present in latest seeds but it was being used as a connection so it should
+    // not be deleted
+    assertTrue(sourceDefinitions.containsKey(DEPRECATED_SOURCE_DEFINITION_BEING_USED));
+
     final StandardSourceDefinition mysqlDefinition = sourceDefinitions.get("435bb9a5-7887-4809-aa58-28c27df0d7ad");
     assertEquals("0.2.0", mysqlDefinition.getDockerImageTag());
     assertEquals("MySQL", mysqlDefinition.getName());
@@ -210,18 +227,21 @@ private StandardSyncOperation assertSyncOperations(ConfigRepository configReposi
   }
 
   private void assertSources(ConfigRepository configRepository) throws JsonValidationException, IOException {
-    final List<SourceConnection> sourceConnections = configRepository.listSourceConnection();
-    assertEquals(sourceConnections.size(), 1);
-    final SourceConnection sourceConnection = sourceConnections.get(0);
-    assertEquals(sourceConnection.getName(), "MySQL localhost");
-    assertEquals(sourceConnection.getSourceDefinitionId().toString(), "435bb9a5-7887-4809-aa58-28c27df0d7ad");
-    assertEquals(sourceConnection.getWorkspaceId().toString(), "5ae6b09b-fdec-41af-aaf7-7d94cfc33ef6");
-    assertEquals(sourceConnection.getSourceId().toString(), "28ffee2b-372a-4f72-9b95-8ed56a8b99c5");
-    assertEquals(sourceConnection.getConfiguration().get("username").asText(), "root");
-    assertEquals(sourceConnection.getConfiguration().get("password").asText(), "password");
-    assertEquals(sourceConnection.getConfiguration().get("database").asText(), "localhost_test");
-    assertEquals(sourceConnection.getConfiguration().get("port").asInt(), 3306);
-    assertEquals(sourceConnection.getConfiguration().get("host").asText(), "host.docker.internal");
+    final Map<String, SourceConnection> sources = configRepository.listSourceConnection().stream()
+        .collect(Collectors.toMap(sourceConnection -> sourceConnection.getSourceId().toString(), sourceConnection -> sourceConnection));
+    assertEquals(sources.size(), 2);
+    final SourceConnection mysqlConnection = sources.get("28ffee2b-372a-4f72-9b95-8ed56a8b99c5");
+    assertEquals(mysqlConnection.getName(), "MySQL localhost");
+    assertEquals(mysqlConnection.getSourceDefinitionId().toString(), "435bb9a5-7887-4809-aa58-28c27df0d7ad");
+    assertEquals(mysqlConnection.getWorkspaceId().toString(), "5ae6b09b-fdec-41af-aaf7-7d94cfc33ef6");
+    assertEquals(mysqlConnection.getSourceId().toString(), "28ffee2b-372a-4f72-9b95-8ed56a8b99c5");
+    assertEquals(mysqlConnection.getConfiguration().get("username").asText(), "root");
+    assertEquals(mysqlConnection.getConfiguration().get("password").asText(), "password");
+    assertEquals(mysqlConnection.getConfiguration().get("database").asText(), "localhost_test");
+    assertEquals(mysqlConnection.getConfiguration().get("port").asInt(), 3306);
+    assertEquals(mysqlConnection.getConfiguration().get("host").asText(), "host.docker.internal");
+    assertTrue(sources.containsKey("e48cae1a-1f5c-42cc-9ec1-a44ff7fb4969"));
+
   }
 
   private void assertWorkspace(ConfigRepository configRepository) throws JsonValidationException, IOException {

airbyte-server/src/test/resources/migration/dummy_data/config/SOURCE_CONNECTION/e48cae1a-1f5c-42cc-9ec1-a44ff7fb4969.json

@@ -0,0 +1,14 @@
+{
+  "name": "Using a source definition deleted",
+  "sourceDefinitionId": "4eb22946-2a79-4d20-a3e6-effd234613c3",
+  "workspaceId": "5ae6b09b-fdec-41af-aaf7-7d94cfc33ef6",
+  "sourceId": "e48cae1a-1f5c-42cc-9ec1-a44ff7fb4969",
+  "configuration": {
+    "username": "root",
+    "password": "password",
+    "database": "localhost_test",
+    "port": 3306,
+    "host": "host.docker.internal"
+  },
+  "tombstone": false
+}

airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/4eb22946-2a79-4d20-a3e6-effd234613c3.json

@@ -0,0 +1,7 @@
+{
+  "sourceDefinitionId": "4eb22946-2a79-4d20-a3e6-effd234613c3",
+  "name": "Old connector still being used",
+  "dockerRepository": "airbyte/source-mysql",
+  "dockerImageTag": "0.2.0",
+  "documentationUrl": "https://docs.airbyte.io/integrations/sources/mysql"
+}

airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/d2147be5-fa36-4936-977e-f031affa5895.json

@@ -0,0 +1,7 @@
+{
+  "sourceDefinitionId": "d2147be5-fa36-4936-977e-f031affa5895",
+  "name": "Old Connector",
+  "dockerRepository": "airbyte/source-appstore-singer",
+  "dockerImageTag": "0.2.0",
+  "documentationUrl": "https://hub.docker.com/r/airbyte/source-appstore-singer"
+}