Skip to content

Commit

Permalink
remove foreign key from actor_oauth_parameter table (#9112)
Browse files Browse the repository at this point in the history 
* remove foreign key from actor_oauth_parameter table

* add test

* fix test
  • Loading branch information
@subodh1810
subodh1810 authored Dec 27, 2021
1 parent e17fa5b commit 0a1dc54
Show file tree
Hide file tree
Showing 9 changed files with 305 additions and 50 deletions.
2 changes: 1 addition & 1 deletion airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderAppTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ void testBootloaderAppBlankDb() throws Exception {
mockedConfigs.getConfigDatabaseUrl())
.getAndInitialize();
val configsMigrator = new ConfigsDatabaseMigrator(configDatabase, this.getClass().getName());
assertEquals("0.32.8.001", configsMigrator.getLatestMigration().getVersion().getVersion());
assertEquals("0.35.1.001", configsMigrator.getLatestMigration().getVersion().getVersion());

val jobsPersistence = new DefaultJobPersistence(jobDatabase);
assertEquals(version, jobsPersistence.getVersion().get());
Expand Down
81 changes: 46 additions & 35 deletions ...rc/test/java/io/airbyte/config/persistence/DatabaseConfigPersistenceE2EReadWriteTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,9 @@
package io.airbyte.config.persistence;

import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.*;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.spy;

import io.airbyte.config.ConfigSchema;
Expand All @@ -26,7 +29,6 @@
import java.io.IOException;
import java.util.List;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;

Expand Down Expand Up @@ -67,28 +69,37 @@ public void test() throws JsonValidationException, IOException, ConfigNotFoundEx
private void deletion() throws ConfigNotFoundException, IOException, JsonValidationException {
// Deleting the workspace should delete everything except for definitions
configPersistence.deleteConfig(ConfigSchema.STANDARD_WORKSPACE, MockData.standardWorkspace().getWorkspaceId().toString());
Assertions.assertTrue(configPersistence.listConfigs(ConfigSchema.STANDARD_SYNC_STATE, StandardSyncState.class).isEmpty());
Assertions.assertTrue(configPersistence.listConfigs(ConfigSchema.STANDARD_SYNC, StandardSync.class).isEmpty());
Assertions.assertTrue(configPersistence.listConfigs(ConfigSchema.STANDARD_SYNC_OPERATION, StandardSyncOperation.class).isEmpty());
Assertions.assertTrue(configPersistence.listConfigs(ConfigSchema.DESTINATION_OAUTH_PARAM, DestinationOAuthParameter.class).isEmpty());
Assertions.assertTrue(configPersistence.listConfigs(ConfigSchema.SOURCE_OAUTH_PARAM, SourceOAuthParameter.class).isEmpty());
Assertions.assertTrue(configPersistence.listConfigs(ConfigSchema.DESTINATION_CONNECTION, SourceConnection.class).isEmpty());
Assertions.assertTrue(configPersistence.listConfigs(ConfigSchema.STANDARD_WORKSPACE, StandardWorkspace.class).isEmpty());

Assertions.assertFalse(configPersistence.listConfigs(ConfigSchema.STANDARD_SOURCE_DEFINITION, StandardSourceDefinition.class).isEmpty());
Assertions
.assertFalse(configPersistence.listConfigs(ConfigSchema.STANDARD_DESTINATION_DEFINITION, StandardDestinationDefinition.class).isEmpty());
assertTrue(configPersistence.listConfigs(ConfigSchema.STANDARD_SYNC_STATE, StandardSyncState.class).isEmpty());
assertTrue(configPersistence.listConfigs(ConfigSchema.STANDARD_SYNC, StandardSync.class).isEmpty());
assertTrue(configPersistence.listConfigs(ConfigSchema.STANDARD_SYNC_OPERATION, StandardSyncOperation.class).isEmpty());
assertTrue(configPersistence.listConfigs(ConfigSchema.DESTINATION_CONNECTION, SourceConnection.class).isEmpty());
assertTrue(configPersistence.listConfigs(ConfigSchema.STANDARD_WORKSPACE, StandardWorkspace.class).isEmpty());

assertFalse(configPersistence.listConfigs(ConfigSchema.SOURCE_OAUTH_PARAM, SourceOAuthParameter.class).isEmpty());
assertFalse(configPersistence.listConfigs(ConfigSchema.DESTINATION_OAUTH_PARAM, DestinationOAuthParameter.class).isEmpty());
assertFalse(configPersistence.listConfigs(ConfigSchema.STANDARD_SOURCE_DEFINITION, StandardSourceDefinition.class).isEmpty());
assertFalse(configPersistence.listConfigs(ConfigSchema.STANDARD_DESTINATION_DEFINITION, StandardDestinationDefinition.class).isEmpty());

for (final SourceOAuthParameter sourceOAuthParameter : MockData.sourceOauthParameters()) {
configPersistence.deleteConfig(ConfigSchema.SOURCE_OAUTH_PARAM, sourceOAuthParameter.getOauthParameterId().toString());
}
assertTrue(configPersistence.listConfigs(ConfigSchema.SOURCE_OAUTH_PARAM, SourceOAuthParameter.class).isEmpty());

for (final DestinationOAuthParameter destinationOAuthParameter : MockData.destinationOauthParameters()) {
configPersistence.deleteConfig(ConfigSchema.DESTINATION_OAUTH_PARAM, destinationOAuthParameter.getOauthParameterId().toString());
}
assertTrue(configPersistence.listConfigs(ConfigSchema.DESTINATION_OAUTH_PARAM, DestinationOAuthParameter.class).isEmpty());

for (final StandardSourceDefinition standardSourceDefinition : MockData.standardSourceDefinitions()) {
configPersistence.deleteConfig(ConfigSchema.STANDARD_SOURCE_DEFINITION, standardSourceDefinition.getSourceDefinitionId().toString());
}
Assertions.assertTrue(configPersistence.listConfigs(ConfigSchema.STANDARD_SOURCE_DEFINITION, StandardSourceDefinition.class).isEmpty());
assertTrue(configPersistence.listConfigs(ConfigSchema.STANDARD_SOURCE_DEFINITION, StandardSourceDefinition.class).isEmpty());

for (final StandardDestinationDefinition standardDestinationDefinition : MockData.standardDestinationDefinitions()) {
configPersistence
.deleteConfig(ConfigSchema.STANDARD_DESTINATION_DEFINITION, standardDestinationDefinition.getDestinationDefinitionId().toString());
}
Assertions.assertTrue(configPersistence.listConfigs(ConfigSchema.STANDARD_DESTINATION_DEFINITION, StandardDestinationDefinition.class).isEmpty());
assertTrue(configPersistence.listConfigs(ConfigSchema.STANDARD_DESTINATION_DEFINITION, StandardDestinationDefinition.class).isEmpty());
}

private void standardSyncState() throws JsonValidationException, IOException, ConfigNotFoundException {
Expand All @@ -99,11 +110,11 @@ private void standardSyncState() throws JsonValidationException, IOException, Co
final StandardSyncState standardSyncStateFromDB = configPersistence.getConfig(ConfigSchema.STANDARD_SYNC_STATE,
standardSyncState.getConnectionId().toString(),
StandardSyncState.class);
Assertions.assertEquals(standardSyncState, standardSyncStateFromDB);
assertEquals(standardSyncState, standardSyncStateFromDB);
}
final List<StandardSyncState> standardSyncStates = configPersistence
.listConfigs(ConfigSchema.STANDARD_SYNC_STATE, StandardSyncState.class);
Assertions.assertEquals(MockData.standardSyncStates().size(), standardSyncStates.size());
assertEquals(MockData.standardSyncStates().size(), standardSyncStates.size());
assertThat(MockData.standardSyncStates()).hasSameElementsAs(standardSyncStates);
}

Expand All @@ -115,11 +126,11 @@ private void standardSync() throws JsonValidationException, IOException, ConfigN
final StandardSync standardSyncFromDB = configPersistence.getConfig(ConfigSchema.STANDARD_SYNC,
standardSync.getConnectionId().toString(),
StandardSync.class);
Assertions.assertEquals(standardSync, standardSyncFromDB);
assertEquals(standardSync, standardSyncFromDB);
}
final List<StandardSync> standardSyncs = configPersistence
.listConfigs(ConfigSchema.STANDARD_SYNC, StandardSync.class);
Assertions.assertEquals(MockData.standardSyncs().size(), standardSyncs.size());
assertEquals(MockData.standardSyncs().size(), standardSyncs.size());
assertThat(MockData.standardSyncs()).hasSameElementsAs(standardSyncs);
}

Expand All @@ -131,11 +142,11 @@ private void standardSyncOperation() throws JsonValidationException, IOException
final StandardSyncOperation standardSyncOperationFromDB = configPersistence.getConfig(ConfigSchema.STANDARD_SYNC_OPERATION,
standardSyncOperation.getOperationId().toString(),
StandardSyncOperation.class);
Assertions.assertEquals(standardSyncOperation, standardSyncOperationFromDB);
assertEquals(standardSyncOperation, standardSyncOperationFromDB);
}
final List<StandardSyncOperation> standardSyncOperations = configPersistence
.listConfigs(ConfigSchema.STANDARD_SYNC_OPERATION, StandardSyncOperation.class);
Assertions.assertEquals(MockData.standardSyncOperations().size(), standardSyncOperations.size());
assertEquals(MockData.standardSyncOperations().size(), standardSyncOperations.size());
assertThat(MockData.standardSyncOperations()).hasSameElementsAs(standardSyncOperations);
}

Expand All @@ -147,11 +158,11 @@ private void destinationOauthParam() throws JsonValidationException, IOException
final DestinationOAuthParameter destinationOAuthParameterFromDB = configPersistence.getConfig(ConfigSchema.DESTINATION_OAUTH_PARAM,
destinationOAuthParameter.getOauthParameterId().toString(),
DestinationOAuthParameter.class);
Assertions.assertEquals(destinationOAuthParameter, destinationOAuthParameterFromDB);
assertEquals(destinationOAuthParameter, destinationOAuthParameterFromDB);
}
final List<DestinationOAuthParameter> destinationOAuthParameters = configPersistence
.listConfigs(ConfigSchema.DESTINATION_OAUTH_PARAM, DestinationOAuthParameter.class);
Assertions.assertEquals(MockData.destinationOauthParameters().size(), destinationOAuthParameters.size());
assertEquals(MockData.destinationOauthParameters().size(), destinationOAuthParameters.size());
assertThat(MockData.destinationOauthParameters()).hasSameElementsAs(destinationOAuthParameters);
}

Expand All @@ -163,11 +174,11 @@ private void sourceOauthParam() throws JsonValidationException, IOException, Con
final SourceOAuthParameter sourceOAuthParameterFromDB = configPersistence.getConfig(ConfigSchema.SOURCE_OAUTH_PARAM,
sourceOAuthParameter.getOauthParameterId().toString(),
SourceOAuthParameter.class);
Assertions.assertEquals(sourceOAuthParameter, sourceOAuthParameterFromDB);
assertEquals(sourceOAuthParameter, sourceOAuthParameterFromDB);
}
final List<SourceOAuthParameter> sourceOAuthParameters = configPersistence
.listConfigs(ConfigSchema.SOURCE_OAUTH_PARAM, SourceOAuthParameter.class);
Assertions.assertEquals(MockData.sourceOauthParameters().size(), sourceOAuthParameters.size());
assertEquals(MockData.sourceOauthParameters().size(), sourceOAuthParameters.size());
assertThat(MockData.sourceOauthParameters()).hasSameElementsAs(sourceOAuthParameters);
}

Expand All @@ -179,11 +190,11 @@ private void destinationConnection() throws JsonValidationException, IOException
final DestinationConnection destinationConnectionFromDB = configPersistence.getConfig(ConfigSchema.DESTINATION_CONNECTION,
destinationConnection.getDestinationId().toString(),
DestinationConnection.class);
Assertions.assertEquals(destinationConnection, destinationConnectionFromDB);
assertEquals(destinationConnection, destinationConnectionFromDB);
}
final List<DestinationConnection> destinationConnections = configPersistence
.listConfigs(ConfigSchema.DESTINATION_CONNECTION, DestinationConnection.class);
Assertions.assertEquals(MockData.destinationConnections().size(), destinationConnections.size());
assertEquals(MockData.destinationConnections().size(), destinationConnections.size());
assertThat(MockData.destinationConnections()).hasSameElementsAs(destinationConnections);
}

Expand All @@ -195,11 +206,11 @@ private void sourceConnection() throws JsonValidationException, IOException, Con
final SourceConnection sourceConnectionFromDB = configPersistence.getConfig(ConfigSchema.SOURCE_CONNECTION,
sourceConnection.getSourceId().toString(),
SourceConnection.class);
Assertions.assertEquals(sourceConnection, sourceConnectionFromDB);
assertEquals(sourceConnection, sourceConnectionFromDB);
}
final List<SourceConnection> sourceConnections = configPersistence
.listConfigs(ConfigSchema.SOURCE_CONNECTION, SourceConnection.class);
Assertions.assertEquals(MockData.sourceConnections().size(), sourceConnections.size());
assertEquals(MockData.sourceConnections().size(), sourceConnections.size());
assertThat(MockData.sourceConnections()).hasSameElementsAs(sourceConnections);
}

Expand All @@ -212,11 +223,11 @@ private void standardDestinationDefinition() throws JsonValidationException, IOE
.getConfig(ConfigSchema.STANDARD_DESTINATION_DEFINITION,
standardDestinationDefinition.getDestinationDefinitionId().toString(),
StandardDestinationDefinition.class);
Assertions.assertEquals(standardDestinationDefinition, standardDestinationDefinitionFromDB);
assertEquals(standardDestinationDefinition, standardDestinationDefinitionFromDB);
}
final List<StandardDestinationDefinition> standardDestinationDefinitions = configPersistence
.listConfigs(ConfigSchema.STANDARD_DESTINATION_DEFINITION, StandardDestinationDefinition.class);
Assertions.assertEquals(MockData.standardDestinationDefinitions().size(), standardDestinationDefinitions.size());
assertEquals(MockData.standardDestinationDefinitions().size(), standardDestinationDefinitions.size());
assertThat(MockData.standardDestinationDefinitions()).hasSameElementsAs(standardDestinationDefinitions);
}

Expand All @@ -228,11 +239,11 @@ private void standardSourceDefinition() throws JsonValidationException, IOExcept
final StandardSourceDefinition standardSourceDefinitionFromDB = configPersistence.getConfig(ConfigSchema.STANDARD_SOURCE_DEFINITION,
standardSourceDefinition.getSourceDefinitionId().toString(),
StandardSourceDefinition.class);
Assertions.assertEquals(standardSourceDefinition, standardSourceDefinitionFromDB);
assertEquals(standardSourceDefinition, standardSourceDefinitionFromDB);
}
final List<StandardSourceDefinition> standardSourceDefinitions = configPersistence
.listConfigs(ConfigSchema.STANDARD_SOURCE_DEFINITION, StandardSourceDefinition.class);
Assertions.assertEquals(MockData.standardSourceDefinitions().size(), standardSourceDefinitions.size());
assertEquals(MockData.standardSourceDefinitions().size(), standardSourceDefinitions.size());
assertThat(MockData.standardSourceDefinitions()).hasSameElementsAs(standardSourceDefinitions);
}

Expand All @@ -244,9 +255,9 @@ private void standardWorkspace() throws JsonValidationException, IOException, Co
MockData.standardWorkspace().getWorkspaceId().toString(),
StandardWorkspace.class);
final List<StandardWorkspace> standardWorkspaces = configPersistence.listConfigs(ConfigSchema.STANDARD_WORKSPACE, StandardWorkspace.class);
Assertions.assertEquals(MockData.standardWorkspace(), standardWorkspace);
Assertions.assertEquals(1, standardWorkspaces.size());
Assertions.assertTrue(standardWorkspaces.contains(MockData.standardWorkspace()));
assertEquals(MockData.standardWorkspace(), standardWorkspace);
assertEquals(1, standardWorkspaces.size());
assertTrue(standardWorkspaces.contains(MockData.standardWorkspace()));
}

}
6 changes: 3 additions & 3 deletions ...yte/db/instance/configs/migrations/V0_32_8_001__AirbyteConfigDatabaseDenormalization.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -727,9 +727,9 @@ private static void populateConnectionOperation(final DSLContext ctx,
LOGGER.info("connection_operation table populated with " + connectionOperationRecords + " records");
}

private static <T> List<ConfigWithMetadata<T>> listConfigsWithMetadata(final AirbyteConfig airbyteConfigType,
final Class<T> clazz,
final DSLContext ctx) {
static <T> List<ConfigWithMetadata<T>> listConfigsWithMetadata(final AirbyteConfig airbyteConfigType,
final Class<T> clazz,
final DSLContext ctx) {
final Field<String> configId = DSL.field("config_id", SQLDataType.VARCHAR(36).nullable(false));
final Field<String> configType = DSL.field("config_type", SQLDataType.VARCHAR(60).nullable(false));
final Field<OffsetDateTime> createdAt =
Expand Down
Loading

0 comments on commit 0a1dc54

Please sign in to comment.