-
Notifications
You must be signed in to change notification settings - Fork 4.1k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
make the seed for YamlSeedConfigPersistence configurable #6409
Changes from 1 commit
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change | ||
---|---|---|---|---|
|
@@ -56,6 +56,7 @@ public abstract class BaseDatabaseConfigPersistenceTest { | |||
|
||||
@BeforeAll | ||||
public static void dbSetup() { | ||||
YamlSeedConfigPersistence.initialize(YamlSeedConfigPersistence.DEFAULT_SEED_DEFINITION_RESOURCE_CLASS); | ||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Can be we have some smiple helper method to construct this class, like There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This
Suggested change
|
||||
container = new PostgreSQLContainer<>("postgres:13-alpine") | ||||
.withDatabaseName("airbyte") | ||||
.withUsername("docker") | ||||
|
@@ -75,7 +76,8 @@ public static void dbDown() { | |||
|
||||
static { | ||||
try { | ||||
ConfigPersistence seedPersistence = YamlSeedConfigPersistence.get(); | ||||
YamlSeedConfigPersistence.initialize(YamlSeedConfigPersistence.DEFAULT_SEED_DEFINITION_RESOURCE_CLASS); | ||||
final ConfigPersistence seedPersistence = YamlSeedConfigPersistence.get(); | ||||
SOURCE_GITHUB = seedPersistence | ||||
.getConfig(ConfigSchema.STANDARD_SOURCE_DEFINITION, "ef69ef6e-aa7f-4af1-a01d-ef775033524e", StandardSourceDefinition.class); | ||||
SOURCE_POSTGRES = seedPersistence | ||||
|
@@ -84,47 +86,49 @@ public static void dbDown() { | |||
.getConfig(ConfigSchema.STANDARD_DESTINATION_DEFINITION, "424892c4-daac-4491-b35d-c6688ba547ba", StandardDestinationDefinition.class); | ||||
DESTINATION_S3 = seedPersistence | ||||
.getConfig(ConfigSchema.STANDARD_DESTINATION_DEFINITION, "4816b78f-1489-44c1-9060-4b19d5fa9362", StandardDestinationDefinition.class); | ||||
} catch (Exception e) { | ||||
} catch (final Exception e) { | ||||
throw new RuntimeException(e); | ||||
} | ||||
} | ||||
|
||||
protected static void writeSource(ConfigPersistence configPersistence, StandardSourceDefinition source) throws Exception { | ||||
protected static void writeSource(final ConfigPersistence configPersistence, final StandardSourceDefinition source) throws Exception { | ||||
configPersistence.writeConfig(ConfigSchema.STANDARD_SOURCE_DEFINITION, source.getSourceDefinitionId().toString(), source); | ||||
} | ||||
|
||||
protected static void writeDestination(ConfigPersistence configPersistence, StandardDestinationDefinition destination) throws Exception { | ||||
protected static void writeDestination(final ConfigPersistence configPersistence, final StandardDestinationDefinition destination) | ||||
throws Exception { | ||||
configPersistence.writeConfig(ConfigSchema.STANDARD_DESTINATION_DEFINITION, destination.getDestinationDefinitionId().toString(), destination); | ||||
} | ||||
|
||||
protected static void deleteDestination(ConfigPersistence configPersistence, StandardDestinationDefinition destination) throws Exception { | ||||
protected static void deleteDestination(final ConfigPersistence configPersistence, final StandardDestinationDefinition destination) | ||||
throws Exception { | ||||
configPersistence.deleteConfig(ConfigSchema.STANDARD_DESTINATION_DEFINITION, destination.getDestinationDefinitionId().toString()); | ||||
} | ||||
|
||||
protected Map<String, Set<JsonNode>> getMapWithSet(Map<String, Stream<JsonNode>> input) { | ||||
protected Map<String, Set<JsonNode>> getMapWithSet(final Map<String, Stream<JsonNode>> input) { | ||||
return input.entrySet().stream().collect(Collectors.toMap( | ||||
Entry::getKey, | ||||
e -> e.getValue().collect(Collectors.toSet()))); | ||||
} | ||||
|
||||
// assertEquals cannot correctly check the equality of two maps with stream values, | ||||
// so streams are converted to sets before being compared. | ||||
protected void assertSameConfigDump(Map<String, Stream<JsonNode>> expected, Map<String, Stream<JsonNode>> actual) { | ||||
protected void assertSameConfigDump(final Map<String, Stream<JsonNode>> expected, final Map<String, Stream<JsonNode>> actual) { | ||||
assertEquals(getMapWithSet(expected), getMapWithSet(actual)); | ||||
} | ||||
|
||||
protected void assertRecordCount(int expectedCount) throws Exception { | ||||
Result<Record1<Integer>> recordCount = database.query(ctx -> ctx.select(count(asterisk())).from(table("airbyte_configs")).fetch()); | ||||
protected void assertRecordCount(final int expectedCount) throws Exception { | ||||
final Result<Record1<Integer>> recordCount = database.query(ctx -> ctx.select(count(asterisk())).from(table("airbyte_configs")).fetch()); | ||||
assertEquals(expectedCount, recordCount.get(0).value1()); | ||||
} | ||||
|
||||
protected void assertHasSource(StandardSourceDefinition source) throws Exception { | ||||
protected void assertHasSource(final StandardSourceDefinition source) throws Exception { | ||||
assertEquals(source, configPersistence | ||||
.getConfig(ConfigSchema.STANDARD_SOURCE_DEFINITION, source.getSourceDefinitionId().toString(), | ||||
StandardSourceDefinition.class)); | ||||
} | ||||
|
||||
protected void assertHasDestination(StandardDestinationDefinition destination) throws Exception { | ||||
protected void assertHasDestination(final StandardDestinationDefinition destination) throws Exception { | ||||
assertEquals(destination, configPersistence | ||||
.getConfig(ConfigSchema.STANDARD_DESTINATION_DEFINITION, destination.getDestinationDefinitionId().toString(), | ||||
StandardDestinationDefinition.class)); | ||||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -61,6 +61,7 @@ public class DatabaseConfigPersistenceLoadDataTest extends BaseDatabaseConfigPer | |
|
||
@BeforeAll | ||
public static void setup() throws Exception { | ||
YamlSeedConfigPersistence.initialize(YamlSeedConfigPersistence.DEFAULT_SEED_DEFINITION_RESOURCE_CLASS); | ||
database = new ConfigsDatabaseInstance(container.getUsername(), container.getPassword(), container.getJdbcUrl()).getAndInitialize(); | ||
configPersistence = spy(new DatabaseConfigPersistence(database)); | ||
database.query(ctx -> ctx.execute("TRUNCATE TABLE airbyte_configs")); | ||
|
@@ -100,23 +101,23 @@ public void testUpdateConfigsInNonEmptyDatabase() throws Exception { | |
@DisplayName("When a connector is in use, its definition should not be updated") | ||
public void testNoUpdateForUsedConnector() throws Exception { | ||
// the seed has a newer version of s3 destination and github source | ||
StandardDestinationDefinition destinationS3V2 = YamlSeedConfigPersistence.get() | ||
final StandardDestinationDefinition destinationS3V2 = YamlSeedConfigPersistence.get() | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This test shows a good example why the static class variable should not be modified. It is very confusing that we need to call |
||
.getConfig(ConfigSchema.STANDARD_DESTINATION_DEFINITION, "4816b78f-1489-44c1-9060-4b19d5fa9362", StandardDestinationDefinition.class) | ||
.withDockerImageTag("10000.1.0"); | ||
when(seedPersistence.listConfigs(ConfigSchema.STANDARD_DESTINATION_DEFINITION, StandardDestinationDefinition.class)) | ||
.thenReturn(Collections.singletonList(destinationS3V2)); | ||
StandardSourceDefinition sourceGithubV2 = YamlSeedConfigPersistence.get() | ||
final StandardSourceDefinition sourceGithubV2 = YamlSeedConfigPersistence.get() | ||
.getConfig(ConfigSchema.STANDARD_SOURCE_DEFINITION, "ef69ef6e-aa7f-4af1-a01d-ef775033524e", StandardSourceDefinition.class) | ||
.withDockerImageTag("10000.15.3"); | ||
when(seedPersistence.listConfigs(ConfigSchema.STANDARD_SOURCE_DEFINITION, StandardSourceDefinition.class)) | ||
.thenReturn(Collections.singletonList(sourceGithubV2)); | ||
|
||
// create connections to mark the source and destination as in use | ||
DestinationConnection s3Connection = new DestinationConnection() | ||
final DestinationConnection s3Connection = new DestinationConnection() | ||
.withDestinationId(UUID.randomUUID()) | ||
.withDestinationDefinitionId(destinationS3V2.getDestinationDefinitionId()); | ||
configPersistence.writeConfig(ConfigSchema.DESTINATION_CONNECTION, s3Connection.getDestinationId().toString(), s3Connection); | ||
SourceConnection githubConnection = new SourceConnection() | ||
final SourceConnection githubConnection = new SourceConnection() | ||
.withSourceId(UUID.randomUUID()) | ||
.withSourceDefinitionId(sourceGithubV2.getSourceDefinitionId()); | ||
configPersistence.writeConfig(ConfigSchema.SOURCE_CONNECTION, githubConnection.getSourceId().toString(), githubConnection); | ||
|
@@ -132,7 +133,7 @@ public void testNoUpdateForUsedConnector() throws Exception { | |
@DisplayName("When a connector is not in use, its definition should be updated") | ||
public void testUpdateForUnusedConnector() throws Exception { | ||
// the seed has a newer version of snowflake destination | ||
StandardDestinationDefinition snowflakeV2 = YamlSeedConfigPersistence.get() | ||
final StandardDestinationDefinition snowflakeV2 = YamlSeedConfigPersistence.get() | ||
.getConfig(ConfigSchema.STANDARD_DESTINATION_DEFINITION, "424892c4-daac-4491-b35d-c6688ba547ba", StandardDestinationDefinition.class) | ||
.withDockerImageTag("10000.2.0"); | ||
when(seedPersistence.listConfigs(ConfigSchema.STANDARD_DESTINATION_DEFINITION, StandardDestinationDefinition.class)) | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I understand why you need to do that but this complex object lifecycle has some code smell.
Why is the YamlSeedConfigPersistence a singleton? Are we ever using it without injecting?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Now that this class can be constructed differently, I think it should not be a singleton any more.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
cool. thanks. this was my intuition too.